From 3c4334cec9f98d4d44c1c618cb894994226d0248 Mon Sep 17 00:00:00 2001 From: retoor Date: Fri, 23 Oct 2026 18:20:04 +0200 Subject: [PATCH] feat: implement MoloApp plans 01-15 with integration Full C++23 framework tree: runtime, observability, security, data, app, realtime, supervisor, clients, CLI, example plus enterprise tenancy, audit, operability, governance, supplychain. Postgres via docker compose for tests. --- .gitignore | 3 + AGENTS.md | 91 + Makefile | 138 ++ docker-compose.yml | 21 + example/README.md | 93 ++ example/SPEC.md | 85 + example/app/App.cpp | 53 + example/app/App.hpp | 69 + example/app/DataDemo.cpp | 141 ++ example/app/Embedded.cpp | 19 + example/app/FormatsDemo.cpp | 131 ++ example/app/Index.cpp | 75 + example/app/OpsDemo.cpp | 224 +++ example/app/RealtimeDemo.cpp | 93 ++ example/app/RoutingDemo.cpp | 80 + example/app/SecurityDemo.cpp | 93 ++ example/app/Seed.cpp | 14 + example/app/Templates.cpp | 136 ++ example/example.mk | 117 ++ example/fw/App.cpp | 123 ++ example/fw/App.hpp | 67 + example/fw/Caps.hpp | 48 + example/fw/Cookies.cpp | 49 + example/fw/Cookies.hpp | 21 + example/fw/Data.cpp | 272 +++ example/fw/Data.hpp | 113 ++ example/fw/Http.cpp | 266 +++ example/fw/Http.hpp | 114 ++ example/fw/Jobs.cpp | 113 ++ example/fw/Jobs.hpp | 75 + example/fw/Kv.cpp | 95 ++ example/fw/Kv.hpp | 61 + example/fw/Negotiate.cpp | 70 + example/fw/Negotiate.hpp | 22 + example/fw/Net.cpp | 82 + example/fw/Net.hpp | 57 + example/fw/Observ.cpp | 86 + example/fw/Observ.hpp | 72 + example/fw/Router.cpp | 129 ++ example/fw/Router.hpp | 52 + example/fw/Server.cpp | 205 +++ example/fw/Server.hpp | 44 + example/fw/StaticFiles.cpp | 74 + example/fw/StaticFiles.hpp | 24 + example/fw/Templates.cpp | 246 +++ example/fw/Templates.hpp | 39 + example/fw/TestClient.cpp | 65 + example/fw/TestClient.hpp | 38 + example/fw/Tls.cpp | 9 + example/fw/Tls.hpp | 25 + example/fw/Validate.cpp | 189 +++ example/fw/Validate.hpp | 72 + example/fw/View.cpp | 26 + example/fw/View.hpp | 29 + example/fw/Workflow.cpp | 143 ++ example/fw/Workflow.hpp | 57 + example/main.cpp | 35 + example/static/app.css | 25 + example/static/app.js | 6 + plan.md | 9 + plans/11-tenancy.md | 187 +++ plans/12-audit-compliance.md | 159 ++ plans/13-operability.md | 175 ++ plans/14-api-governance.md | 162 ++ plans/15-supply-chain.md | 150 ++ plans/README.md | 17 +- src/app/Makefile | 12 + src/app/SPEC.md | 96 ++ src/app/app.mk | 29 + src/app/app/application.h | 528 ++++++ src/app/app/plugin.h | 109 ++ src/app/async/task.h | 123 ++ src/app/caps.h | 52 + src/app/di/container.h | 208 +++ src/app/errors/handler.h | 209 +++ src/app/errors/http_error.h | 178 ++ src/app/forms/form.h | 99 ++ src/app/forms/multipart.h | 127 ++ src/app/forms/urlencoded.h | 65 + src/app/http_msg/cookies.h | 82 + src/app/http_msg/headers.h | 95 ++ src/app/http_msg/html.h | 42 + src/app/http_msg/method.h | 37 + src/app/http_msg/query.h | 122 ++ src/app/http_msg/request.h | 138 ++ src/app/http_msg/response.h | 155 ++ src/app/middleware/builtins.h | 92 ++ src/app/middleware/chain.h | 63 + src/app/middleware/middleware.h | 69 + src/app/negotiate/accept.h | 98 ++ src/app/negotiate/negotiator.h | 137 ++ src/app/routing/path_compiler.h | 227 +++ src/app/routing/route.h | 113 ++ src/app/routing/router.h | 319 ++++ src/app/seams/request_id.h | 23 + src/app/seams/session.h | 62 + src/app/templates/ast.h | 116 ++ src/app/templates/cache.h | 62 + src/app/templates/context.h | 66 + src/app/templates/environment.h | 129 ++ src/app/templates/filters.h | 163 ++ src/app/templates/lexer.h | 139 ++ src/app/templates/loaders.h | 115 ++ src/app/templates/parser.h | 823 ++++++++++ src/app/templates/predicates.h | 49 + src/app/templates/renderer.h | 639 +++++++ src/app/templates/response.h | 23 + src/app/validate/errors.h | 51 + src/app/validate/field.h | 271 +++ src/app/validate/json.h | 298 ++++ src/app/validate/model.h | 88 + src/app/validate/value.h | 115 ++ src/app/views/adapters.h | 34 + src/app/views/flash.h | 42 + src/app/views/view.h | 60 + src/audit.mk | 40 + src/audit/PATCHES.md | 141 ++ src/audit/adapt.h | 670 ++++++++ src/audit/audit.h | 24 + src/audit/caps.h | 40 + src/audit/log/appender.h | 124 ++ src/audit/log/chain.h | 112 ++ src/audit/log/query.h | 107 ++ src/audit/log/record.h | 81 + src/audit/log/sql_store.h | 216 +++ src/audit/log/store.h | 80 + src/audit/mask/fixture.h | 84 + src/audit/mask/masker.h | 53 + src/audit/pii/mark.h | 68 + src/audit/pii/registry.h | 55 + src/audit/redact/pattern.h | 209 +++ src/audit/redact/redactor.h | 67 + src/audit/retention/enforcer.h | 93 ++ src/audit/retention/hold.h | 41 + src/audit/retention/policy.h | 39 + src/audit/rights/eraser.h | 82 + src/audit/rights/exporter.h | 96 ++ src/audit/rights/request.h | 107 ++ src/audit/rights/subject.h | 20 + src/cli/ai/ai_client.cpp | 102 ++ src/cli/ai/ai_client.hpp | 45 + src/cli/ai/ai_session.cpp | 58 + src/cli/ai/ai_session.hpp | 45 + src/cli/ai/ai_transport.cpp | 23 + src/cli/ai/ai_transport.hpp | 51 + src/cli/ai/prompt_pack.cpp | 44 + src/cli/ai/prompt_pack.hpp | 14 + src/cli/ai/redactor.cpp | 79 + src/cli/ai/redactor.hpp | 22 + src/cli/ai/socket_transport.cpp | 127 ++ src/cli/ai/socket_transport.hpp | 26 + src/cli/ai/sse_stream_parser.cpp | 56 + src/cli/ai/sse_stream_parser.hpp | 25 + src/cli/caps.hpp | 56 + src/cli/cli.mk | 34 + src/cli/cli/arg_parser.cpp | 102 ++ src/cli/cli/arg_parser.hpp | 34 + src/cli/cli/cli_app.cpp | 763 +++++++++ src/cli/cli/cli_app.hpp | 36 + src/cli/cli/command.cpp | 57 + src/cli/cli/command.hpp | 53 + src/cli/cli/command_tree.cpp | 45 + src/cli/cli/command_tree.hpp | 29 + src/cli/cli/exit_code.hpp | 31 + src/cli/cli/flag.cpp | 15 + src/cli/cli/flag.hpp | 28 + src/cli/cli/flag_set.cpp | 44 + src/cli/cli/flag_set.hpp | 28 + src/cli/cli/help_renderer.cpp | 59 + src/cli/cli/help_renderer.hpp | 19 + src/cli/cli/json_writer.cpp | 69 + src/cli/cli/json_writer.hpp | 23 + src/cli/cli/migrate_api.cpp | 17 + src/cli/cli/migrate_api.hpp | 26 + src/cli/cli/output.cpp | 40 + src/cli/cli/output.hpp | 35 + src/cli/cli/scaffolder.cpp | 103 ++ src/cli/cli/scaffolder.hpp | 28 + src/cli/cli/supervisor_api.cpp | 57 + src/cli/cli/supervisor_api.hpp | 59 + src/cli/cli/supervisor_cli.cpp | 180 ++ src/cli/cli/supervisor_cli.hpp | 28 + src/cli/cli/table_writer.cpp | 34 + src/cli/cli/table_writer.hpp | 20 + src/cli/csp/csp_builder.cpp | 61 + src/cli/csp/csp_builder.hpp | 31 + src/cli/csp/csp_policy.cpp | 24 + src/cli/csp/csp_policy.hpp | 23 + src/cli/csp/csp_report_handler.cpp | 30 + src/cli/csp/csp_report_handler.hpp | 29 + src/cli/csp/nonce.cpp | 41 + src/cli/csp/nonce.hpp | 15 + src/cli/dev/dev_config.cpp | 85 + src/cli/dev/dev_config.hpp | 49 + src/cli/dev/dev_server.cpp | 131 ++ src/cli/dev/dev_server.hpp | 38 + src/cli/dev/file_watcher.cpp | 16 + src/cli/dev/file_watcher.hpp | 28 + src/cli/dev/inotify_watcher.cpp | 107 ++ src/cli/dev/inotify_watcher.hpp | 28 + src/cli/dev/polling_watcher.cpp | 107 ++ src/cli/dev/polling_watcher.hpp | 28 + src/cli/dev/reloader.cpp | 136 ++ src/cli/dev/reloader.hpp | 30 + src/cli/embed/asset_scanner.cpp | 73 + src/cli/embed/asset_scanner.hpp | 29 + src/cli/embed/embed_emitter.cpp | 179 ++ src/cli/embed/embed_emitter.hpp | 28 + src/cli/embed/embedded_bundle.cpp | 19 + src/cli/embed/embedded_bundle.hpp | 32 + src/cli/embed/gzip_stored.cpp | 44 + src/cli/embed/gzip_stored.hpp | 18 + src/cli/embed/mime.cpp | 34 + src/cli/embed/mime.hpp | 14 + src/cli/embed/sha256.cpp | 100 ++ src/cli/embed/sha256.hpp | 16 + src/cli/i18n/catalog.cpp | 18 + src/cli/i18n/catalog.hpp | 32 + src/cli/i18n/locale.cpp | 51 + src/cli/i18n/locale.hpp | 27 + src/cli/i18n/plural_rule.cpp | 41 + src/cli/i18n/plural_rule.hpp | 14 + src/cli/i18n/po_loader.cpp | 138 ++ src/cli/i18n/po_loader.hpp | 23 + src/cli/i18n/translator.cpp | 91 + src/cli/i18n/translator.hpp | 45 + src/cli/main.cpp | 7 + src/cli/ops/health_check.cpp | 4 + src/cli/ops/health_check.hpp | 36 + src/cli/ops/health_registry.cpp | 35 + src/cli/ops/health_registry.hpp | 28 + src/cli/ops/liveness_check.cpp | 17 + src/cli/ops/liveness_check.hpp | 20 + src/cli/ops/ops_routes.cpp | 147 ++ src/cli/ops/ops_routes.hpp | 43 + src/cli/ops/otlp_bridge.cpp | 32 + src/cli/ops/otlp_bridge.hpp | 26 + src/cli/ops/prometheus_exporter.cpp | 103 ++ src/cli/ops/prometheus_exporter.hpp | 55 + src/cli/ops/readiness_check.cpp | 24 + src/cli/ops/readiness_check.hpp | 30 + src/cli/ops/startup_gate.cpp | 4 + src/cli/ops/startup_gate.hpp | 18 + src/cli/spec.md | 89 + src/cli/version.hpp | 13 + src/clients/SPEC.md | 86 + src/clients/auth.cpp | 58 + src/clients/auth.h | 57 + src/clients/body/form.cpp | 110 ++ src/clients/body/form.h | 69 + src/clients/body/json.cpp | 390 +++++ src/clients/body/json.h | 82 + src/clients/body/stream.cpp | 92 ++ src/clients/body/stream.h | 93 ++ src/clients/body/xml.cpp | 147 ++ src/clients/body/xml.h | 41 + src/clients/caps.h | 60 + src/clients/client.cpp | 399 +++++ src/clients/client.h | 104 ++ src/clients/client_config.cpp | 19 + src/clients/client_config.h | 90 + src/clients/clients.mk | 47 + src/clients/cookies.cpp | 173 ++ src/clients/cookies.h | 50 + src/clients/headers.cpp | 87 + src/clients/headers.h | 56 + src/clients/openapi/builder.cpp | 97 ++ src/clients/openapi/builder.h | 73 + src/clients/openapi/document.cpp | 168 ++ src/clients/openapi/document.h | 121 ++ src/clients/openapi/extensions.cpp | 13 + src/clients/openapi/extensions.h | 23 + src/clients/openapi/routes.cpp | 59 + src/clients/openapi/routes.h | 37 + src/clients/openapi/schema.cpp | 58 + src/clients/openapi/schema.h | 50 + src/clients/pool.cpp | 144 ++ src/clients/pool.h | 65 + src/clients/proxy.cpp | 143 ++ src/clients/proxy.h | 40 + src/clients/request.cpp | 139 ++ src/clients/request.h | 110 ++ src/clients/response.cpp | 27 + src/clients/response.h | 53 + src/clients/rest/errors.cpp | 124 ++ src/clients/rest/errors.h | 53 + src/clients/rest/idempotency.cpp | 47 + src/clients/rest/idempotency.h | 48 + src/clients/rest/paginator.cpp | 112 ++ src/clients/rest/paginator.h | 51 + src/clients/rest/resource.cpp | 111 ++ src/clients/rest/resource.h | 57 + src/clients/retry.cpp | 152 ++ src/clients/retry.h | 61 + src/clients/task.h | 127 ++ src/clients/testing/assertions.cpp | 166 ++ src/clients/testing/assertions.h | 35 + src/clients/testing/clock.cpp | 50 + src/clients/testing/clock.h | 23 + src/clients/testing/fixtures.cpp | 73 + src/clients/testing/fixtures.h | 67 + src/clients/testing/test_client.cpp | 33 + src/clients/testing/test_client.h | 42 + src/clients/timeouts.h | 102 ++ src/clients/transport.cpp | 12 + src/clients/transport.h | 40 + src/clients/transports/app_transport.cpp | 61 + src/clients/transports/app_transport.h | 50 + src/clients/transports/http_transport.cpp | 682 ++++++++ src/clients/transports/http_transport.h | 64 + src/clients/transports/mock_transport.cpp | 132 ++ src/clients/transports/mock_transport.h | 79 + src/clients/transports/tls.h | 58 + src/clients/transports/tls_stub.cpp | 25 + src/clients/url.cpp | 290 ++++ src/clients/url.h | 65 + src/data/Makefile | 27 + src/data/cache/cache.hpp | 216 +++ src/data/cache/policy.hpp | 37 + src/data/cache/singleflight.hpp | 76 + src/data/caps.hpp | 59 + src/data/connection.hpp | 58 + src/data/context.hpp | 83 + src/data/data.mk | 6 + src/data/drivers/pg/auth.hpp | 186 +++ src/data/drivers/pg/codecs.hpp | 209 +++ src/data/drivers/pg/conn.hpp | 467 ++++++ src/data/drivers/pg/extended.hpp | 66 + src/data/drivers/pg/message.hpp | 307 ++++ src/data/drivers/pg/secret.hpp | 74 + src/data/drivers/pg/sha256.hpp | 188 +++ src/data/drivers/sqlite/busy.hpp | 47 + src/data/drivers/sqlite/db.hpp | 153 ++ src/data/drivers/sqlite/gate.hpp | 30 + src/data/drivers/sqlite/store.hpp | 1463 +++++++++++++++++ src/data/drivers/sqlite/wal.hpp | 61 + src/data/error.hpp | 128 ++ src/data/kv/memory.hpp | 139 ++ src/data/kv/store.hpp | 52 + src/data/obs.hpp | 55 + src/data/orm/field.hpp | 163 ++ src/data/orm/healer.hpp | 396 +++++ src/data/orm/invalidation.hpp | 54 + src/data/orm/migration.hpp | 86 + src/data/orm/migrator.hpp | 115 ++ src/data/orm/model.hpp | 254 +++ src/data/orm/query.hpp | 218 +++ src/data/orm/relations.hpp | 80 + src/data/orm/schema.hpp | 159 ++ src/data/orm/sync.hpp | 122 ++ src/data/orm/types.md | 24 + src/data/pool.hpp | 268 +++ src/data/pool_config.hpp | 41 + src/data/pubsub/broker.hpp | 140 ++ src/data/pubsub/channel.hpp | 53 + src/data/pubsub/subscription.hpp | 77 + src/data/result.hpp | 50 + src/data/row.hpp | 45 + src/data/statement.hpp | 62 + src/data/transaction.hpp | 116 ++ src/data/value.hpp | 46 + src/governance.mk | 39 + src/governance/PATCHES.md | 248 +++ src/governance/adapters/BaseAdapters.h | 40 + src/governance/caps.h | 34 + src/governance/diff/ApiDiffCli.h | 94 ++ src/governance/diff/BreakingRules.h | 41 + src/governance/diff/GovernGate.cpp | 76 + src/governance/diff/GovernGate.h | 23 + src/governance/diff/SpecDiffer.h | 159 ++ src/governance/diff/SpecSnapshot.h | 392 +++++ src/governance/diff/apidiff_main.cpp | 4 + src/governance/keys/ApiKey.h | 34 + src/governance/keys/KeyAuth.h | 109 ++ src/governance/keys/KeyIssuer.h | 226 +++ src/governance/keys/KeyStore.h | 22 + src/governance/keys/MemoryKeyStore.h | 48 + src/governance/keys/Scope.h | 95 ++ src/governance/problem.h | 105 ++ src/governance/surface/RouteLinter.h | 215 +++ src/governance/surface/SurfaceRules.h | 46 + src/governance/tiers/QuotaLedger.h | 105 ++ src/governance/tiers/TierHeaders.h | 47 + src/governance/tiers/TierPolicy.h | 27 + src/governance/tiers/TierTable.h | 36 + src/governance/versions/ApiVersion.h | 54 + .../versions/DeprecationMiddleware.h | 86 + src/governance/versions/VersionedRouter.h | 120 ++ src/observability/README.md | 58 + src/observability/config.h | 100 ++ src/observability/correlation/correlation.h | 5 + src/observability/correlation/redaction.h | 70 + .../correlation/request_context.cpp | 20 + .../correlation/request_context.h | 28 + src/observability/correlation/request_id.cpp | 39 + src/observability/correlation/request_id.h | 14 + src/observability/events/dispatcher.cpp | 161 ++ src/observability/events/dispatcher.h | 78 + src/observability/events/event.h | 67 + src/observability/events/lifecycle_events.h | 115 ++ src/observability/events/listener.h | 20 + src/observability/hooks.h | 64 + src/observability/logging/async_queue.cpp | 69 + src/observability/logging/async_queue.h | 52 + src/observability/logging/async_sink.h | 36 + src/observability/logging/console_sink.h | 38 + src/observability/logging/file_sink.cpp | 60 + src/observability/logging/file_sink.h | 36 + src/observability/logging/json_format.cpp | 126 ++ src/observability/logging/json_format.h | 21 + src/observability/logging/level.h | 57 + src/observability/logging/logger.cpp | 94 ++ src/observability/logging/logger.h | 52 + src/observability/logging/macros.h | 107 ++ src/observability/logging/null_sink.h | 13 + src/observability/logging/record.h | 38 + src/observability/logging/registry.cpp | 88 + src/observability/logging/registry.h | 57 + src/observability/logging/ring_sink.h | 45 + src/observability/logging/sink.h | 16 + src/observability/metrics/counter.h | 38 + src/observability/metrics/gauge.h | 44 + src/observability/metrics/histogram.h | 79 + src/observability/metrics/labels.h | 55 + src/observability/metrics/prometheus.cpp | 141 ++ src/observability/metrics/prometheus.h | 20 + src/observability/metrics/registry.cpp | 193 +++ src/observability/metrics/registry.h | 90 + src/observability/metrics/timer.h | 48 + src/observability/observability.h | 48 + src/observability/observability.mk | 59 + src/observability/tracing/batch_processor.cpp | 96 ++ src/observability/tracing/batch_processor.h | 62 + src/observability/tracing/exporter.h | 19 + src/observability/tracing/http_sender.h | 56 + src/observability/tracing/ids.h | 170 ++ src/observability/tracing/memory_exporter.h | 38 + src/observability/tracing/otlp_http.cpp | 192 +++ src/observability/tracing/otlp_http.h | 55 + src/observability/tracing/propagator.cpp | 100 ++ src/observability/tracing/propagator.h | 26 + src/observability/tracing/sampler.h | 57 + src/observability/tracing/span.cpp | 120 ++ src/observability/tracing/span.h | 85 + src/observability/tracing/span_context.h | 22 + src/observability/tracing/tracer.cpp | 75 + src/observability/tracing/tracer.h | 47 + src/operability.mk | 71 + src/operability/PATCHES.md | 334 ++++ .../include/supervisor/common/clock.hpp | 59 + .../include/supervisor/common/kv_store.hpp | 124 ++ .../include/supervisor/common/observer.hpp | 80 + src/operability/caps.h | 57 + src/operability/chaos/experiment.cpp | 87 + src/operability/chaos/experiment.h | 56 + src/operability/chaos/hook.h | 57 + src/operability/chaos/injector.cpp | 86 + src/operability/chaos/injector.h | 46 + src/operability/coord/dist_lock.cpp | 63 + src/operability/coord/dist_lock.h | 44 + src/operability/coord/election.cpp | 74 + src/operability/coord/election.h | 47 + src/operability/coord/fence.h | 77 + src/operability/coord/lease.cpp | 74 + src/operability/coord/lease.h | 50 + src/operability/debug/endpoint.cpp | 63 + src/operability/debug/endpoint.h | 39 + src/operability/debug/snapshot.cpp | 89 + src/operability/debug/snapshot.h | 40 + src/operability/deploy/blue_green.cpp | 205 +++ src/operability/deploy/blue_green.h | 74 + src/operability/deploy/lifecycle.cpp | 137 ++ src/operability/deploy/lifecycle.h | 71 + src/operability/deploy/rolling.cpp | 152 ++ src/operability/deploy/rolling.h | 62 + src/operability/deploy/strategy.h | 29 + src/operability/resilience/breaker.cpp | 114 ++ src/operability/resilience/breaker.h | 55 + src/operability/resilience/bulkhead.cpp | 75 + src/operability/resilience/bulkhead.h | 49 + src/operability/resilience/deadline.cpp | 36 + src/operability/resilience/deadline.h | 41 + src/operability/resilience/shedder.cpp | 44 + src/operability/resilience/shedder.h | 44 + src/operability/resilience/slo.cpp | 182 ++ src/operability/resilience/slo.h | 63 + src/realtime/SPEC.md | 135 ++ src/realtime/engineio/engine_session.cpp | 59 + src/realtime/engineio/engine_session.h | 79 + src/realtime/engineio/heartbeat_scheduler.cpp | 43 + src/realtime/engineio/heartbeat_scheduler.h | 37 + src/realtime/engineio/packet_codec.cpp | 74 + src/realtime/engineio/packet_codec.h | 48 + src/realtime/engineio/polling_transport.cpp | 71 + src/realtime/engineio/polling_transport.h | 43 + src/realtime/engineio/upgrade_manager.cpp | 26 + src/realtime/engineio/upgrade_manager.h | 43 + src/realtime/engineio/websocket_transport.cpp | 42 + src/realtime/engineio/websocket_transport.h | 35 + src/realtime/realtime.mk | 90 + src/realtime/shared/backpressure_valve.cpp | 32 + src/realtime/shared/backpressure_valve.h | 33 + src/realtime/shared/base64.cpp | 110 ++ src/realtime/shared/base64.h | 21 + src/realtime/shared/base_realtime_client.cpp | 63 + src/realtime/shared/base_realtime_client.h | 82 + src/realtime/shared/caps.h | 57 + src/realtime/shared/connection_context.h | 37 + src/realtime/shared/http_seam.cpp | 119 ++ src/realtime/shared/http_seam.h | 50 + src/realtime/shared/mini_json.cpp | 409 +++++ src/realtime/shared/mini_json.h | 61 + src/realtime/shared/random_token.cpp | 63 + src/realtime/shared/random_token.h | 23 + src/realtime/shared/realtime_log.cpp | 69 + src/realtime/shared/realtime_log.h | 41 + src/realtime/shared/realtime_metrics.cpp | 76 + src/realtime/shared/realtime_metrics.h | 53 + src/realtime/shared/reconnect_policy.cpp | 51 + src/realtime/shared/reconnect_policy.h | 43 + src/realtime/shared/send_queue.cpp | 49 + src/realtime/shared/send_queue.h | 68 + src/realtime/shared/sha1.cpp | 128 ++ src/realtime/shared/sha1.h | 34 + src/realtime/shared/transport.cpp | 48 + src/realtime/shared/transport.h | 69 + src/realtime/shared/utf8.cpp | 58 + src/realtime/shared/utf8.h | 15 + src/realtime/socketio/ack_tracker.cpp | 40 + src/realtime/socketio/ack_tracker.h | 45 + .../socketio/client/socketio_client.cpp | 194 +++ .../socketio/client/socketio_client.h | 80 + src/realtime/socketio/event_dispatcher.cpp | 36 + src/realtime/socketio/event_dispatcher.h | 38 + src/realtime/socketio/namespace.cpp | 30 + src/realtime/socketio/namespace.h | 42 + src/realtime/socketio/packet_codec.cpp | 181 ++ src/realtime/socketio/packet_codec.h | 60 + src/realtime/socketio/room_manager.cpp | 128 ++ src/realtime/socketio/room_manager.h | 46 + src/realtime/socketio/socketio_server.cpp | 184 +++ src/realtime/socketio/socketio_server.h | 78 + src/realtime/sse/event_encoder.cpp | 70 + src/realtime/sse/event_encoder.h | 33 + src/realtime/sse/event_store.cpp | 58 + src/realtime/sse/event_store.h | 56 + src/realtime/sse/sse_client.cpp | 133 ++ src/realtime/sse/sse_client.h | 67 + src/realtime/sse/sse_response.cpp | 53 + src/realtime/sse/sse_response.h | 50 + src/realtime/streaming/chunked_body.cpp | 58 + src/realtime/streaming/chunked_body.h | 43 + src/realtime/streaming/flush_policy.cpp | 35 + src/realtime/streaming/flush_policy.h | 32 + src/realtime/streaming/passthrough_body.cpp | 83 + src/realtime/streaming/passthrough_body.h | 58 + src/realtime/webdav/client/webdav_client.cpp | 165 ++ src/realtime/webdav/client/webdav_client.h | 83 + .../webdav/server/copy_move_handler.cpp | 89 + .../webdav/server/copy_move_handler.h | 28 + src/realtime/webdav/server/dav_view.cpp | 257 +++ src/realtime/webdav/server/dav_view.h | 52 + src/realtime/webdav/server/dav_xml.cpp | 179 ++ src/realtime/webdav/server/dav_xml.h | 31 + src/realtime/webdav/server/lock_manager.cpp | 163 ++ src/realtime/webdav/server/lock_manager.h | 68 + src/realtime/webdav/server/mkcol_handler.cpp | 40 + src/realtime/webdav/server/mkcol_handler.h | 18 + .../webdav/server/multistatus_builder.cpp | 104 ++ .../webdav/server/multistatus_builder.h | 43 + .../webdav/server/propfind_handler.cpp | 159 ++ src/realtime/webdav/server/propfind_handler.h | 38 + .../webdav/server/proppatch_handler.cpp | 70 + .../webdav/server/proppatch_handler.h | 27 + src/realtime/webdav/server/quota_guard.cpp | 21 + src/realtime/webdav/server/quota_guard.h | 30 + src/realtime/webdav/server/resource_tree.cpp | 237 +++ src/realtime/webdav/server/resource_tree.h | 74 + .../websocket/client/client_handshake.cpp | 68 + .../websocket/client/client_handshake.h | 38 + .../websocket/client/websocket_client.cpp | 153 ++ .../websocket/client/websocket_client.h | 67 + .../websocket/server/frame_parser.cpp | 125 ++ src/realtime/websocket/server/frame_parser.h | 62 + .../websocket/server/frame_serializer.cpp | 92 ++ .../websocket/server/frame_serializer.h | 40 + .../websocket/server/handshake_validator.cpp | 94 ++ .../websocket/server/handshake_validator.h | 46 + .../websocket/server/heartbeat_manager.cpp | 48 + .../websocket/server/heartbeat_manager.h | 49 + src/realtime/websocket/server/opcode.h | 40 + .../websocket/server/websocket_handler.cpp | 21 + .../websocket/server/websocket_handler.h | 46 + .../websocket/server/websocket_response.cpp | 234 +++ .../websocket/server/websocket_response.h | 103 ++ .../websocket/server/websocket_view.cpp | 51 + .../websocket/server/websocket_view.h | 58 + src/runtime/base64.h | 92 ++ src/runtime/caps.h | 80 + src/runtime/channel.h | 162 ++ src/runtime/compress/brotli.cpp | 345 ++++ src/runtime/compress/brotli.h | 65 + src/runtime/compress/compressor.h | 46 + src/runtime/compress/gzip.cpp | 773 +++++++++ src/runtime/compress/gzip.h | 141 ++ src/runtime/compress/negotiate.cpp | 167 ++ src/runtime/compress/negotiate.h | 33 + src/runtime/cookie/cookie.cpp | 153 ++ src/runtime/cookie/cookie.h | 49 + src/runtime/cookie/jar.cpp | 62 + src/runtime/cookie/jar.h | 53 + src/runtime/cookie/signer.h | 45 + src/runtime/event_loop.cpp | 137 ++ src/runtime/event_loop.h | 98 ++ src/runtime/executor.cpp | 76 + src/runtime/executor.h | 44 + src/runtime/http/connection.cpp | 672 ++++++++ src/runtime/http/connection.h | 118 ++ src/runtime/http/fd_waiter.h | 127 ++ src/runtime/http/listener.cpp | 175 ++ src/runtime/http/listener.h | 53 + src/runtime/http/message.cpp | 147 ++ src/runtime/http/message.h | 91 + src/runtime/http/protocol_detect.cpp | 41 + src/runtime/http/protocol_detect.h | 30 + src/runtime/http/v1/parser.cpp | 425 +++++ src/runtime/http/v1/parser.h | 92 ++ src/runtime/http/v1/serializer.cpp | 83 + src/runtime/http/v1/serializer.h | 28 + src/runtime/http/v2/connection.cpp | 962 +++++++++++ src/runtime/http/v2/connection.h | 152 ++ src/runtime/http/v2/flow_control.cpp | 82 + src/runtime/http/v2/flow_control.h | 43 + src/runtime/http/v2/hpack.cpp | 344 ++++ src/runtime/http/v2/hpack.h | 76 + src/runtime/http/v2/huffman_table.inc | 259 +++ src/runtime/http/v2/settings.h | 102 ++ src/runtime/http/v2/static_table.inc | 64 + src/runtime/http/v2/stream.cpp | 145 ++ src/runtime/http/v2/stream.h | 81 + src/runtime/observability_hooks.h | 39 + src/runtime/poller.h | 36 + src/runtime/poller_epoll.cpp | 148 ++ src/runtime/poller_epoll.h | 52 + src/runtime/poller_iocp.cpp | 130 ++ src/runtime/poller_iocp.h | 39 + src/runtime/poller_kqueue.cpp | 169 ++ src/runtime/poller_kqueue.h | 36 + src/runtime/runtime.mk | 94 ++ src/runtime/signal_handler.cpp | 111 ++ src/runtime/signal_handler.h | 45 + src/runtime/static/asset_provider.cpp | 75 + src/runtime/static/asset_provider.h | 59 + src/runtime/static/disk_provider.cpp | 84 + src/runtime/static/disk_provider.h | 31 + src/runtime/static/embedded_provider.cpp | 36 + src/runtime/static/embedded_provider.h | 35 + src/runtime/static/etag.cpp | 129 ++ src/runtime/static/etag.h | 26 + src/runtime/static/file_server.cpp | 280 ++++ src/runtime/static/file_server.h | 45 + src/runtime/static/path_guard.cpp | 69 + src/runtime/static/path_guard.h | 16 + src/runtime/strand.h | 56 + src/runtime/task.h | 216 +++ src/runtime/timer_wheel.cpp | 132 ++ src/runtime/timer_wheel.h | 56 + src/runtime/tls_handle.h | 74 + src/runtime/transport.h | 27 + src/runtime/transport_tcp.cpp | 40 + src/runtime/transport_tcp.h | 34 + src/security/auth/Authenticator.cpp | 108 ++ src/security/auth/Authenticator.h | 80 + src/security/auth/Guard.cpp | 56 + src/security/auth/Guard.h | 44 + src/security/auth/Permission.h | 36 + src/security/crypto/AesGcm.cpp | 237 +++ src/security/crypto/AesGcm.h | 37 + src/security/crypto/Base64.cpp | 125 ++ src/security/crypto/Base64.h | 32 + src/security/crypto/Blake2b.cpp | 120 ++ src/security/crypto/Blake2b.h | 41 + src/security/crypto/ChaCha20Poly1305.cpp | 330 ++++ src/security/crypto/ChaCha20Poly1305.h | 35 + src/security/crypto/CryptoBox.cpp | 128 ++ src/security/crypto/CryptoBox.h | 41 + src/security/crypto/Hmac.cpp | 108 ++ src/security/crypto/Hmac.h | 34 + src/security/crypto/KeyHierarchy.cpp | 144 ++ src/security/crypto/KeyHierarchy.h | 67 + src/security/crypto/PasswordHasher.cpp | 442 +++++ src/security/crypto/PasswordHasher.h | 69 + src/security/crypto/SecureRandom.cpp | 92 ++ src/security/crypto/SecureRandom.h | 37 + src/security/crypto/Sha256.cpp | 121 ++ src/security/crypto/Sha256.h | 40 + src/security/guard/BodyLimits.cpp | 94 ++ src/security/guard/BodyLimits.h | 57 + src/security/guard/CorsPolicy.cpp | 91 + src/security/guard/CorsPolicy.h | 52 + src/security/guard/CsrfGuard.cpp | 100 ++ src/security/guard/CsrfGuard.h | 65 + src/security/guard/DenialSink.cpp | 18 + src/security/guard/DenialSink.h | 41 + src/security/guard/HttpMessageView.cpp | 50 + src/security/guard/HttpMessageView.h | 69 + src/security/guard/PathGuard.cpp | 100 ++ src/security/guard/PathGuard.h | 34 + src/security/guard/RateLimiter.cpp | 147 ++ src/security/guard/RateLimiter.h | 91 + src/security/guard/SecurityHeaders.cpp | 24 + src/security/guard/SecurityHeaders.h | 45 + src/security/guard/SqlSafety.cpp | 57 + src/security/guard/SqlSafety.h | 47 + src/security/secret/SecretsProvider.cpp | 303 ++++ src/security/secret/SecretsProvider.h | 70 + src/security/security.mk | 226 +++ src/security/session/DbStore.cpp | 42 + src/security/session/DbStore.h | 43 + src/security/session/MemoryStore.cpp | 74 + src/security/session/MemoryStore.h | 37 + src/security/session/Session.cpp | 24 + src/security/session/Session.h | 40 + src/security/session/SessionManager.cpp | 143 ++ src/security/session/SessionManager.h | 71 + src/security/session/SessionStore.h | 31 + src/security/tls/TlsAcceptor.cpp | 165 ++ src/security/tls/TlsAcceptor.h | 117 ++ src/security/tls/TlsBackend.h | 68 + src/security/tls/TlsConfig.cpp | 33 + src/security/tls/TlsConfig.h | 50 + src/security/tls/TlsContext.cpp | 51 + src/security/tls/TlsContext.h | 48 + src/security/tls/WolfSslBackend.cpp | 97 ++ src/security/tls/WolfSslBackend.h | 44 + src/security/upload/UploadStore.cpp | 134 ++ src/security/upload/UploadStore.h | 77 + src/security/upload/UploadValidator.cpp | 89 + src/security/upload/UploadValidator.h | 67 + src/supervisor/TOPOLOGY.md | 98 ++ src/supervisor/bus/ack_tracker.cpp | 52 + src/supervisor/bus/ack_tracker.hpp | 32 + src/supervisor/bus/bus_error.hpp | 17 + src/supervisor/bus/consumer_cursor.cpp | 57 + src/supervisor/bus/consumer_cursor.hpp | 32 + src/supervisor/bus/segment_log.cpp | 374 +++++ src/supervisor/bus/segment_log.hpp | 93 ++ src/supervisor/bus/stream_bus.cpp | 281 ++++ src/supervisor/bus/stream_bus.hpp | 102 ++ src/supervisor/bus/topic_bus.cpp | 103 ++ src/supervisor/bus/topic_bus.hpp | 62 + src/supervisor/common/caps.hpp | 49 + src/supervisor/common/clock.cpp | 109 ++ src/supervisor/common/clock.hpp | 50 + src/supervisor/common/json.cpp | 382 +++++ src/supervisor/common/json.hpp | 61 + src/supervisor/common/kv_store.cpp | 118 ++ src/supervisor/common/kv_store.hpp | 79 + src/supervisor/common/observer.cpp | 61 + src/supervisor/common/observer.hpp | 60 + .../replication/conflict_resolver.cpp | 230 +++ .../replication/conflict_resolver.hpp | 85 + src/supervisor/replication/hybrid_clock.cpp | 79 + src/supervisor/replication/hybrid_clock.hpp | 43 + src/supervisor/replication/oplog_entry.cpp | 86 + src/supervisor/replication/oplog_entry.hpp | 62 + src/supervisor/replication/oplog_reader.cpp | 47 + src/supervisor/replication/oplog_reader.hpp | 36 + src/supervisor/replication/oplog_writer.cpp | 46 + src/supervisor/replication/oplog_writer.hpp | 40 + src/supervisor/replication/replicator.cpp | 122 ++ src/supervisor/replication/replicator.hpp | 73 + src/supervisor/replication/sync_session.cpp | 73 + src/supervisor/replication/sync_session.hpp | 63 + src/supervisor/replication/version_vector.cpp | 96 ++ src/supervisor/replication/version_vector.hpp | 34 + src/supervisor/supervisor.mk | 78 + src/supervisor/supervisor/health_check.cpp | 150 ++ src/supervisor/supervisor/health_check.hpp | 58 + src/supervisor/supervisor/restart_policy.cpp | 38 + src/supervisor/supervisor/restart_policy.hpp | 28 + .../supervisor/shutdown_coordinator.cpp | 50 + .../supervisor/shutdown_coordinator.hpp | 40 + src/supervisor/supervisor/supervisor.cpp | 153 ++ src/supervisor/supervisor/supervisor.hpp | 68 + .../supervisor/supervisor_error.hpp | 13 + src/supervisor/supervisor/worker_group.cpp | 481 ++++++ src/supervisor/supervisor/worker_group.hpp | 102 ++ src/supervisor/supervisor/worker_process.cpp | 150 ++ src/supervisor/supervisor/worker_process.hpp | 71 + src/supervisor/supervisor/worker_state.hpp | 32 + src/supervisor/tasks/cron.cpp | 187 +++ src/supervisor/tasks/cron.hpp | 35 + src/supervisor/tasks/dead_letter_queue.cpp | 91 + src/supervisor/tasks/dead_letter_queue.hpp | 52 + src/supervisor/tasks/retry_policy.cpp | 56 + src/supervisor/tasks/retry_policy.hpp | 31 + src/supervisor/tasks/scheduler.cpp | 380 +++++ src/supervisor/tasks/scheduler.hpp | 114 ++ src/supervisor/tasks/task_envelope.cpp | 150 ++ src/supervisor/tasks/task_envelope.hpp | 57 + src/supervisor/tasks/task_handler.cpp | 14 + src/supervisor/tasks/task_handler.hpp | 40 + src/supervisor/tasks/task_queue.cpp | 683 ++++++++ src/supervisor/tasks/task_queue.hpp | 158 ++ src/supervisor/tasks/task_registry.cpp | 36 + src/supervisor/tasks/task_registry.hpp | 31 + src/supplychain.mk | 142 ++ src/supplychain/PATCHES.md | 131 ++ .../adapters/cli/cli/arg_parser.hpp | 28 + .../adapters/cli/cli/exit_code.hpp | 12 + src/supplychain/adapters/cli/cli/flag.hpp | 34 + .../adapters/cli/cli/json_writer.hpp | 72 + src/supplychain/adapters/cli/cli/output.hpp | 63 + src/supplychain/adapters/cli/embed/sha256.hpp | 186 +++ src/supplychain/adapters/cli/version.hpp | 16 + src/supplychain/attest/Attestor.cpp | 79 + src/supplychain/attest/Attestor.hpp | 23 + src/supplychain/attest/ProvenanceBuilder.hpp | 61 + src/supplychain/attest/Signer.cpp | 79 + src/supplychain/attest/Signer.hpp | 21 + src/supplychain/attest/Verify.cpp | 79 + src/supplychain/attest/Verify.hpp | 27 + src/supplychain/caps.hpp | 33 + src/supplychain/cli/SupplyCli.cpp | 502 ++++++ src/supplychain/cli/SupplyCli.hpp | 44 + src/supplychain/common/Json.cpp | 368 +++++ src/supplychain/common/Json.hpp | 44 + src/supplychain/policy/DependencyPolicy.hpp | 92 ++ src/supplychain/policy/WolfPin.cpp | 91 + src/supplychain/policy/WolfPin.hpp | 32 + src/supplychain/sbom/CycloneDxEmitter.cpp | 98 ++ src/supplychain/sbom/CycloneDxEmitter.hpp | 20 + src/supplychain/sbom/SbomCheck.cpp | 23 + src/supplychain/sbom/SbomCheck.hpp | 20 + src/supplychain/sbom/SbomGenerator.cpp | 122 ++ src/supplychain/sbom/SbomGenerator.hpp | 32 + src/supplychain/sbom/SpdxEmitter.cpp | 97 ++ src/supplychain/sbom/SpdxEmitter.hpp | 19 + src/supplychain/sbom/Types.hpp | 33 + src/supplychain/scan/OsvClient.cpp | 130 ++ src/supplychain/scan/OsvClient.hpp | 46 + src/supplychain/scan/VulnHook.cpp | 154 ++ src/supplychain/scan/VulnHook.hpp | 39 + src/supplychain/spec.md | 161 ++ src/tenancy.mk | 24 + src/tenancy/PATCHES.md | 346 ++++ src/tenancy/SPEC.md | 75 + src/tenancy/adapters/db.h | 113 ++ src/tenancy/adapters/http.h | 130 ++ src/tenancy/adapters/task.h | 63 + src/tenancy/bridge.h | 146 ++ src/tenancy/caps.h | 30 + src/tenancy/catalog.h | 124 ++ src/tenancy/config_overlay.h | 82 + src/tenancy/context.h | 35 + src/tenancy/feature_flags.h | 51 + src/tenancy/metrics.h | 66 + src/tenancy/middleware.h | 61 + src/tenancy/orm_scope.h | 94 ++ src/tenancy/pool_factory.h | 160 ++ src/tenancy/provisioner.h | 78 + src/tenancy/quota.h | 112 ++ src/tenancy/resolver.h | 113 ++ src/tenancy/rls_policy.h | 40 + src/tenancy/store/catalog_store.h | 24 + src/tenancy/store/db_catalog.h | 119 ++ src/tenancy/store/memory_catalog.h | 81 + src/tenancy/strategy.h | 46 + src/tenancy/tenant.h | 80 + src/tenancy/tenant_buckets.h | 78 + src/tenancy/tenant_id.h | 59 + tests/README.md | 65 + tests/example/auth/login_test.cpp | 22 + tests/example/auth/me_test.cpp | 21 + tests/example/cache/lookup_test.cpp | 14 + tests/example/chain/deep_test.cpp | 14 + tests/example/cli/ai_test.cpp | 16 + tests/example/embedded/assets_test.cpp | 16 + tests/example/events/fire_test.cpp | 19 + tests/example/formats/data_test.cpp | 26 + tests/example/forms/contact_test.cpp | 21 + tests/example/health_test.cpp | 26 + tests/example/index_test.cpp | 18 + tests/example/kv/items_test.cpp | 19 + tests/example/metrics/counters_test.cpp | 15 + tests/example/openapi/docs_test.cpp | 16 + tests/example/orm/users_test.cpp | 24 + tests/example/pubsub/publish_test.cpp | 20 + tests/example/redirect/here_test.cpp | 14 + tests/example/replication/state_test.cpp | 18 + tests/example/restclient/self_test.cpp | 12 + tests/example/routing/hello_test.cpp | 16 + tests/example/routing/routers_test.cpp | 14 + tests/example/routing/views_test.cpp | 21 + tests/example/sessions/counter_test.cpp | 17 + tests/example/socketio/rooms_test.cpp | 17 + tests/example/sse/events_test.cpp | 14 + tests/example/static/files_test.cpp | 17 + tests/example/streaming/feed_test.cpp | 15 + tests/example/streaming/passthrough_test.cpp | 14 + tests/example/supervisor/tasks_test.cpp | 18 + tests/example/supervisor/workers_test.cpp | 14 + tests/example/support/Assert.hpp | 49 + tests/example/telemetry/trace_test.cpp | 16 + tests/example/templates/page_test.cpp | 18 + tests/example/uploads/file_test.cpp | 23 + tests/example/validate/user_test.cpp | 21 + tests/example/webdav/files_test.cpp | 23 + tests/example/workflow/release_test.cpp | 24 + tests/example/ws/chat_test.cpp | 15 + tests/src/app/00_compile_all_test.cpp | 62 + tests/src/app/_testlib.h | 92 ++ tests/src/app/di/scopes_test.cpp | 63 + tests/src/app/errors/render_test.cpp | 47 + tests/src/app/forms/bind_test.cpp | 92 ++ tests/src/app/health/ping_test.cpp | 65 + tests/src/app/middleware/order_test.cpp | 96 ++ tests/src/app/mount/nested_test.cpp | 102 ++ tests/src/app/negotiate/select_test.cpp | 74 + tests/src/app/plugins/hooks_test.cpp | 75 + tests/src/app/templates/render_test.cpp | 220 +++ tests/src/app/users/detail_test.cpp | 75 + tests/src/app/validate/input_test.cpp | 75 + tests/src/app/views/dispatch_test.cpp | 54 + tests/src/audit/chain_tamper.cpp | 61 + tests/src/audit/erasure_cascade.cpp | 82 + tests/src/audit/export_bundle.cpp | 59 + tests/src/audit/mask_fixture.cpp | 47 + tests/src/audit/orm_hook.cpp | 94 ++ tests/src/audit/pii_boot.cpp | 46 + tests/src/audit/query_guard.cpp | 64 + tests/src/audit/redactor.cpp | 50 + tests/src/audit/retention_hold.cpp | 59 + tests/src/audit/sql_store.cpp | 107 ++ tests/src/cli/test_ai.cpp | 73 + tests/src/cli/test_cli.cpp | 82 + tests/src/cli/test_cli_binary.sh | 86 + tests/src/cli/test_csp.cpp | 48 + tests/src/cli/test_dev.cpp | 67 + tests/src/cli/test_embed.cpp | 81 + tests/src/cli/test_healthz.cpp | 38 + tests/src/cli/test_i18n.cpp | 60 + tests/src/cli/test_metrics.cpp | 48 + tests/src/cli/test_readyz.cpp | 72 + tests/src/cli/test_startupz.cpp | 37 + tests/src/clients/framework.h | 106 ++ tests/src/clients/main.cpp | 44 + tests/src/clients/test_bodies.cpp | 141 ++ tests/src/clients/test_client_mock.cpp | 375 +++++ tests/src/clients/test_http_transport.cpp | 375 +++++ tests/src/clients/test_openapi.cpp | 161 ++ tests/src/clients/test_proxy.cpp | 67 + tests/src/clients/test_rest.cpp | 143 ++ tests/src/clients/test_testing.cpp | 150 ++ tests/src/clients/test_url.cpp | 75 + tests/src/data/mini.hpp | 33 + tests/src/data/test_kv_cache.cpp | 217 +++ tests/src/data/test_orm.cpp | 218 +++ tests/src/data/test_pg_wire.cpp | 313 ++++ tests/src/data/test_pool_txn.cpp | 187 +++ tests/src/data/test_pubsub.cpp | 137 ++ tests/src/data/test_sqlite.cpp | 235 +++ tests/src/data/test_sync_heal_migrate.cpp | 232 +++ tests/src/data/test_value_error.cpp | 102 ++ .../governance/deprecation_headers_test.cpp | 65 + tests/src/governance/differ_vectors_test.cpp | 139 ++ tests/src/governance/gate_test.cpp | 120 ++ tests/src/governance/keyauth_http_test.cpp | 101 ++ tests/src/governance/keys_issuer_test.cpp | 85 + tests/src/governance/keys_scope_test.cpp | 63 + tests/src/governance/surface_lint_test.cpp | 115 ++ tests/src/governance/tiers_test.cpp | 95 ++ tests/src/governance/versions_test.cpp | 81 + tests/src/observability/correlation_test.cpp | 140 ++ tests/src/observability/events_test.cpp | 120 ++ tests/src/observability/logging_test.cpp | 208 +++ .../observability/metrics_endpoint_test.cpp | 203 +++ tests/src/observability/otlp_export_test.cpp | 217 +++ tests/src/observability/propagation_test.cpp | 108 ++ tests/src/observability/test_helper.h | 29 + tests/src/observability/tracing_test.cpp | 234 +++ tests/src/observability/zero_cost_probe.cpp | 47 + tests/src/operability/main.cpp | 30 + .../src/operability/test_breaker_bulkhead.cpp | 154 ++ tests/src/operability/test_chaos.cpp | 122 ++ tests/src/operability/test_coord.cpp | 155 ++ tests/src/operability/test_debug.cpp | 103 ++ tests/src/operability/test_deploy.cpp | 176 ++ tests/src/operability/test_lifecycle.cpp | 94 ++ tests/src/operability/test_shed_deadline.cpp | 78 + tests/src/operability/test_slo.cpp | 101 ++ tests/src/realtime/dav/files_test.cpp | 185 +++ .../src/realtime/engineio/handshake_test.cpp | 160 ++ tests/src/realtime/shared/foundation_test.cpp | 149 ++ tests/src/realtime/shared/robustness_test.cpp | 95 ++ tests/src/realtime/socketio/chat_test.cpp | 221 +++ tests/src/realtime/sse/client_test.cpp | 82 + tests/src/realtime/sse/feed_test.cpp | 102 ++ .../src/realtime/stream/passthrough_test.cpp | 107 ++ tests/src/realtime/test_main.h | 45 + tests/src/realtime/ws/echo_test.cpp | 139 ++ tests/src/realtime/ws/frames_test.cpp | 169 ++ tests/src/realtime/ws/handshake_test.cpp | 86 + tests/src/realtime/ws/heartbeat_test.cpp | 51 + tests/src/runtime/compress/test_brotli.cpp | 119 ++ tests/src/runtime/compress/test_gzip.cpp | 139 ++ tests/src/runtime/compress/test_negotiate.cpp | 61 + tests/src/runtime/cookie/test_cookie.cpp | 118 ++ tests/src/runtime/fixtures/www/.hidden | 1 + tests/src/runtime/fixtures/www/hello.txt | 1 + tests/src/runtime/fixtures/www/index.html | 1 + tests/src/runtime/fixtures/www/sub/file.txt | 1 + tests/src/runtime/http/test_integration.cpp | 649 ++++++++ tests/src/runtime/http/test_message.cpp | 87 + .../src/runtime/http/test_protocol_detect.cpp | 34 + tests/src/runtime/http/v1/test_parser.cpp | 156 ++ tests/src/runtime/http/v1/test_serializer.cpp | 58 + tests/src/runtime/http/v2/test_flow.cpp | 45 + .../src/runtime/http/v2/test_h2connection.cpp | 526 ++++++ tests/src/runtime/http/v2/test_hpack.cpp | 174 ++ tests/src/runtime/http/v2/test_settings.cpp | 42 + tests/src/runtime/http/v2/test_stream.cpp | 103 ++ tests/src/runtime/interop/interop.py | 289 ++++ tests/src/runtime/interop/test_server.cpp | 255 +++ tests/src/runtime/interop/www/hello.txt | 1 + tests/src/runtime/main.cpp | 3 + tests/src/runtime/static/test_file_server.cpp | 266 +++ tests/src/runtime/static/test_path_guard.cpp | 39 + tests/src/runtime/test_base64.cpp | 38 + tests/src/runtime/test_channel_strand.cpp | 70 + tests/src/runtime/test_event_loop.cpp | 126 ++ tests/src/runtime/test_executor.cpp | 47 + tests/src/runtime/test_framework.h | 114 ++ tests/src/runtime/test_task.cpp | 92 ++ tests/src/runtime/test_timer.cpp | 57 + tests/src/security/auth/auth_test.cpp | 158 ++ tests/src/security/crypto/crypto_test.cpp | 308 ++++ tests/src/security/guard/guard_test.cpp | 256 +++ tests/src/security/secret/secrets_test.cpp | 88 + tests/src/security/session/session_test.cpp | 155 ++ tests/src/security/tls/tls_test.cpp | 198 +++ tests/src/security/upload/upload_test.cpp | 117 ++ tests/src/supervisor/test_bus.cpp | 205 +++ tests/src/supervisor/test_durability.cpp | 253 +++ tests/src/supervisor/test_multi_instance.cpp | 111 ++ tests/src/supervisor/test_replication.cpp | 304 ++++ tests/src/supervisor/test_scheduler.cpp | 258 +++ tests/src/supervisor/test_supervisor.cpp | 262 +++ tests/src/supervisor/test_tasks.cpp | 348 ++++ tests/src/supplychain/golden/advisories.json | 18 + tests/src/supplychain/golden/provenance.json | 50 + tests/src/supplychain/golden/sbom.cdx.json | 95 ++ tests/src/supplychain/golden/sbom.spdx.json | 98 ++ tests/src/supplychain/test_cdx_equiv.cpp | 145 ++ tests/src/supplychain/test_pin.cpp | 105 ++ tests/src/supplychain/test_provenance.cpp | 112 ++ tests/src/supplychain/test_scan.cpp | 156 ++ tests/src/supplychain/test_spdx.cpp | 139 ++ tests/src/supplychain/test_verify.cpp | 243 +++ tests/src/tenancy/mini.hpp | 63 + tests/src/tenancy/test_bridge.cpp | 75 + tests/src/tenancy/test_flags_config.cpp | 76 + tests/src/tenancy/test_metrics.cpp | 47 + tests/src/tenancy/test_pool_scope.cpp | 151 ++ tests/src/tenancy/test_provision.cpp | 194 +++ tests/src/tenancy/test_quota.cpp | 79 + tests/src/tenancy/test_tenant_id_resolve.cpp | 185 +++ 1068 files changed, 100899 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 Makefile create mode 100644 docker-compose.yml create mode 100644 example/README.md create mode 100644 example/SPEC.md create mode 100644 example/app/App.cpp create mode 100644 example/app/App.hpp create mode 100644 example/app/DataDemo.cpp create mode 100644 example/app/Embedded.cpp create mode 100644 example/app/FormatsDemo.cpp create mode 100644 example/app/Index.cpp create mode 100644 example/app/OpsDemo.cpp create mode 100644 example/app/RealtimeDemo.cpp create mode 100644 example/app/RoutingDemo.cpp create mode 100644 example/app/SecurityDemo.cpp create mode 100644 example/app/Seed.cpp create mode 100644 example/app/Templates.cpp create mode 100644 example/example.mk create mode 100644 example/fw/App.cpp create mode 100644 example/fw/App.hpp create mode 100644 example/fw/Caps.hpp create mode 100644 example/fw/Cookies.cpp create mode 100644 example/fw/Cookies.hpp create mode 100644 example/fw/Data.cpp create mode 100644 example/fw/Data.hpp create mode 100644 example/fw/Http.cpp create mode 100644 example/fw/Http.hpp create mode 100644 example/fw/Jobs.cpp create mode 100644 example/fw/Jobs.hpp create mode 100644 example/fw/Kv.cpp create mode 100644 example/fw/Kv.hpp create mode 100644 example/fw/Negotiate.cpp create mode 100644 example/fw/Negotiate.hpp create mode 100644 example/fw/Net.cpp create mode 100644 example/fw/Net.hpp create mode 100644 example/fw/Observ.cpp create mode 100644 example/fw/Observ.hpp create mode 100644 example/fw/Router.cpp create mode 100644 example/fw/Router.hpp create mode 100644 example/fw/Server.cpp create mode 100644 example/fw/Server.hpp create mode 100644 example/fw/StaticFiles.cpp create mode 100644 example/fw/StaticFiles.hpp create mode 100644 example/fw/Templates.cpp create mode 100644 example/fw/Templates.hpp create mode 100644 example/fw/TestClient.cpp create mode 100644 example/fw/TestClient.hpp create mode 100644 example/fw/Tls.cpp create mode 100644 example/fw/Tls.hpp create mode 100644 example/fw/Validate.cpp create mode 100644 example/fw/Validate.hpp create mode 100644 example/fw/View.cpp create mode 100644 example/fw/View.hpp create mode 100644 example/fw/Workflow.cpp create mode 100644 example/fw/Workflow.hpp create mode 100644 example/main.cpp create mode 100644 example/static/app.css create mode 100644 example/static/app.js create mode 100644 plans/11-tenancy.md create mode 100644 plans/12-audit-compliance.md create mode 100644 plans/13-operability.md create mode 100644 plans/14-api-governance.md create mode 100644 plans/15-supply-chain.md create mode 100644 src/app/Makefile create mode 100644 src/app/SPEC.md create mode 100644 src/app/app.mk create mode 100644 src/app/app/application.h create mode 100644 src/app/app/plugin.h create mode 100644 src/app/async/task.h create mode 100644 src/app/caps.h create mode 100644 src/app/di/container.h create mode 100644 src/app/errors/handler.h create mode 100644 src/app/errors/http_error.h create mode 100644 src/app/forms/form.h create mode 100644 src/app/forms/multipart.h create mode 100644 src/app/forms/urlencoded.h create mode 100644 src/app/http_msg/cookies.h create mode 100644 src/app/http_msg/headers.h create mode 100644 src/app/http_msg/html.h create mode 100644 src/app/http_msg/method.h create mode 100644 src/app/http_msg/query.h create mode 100644 src/app/http_msg/request.h create mode 100644 src/app/http_msg/response.h create mode 100644 src/app/middleware/builtins.h create mode 100644 src/app/middleware/chain.h create mode 100644 src/app/middleware/middleware.h create mode 100644 src/app/negotiate/accept.h create mode 100644 src/app/negotiate/negotiator.h create mode 100644 src/app/routing/path_compiler.h create mode 100644 src/app/routing/route.h create mode 100644 src/app/routing/router.h create mode 100644 src/app/seams/request_id.h create mode 100644 src/app/seams/session.h create mode 100644 src/app/templates/ast.h create mode 100644 src/app/templates/cache.h create mode 100644 src/app/templates/context.h create mode 100644 src/app/templates/environment.h create mode 100644 src/app/templates/filters.h create mode 100644 src/app/templates/lexer.h create mode 100644 src/app/templates/loaders.h create mode 100644 src/app/templates/parser.h create mode 100644 src/app/templates/predicates.h create mode 100644 src/app/templates/renderer.h create mode 100644 src/app/templates/response.h create mode 100644 src/app/validate/errors.h create mode 100644 src/app/validate/field.h create mode 100644 src/app/validate/json.h create mode 100644 src/app/validate/model.h create mode 100644 src/app/validate/value.h create mode 100644 src/app/views/adapters.h create mode 100644 src/app/views/flash.h create mode 100644 src/app/views/view.h create mode 100644 src/audit.mk create mode 100644 src/audit/PATCHES.md create mode 100644 src/audit/adapt.h create mode 100644 src/audit/audit.h create mode 100644 src/audit/caps.h create mode 100644 src/audit/log/appender.h create mode 100644 src/audit/log/chain.h create mode 100644 src/audit/log/query.h create mode 100644 src/audit/log/record.h create mode 100644 src/audit/log/sql_store.h create mode 100644 src/audit/log/store.h create mode 100644 src/audit/mask/fixture.h create mode 100644 src/audit/mask/masker.h create mode 100644 src/audit/pii/mark.h create mode 100644 src/audit/pii/registry.h create mode 100644 src/audit/redact/pattern.h create mode 100644 src/audit/redact/redactor.h create mode 100644 src/audit/retention/enforcer.h create mode 100644 src/audit/retention/hold.h create mode 100644 src/audit/retention/policy.h create mode 100644 src/audit/rights/eraser.h create mode 100644 src/audit/rights/exporter.h create mode 100644 src/audit/rights/request.h create mode 100644 src/audit/rights/subject.h create mode 100644 src/cli/ai/ai_client.cpp create mode 100644 src/cli/ai/ai_client.hpp create mode 100644 src/cli/ai/ai_session.cpp create mode 100644 src/cli/ai/ai_session.hpp create mode 100644 src/cli/ai/ai_transport.cpp create mode 100644 src/cli/ai/ai_transport.hpp create mode 100644 src/cli/ai/prompt_pack.cpp create mode 100644 src/cli/ai/prompt_pack.hpp create mode 100644 src/cli/ai/redactor.cpp create mode 100644 src/cli/ai/redactor.hpp create mode 100644 src/cli/ai/socket_transport.cpp create mode 100644 src/cli/ai/socket_transport.hpp create mode 100644 src/cli/ai/sse_stream_parser.cpp create mode 100644 src/cli/ai/sse_stream_parser.hpp create mode 100644 src/cli/caps.hpp create mode 100644 src/cli/cli.mk create mode 100644 src/cli/cli/arg_parser.cpp create mode 100644 src/cli/cli/arg_parser.hpp create mode 100644 src/cli/cli/cli_app.cpp create mode 100644 src/cli/cli/cli_app.hpp create mode 100644 src/cli/cli/command.cpp create mode 100644 src/cli/cli/command.hpp create mode 100644 src/cli/cli/command_tree.cpp create mode 100644 src/cli/cli/command_tree.hpp create mode 100644 src/cli/cli/exit_code.hpp create mode 100644 src/cli/cli/flag.cpp create mode 100644 src/cli/cli/flag.hpp create mode 100644 src/cli/cli/flag_set.cpp create mode 100644 src/cli/cli/flag_set.hpp create mode 100644 src/cli/cli/help_renderer.cpp create mode 100644 src/cli/cli/help_renderer.hpp create mode 100644 src/cli/cli/json_writer.cpp create mode 100644 src/cli/cli/json_writer.hpp create mode 100644 src/cli/cli/migrate_api.cpp create mode 100644 src/cli/cli/migrate_api.hpp create mode 100644 src/cli/cli/output.cpp create mode 100644 src/cli/cli/output.hpp create mode 100644 src/cli/cli/scaffolder.cpp create mode 100644 src/cli/cli/scaffolder.hpp create mode 100644 src/cli/cli/supervisor_api.cpp create mode 100644 src/cli/cli/supervisor_api.hpp create mode 100644 src/cli/cli/supervisor_cli.cpp create mode 100644 src/cli/cli/supervisor_cli.hpp create mode 100644 src/cli/cli/table_writer.cpp create mode 100644 src/cli/cli/table_writer.hpp create mode 100644 src/cli/csp/csp_builder.cpp create mode 100644 src/cli/csp/csp_builder.hpp create mode 100644 src/cli/csp/csp_policy.cpp create mode 100644 src/cli/csp/csp_policy.hpp create mode 100644 src/cli/csp/csp_report_handler.cpp create mode 100644 src/cli/csp/csp_report_handler.hpp create mode 100644 src/cli/csp/nonce.cpp create mode 100644 src/cli/csp/nonce.hpp create mode 100644 src/cli/dev/dev_config.cpp create mode 100644 src/cli/dev/dev_config.hpp create mode 100644 src/cli/dev/dev_server.cpp create mode 100644 src/cli/dev/dev_server.hpp create mode 100644 src/cli/dev/file_watcher.cpp create mode 100644 src/cli/dev/file_watcher.hpp create mode 100644 src/cli/dev/inotify_watcher.cpp create mode 100644 src/cli/dev/inotify_watcher.hpp create mode 100644 src/cli/dev/polling_watcher.cpp create mode 100644 src/cli/dev/polling_watcher.hpp create mode 100644 src/cli/dev/reloader.cpp create mode 100644 src/cli/dev/reloader.hpp create mode 100644 src/cli/embed/asset_scanner.cpp create mode 100644 src/cli/embed/asset_scanner.hpp create mode 100644 src/cli/embed/embed_emitter.cpp create mode 100644 src/cli/embed/embed_emitter.hpp create mode 100644 src/cli/embed/embedded_bundle.cpp create mode 100644 src/cli/embed/embedded_bundle.hpp create mode 100644 src/cli/embed/gzip_stored.cpp create mode 100644 src/cli/embed/gzip_stored.hpp create mode 100644 src/cli/embed/mime.cpp create mode 100644 src/cli/embed/mime.hpp create mode 100644 src/cli/embed/sha256.cpp create mode 100644 src/cli/embed/sha256.hpp create mode 100644 src/cli/i18n/catalog.cpp create mode 100644 src/cli/i18n/catalog.hpp create mode 100644 src/cli/i18n/locale.cpp create mode 100644 src/cli/i18n/locale.hpp create mode 100644 src/cli/i18n/plural_rule.cpp create mode 100644 src/cli/i18n/plural_rule.hpp create mode 100644 src/cli/i18n/po_loader.cpp create mode 100644 src/cli/i18n/po_loader.hpp create mode 100644 src/cli/i18n/translator.cpp create mode 100644 src/cli/i18n/translator.hpp create mode 100644 src/cli/main.cpp create mode 100644 src/cli/ops/health_check.cpp create mode 100644 src/cli/ops/health_check.hpp create mode 100644 src/cli/ops/health_registry.cpp create mode 100644 src/cli/ops/health_registry.hpp create mode 100644 src/cli/ops/liveness_check.cpp create mode 100644 src/cli/ops/liveness_check.hpp create mode 100644 src/cli/ops/ops_routes.cpp create mode 100644 src/cli/ops/ops_routes.hpp create mode 100644 src/cli/ops/otlp_bridge.cpp create mode 100644 src/cli/ops/otlp_bridge.hpp create mode 100644 src/cli/ops/prometheus_exporter.cpp create mode 100644 src/cli/ops/prometheus_exporter.hpp create mode 100644 src/cli/ops/readiness_check.cpp create mode 100644 src/cli/ops/readiness_check.hpp create mode 100644 src/cli/ops/startup_gate.cpp create mode 100644 src/cli/ops/startup_gate.hpp create mode 100644 src/cli/spec.md create mode 100644 src/cli/version.hpp create mode 100644 src/clients/SPEC.md create mode 100644 src/clients/auth.cpp create mode 100644 src/clients/auth.h create mode 100644 src/clients/body/form.cpp create mode 100644 src/clients/body/form.h create mode 100644 src/clients/body/json.cpp create mode 100644 src/clients/body/json.h create mode 100644 src/clients/body/stream.cpp create mode 100644 src/clients/body/stream.h create mode 100644 src/clients/body/xml.cpp create mode 100644 src/clients/body/xml.h create mode 100644 src/clients/caps.h create mode 100644 src/clients/client.cpp create mode 100644 src/clients/client.h create mode 100644 src/clients/client_config.cpp create mode 100644 src/clients/client_config.h create mode 100644 src/clients/clients.mk create mode 100644 src/clients/cookies.cpp create mode 100644 src/clients/cookies.h create mode 100644 src/clients/headers.cpp create mode 100644 src/clients/headers.h create mode 100644 src/clients/openapi/builder.cpp create mode 100644 src/clients/openapi/builder.h create mode 100644 src/clients/openapi/document.cpp create mode 100644 src/clients/openapi/document.h create mode 100644 src/clients/openapi/extensions.cpp create mode 100644 src/clients/openapi/extensions.h create mode 100644 src/clients/openapi/routes.cpp create mode 100644 src/clients/openapi/routes.h create mode 100644 src/clients/openapi/schema.cpp create mode 100644 src/clients/openapi/schema.h create mode 100644 src/clients/pool.cpp create mode 100644 src/clients/pool.h create mode 100644 src/clients/proxy.cpp create mode 100644 src/clients/proxy.h create mode 100644 src/clients/request.cpp create mode 100644 src/clients/request.h create mode 100644 src/clients/response.cpp create mode 100644 src/clients/response.h create mode 100644 src/clients/rest/errors.cpp create mode 100644 src/clients/rest/errors.h create mode 100644 src/clients/rest/idempotency.cpp create mode 100644 src/clients/rest/idempotency.h create mode 100644 src/clients/rest/paginator.cpp create mode 100644 src/clients/rest/paginator.h create mode 100644 src/clients/rest/resource.cpp create mode 100644 src/clients/rest/resource.h create mode 100644 src/clients/retry.cpp create mode 100644 src/clients/retry.h create mode 100644 src/clients/task.h create mode 100644 src/clients/testing/assertions.cpp create mode 100644 src/clients/testing/assertions.h create mode 100644 src/clients/testing/clock.cpp create mode 100644 src/clients/testing/clock.h create mode 100644 src/clients/testing/fixtures.cpp create mode 100644 src/clients/testing/fixtures.h create mode 100644 src/clients/testing/test_client.cpp create mode 100644 src/clients/testing/test_client.h create mode 100644 src/clients/timeouts.h create mode 100644 src/clients/transport.cpp create mode 100644 src/clients/transport.h create mode 100644 src/clients/transports/app_transport.cpp create mode 100644 src/clients/transports/app_transport.h create mode 100644 src/clients/transports/http_transport.cpp create mode 100644 src/clients/transports/http_transport.h create mode 100644 src/clients/transports/mock_transport.cpp create mode 100644 src/clients/transports/mock_transport.h create mode 100644 src/clients/transports/tls.h create mode 100644 src/clients/transports/tls_stub.cpp create mode 100644 src/clients/url.cpp create mode 100644 src/clients/url.h create mode 100644 src/data/Makefile create mode 100644 src/data/cache/cache.hpp create mode 100644 src/data/cache/policy.hpp create mode 100644 src/data/cache/singleflight.hpp create mode 100644 src/data/caps.hpp create mode 100644 src/data/connection.hpp create mode 100644 src/data/context.hpp create mode 100644 src/data/data.mk create mode 100644 src/data/drivers/pg/auth.hpp create mode 100644 src/data/drivers/pg/codecs.hpp create mode 100644 src/data/drivers/pg/conn.hpp create mode 100644 src/data/drivers/pg/extended.hpp create mode 100644 src/data/drivers/pg/message.hpp create mode 100644 src/data/drivers/pg/secret.hpp create mode 100644 src/data/drivers/pg/sha256.hpp create mode 100644 src/data/drivers/sqlite/busy.hpp create mode 100644 src/data/drivers/sqlite/db.hpp create mode 100644 src/data/drivers/sqlite/gate.hpp create mode 100644 src/data/drivers/sqlite/store.hpp create mode 100644 src/data/drivers/sqlite/wal.hpp create mode 100644 src/data/error.hpp create mode 100644 src/data/kv/memory.hpp create mode 100644 src/data/kv/store.hpp create mode 100644 src/data/obs.hpp create mode 100644 src/data/orm/field.hpp create mode 100644 src/data/orm/healer.hpp create mode 100644 src/data/orm/invalidation.hpp create mode 100644 src/data/orm/migration.hpp create mode 100644 src/data/orm/migrator.hpp create mode 100644 src/data/orm/model.hpp create mode 100644 src/data/orm/query.hpp create mode 100644 src/data/orm/relations.hpp create mode 100644 src/data/orm/schema.hpp create mode 100644 src/data/orm/sync.hpp create mode 100644 src/data/orm/types.md create mode 100644 src/data/pool.hpp create mode 100644 src/data/pool_config.hpp create mode 100644 src/data/pubsub/broker.hpp create mode 100644 src/data/pubsub/channel.hpp create mode 100644 src/data/pubsub/subscription.hpp create mode 100644 src/data/result.hpp create mode 100644 src/data/row.hpp create mode 100644 src/data/statement.hpp create mode 100644 src/data/transaction.hpp create mode 100644 src/data/value.hpp create mode 100644 src/governance.mk create mode 100644 src/governance/PATCHES.md create mode 100644 src/governance/adapters/BaseAdapters.h create mode 100644 src/governance/caps.h create mode 100644 src/governance/diff/ApiDiffCli.h create mode 100644 src/governance/diff/BreakingRules.h create mode 100644 src/governance/diff/GovernGate.cpp create mode 100644 src/governance/diff/GovernGate.h create mode 100644 src/governance/diff/SpecDiffer.h create mode 100644 src/governance/diff/SpecSnapshot.h create mode 100644 src/governance/diff/apidiff_main.cpp create mode 100644 src/governance/keys/ApiKey.h create mode 100644 src/governance/keys/KeyAuth.h create mode 100644 src/governance/keys/KeyIssuer.h create mode 100644 src/governance/keys/KeyStore.h create mode 100644 src/governance/keys/MemoryKeyStore.h create mode 100644 src/governance/keys/Scope.h create mode 100644 src/governance/problem.h create mode 100644 src/governance/surface/RouteLinter.h create mode 100644 src/governance/surface/SurfaceRules.h create mode 100644 src/governance/tiers/QuotaLedger.h create mode 100644 src/governance/tiers/TierHeaders.h create mode 100644 src/governance/tiers/TierPolicy.h create mode 100644 src/governance/tiers/TierTable.h create mode 100644 src/governance/versions/ApiVersion.h create mode 100644 src/governance/versions/DeprecationMiddleware.h create mode 100644 src/governance/versions/VersionedRouter.h create mode 100644 src/observability/README.md create mode 100644 src/observability/config.h create mode 100644 src/observability/correlation/correlation.h create mode 100644 src/observability/correlation/redaction.h create mode 100644 src/observability/correlation/request_context.cpp create mode 100644 src/observability/correlation/request_context.h create mode 100644 src/observability/correlation/request_id.cpp create mode 100644 src/observability/correlation/request_id.h create mode 100644 src/observability/events/dispatcher.cpp create mode 100644 src/observability/events/dispatcher.h create mode 100644 src/observability/events/event.h create mode 100644 src/observability/events/lifecycle_events.h create mode 100644 src/observability/events/listener.h create mode 100644 src/observability/hooks.h create mode 100644 src/observability/logging/async_queue.cpp create mode 100644 src/observability/logging/async_queue.h create mode 100644 src/observability/logging/async_sink.h create mode 100644 src/observability/logging/console_sink.h create mode 100644 src/observability/logging/file_sink.cpp create mode 100644 src/observability/logging/file_sink.h create mode 100644 src/observability/logging/json_format.cpp create mode 100644 src/observability/logging/json_format.h create mode 100644 src/observability/logging/level.h create mode 100644 src/observability/logging/logger.cpp create mode 100644 src/observability/logging/logger.h create mode 100644 src/observability/logging/macros.h create mode 100644 src/observability/logging/null_sink.h create mode 100644 src/observability/logging/record.h create mode 100644 src/observability/logging/registry.cpp create mode 100644 src/observability/logging/registry.h create mode 100644 src/observability/logging/ring_sink.h create mode 100644 src/observability/logging/sink.h create mode 100644 src/observability/metrics/counter.h create mode 100644 src/observability/metrics/gauge.h create mode 100644 src/observability/metrics/histogram.h create mode 100644 src/observability/metrics/labels.h create mode 100644 src/observability/metrics/prometheus.cpp create mode 100644 src/observability/metrics/prometheus.h create mode 100644 src/observability/metrics/registry.cpp create mode 100644 src/observability/metrics/registry.h create mode 100644 src/observability/metrics/timer.h create mode 100644 src/observability/observability.h create mode 100644 src/observability/observability.mk create mode 100644 src/observability/tracing/batch_processor.cpp create mode 100644 src/observability/tracing/batch_processor.h create mode 100644 src/observability/tracing/exporter.h create mode 100644 src/observability/tracing/http_sender.h create mode 100644 src/observability/tracing/ids.h create mode 100644 src/observability/tracing/memory_exporter.h create mode 100644 src/observability/tracing/otlp_http.cpp create mode 100644 src/observability/tracing/otlp_http.h create mode 100644 src/observability/tracing/propagator.cpp create mode 100644 src/observability/tracing/propagator.h create mode 100644 src/observability/tracing/sampler.h create mode 100644 src/observability/tracing/span.cpp create mode 100644 src/observability/tracing/span.h create mode 100644 src/observability/tracing/span_context.h create mode 100644 src/observability/tracing/tracer.cpp create mode 100644 src/observability/tracing/tracer.h create mode 100644 src/operability.mk create mode 100644 src/operability/PATCHES.md create mode 100644 src/operability/adapters/include/supervisor/common/clock.hpp create mode 100644 src/operability/adapters/include/supervisor/common/kv_store.hpp create mode 100644 src/operability/adapters/include/supervisor/common/observer.hpp create mode 100644 src/operability/caps.h create mode 100644 src/operability/chaos/experiment.cpp create mode 100644 src/operability/chaos/experiment.h create mode 100644 src/operability/chaos/hook.h create mode 100644 src/operability/chaos/injector.cpp create mode 100644 src/operability/chaos/injector.h create mode 100644 src/operability/coord/dist_lock.cpp create mode 100644 src/operability/coord/dist_lock.h create mode 100644 src/operability/coord/election.cpp create mode 100644 src/operability/coord/election.h create mode 100644 src/operability/coord/fence.h create mode 100644 src/operability/coord/lease.cpp create mode 100644 src/operability/coord/lease.h create mode 100644 src/operability/debug/endpoint.cpp create mode 100644 src/operability/debug/endpoint.h create mode 100644 src/operability/debug/snapshot.cpp create mode 100644 src/operability/debug/snapshot.h create mode 100644 src/operability/deploy/blue_green.cpp create mode 100644 src/operability/deploy/blue_green.h create mode 100644 src/operability/deploy/lifecycle.cpp create mode 100644 src/operability/deploy/lifecycle.h create mode 100644 src/operability/deploy/rolling.cpp create mode 100644 src/operability/deploy/rolling.h create mode 100644 src/operability/deploy/strategy.h create mode 100644 src/operability/resilience/breaker.cpp create mode 100644 src/operability/resilience/breaker.h create mode 100644 src/operability/resilience/bulkhead.cpp create mode 100644 src/operability/resilience/bulkhead.h create mode 100644 src/operability/resilience/deadline.cpp create mode 100644 src/operability/resilience/deadline.h create mode 100644 src/operability/resilience/shedder.cpp create mode 100644 src/operability/resilience/shedder.h create mode 100644 src/operability/resilience/slo.cpp create mode 100644 src/operability/resilience/slo.h create mode 100644 src/realtime/SPEC.md create mode 100644 src/realtime/engineio/engine_session.cpp create mode 100644 src/realtime/engineio/engine_session.h create mode 100644 src/realtime/engineio/heartbeat_scheduler.cpp create mode 100644 src/realtime/engineio/heartbeat_scheduler.h create mode 100644 src/realtime/engineio/packet_codec.cpp create mode 100644 src/realtime/engineio/packet_codec.h create mode 100644 src/realtime/engineio/polling_transport.cpp create mode 100644 src/realtime/engineio/polling_transport.h create mode 100644 src/realtime/engineio/upgrade_manager.cpp create mode 100644 src/realtime/engineio/upgrade_manager.h create mode 100644 src/realtime/engineio/websocket_transport.cpp create mode 100644 src/realtime/engineio/websocket_transport.h create mode 100644 src/realtime/realtime.mk create mode 100644 src/realtime/shared/backpressure_valve.cpp create mode 100644 src/realtime/shared/backpressure_valve.h create mode 100644 src/realtime/shared/base64.cpp create mode 100644 src/realtime/shared/base64.h create mode 100644 src/realtime/shared/base_realtime_client.cpp create mode 100644 src/realtime/shared/base_realtime_client.h create mode 100644 src/realtime/shared/caps.h create mode 100644 src/realtime/shared/connection_context.h create mode 100644 src/realtime/shared/http_seam.cpp create mode 100644 src/realtime/shared/http_seam.h create mode 100644 src/realtime/shared/mini_json.cpp create mode 100644 src/realtime/shared/mini_json.h create mode 100644 src/realtime/shared/random_token.cpp create mode 100644 src/realtime/shared/random_token.h create mode 100644 src/realtime/shared/realtime_log.cpp create mode 100644 src/realtime/shared/realtime_log.h create mode 100644 src/realtime/shared/realtime_metrics.cpp create mode 100644 src/realtime/shared/realtime_metrics.h create mode 100644 src/realtime/shared/reconnect_policy.cpp create mode 100644 src/realtime/shared/reconnect_policy.h create mode 100644 src/realtime/shared/send_queue.cpp create mode 100644 src/realtime/shared/send_queue.h create mode 100644 src/realtime/shared/sha1.cpp create mode 100644 src/realtime/shared/sha1.h create mode 100644 src/realtime/shared/transport.cpp create mode 100644 src/realtime/shared/transport.h create mode 100644 src/realtime/shared/utf8.cpp create mode 100644 src/realtime/shared/utf8.h create mode 100644 src/realtime/socketio/ack_tracker.cpp create mode 100644 src/realtime/socketio/ack_tracker.h create mode 100644 src/realtime/socketio/client/socketio_client.cpp create mode 100644 src/realtime/socketio/client/socketio_client.h create mode 100644 src/realtime/socketio/event_dispatcher.cpp create mode 100644 src/realtime/socketio/event_dispatcher.h create mode 100644 src/realtime/socketio/namespace.cpp create mode 100644 src/realtime/socketio/namespace.h create mode 100644 src/realtime/socketio/packet_codec.cpp create mode 100644 src/realtime/socketio/packet_codec.h create mode 100644 src/realtime/socketio/room_manager.cpp create mode 100644 src/realtime/socketio/room_manager.h create mode 100644 src/realtime/socketio/socketio_server.cpp create mode 100644 src/realtime/socketio/socketio_server.h create mode 100644 src/realtime/sse/event_encoder.cpp create mode 100644 src/realtime/sse/event_encoder.h create mode 100644 src/realtime/sse/event_store.cpp create mode 100644 src/realtime/sse/event_store.h create mode 100644 src/realtime/sse/sse_client.cpp create mode 100644 src/realtime/sse/sse_client.h create mode 100644 src/realtime/sse/sse_response.cpp create mode 100644 src/realtime/sse/sse_response.h create mode 100644 src/realtime/streaming/chunked_body.cpp create mode 100644 src/realtime/streaming/chunked_body.h create mode 100644 src/realtime/streaming/flush_policy.cpp create mode 100644 src/realtime/streaming/flush_policy.h create mode 100644 src/realtime/streaming/passthrough_body.cpp create mode 100644 src/realtime/streaming/passthrough_body.h create mode 100644 src/realtime/webdav/client/webdav_client.cpp create mode 100644 src/realtime/webdav/client/webdav_client.h create mode 100644 src/realtime/webdav/server/copy_move_handler.cpp create mode 100644 src/realtime/webdav/server/copy_move_handler.h create mode 100644 src/realtime/webdav/server/dav_view.cpp create mode 100644 src/realtime/webdav/server/dav_view.h create mode 100644 src/realtime/webdav/server/dav_xml.cpp create mode 100644 src/realtime/webdav/server/dav_xml.h create mode 100644 src/realtime/webdav/server/lock_manager.cpp create mode 100644 src/realtime/webdav/server/lock_manager.h create mode 100644 src/realtime/webdav/server/mkcol_handler.cpp create mode 100644 src/realtime/webdav/server/mkcol_handler.h create mode 100644 src/realtime/webdav/server/multistatus_builder.cpp create mode 100644 src/realtime/webdav/server/multistatus_builder.h create mode 100644 src/realtime/webdav/server/propfind_handler.cpp create mode 100644 src/realtime/webdav/server/propfind_handler.h create mode 100644 src/realtime/webdav/server/proppatch_handler.cpp create mode 100644 src/realtime/webdav/server/proppatch_handler.h create mode 100644 src/realtime/webdav/server/quota_guard.cpp create mode 100644 src/realtime/webdav/server/quota_guard.h create mode 100644 src/realtime/webdav/server/resource_tree.cpp create mode 100644 src/realtime/webdav/server/resource_tree.h create mode 100644 src/realtime/websocket/client/client_handshake.cpp create mode 100644 src/realtime/websocket/client/client_handshake.h create mode 100644 src/realtime/websocket/client/websocket_client.cpp create mode 100644 src/realtime/websocket/client/websocket_client.h create mode 100644 src/realtime/websocket/server/frame_parser.cpp create mode 100644 src/realtime/websocket/server/frame_parser.h create mode 100644 src/realtime/websocket/server/frame_serializer.cpp create mode 100644 src/realtime/websocket/server/frame_serializer.h create mode 100644 src/realtime/websocket/server/handshake_validator.cpp create mode 100644 src/realtime/websocket/server/handshake_validator.h create mode 100644 src/realtime/websocket/server/heartbeat_manager.cpp create mode 100644 src/realtime/websocket/server/heartbeat_manager.h create mode 100644 src/realtime/websocket/server/opcode.h create mode 100644 src/realtime/websocket/server/websocket_handler.cpp create mode 100644 src/realtime/websocket/server/websocket_handler.h create mode 100644 src/realtime/websocket/server/websocket_response.cpp create mode 100644 src/realtime/websocket/server/websocket_response.h create mode 100644 src/realtime/websocket/server/websocket_view.cpp create mode 100644 src/realtime/websocket/server/websocket_view.h create mode 100644 src/runtime/base64.h create mode 100644 src/runtime/caps.h create mode 100644 src/runtime/channel.h create mode 100644 src/runtime/compress/brotli.cpp create mode 100644 src/runtime/compress/brotli.h create mode 100644 src/runtime/compress/compressor.h create mode 100644 src/runtime/compress/gzip.cpp create mode 100644 src/runtime/compress/gzip.h create mode 100644 src/runtime/compress/negotiate.cpp create mode 100644 src/runtime/compress/negotiate.h create mode 100644 src/runtime/cookie/cookie.cpp create mode 100644 src/runtime/cookie/cookie.h create mode 100644 src/runtime/cookie/jar.cpp create mode 100644 src/runtime/cookie/jar.h create mode 100644 src/runtime/cookie/signer.h create mode 100644 src/runtime/event_loop.cpp create mode 100644 src/runtime/event_loop.h create mode 100644 src/runtime/executor.cpp create mode 100644 src/runtime/executor.h create mode 100644 src/runtime/http/connection.cpp create mode 100644 src/runtime/http/connection.h create mode 100644 src/runtime/http/fd_waiter.h create mode 100644 src/runtime/http/listener.cpp create mode 100644 src/runtime/http/listener.h create mode 100644 src/runtime/http/message.cpp create mode 100644 src/runtime/http/message.h create mode 100644 src/runtime/http/protocol_detect.cpp create mode 100644 src/runtime/http/protocol_detect.h create mode 100644 src/runtime/http/v1/parser.cpp create mode 100644 src/runtime/http/v1/parser.h create mode 100644 src/runtime/http/v1/serializer.cpp create mode 100644 src/runtime/http/v1/serializer.h create mode 100644 src/runtime/http/v2/connection.cpp create mode 100644 src/runtime/http/v2/connection.h create mode 100644 src/runtime/http/v2/flow_control.cpp create mode 100644 src/runtime/http/v2/flow_control.h create mode 100644 src/runtime/http/v2/hpack.cpp create mode 100644 src/runtime/http/v2/hpack.h create mode 100644 src/runtime/http/v2/huffman_table.inc create mode 100644 src/runtime/http/v2/settings.h create mode 100644 src/runtime/http/v2/static_table.inc create mode 100644 src/runtime/http/v2/stream.cpp create mode 100644 src/runtime/http/v2/stream.h create mode 100644 src/runtime/observability_hooks.h create mode 100644 src/runtime/poller.h create mode 100644 src/runtime/poller_epoll.cpp create mode 100644 src/runtime/poller_epoll.h create mode 100644 src/runtime/poller_iocp.cpp create mode 100644 src/runtime/poller_iocp.h create mode 100644 src/runtime/poller_kqueue.cpp create mode 100644 src/runtime/poller_kqueue.h create mode 100644 src/runtime/runtime.mk create mode 100644 src/runtime/signal_handler.cpp create mode 100644 src/runtime/signal_handler.h create mode 100644 src/runtime/static/asset_provider.cpp create mode 100644 src/runtime/static/asset_provider.h create mode 100644 src/runtime/static/disk_provider.cpp create mode 100644 src/runtime/static/disk_provider.h create mode 100644 src/runtime/static/embedded_provider.cpp create mode 100644 src/runtime/static/embedded_provider.h create mode 100644 src/runtime/static/etag.cpp create mode 100644 src/runtime/static/etag.h create mode 100644 src/runtime/static/file_server.cpp create mode 100644 src/runtime/static/file_server.h create mode 100644 src/runtime/static/path_guard.cpp create mode 100644 src/runtime/static/path_guard.h create mode 100644 src/runtime/strand.h create mode 100644 src/runtime/task.h create mode 100644 src/runtime/timer_wheel.cpp create mode 100644 src/runtime/timer_wheel.h create mode 100644 src/runtime/tls_handle.h create mode 100644 src/runtime/transport.h create mode 100644 src/runtime/transport_tcp.cpp create mode 100644 src/runtime/transport_tcp.h create mode 100644 src/security/auth/Authenticator.cpp create mode 100644 src/security/auth/Authenticator.h create mode 100644 src/security/auth/Guard.cpp create mode 100644 src/security/auth/Guard.h create mode 100644 src/security/auth/Permission.h create mode 100644 src/security/crypto/AesGcm.cpp create mode 100644 src/security/crypto/AesGcm.h create mode 100644 src/security/crypto/Base64.cpp create mode 100644 src/security/crypto/Base64.h create mode 100644 src/security/crypto/Blake2b.cpp create mode 100644 src/security/crypto/Blake2b.h create mode 100644 src/security/crypto/ChaCha20Poly1305.cpp create mode 100644 src/security/crypto/ChaCha20Poly1305.h create mode 100644 src/security/crypto/CryptoBox.cpp create mode 100644 src/security/crypto/CryptoBox.h create mode 100644 src/security/crypto/Hmac.cpp create mode 100644 src/security/crypto/Hmac.h create mode 100644 src/security/crypto/KeyHierarchy.cpp create mode 100644 src/security/crypto/KeyHierarchy.h create mode 100644 src/security/crypto/PasswordHasher.cpp create mode 100644 src/security/crypto/PasswordHasher.h create mode 100644 src/security/crypto/SecureRandom.cpp create mode 100644 src/security/crypto/SecureRandom.h create mode 100644 src/security/crypto/Sha256.cpp create mode 100644 src/security/crypto/Sha256.h create mode 100644 src/security/guard/BodyLimits.cpp create mode 100644 src/security/guard/BodyLimits.h create mode 100644 src/security/guard/CorsPolicy.cpp create mode 100644 src/security/guard/CorsPolicy.h create mode 100644 src/security/guard/CsrfGuard.cpp create mode 100644 src/security/guard/CsrfGuard.h create mode 100644 src/security/guard/DenialSink.cpp create mode 100644 src/security/guard/DenialSink.h create mode 100644 src/security/guard/HttpMessageView.cpp create mode 100644 src/security/guard/HttpMessageView.h create mode 100644 src/security/guard/PathGuard.cpp create mode 100644 src/security/guard/PathGuard.h create mode 100644 src/security/guard/RateLimiter.cpp create mode 100644 src/security/guard/RateLimiter.h create mode 100644 src/security/guard/SecurityHeaders.cpp create mode 100644 src/security/guard/SecurityHeaders.h create mode 100644 src/security/guard/SqlSafety.cpp create mode 100644 src/security/guard/SqlSafety.h create mode 100644 src/security/secret/SecretsProvider.cpp create mode 100644 src/security/secret/SecretsProvider.h create mode 100644 src/security/security.mk create mode 100644 src/security/session/DbStore.cpp create mode 100644 src/security/session/DbStore.h create mode 100644 src/security/session/MemoryStore.cpp create mode 100644 src/security/session/MemoryStore.h create mode 100644 src/security/session/Session.cpp create mode 100644 src/security/session/Session.h create mode 100644 src/security/session/SessionManager.cpp create mode 100644 src/security/session/SessionManager.h create mode 100644 src/security/session/SessionStore.h create mode 100644 src/security/tls/TlsAcceptor.cpp create mode 100644 src/security/tls/TlsAcceptor.h create mode 100644 src/security/tls/TlsBackend.h create mode 100644 src/security/tls/TlsConfig.cpp create mode 100644 src/security/tls/TlsConfig.h create mode 100644 src/security/tls/TlsContext.cpp create mode 100644 src/security/tls/TlsContext.h create mode 100644 src/security/tls/WolfSslBackend.cpp create mode 100644 src/security/tls/WolfSslBackend.h create mode 100644 src/security/upload/UploadStore.cpp create mode 100644 src/security/upload/UploadStore.h create mode 100644 src/security/upload/UploadValidator.cpp create mode 100644 src/security/upload/UploadValidator.h create mode 100644 src/supervisor/TOPOLOGY.md create mode 100644 src/supervisor/bus/ack_tracker.cpp create mode 100644 src/supervisor/bus/ack_tracker.hpp create mode 100644 src/supervisor/bus/bus_error.hpp create mode 100644 src/supervisor/bus/consumer_cursor.cpp create mode 100644 src/supervisor/bus/consumer_cursor.hpp create mode 100644 src/supervisor/bus/segment_log.cpp create mode 100644 src/supervisor/bus/segment_log.hpp create mode 100644 src/supervisor/bus/stream_bus.cpp create mode 100644 src/supervisor/bus/stream_bus.hpp create mode 100644 src/supervisor/bus/topic_bus.cpp create mode 100644 src/supervisor/bus/topic_bus.hpp create mode 100644 src/supervisor/common/caps.hpp create mode 100644 src/supervisor/common/clock.cpp create mode 100644 src/supervisor/common/clock.hpp create mode 100644 src/supervisor/common/json.cpp create mode 100644 src/supervisor/common/json.hpp create mode 100644 src/supervisor/common/kv_store.cpp create mode 100644 src/supervisor/common/kv_store.hpp create mode 100644 src/supervisor/common/observer.cpp create mode 100644 src/supervisor/common/observer.hpp create mode 100644 src/supervisor/replication/conflict_resolver.cpp create mode 100644 src/supervisor/replication/conflict_resolver.hpp create mode 100644 src/supervisor/replication/hybrid_clock.cpp create mode 100644 src/supervisor/replication/hybrid_clock.hpp create mode 100644 src/supervisor/replication/oplog_entry.cpp create mode 100644 src/supervisor/replication/oplog_entry.hpp create mode 100644 src/supervisor/replication/oplog_reader.cpp create mode 100644 src/supervisor/replication/oplog_reader.hpp create mode 100644 src/supervisor/replication/oplog_writer.cpp create mode 100644 src/supervisor/replication/oplog_writer.hpp create mode 100644 src/supervisor/replication/replicator.cpp create mode 100644 src/supervisor/replication/replicator.hpp create mode 100644 src/supervisor/replication/sync_session.cpp create mode 100644 src/supervisor/replication/sync_session.hpp create mode 100644 src/supervisor/replication/version_vector.cpp create mode 100644 src/supervisor/replication/version_vector.hpp create mode 100644 src/supervisor/supervisor.mk create mode 100644 src/supervisor/supervisor/health_check.cpp create mode 100644 src/supervisor/supervisor/health_check.hpp create mode 100644 src/supervisor/supervisor/restart_policy.cpp create mode 100644 src/supervisor/supervisor/restart_policy.hpp create mode 100644 src/supervisor/supervisor/shutdown_coordinator.cpp create mode 100644 src/supervisor/supervisor/shutdown_coordinator.hpp create mode 100644 src/supervisor/supervisor/supervisor.cpp create mode 100644 src/supervisor/supervisor/supervisor.hpp create mode 100644 src/supervisor/supervisor/supervisor_error.hpp create mode 100644 src/supervisor/supervisor/worker_group.cpp create mode 100644 src/supervisor/supervisor/worker_group.hpp create mode 100644 src/supervisor/supervisor/worker_process.cpp create mode 100644 src/supervisor/supervisor/worker_process.hpp create mode 100644 src/supervisor/supervisor/worker_state.hpp create mode 100644 src/supervisor/tasks/cron.cpp create mode 100644 src/supervisor/tasks/cron.hpp create mode 100644 src/supervisor/tasks/dead_letter_queue.cpp create mode 100644 src/supervisor/tasks/dead_letter_queue.hpp create mode 100644 src/supervisor/tasks/retry_policy.cpp create mode 100644 src/supervisor/tasks/retry_policy.hpp create mode 100644 src/supervisor/tasks/scheduler.cpp create mode 100644 src/supervisor/tasks/scheduler.hpp create mode 100644 src/supervisor/tasks/task_envelope.cpp create mode 100644 src/supervisor/tasks/task_envelope.hpp create mode 100644 src/supervisor/tasks/task_handler.cpp create mode 100644 src/supervisor/tasks/task_handler.hpp create mode 100644 src/supervisor/tasks/task_queue.cpp create mode 100644 src/supervisor/tasks/task_queue.hpp create mode 100644 src/supervisor/tasks/task_registry.cpp create mode 100644 src/supervisor/tasks/task_registry.hpp create mode 100644 src/supplychain.mk create mode 100644 src/supplychain/PATCHES.md create mode 100644 src/supplychain/adapters/cli/cli/arg_parser.hpp create mode 100644 src/supplychain/adapters/cli/cli/exit_code.hpp create mode 100644 src/supplychain/adapters/cli/cli/flag.hpp create mode 100644 src/supplychain/adapters/cli/cli/json_writer.hpp create mode 100644 src/supplychain/adapters/cli/cli/output.hpp create mode 100644 src/supplychain/adapters/cli/embed/sha256.hpp create mode 100644 src/supplychain/adapters/cli/version.hpp create mode 100644 src/supplychain/attest/Attestor.cpp create mode 100644 src/supplychain/attest/Attestor.hpp create mode 100644 src/supplychain/attest/ProvenanceBuilder.hpp create mode 100644 src/supplychain/attest/Signer.cpp create mode 100644 src/supplychain/attest/Signer.hpp create mode 100644 src/supplychain/attest/Verify.cpp create mode 100644 src/supplychain/attest/Verify.hpp create mode 100644 src/supplychain/caps.hpp create mode 100644 src/supplychain/cli/SupplyCli.cpp create mode 100644 src/supplychain/cli/SupplyCli.hpp create mode 100644 src/supplychain/common/Json.cpp create mode 100644 src/supplychain/common/Json.hpp create mode 100644 src/supplychain/policy/DependencyPolicy.hpp create mode 100644 src/supplychain/policy/WolfPin.cpp create mode 100644 src/supplychain/policy/WolfPin.hpp create mode 100644 src/supplychain/sbom/CycloneDxEmitter.cpp create mode 100644 src/supplychain/sbom/CycloneDxEmitter.hpp create mode 100644 src/supplychain/sbom/SbomCheck.cpp create mode 100644 src/supplychain/sbom/SbomCheck.hpp create mode 100644 src/supplychain/sbom/SbomGenerator.cpp create mode 100644 src/supplychain/sbom/SbomGenerator.hpp create mode 100644 src/supplychain/sbom/SpdxEmitter.cpp create mode 100644 src/supplychain/sbom/SpdxEmitter.hpp create mode 100644 src/supplychain/sbom/Types.hpp create mode 100644 src/supplychain/scan/OsvClient.cpp create mode 100644 src/supplychain/scan/OsvClient.hpp create mode 100644 src/supplychain/scan/VulnHook.cpp create mode 100644 src/supplychain/scan/VulnHook.hpp create mode 100644 src/supplychain/spec.md create mode 100644 src/tenancy.mk create mode 100644 src/tenancy/PATCHES.md create mode 100644 src/tenancy/SPEC.md create mode 100644 src/tenancy/adapters/db.h create mode 100644 src/tenancy/adapters/http.h create mode 100644 src/tenancy/adapters/task.h create mode 100644 src/tenancy/bridge.h create mode 100644 src/tenancy/caps.h create mode 100644 src/tenancy/catalog.h create mode 100644 src/tenancy/config_overlay.h create mode 100644 src/tenancy/context.h create mode 100644 src/tenancy/feature_flags.h create mode 100644 src/tenancy/metrics.h create mode 100644 src/tenancy/middleware.h create mode 100644 src/tenancy/orm_scope.h create mode 100644 src/tenancy/pool_factory.h create mode 100644 src/tenancy/provisioner.h create mode 100644 src/tenancy/quota.h create mode 100644 src/tenancy/resolver.h create mode 100644 src/tenancy/rls_policy.h create mode 100644 src/tenancy/store/catalog_store.h create mode 100644 src/tenancy/store/db_catalog.h create mode 100644 src/tenancy/store/memory_catalog.h create mode 100644 src/tenancy/strategy.h create mode 100644 src/tenancy/tenant.h create mode 100644 src/tenancy/tenant_buckets.h create mode 100644 src/tenancy/tenant_id.h create mode 100644 tests/README.md create mode 100644 tests/example/auth/login_test.cpp create mode 100644 tests/example/auth/me_test.cpp create mode 100644 tests/example/cache/lookup_test.cpp create mode 100644 tests/example/chain/deep_test.cpp create mode 100644 tests/example/cli/ai_test.cpp create mode 100644 tests/example/embedded/assets_test.cpp create mode 100644 tests/example/events/fire_test.cpp create mode 100644 tests/example/formats/data_test.cpp create mode 100644 tests/example/forms/contact_test.cpp create mode 100644 tests/example/health_test.cpp create mode 100644 tests/example/index_test.cpp create mode 100644 tests/example/kv/items_test.cpp create mode 100644 tests/example/metrics/counters_test.cpp create mode 100644 tests/example/openapi/docs_test.cpp create mode 100644 tests/example/orm/users_test.cpp create mode 100644 tests/example/pubsub/publish_test.cpp create mode 100644 tests/example/redirect/here_test.cpp create mode 100644 tests/example/replication/state_test.cpp create mode 100644 tests/example/restclient/self_test.cpp create mode 100644 tests/example/routing/hello_test.cpp create mode 100644 tests/example/routing/routers_test.cpp create mode 100644 tests/example/routing/views_test.cpp create mode 100644 tests/example/sessions/counter_test.cpp create mode 100644 tests/example/socketio/rooms_test.cpp create mode 100644 tests/example/sse/events_test.cpp create mode 100644 tests/example/static/files_test.cpp create mode 100644 tests/example/streaming/feed_test.cpp create mode 100644 tests/example/streaming/passthrough_test.cpp create mode 100644 tests/example/supervisor/tasks_test.cpp create mode 100644 tests/example/supervisor/workers_test.cpp create mode 100644 tests/example/support/Assert.hpp create mode 100644 tests/example/telemetry/trace_test.cpp create mode 100644 tests/example/templates/page_test.cpp create mode 100644 tests/example/uploads/file_test.cpp create mode 100644 tests/example/validate/user_test.cpp create mode 100644 tests/example/webdav/files_test.cpp create mode 100644 tests/example/workflow/release_test.cpp create mode 100644 tests/example/ws/chat_test.cpp create mode 100644 tests/src/app/00_compile_all_test.cpp create mode 100644 tests/src/app/_testlib.h create mode 100644 tests/src/app/di/scopes_test.cpp create mode 100644 tests/src/app/errors/render_test.cpp create mode 100644 tests/src/app/forms/bind_test.cpp create mode 100644 tests/src/app/health/ping_test.cpp create mode 100644 tests/src/app/middleware/order_test.cpp create mode 100644 tests/src/app/mount/nested_test.cpp create mode 100644 tests/src/app/negotiate/select_test.cpp create mode 100644 tests/src/app/plugins/hooks_test.cpp create mode 100644 tests/src/app/templates/render_test.cpp create mode 100644 tests/src/app/users/detail_test.cpp create mode 100644 tests/src/app/validate/input_test.cpp create mode 100644 tests/src/app/views/dispatch_test.cpp create mode 100644 tests/src/audit/chain_tamper.cpp create mode 100644 tests/src/audit/erasure_cascade.cpp create mode 100644 tests/src/audit/export_bundle.cpp create mode 100644 tests/src/audit/mask_fixture.cpp create mode 100644 tests/src/audit/orm_hook.cpp create mode 100644 tests/src/audit/pii_boot.cpp create mode 100644 tests/src/audit/query_guard.cpp create mode 100644 tests/src/audit/redactor.cpp create mode 100644 tests/src/audit/retention_hold.cpp create mode 100644 tests/src/audit/sql_store.cpp create mode 100644 tests/src/cli/test_ai.cpp create mode 100644 tests/src/cli/test_cli.cpp create mode 100755 tests/src/cli/test_cli_binary.sh create mode 100644 tests/src/cli/test_csp.cpp create mode 100644 tests/src/cli/test_dev.cpp create mode 100644 tests/src/cli/test_embed.cpp create mode 100644 tests/src/cli/test_healthz.cpp create mode 100644 tests/src/cli/test_i18n.cpp create mode 100644 tests/src/cli/test_metrics.cpp create mode 100644 tests/src/cli/test_readyz.cpp create mode 100644 tests/src/cli/test_startupz.cpp create mode 100644 tests/src/clients/framework.h create mode 100644 tests/src/clients/main.cpp create mode 100644 tests/src/clients/test_bodies.cpp create mode 100644 tests/src/clients/test_client_mock.cpp create mode 100644 tests/src/clients/test_http_transport.cpp create mode 100644 tests/src/clients/test_openapi.cpp create mode 100644 tests/src/clients/test_proxy.cpp create mode 100644 tests/src/clients/test_rest.cpp create mode 100644 tests/src/clients/test_testing.cpp create mode 100644 tests/src/clients/test_url.cpp create mode 100644 tests/src/data/mini.hpp create mode 100644 tests/src/data/test_kv_cache.cpp create mode 100644 tests/src/data/test_orm.cpp create mode 100644 tests/src/data/test_pg_wire.cpp create mode 100644 tests/src/data/test_pool_txn.cpp create mode 100644 tests/src/data/test_pubsub.cpp create mode 100644 tests/src/data/test_sqlite.cpp create mode 100644 tests/src/data/test_sync_heal_migrate.cpp create mode 100644 tests/src/data/test_value_error.cpp create mode 100644 tests/src/governance/deprecation_headers_test.cpp create mode 100644 tests/src/governance/differ_vectors_test.cpp create mode 100644 tests/src/governance/gate_test.cpp create mode 100644 tests/src/governance/keyauth_http_test.cpp create mode 100644 tests/src/governance/keys_issuer_test.cpp create mode 100644 tests/src/governance/keys_scope_test.cpp create mode 100644 tests/src/governance/surface_lint_test.cpp create mode 100644 tests/src/governance/tiers_test.cpp create mode 100644 tests/src/governance/versions_test.cpp create mode 100644 tests/src/observability/correlation_test.cpp create mode 100644 tests/src/observability/events_test.cpp create mode 100644 tests/src/observability/logging_test.cpp create mode 100644 tests/src/observability/metrics_endpoint_test.cpp create mode 100644 tests/src/observability/otlp_export_test.cpp create mode 100644 tests/src/observability/propagation_test.cpp create mode 100644 tests/src/observability/test_helper.h create mode 100644 tests/src/observability/tracing_test.cpp create mode 100644 tests/src/observability/zero_cost_probe.cpp create mode 100644 tests/src/operability/main.cpp create mode 100644 tests/src/operability/test_breaker_bulkhead.cpp create mode 100644 tests/src/operability/test_chaos.cpp create mode 100644 tests/src/operability/test_coord.cpp create mode 100644 tests/src/operability/test_debug.cpp create mode 100644 tests/src/operability/test_deploy.cpp create mode 100644 tests/src/operability/test_lifecycle.cpp create mode 100644 tests/src/operability/test_shed_deadline.cpp create mode 100644 tests/src/operability/test_slo.cpp create mode 100644 tests/src/realtime/dav/files_test.cpp create mode 100644 tests/src/realtime/engineio/handshake_test.cpp create mode 100644 tests/src/realtime/shared/foundation_test.cpp create mode 100644 tests/src/realtime/shared/robustness_test.cpp create mode 100644 tests/src/realtime/socketio/chat_test.cpp create mode 100644 tests/src/realtime/sse/client_test.cpp create mode 100644 tests/src/realtime/sse/feed_test.cpp create mode 100644 tests/src/realtime/stream/passthrough_test.cpp create mode 100644 tests/src/realtime/test_main.h create mode 100644 tests/src/realtime/ws/echo_test.cpp create mode 100644 tests/src/realtime/ws/frames_test.cpp create mode 100644 tests/src/realtime/ws/handshake_test.cpp create mode 100644 tests/src/realtime/ws/heartbeat_test.cpp create mode 100644 tests/src/runtime/compress/test_brotli.cpp create mode 100644 tests/src/runtime/compress/test_gzip.cpp create mode 100644 tests/src/runtime/compress/test_negotiate.cpp create mode 100644 tests/src/runtime/cookie/test_cookie.cpp create mode 100644 tests/src/runtime/fixtures/www/.hidden create mode 100644 tests/src/runtime/fixtures/www/hello.txt create mode 100644 tests/src/runtime/fixtures/www/index.html create mode 100644 tests/src/runtime/fixtures/www/sub/file.txt create mode 100644 tests/src/runtime/http/test_integration.cpp create mode 100644 tests/src/runtime/http/test_message.cpp create mode 100644 tests/src/runtime/http/test_protocol_detect.cpp create mode 100644 tests/src/runtime/http/v1/test_parser.cpp create mode 100644 tests/src/runtime/http/v1/test_serializer.cpp create mode 100644 tests/src/runtime/http/v2/test_flow.cpp create mode 100644 tests/src/runtime/http/v2/test_h2connection.cpp create mode 100644 tests/src/runtime/http/v2/test_hpack.cpp create mode 100644 tests/src/runtime/http/v2/test_settings.cpp create mode 100644 tests/src/runtime/http/v2/test_stream.cpp create mode 100644 tests/src/runtime/interop/interop.py create mode 100644 tests/src/runtime/interop/test_server.cpp create mode 100644 tests/src/runtime/interop/www/hello.txt create mode 100644 tests/src/runtime/main.cpp create mode 100644 tests/src/runtime/static/test_file_server.cpp create mode 100644 tests/src/runtime/static/test_path_guard.cpp create mode 100644 tests/src/runtime/test_base64.cpp create mode 100644 tests/src/runtime/test_channel_strand.cpp create mode 100644 tests/src/runtime/test_event_loop.cpp create mode 100644 tests/src/runtime/test_executor.cpp create mode 100644 tests/src/runtime/test_framework.h create mode 100644 tests/src/runtime/test_task.cpp create mode 100644 tests/src/runtime/test_timer.cpp create mode 100644 tests/src/security/auth/auth_test.cpp create mode 100644 tests/src/security/crypto/crypto_test.cpp create mode 100644 tests/src/security/guard/guard_test.cpp create mode 100644 tests/src/security/secret/secrets_test.cpp create mode 100644 tests/src/security/session/session_test.cpp create mode 100644 tests/src/security/tls/tls_test.cpp create mode 100644 tests/src/security/upload/upload_test.cpp create mode 100644 tests/src/supervisor/test_bus.cpp create mode 100644 tests/src/supervisor/test_durability.cpp create mode 100644 tests/src/supervisor/test_multi_instance.cpp create mode 100644 tests/src/supervisor/test_replication.cpp create mode 100644 tests/src/supervisor/test_scheduler.cpp create mode 100644 tests/src/supervisor/test_supervisor.cpp create mode 100644 tests/src/supervisor/test_tasks.cpp create mode 100644 tests/src/supplychain/golden/advisories.json create mode 100644 tests/src/supplychain/golden/provenance.json create mode 100644 tests/src/supplychain/golden/sbom.cdx.json create mode 100644 tests/src/supplychain/golden/sbom.spdx.json create mode 100644 tests/src/supplychain/test_cdx_equiv.cpp create mode 100644 tests/src/supplychain/test_pin.cpp create mode 100644 tests/src/supplychain/test_provenance.cpp create mode 100644 tests/src/supplychain/test_scan.cpp create mode 100644 tests/src/supplychain/test_spdx.cpp create mode 100644 tests/src/supplychain/test_verify.cpp create mode 100644 tests/src/tenancy/mini.hpp create mode 100644 tests/src/tenancy/test_bridge.cpp create mode 100644 tests/src/tenancy/test_flags_config.cpp create mode 100644 tests/src/tenancy/test_metrics.cpp create mode 100644 tests/src/tenancy/test_pool_scope.cpp create mode 100644 tests/src/tenancy/test_provision.cpp create mode 100644 tests/src/tenancy/test_quota.cpp create mode 100644 tests/src/tenancy/test_tenant_id_resolve.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..79b039d --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +build/ +*.o +*.a diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a4fdd0c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,91 @@ +# AGENTS.md (DRAFT, owned by plan 10-workflow-example.md) + +Normative agent workflow for MoloApp. Human quickstart lives in README.md; +this file is the single entry point for autonomous work. Status: draft until +the root Makefile and CLI land (09-cli-ops.md); per-part commands below are +live today via `example/example.mk`. + +## Setup commands + +```sh +g++ --version # expect g++ 15.2 with -std=c++23 +make --version # GNU or BSD make +``` + +Expected: both print versions. Failure action: stop, page a human. + +## Build + +```sh +make # whole repo (root Makefile includes per-part .mk files) +make -f example/example.mk example # this part only, standalone +``` + +Expected: warning-free build, outputs only under `build/`. Failure action: +fix at root cause, never silence warnings. + +## Test + +```sh +make test # whole repo +make -f example/example.mk example-test # this part only, standalone +``` + +Expected: `example suite: 37 passed, 0 failed`. Failure action: reproduce +through the real stack first (failing route-level test), then fix; the +reproduction stays in the suite. Conventions: `tests/README.md`. + +## Lint + +```sh +make lint # configured gate (falls back to -Wall -Wextra clean build) +``` + +Expected: zero findings. Failure action: fix, never skip the gate. + +## Docs update + +After behavior changes, in the same session: spec delta first, then code, +then example route, then tests, then changelog entry. Docs state what the +system does today; gaps list the owning file next to the code. + +## Version bump + +Never hand-chosen. From commits since the last tag: any `!` or +`BREAKING CHANGE` -> MAJOR; else any `feat` -> MINOR; else any `fix` -> +PATCH; else no release. `VERSION` plus annotated tag `vMAJOR.MINOR.PATCH` +update atomically with the changelog entry. Demo: `POST /workflow/release`. + +## Commit + +Format: `[(scope)][!]: `, subject max 100 chars. +Allowed types: feat, fix, docs, style, refactor, perf, test, build, ci, +chore. Scope is the area slug: runtime, security, data, routing, +realtime, supervisor, clients, cli, workflow. Malformed commits are +rejected before version analysis. + +## Tag + +Tagging requires a green suite plus live verification after deploy, same +session. Failed verification blocks the tag. Max 1 release per green run. + +## Deploy + +```sh +make deploy # defined by 09-cli-ops.md; loopback single binary for v1 +``` + +## Verify live + +After deploy: health probe, metrics scrape, example index load on a 360px +viewport, click the matrix top 10 routes. Any red step halts the pipeline; +report, never force-push past red. + +## Floors and escalation + +Never cross alone: `specs/`, `CHANGELOG.md`, `VERSION`, tags, retention +windows, destructive file deletes, history rewrite, force push, hard +reset. On a floor hit: dry-run, stop, page a human. Pipeline order is +lint -> suite -> bump -> changelog -> commit -> tag -> deploy -> verify; +timeout 30 min, then escalate. On code-vs-spec conflict the spec wins +until formally amended in the same change. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c9903a4 --- /dev/null +++ b/Makefile @@ -0,0 +1,138 @@ +# MoloApp root Makefile. Aggregates the per-part subdir fragments: +# each part ships src//.mk (plus example/example.mk). +# Targets: all, test, clean, interop, lint. +# +# Composition rules used here: +# - Double-colon all::/test::/clean:: so supervisor.mk and example.mk can +# append their own branches without editing this file. +# - Fragments are included byte-exact from the parts; integration-only +# wiring (missing rules, missing pattern edges) lives below the includes +# and is marked INTEGRATION. + +CXX ?= g++ +CXXFLAGS ?= -std=c++23 -O2 -Wall -Wextra -pthread +AR ?= ar + +# Consumed by src/runtime/runtime.mk (verified runtime flags). +MOLO_CXXFLAGS ?= -std=c++23 -O1 -g -Wall -Wextra -Werror -Isrc +MOLO_ALL = +MOLO_TEST_BINS = +MOLO_INTEROP = + +# Default goal is `all`: these are the first ordinary targets in this file. +all:: +test:: +clean:: + rm -rf build + +# Operability first: supervisor.mk and cli.mk reference OPERABILITY_LIB. +-include src/operability.mk +-include src/runtime/runtime.mk +-include src/observability/observability.mk +-include src/security/security.mk +-include src/data/data.mk +-include src/app/app.mk +-include src/realtime/realtime.mk +-include src/supervisor/supervisor.mk +-include src/clients/clients.mk +# NOTE: supplychain.mk and operability.mk precede cli.mk and supervisor.mk +# so SUPPLY_LIB / OPERABILITY_LIB are defined when those add link edges +# (supply PATCHES section 2, operability PATCHES sections 5-6 and 10). +-include src/supplychain.mk +-include src/cli/cli.mk +-include src/tenancy.mk +-include src/audit.mk +-include src/governance.mk +-include example/example.mk + +# --- all wiring (INTEGRATION; supervisor/example self-append via all::) --- +all:: $(MOLO_ALL) +all:: $(OBS_LIB) +all:: security-all +all:: data-all +all:: app-build +all:: $(REALTIME_LIB) +all:: clients +all:: $(MOLO_BIN) +all:: tenancy-tests +all:: audit-tests +all:: operability-tests +all:: govern-build +all:: supply-all + +# --- test wiring (INTEGRATION; supervisor/example self-append via test::) -- +test:: $(MOLO_TEST_BINS) + set -e; for t in $(MOLO_TEST_BINS); do ./$$t; done +test:: obs-test +test:: security-test +test:: data-test +test:: app-test +test:: realtime-test +test:: clients-test +test:: cli-test +test:: tenancy-check +test:: audit-check +test:: operability-check +test:: govern-test +test:: supply-test + +# --- data rules (INTEGRATION: data.mk ships variables only, no rules) ------ +# Mirrors the verified standalone flags (src/data/Makefile): header-only +# lib, one binary per suite, -Isrc/data -Itests/src/data. +DATA_BINS = $(addprefix build/data/test_,$(DATA_TESTS)) + +data-all: $(DATA_BINS) + +build/data/test_%: tests/src/data/test_%.cpp $(DATA_HEADERS) tests/src/data/mini.hpp + @mkdir -p build/data + $(CXX) $(CXXFLAGS) -Isrc/data -Itests/src/data $< -o $@ + +data-test: data-all + @for t in $(DATA_BINS); do echo "== $$t"; ./$$t || exit 1; done + @echo "data: all suites green" + +data-clean: + rm -rf build/data + +# --- realtime test runner (INTEGRATION: realtime.mk ships bins, no runner) -- +realtime-test: $(REALTIME_TESTS) + @pass=0; fail=0; \ + for t in $(REALTIME_TESTS); do \ + if ./$$t; then pass=$$((pass+1)); else fail=$$((fail+1)); \ + echo "FAIL: $$t"; fi; \ + done; \ + echo "realtime tests passed=$$pass failed=$$fail"; \ + test $$fail -eq 0 + +# --- cli main object (INTEGRATION: cli.mk's build/obj/%.o pattern cannot ---- +# derive build/obj/src/cli/main.cpp.o from src/cli/main.cpp, since the stem +# would require a nonexistent src/cli/main.cpp.cpp. Explicit edge fixes it; +# `make -f src/cli/cli.mk -n build/obj/src/cli/main.cpp.o` fails without it. +$(CLI_MAIN_OBJ): src/cli/main.cpp + @mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) $(CLI_INC) -c $< -o $@ + +# --- interop (runtime loopback servers; kept out of `test` per the --------- +# runtime author's own split between `test` and `interop`) ------------------ +interop: $(MOLO_ALL) + set -e; for t in $(MOLO_INTEROP); do $(MAKE) $$t; done + +# --- lint (referenced by AGENTS.md; falls back to the warning-free build) -- +lint: all + @if grep -rl --exclude-dir=build "$$(printf '\342\200\224')" src tests example Makefile 2>/dev/null; then \ + echo "lint: em dash found"; exit 1; \ + else echo "lint: no em dash, build warning-free"; fi + +# --- audit test runner (INTEGRATION: audit.mk builds bins, no runner) --------- +audit-check: audit-tests + @for t in $(AUDIT_TEST_BINS); do echo "== $$t"; ./$$t || exit 1; done + @echo "audit: all suites green" + +# --- operability production link (INTEGRATION: with the base tree present --- +# the real supervisor/common Clock/KvStore/Observer satisfy the includes, so +# their TU definitions ride along on the test link lines). +OPERABILITY_TEST_SRCS += src/supervisor/common/clock.cpp +OPERABILITY_TEST_SRCS += src/supervisor/common/kv_store.cpp +OPERABILITY_TEST_SRCS += src/supervisor/common/observer.cpp + +.PHONY: all test clean interop lint data-all data-test data-clean realtime-test cli-test audit-check diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1085660 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +name: moloapp +services: + postgres: + image: postgres:18 + container_name: moloapp-postgres + restart: unless-stopped + environment: + POSTGRES_USER: molo + POSTGRES_PASSWORD: molo + POSTGRES_DB: molo_test + ports: + - "127.0.0.1:5432:5432" + volumes: + - moloapp-pgdata:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U molo -d molo_test"] + interval: 5s + timeout: 3s + retries: 12 +volumes: + moloapp-pgdata: diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..181db8a --- /dev/null +++ b/example/README.md @@ -0,0 +1,93 @@ +# Living example + +Big living example project proving every MoloApp feature end to end. +Plan: `plans/10-workflow-example.md`, section 6.4. + +Run it: + +```sh +make -f example/example.mk example +./build/example/living-server 8080 +``` + +Open `http://127.0.0.1:8080/` and click through the matrix. Narrow the +window to 360px: the top nav collapses into a dropdown menu. + +Run the suite (37 route-level integration tests, real stack, no mocks): + +```sh +make -f example/example.mk example-test +``` + +## Framework status + +The framework is skeletal: `example/fw/` holds a compiling skeleton of the +public API (request, response hierarchy, router, application, views, +validation, forms, negotiation, templates, middleware, DI-shaped services, +errors, cookies, static, TLS stub, KV, cache, pubsub, events, metrics, +telemetry, sessions, auth, ORM stub, uploads, WebDAV, supervisor, tasks, +replication, REST client, OpenAPI, AI harness sample, release pipeline, +test client, loopback server). Gaps are marked in `example/SPEC.md` and in +each header. As siblings 01 through 09 land, this app re-binds to the real +namespaces route by route; the URLs, tests, and matrix stay stable. + +## Coverage matrix + +Feature -> route -> test file -> plan link. One row per feature; CI fails +on a missing row. + +| Feature | Route | Test file | Plan | +|---|---|---|---| +| index | `/` | `tests/example/index_test.cpp` | `plans/10-workflow-example.md` 6.4 | +| health probe | `/health` | `tests/example/health_test.cpp` | `plans/09-cli-ops.md` health | +| routing | `/routing/hello/{name}` | `tests/example/routing/hello_test.cpp` | `plans/05-request-routing.md` | +| class views | `/routing/views/greet` | `tests/example/routing/views_test.cpp` | `plans/05-request-routing.md` | +| routers | `/routing/routers/nested/deep` | `tests/example/routing/routers_test.cpp` | `plans/05-request-routing.md` | +| app chaining | `/app1/app1/app1/whoami` | `tests/example/chain/deep_test.cpp` | `plans/05-request-routing.md` | +| orm sqlite | `/orm/users?backend=sqlite` | `tests/example/orm/users_test.cpp` | `plans/04-data.md` | +| orm postgres | `/orm/users?backend=postgres` | `tests/example/orm/users_test.cpp` | `plans/04-data.md` | +| basic auth | `/auth/me` | `tests/example/auth/me_test.cpp` | `plans/03-security.md` | +| login | `/auth/login` | `tests/example/auth/login_test.cpp` | `plans/03-security.md` | +| sessions | `/sessions/counter` | `tests/example/sessions/counter_test.cpp` | `plans/03-security.md` | +| uploads | `/uploads/file` | `tests/example/uploads/file_test.cpp` | `plans/03-security.md` | +| forms | `/forms/contact` | `tests/example/forms/contact_test.cpp` | `plans/05-request-routing.md` | +| validation | `/validate/user` | `tests/example/validate/user_test.cpp` | `plans/05-request-routing.md` | +| templates | `/templates/page` | `tests/example/templates/page_test.cpp` | `plans/05-request-routing.md` | +| formats | `/formats/data` | `tests/example/formats/data_test.cpp` | `plans/05-request-routing.md` | +| static disk | `/static/{name}` | `tests/example/static/files_test.cpp` | `plans/01-runtime.md` | +| embedded | `/embedded/{name}` | `tests/example/embedded/assets_test.cpp` | `plans/09-cli-ops.md` | +| streaming | `/streaming/feed` | `tests/example/streaming/feed_test.cpp` | `plans/06-realtime.md` | +| passthrough | `/streaming/passthrough` | `tests/example/streaming/passthrough_test.cpp` | `plans/06-realtime.md` | +| websocket | `/ws/chat` | `tests/example/ws/chat_test.cpp` | `plans/06-realtime.md` | +| socket.io | `/socketio/rooms` | `tests/example/socketio/rooms_test.cpp` | `plans/06-realtime.md` | +| sse | `/sse/events` | `tests/example/sse/events_test.cpp` | `plans/06-realtime.md` | +| webdav | `/webdav/files/` | `tests/example/webdav/files_test.cpp` | `plans/06-realtime.md` | +| pubsub | `/pubsub/publish` | `tests/example/pubsub/publish_test.cpp` | `plans/04-data.md` | +| events | `/events/fire` | `tests/example/events/fire_test.cpp` | `plans/02-observability.md` | +| metrics | `/metrics/counters` | `tests/example/metrics/counters_test.cpp` | `plans/02-observability.md` | +| telemetry | `/telemetry/trace` | `tests/example/telemetry/trace_test.cpp` | `plans/02-observability.md` | +| kv store | `/kv/items` | `tests/example/kv/items_test.cpp` | `plans/04-data.md` | +| cache | `/cache/lookup` | `tests/example/cache/lookup_test.cpp` | `plans/04-data.md` | +| tasks | `/supervisor/tasks` | `tests/example/supervisor/tasks_test.cpp` | `plans/07-supervisor-replication.md` | +| workers | `/supervisor/workers` | `tests/example/supervisor/workers_test.cpp` | `plans/07-supervisor-replication.md` | +| replication | `/replication/state` | `tests/example/replication/state_test.cpp` | `plans/07-supervisor-replication.md` | +| rest client | `/restclient/self` | `tests/example/restclient/self_test.cpp` | `plans/08-clients-openapi.md` | +| openapi | `/openapi.json` | `tests/example/openapi/docs_test.cpp` | `plans/08-clients-openapi.md` | +| docs page | `/docs` | `tests/example/openapi/docs_test.cpp` | `plans/08-clients-openapi.md` | +| ai harness | `/cli/ai` | `tests/example/cli/ai_test.cpp` | `plans/09-cli-ops.md` | +| workflow | `/workflow/release` | `tests/example/workflow/release_test.cpp` | `plans/10-workflow-example.md` 6.2 | +| redirects | `/redirect/here` | `tests/example/redirect/here_test.cpp` | `plans/05-request-routing.md` | + +## Layout + +```text +example/ + example.mk # per-part make include (build, test, clean) + SPEC.md # mini-spec: behaviors, caps, formats, errors, gaps + README.md # this file + main.cpp # loopback server entry + fw/ # skeletal framework public API (namespace molo::example::fw) + app/ # demo routes (namespace molo::example::app) + static/ # disk static assets (responsive CSS, tiny JS) +tests/example/ # route-level integration tests mirroring URLs +``` diff --git a/example/SPEC.md b/example/SPEC.md new file mode 100644 index 0000000..43b4875 --- /dev/null +++ b/example/SPEC.md @@ -0,0 +1,85 @@ +# SPEC: living example (plan 10-workflow-example.md, section 6.4) + +Status: accepted skeleton, 2026-10-23. Follows `plans/10-workflow-example.md` +rules 1-10 to the extent the framework exists; every gap below names the +owning sibling plan and the file that will close it. + +## Behaviors + +1. `GET /` renders the coverage matrix from a template with inheritance. +2. Every matrix row resolves to a live route, a test file, and a plan link. +3. Every test drives the real stack: TestClient -> middleware -> router -> + view -> in-memory services, plus one real loopback HTTP round-trip. +4. Validation failures list ALL field failures with status 422. +5. Unknown routes answer 404 JSON; wrong methods answer 405 or 404. +6. Over-cap request bodies answer 413; unnegotiated formats answer 406. +7. Sessions persist across requests via signed cookies in the test jar. +8. ORM endpoints self-heal: unknown tables and columns are created inline + and the response reports what healed. +9. The release demo parses Conventional Commits, maps the bump, renders a + changelog entry, and gates on checks (dry-run: never writes). +10. All pages render on a 360px viewport; wide nav collapses to a dropdown. + +## Caps + +All ceilings live as literals in `example/fw/Caps.hpp`: 1 MiB request body, +100 headers, 200 form fields, 32 JSON depth, 8 MiB upload, 4096 routes, +8 mount depth, 100k KV entries, 10k cache entries, 1000 pubsub topics, +1000 metrics, 10k spans, 100k sessions, 100k ORM rows, 10k tasks, +64 workers, 1000 stream chunks, 64 listen backlog, 30 s socket timeout, +512 connections, 600 s suite cap, 30 s single-test cap, 1 release per +green run, 20 changelog lines, 100-char commit subjects, 30 min pipeline +timeout. + +## Formats + +- JSON errors: `{"error":"","message":""}`. +- Validation errors: `{"error":"validation_error","failures":[{"field", + "message"}]}`. +- Prometheus: one ` ` line per metric. +- SSE: `data: ` frames separated by blank lines. +- WebDAV PROPFIND: `207` with `D:multistatus` XML. +- OpenAPI: `{"openapi":"3.0.0","info":{...},"paths":{...}}`. + +## Errors + +`HttpError(status, code, message)` maps to the status with a JSON body. +`ValidationError` maps to 422. Unmapped exceptions become 500 JSON without +stack details. Route table: 301/302 redirects, 400 bad input, 401 missing +auth, 403 bad credentials, 404 unknown, 405 wrong method, 406 no format, +413 over cap, 415 blocked upload type, 422 validation, 500 internal. + +## Isolation + +May depend on: C++23 stdlib, POSIX sockets, `example/fw/` only. May NOT +depend on: any `src/` tree (does not exist yet in this worktree), wolfSSL +(stub behind `ITlsContext`), real Postgres/SQLite (in-memory doubles), +real network except loopback. Namespaces mirror directories: +`molo::example::fw` and `molo::example::app`. + +## Gaps + +- Real namespaces (`molo::app`, `molo::routing`, ...) replace `fw/` + skeletons as siblings 01-09 land. Owner: each sibling plan. +- Coroutine handlers, event loop, HTTP/2, compression, keep-alive: + owner 01-runtime.md, file `example/fw/Server.*`. +- Real log format, OTel/Prometheus export: owner 02-observability.md, + file `example/fw/Observ.*`. +- wolfSSL TLS, Argon2id, AES-GCM, CSRF, CORS, rate limits: + owner 03-security.md, files `example/fw/Tls.*`, `example/fw/Cookies.*`. +- Postgres wire client, SQLite handling, pooling, transactions: + owner 04-data.md, file `example/fw/Data.*`. +- Full validation coercions, template sandbox, DI container, flash + messages: owner 05-request-routing.md, files `example/fw/Validate.*`, + `example/fw/Templates.*`, `example/app/App.*`. +- Real WS/Socket.IO/SSE framing, rooms, namespaces, realtime clients: + owner 06-realtime.md, file `example/app/RealtimeDemo.*`. +- Worker processes, durable streams, conflict hooks, multi-instance: + owner 07-supervisor-replication.md, file `example/fw/Jobs.*`. +- Real HTTP client, test fixtures, time freezing: owner + 08-clients-openapi.md, files `example/fw/Net.*`, `example/fw/TestClient.*`. +- Root Makefile, CLI, dev server, embedder codegen, AI CLI loop: + owner 09-cli-ops.md, files `example/example.mk`, `example/app/Embedded.*`. +- `specs/`, root `AGENTS.md` final, `CHANGELOG.md`, `VERSION`, tags: + shared floors, see `/AGENTS.md` draft; this part must not write them + alone except the drafts it owns. diff --git a/example/app/App.cpp b/example/app/App.cpp new file mode 100644 index 0000000..6b54542 --- /dev/null +++ b/example/app/App.cpp @@ -0,0 +1,53 @@ +#include "App.hpp" + +namespace molo::example::app { + +std::shared_ptr build_application(Services& services) { + seed_services(services); + load_templates(services); + load_embedded(services); + + auto app = std::make_shared(); + app->use(fw::MiddlewareSlot::Errors, fw::errors_middleware()); + app->use(fw::MiddlewareSlot::Observability, fw::request_id_middleware()); + app->use(fw::MiddlewareSlot::SessionAuth, + [](fw::Request req, fw::Next next) -> fw::Response { + fw::Response res = next(std::move(req)); + res.headers.set("X-Frame-Options", "DENY"); + res.headers.set("X-Content-Type-Options", "nosniff"); + return res; + }); + + fw::Router index; + register_index(index, services); + app->include_router("", index); + + fw::Router routing; + register_routing_demo(routing, services); + app->include_router("/routing", routing, "routing."); + + fw::Router data; + register_data_demo(data, services); + app->include_router("", data); + + fw::Router security; + register_security_demo(security, services); + app->include_router("", security); + + fw::Router formats; + register_formats_demo(formats, services); + app->include_router("", formats); + + fw::Router realtime; + register_realtime_demo(realtime, services); + app->include_router("", realtime); + + fw::Router ops; + register_ops_demo(ops, services); + app->include_router("", ops); + + register_chain_demo(app, services); + return app; +} + +} // namespace molo::example::app diff --git a/example/app/App.hpp b/example/app/App.hpp new file mode 100644 index 0000000..0c9e2a9 --- /dev/null +++ b/example/app/App.hpp @@ -0,0 +1,69 @@ +#pragma once + +// Demo application assembly: shared Services (the DI skeleton), the root +// Application with middleware, routers, and mounted sub-applications. +// Gap: real molo::di::Container and plugin registry in 05-request-routing.md. + +#include +#include +#include + +#include "../fw/App.hpp" +#include "../fw/Data.hpp" +#include "../fw/Jobs.hpp" +#include "../fw/Kv.hpp" +#include "../fw/Net.hpp" +#include "../fw/Observ.hpp" +#include "../fw/Templates.hpp" + +namespace molo::example::app { + +struct Services { + std::shared_ptr kv = std::make_shared(); + std::shared_ptr cache = std::make_shared(); + std::shared_ptr pubsub = std::make_shared(); + std::shared_ptr events = std::make_shared(); + std::shared_ptr metrics = std::make_shared(); + std::shared_ptr telemetry = + std::make_shared(); + std::shared_ptr sessions = + std::make_shared(); + std::shared_ptr auth = std::make_shared(); + std::shared_ptr sqlite = + std::make_shared(fw::OrmBackend::Sqlite); + std::shared_ptr postgres = + std::make_shared(fw::OrmBackend::Postgres); + std::shared_ptr uploads = + std::make_shared(); + std::shared_ptr webdav = + std::make_shared(); + std::shared_ptr supervisor = + std::make_shared(); + std::shared_ptr tasks = std::make_shared(); + std::shared_ptr repl = + std::make_shared(); + std::shared_ptr openapi = std::make_shared(); + std::shared_ptr templates = + std::make_shared(); + std::map embedded; + std::string static_root = "example/static"; + std::string cookie_secret = "living-example-secret"; +}; + +std::shared_ptr build_application(Services& services); + +void register_index(fw::Router& router, Services& services); +void register_routing_demo(fw::Router& router, Services& services); +void register_chain_demo(std::shared_ptr app, + Services& services); +void register_data_demo(fw::Router& router, Services& services); +void register_security_demo(fw::Router& router, Services& services); +void register_formats_demo(fw::Router& router, Services& services); +void register_realtime_demo(fw::Router& router, Services& services); +void register_ops_demo(fw::Router& router, Services& services); + +void load_templates(Services& services); +void load_embedded(Services& services); +void seed_services(Services& services); + +} // namespace molo::example::app diff --git a/example/app/DataDemo.cpp b/example/app/DataDemo.cpp new file mode 100644 index 0000000..d9733bd --- /dev/null +++ b/example/app/DataDemo.cpp @@ -0,0 +1,141 @@ +#include "App.hpp" + +#include "../fw/Http.hpp" +#include "../fw/Router.hpp" +#include "../fw/Validate.hpp" + +namespace molo::example::app { + +namespace { + +fw::OrmDb* pick_db(Services& services, const fw::Request& req) { + if (fw::parse_backend(req.query_or("backend", "sqlite")) == + fw::OrmBackend::Postgres) + return services.postgres.get(); + return services.sqlite.get(); +} + +std::string rows_json(const std::vector& rows) { + std::string out = "["; + bool first = true; + for (const auto& r : rows) { + if (!first) out += ","; + first = false; + out += "{"; + bool fc = true; + for (const auto& [k, v] : r.columns) { + if (!fc) out += ","; + fc = false; + out += "\"" + fw::json_escape(k) + "\":\"" + fw::json_escape(v) + "\""; + } + out += "}"; + } + return out + "]"; +} + +} // namespace + +void register_data_demo(fw::Router& router, Services& services) { + fw::OrmDb* sqlite = services.sqlite.get(); + fw::OrmDb* postgres = services.postgres.get(); + fw::KvStore* kv = services.kv.get(); + fw::Cache* cache = services.cache.get(); + fw::Pubsub* pubsub = services.pubsub.get(); + + router.get("/orm/users", "orm-users", [sqlite, postgres](fw::Request req) { + fw::OrmDb* db = req.query_or("backend", "sqlite") == "postgres" ? postgres + : sqlite; + fw::OrmResult res = db->select_all("users"); + std::string body = "{\"backend\":\"" + fw::backend_name(db->backend()) + + "\",\"healed\":" + (res.healed ? "true" : "false") + + ",\"note\":\"" + fw::json_escape(res.healed_note) + + "\",\"rows\":" + rows_json(res.rows) + "}"; + return fw::JsonResponse(body, 200); + }); + router.post("/orm/users", "orm-users-post", + [sqlite, postgres](fw::Request req) { + fw::OrmDb* db = req.query_or("backend", "sqlite") == "postgres" + ? postgres + : sqlite; + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + fw::OrmResult res = db->insert("users", input); + if (!res.ok) + return fw::JsonResponse("{\"error\":\"store full\"}", 500); + std::string body = + "{\"backend\":\"" + fw::backend_name(db->backend()) + + "\",\"healed\":" + (res.healed ? "true" : "false") + + ",\"note\":\"" + fw::json_escape(res.healed_note) + + "\",\"rows\":" + rows_json(res.rows) + "}"; + return fw::JsonResponse(body, 201); + }); + + router.get("/kv/items", "kv-get", [kv](fw::Request req) { + std::string key = req.query_or("key", ""); + if (key.empty()) return fw::JsonResponse("{\"error\":\"key required\"}", 400); + std::string value; + if (!kv->get(key, value)) + return fw::JsonResponse("{\"error\":\"not found\"}", 404); + return fw::JsonResponse("{\"key\":\"" + fw::json_escape(key) + + "\",\"value\":\"" + fw::json_escape(value) + + "\"}", + 200); + }); + router.post("/kv/items", "kv-set", [kv](fw::Request req) { + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + if (!input.count("key") || !input.count("value")) + return fw::JsonResponse("{\"error\":\"key and value required\"}", 400); + if (!kv->set(input["key"], input["value"])) + return fw::JsonResponse("{\"error\":\"store full\"}", 500); + return fw::JsonResponse("{\"stored\":\"" + fw::json_escape(input["key"]) + + "\"}", + 201); + }); + + router.get("/cache/lookup", "cache", [cache](fw::Request req) { + std::string key = req.query_or("key", "demo"); + std::string value; + bool hit = cache->lookup(key, value, 1000); + if (!hit) { + value = "computed-for-" + key; + cache->put(key, value, 60, 1000); + return fw::JsonResponse( + "{\"key\":\"" + fw::json_escape(key) + "\",\"hit\":false,\"value\":\"" + + fw::json_escape(value) + "\"}", + 200); + } + return fw::JsonResponse( + "{\"key\":\"" + fw::json_escape(key) + "\",\"hit\":true,\"value\":\"" + + fw::json_escape(value) + "\"}", + 200); + }); + + router.post("/pubsub/publish", "pubsub", [pubsub](fw::Request req) { + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + std::string topic = input.count("topic") ? input["topic"] : "news"; + std::string message = input.count("message") ? input["message"] : "hello"; + std::size_t delivered = pubsub->publish(topic, message); + return fw::JsonResponse("{\"topic\":\"" + fw::json_escape(topic) + + "\",\"delivered\":" + + std::to_string(delivered) + "}", + 200); + }); + router.get("/pubsub/publish", "pubsub-history", [pubsub](fw::Request req) { + std::string topic = req.query_or("topic", "news"); + auto hist = pubsub->history(topic); + std::string body = "{\"topic\":\"" + fw::json_escape(topic) + + "\",\"history\":["; + bool first = true; + for (const auto& m : hist) { + if (!first) body += ","; + first = false; + body += "\"" + fw::json_escape(m) + "\""; + } + return fw::JsonResponse(body + "]}", 200); + }); + (void)pick_db; +} + +} // namespace molo::example::app diff --git a/example/app/Embedded.cpp b/example/app/Embedded.cpp new file mode 100644 index 0000000..08d12fc --- /dev/null +++ b/example/app/Embedded.cpp @@ -0,0 +1,19 @@ +#include "App.hpp" + +namespace molo::example::app { + +namespace { + +// Byte-identical reproducible bundle. Production bundles are generated by +// the asset embedder CLI (09-cli-ops.md); this hand-written copy proves the +// serving path with sorted names and no timestamps. + +} // namespace + +void load_embedded(Services& services) { + services.embedded["/app.css"] = + "/* embedded bundle */\nbody{font-family:sans-serif}\n"; + services.embedded["/hello.txt"] = "hello from embedded assets\n"; +} + +} // namespace molo::example::app diff --git a/example/app/FormatsDemo.cpp b/example/app/FormatsDemo.cpp new file mode 100644 index 0000000..851491e --- /dev/null +++ b/example/app/FormatsDemo.cpp @@ -0,0 +1,131 @@ +#include "App.hpp" + +#include "../fw/Negotiate.hpp" +#include "../fw/Router.hpp" +#include "../fw/StaticFiles.hpp" +#include "../fw/Templates.hpp" +#include "../fw/Validate.hpp" + +namespace molo::example::app { + +namespace { + +fw::Model contact_model() { + return fw::Model( + {{"name", fw::FieldType::String, true, 1, 80, 0, 0, false, ""}, + {"email", fw::FieldType::Email, true, 3, 120, 0, 0, false, ""}, + {"message", fw::FieldType::String, true, 1, 2000, 0, 0, false, ""}}); +} + +fw::Model user_model() { + fw::Field age{"age", fw::FieldType::Integer, true, 0, 0, 0, 150, true, ""}; + return fw::Model( + {{"username", fw::FieldType::String, true, 3, 40, 0, 0, false, ""}, + {"email", fw::FieldType::Email, true, 3, 120, 0, 0, false, ""}, + age}); +} + +} // namespace + +void register_formats_demo(fw::Router& router, Services& services) { + fw::Environment* env = services.templates.get(); + std::map embedded = services.embedded; + std::string static_root = services.static_root; + + router.get("/templates/page", "template-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Templates"); + ctx["heading"] = std::string("Template demo"); + ctx["author"] = std::string("ada"); + ctx["snippet"] = std::string("bold"); + fw::TemplateList items; + items.push_back({{"name", "extends"}, {"kind", "inheritance"}}); + items.push_back({{"name", "include"}, {"kind", "partial"}}); + items.push_back({{"name", "for"}, {"kind", "loop"}}); + ctx["items"] = items; + return fw::Response(200, env->render("page.html", ctx), + "text/html; charset=utf-8"); + }); + + router.get("/formats/data", "formats", [](fw::Request req) { + std::vector offers = {"application/json", "application/xml", + "text/html", "text/plain"}; + std::string pick = + fw::negotiate(req.headers.get("Accept"), offers); + if (pick == "application/json") + return fw::Response( + 200, "{\"format\":\"json\",\"value\":42}", "application/json"); + if (pick == "application/xml") + return fw::Response(200, "42", + "application/xml"); + if (pick == "text/html") + return fw::Response(200, "

value: 42

", + "text/html; charset=utf-8"); + return fw::Response(200, "value: 42", "text/plain; charset=utf-8"); + }); + + router.get("/forms/contact", "contact-get", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Contact"); + ctx["posted"] = std::string(""); + ctx["name"] = std::string(""); + ctx["email"] = std::string(""); + ctx["message"] = std::string(""); + return fw::Response(200, env->render("contact.html", ctx), + "text/html; charset=utf-8"); + }); + router.post("/forms/contact", "contact-post", [env](fw::Request req) { + fw::Form form(contact_model()); + fw::TemplateCtx ctx; + ctx["title"] = std::string("Contact"); + try { + auto clean = form.bind(req); + ctx["posted"] = std::string("true"); + ctx["name"] = clean["name"]; + ctx["email"] = clean["email"]; + ctx["message"] = clean["message"]; + } catch (const fw::ValidationError& e) { + ctx["posted"] = std::string(""); + auto input = fw::parse_form(req.body); + ctx["name"] = input.count("name") ? input["name"] : ""; + ctx["email"] = input.count("email") ? input["email"] : ""; + ctx["message"] = input.count("message") ? input["message"] : ""; + ctx["error"] = e.to_json(); + return fw::Response(422, env->render("contact.html", ctx), + "text/html; charset=utf-8"); + } + return fw::Response(200, env->render("contact.html", ctx), + "text/html; charset=utf-8"); + }); + + router.post("/validate/user", "validate-user", [](fw::Request req) { + fw::Model model = user_model(); + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + try { + auto clean = model.validate(input); + return fw::JsonResponse( + "{\"ok\":true,\"username\":\"" + + fw::json_escape(clean["username"]) + "\"}", + 200); + } catch (const fw::ValidationError& e) { + return fw::JsonResponse(e.to_json(), 422); + } + }); + + router.get("/static/{name}", "static", [static_root](fw::Request req) { + return fw::serve_disk_file(static_root, req.param_or("name", "")); + }); + router.get("/embedded/{name}", "embedded", [embedded](fw::Request req) { + return fw::serve_embedded(embedded, req.param_or("name", "")); + }); + + router.get("/redirect/here", "redirect", [](fw::Request req) { + bool permanent = req.query_or("mode", "temp") == "perm"; + return fw::Response(fw::RedirectResponse("/health", permanent)); + }); +} + +} // namespace molo::example::app diff --git a/example/app/Index.cpp b/example/app/Index.cpp new file mode 100644 index 0000000..7d0b536 --- /dev/null +++ b/example/app/Index.cpp @@ -0,0 +1,75 @@ +#include "App.hpp" + +#include "../fw/Http.hpp" +#include "../fw/Router.hpp" +#include "../fw/Templates.hpp" + +namespace molo::example::app { + +namespace { + +const char* kFeatures[][2] = { + {"routing", "/routing/hello/world"}, + {"views", "/views/greet"}, + {"routers", "/routers/nested/deep"}, + {"chaining", "/app1/app1/app1/whoami"}, + {"orm-sqlite", "/orm/users?backend=sqlite"}, + {"orm-postgres", "/orm/users?backend=postgres"}, + {"auth", "/auth/me"}, + {"sessions", "/sessions/counter"}, + {"forms", "/forms/contact"}, + {"validation", "/validate/user"}, + {"templates", "/templates/page"}, + {"formats", "/formats/data"}, + {"static", "/static/app.css"}, + {"embedded", "/embedded/hello.txt"}, + {"streaming", "/streaming/feed"}, + {"passthrough", "/streaming/passthrough"}, + {"websocket", "/ws/chat"}, + {"socketio", "/socketio/rooms"}, + {"sse", "/sse/events"}, + {"pubsub", "/pubsub/publish"}, + {"events", "/events/fire"}, + {"metrics", "/metrics/counters"}, + {"telemetry", "/telemetry/trace"}, + {"kv", "/kv/items"}, + {"cache", "/cache/lookup"}, + {"uploads", "/uploads/file"}, + {"webdav", "/webdav/files/"}, + {"replication", "/replication/state"}, + {"tasks", "/supervisor/tasks"}, + {"workers", "/supervisor/workers"}, + {"restclient", "/restclient/self"}, + {"openapi", "/openapi.json"}, + {"docs", "/docs"}, + {"ai", "/cli/ai"}, + {"workflow", "/workflow/release"}, + {"redirect", "/redirect/here"}, + {"health", "/health"}, +}; + +} // namespace + +void register_index(fw::Router& router, Services& services) { + fw::Environment* env = services.templates.get(); + router.get("/", "index", [env](fw::Request req) { + fw::TemplateCtx ctx; + ctx["title"] = std::string("MoloApp living example"); + ctx["show_matrix"] = std::string("true"); + fw::TemplateList rows; + for (const auto& [feature, route] : kFeatures) { + rows.push_back({{"feature", feature}, {"route", route}}); + } + ctx["matrix"] = rows; + (void)req; + return fw::Response(200, env->render("index.html", ctx), + "text/html; charset=utf-8"); + }); + router.get("/health", "health", [](fw::Request req) { + (void)req; + return fw::JsonResponse( + "{\"status\":\"ok\",\"tls\":\"stub-no-wolfssl\"}", 200); + }); +} + +} // namespace molo::example::app diff --git a/example/app/OpsDemo.cpp b/example/app/OpsDemo.cpp new file mode 100644 index 0000000..cd38569 --- /dev/null +++ b/example/app/OpsDemo.cpp @@ -0,0 +1,224 @@ +#include "App.hpp" + +#include "../fw/Jobs.hpp" +#include "../fw/Net.hpp" +#include "../fw/Observ.hpp" +#include "../fw/Router.hpp" +#include "../fw/Templates.hpp" +#include "../fw/Validate.hpp" +#include "../fw/Workflow.hpp" + +namespace molo::example::app { + +void register_ops_demo(fw::Router& router, Services& services) { + fw::EventBus* events = services.events.get(); + fw::Metrics* metrics = services.metrics.get(); + fw::Telemetry* telemetry = services.telemetry.get(); + fw::Supervisor* supervisor = services.supervisor.get(); + fw::TaskQueue* tasks = services.tasks.get(); + fw::ReplicationLog* repl = services.repl.get(); + fw::OpenApi* openapi = services.openapi.get(); + fw::Environment* env = services.templates.get(); + + router.post("/events/fire", "events-fire", [events](fw::Request req) { + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + std::string type = input.count("type") ? input["type"] : "demo.ping"; + std::string payload = + input.count("payload") ? input["payload"] : "ping"; + events->emit({type, payload, req.correlation_id}); + return fw::JsonResponse( + "{\"fired\":\"" + fw::json_escape(type) + "\"}", 200); + }); + router.get("/events/fire", "events-list", [events](fw::Request req) { + (void)req; + auto delivered = events->delivered(); + std::string body = "{\"delivered\":["; + bool first = true; + for (const auto& e : delivered) { + if (!first) body += ","; + first = false; + body += "{\"type\":\"" + fw::json_escape(e.type) + "\",\"payload\":\"" + + fw::json_escape(e.payload) + "\"}"; + } + return fw::JsonResponse(body + "]}", 200); + }); + + router.get("/metrics/counters", "metrics", + [metrics](fw::Request req) { + (void)req; + metrics->inc("example.page.views"); + fw::Response res(200, metrics->render_prometheus(), + "text/plain; version=0.0.4"); + return res; + }); + + router.get("/telemetry/trace", "telemetry", + [telemetry](fw::Request req) { + fw::Span span = + telemetry->start_span("demo", req.correlation_id); + telemetry->finish_span(span); + return fw::JsonResponse( + "{\"trace\":\"" + fw::json_escape(span.trace_id) + + "\",\"span\":\"" + fw::json_escape(span.span_id) + + "\",\"correlation\":\"" + + fw::json_escape(req.correlation_id) + "\"}", + 200); + }); + + router.get("/replication/state", "repl", [repl](fw::Request req) { + unsigned long since = 0; + try { + since = (unsigned long)std::stoul(req.query_or("since", "0")); + } catch (...) { + since = 0; + } + auto entries = repl->since(since); + std::string body = "{\"latest\":" + std::to_string(repl->latest()) + + ",\"entries\":["; + bool first = true; + for (const auto& e : entries) { + if (!first) body += ","; + first = false; + body += "{\"seq\":" + std::to_string(e.seq) + ",\"object\":\"" + + fw::json_escape(e.object) + "\",\"resolution\":\"" + + fw::json_escape(e.resolution) + "\"}"; + } + return fw::JsonResponse(body + "]}", 200); + }); + router.post("/replication/state", "repl-append", [repl](fw::Request req) { + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + std::string object = input.count("object") ? input["object"] : "doc-1"; + std::string data = input.count("data") ? input["data"] : "{}"; + fw::ReplEntry e = repl->append(object, data); + return fw::JsonResponse("{\"seq\":" + std::to_string(e.seq) + + ",\"resolution\":\"" + + fw::json_escape(e.resolution) + "\"}", + 201); + }); + + router.get("/supervisor/tasks", "tasks-list", [tasks](fw::Request req) { + (void)req; + std::string body = "{\"tasks\":["; + bool first = true; + for (const auto& t : tasks->list()) { + if (!first) body += ","; + first = false; + body += "{\"id\":\"" + fw::json_escape(t.id) + "\",\"state\":\"" + + fw::json_escape(t.state) + "\"}"; + } + return fw::JsonResponse(body + "]}", 200); + }); + router.post("/supervisor/tasks", "tasks-enqueue", [tasks](fw::Request req) { + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + std::string kind = input.count("kind") ? input["kind"] : "email"; + std::string payload = + input.count("payload") ? input["payload"] : "{}"; + std::string id = tasks->enqueue(kind, payload, 0); + if (id.empty()) return fw::JsonResponse("{\"error\":\"queue full\"}", 500); + return fw::JsonResponse("{\"id\":\"" + fw::json_escape(id) + "\"}", 201); + }); + router.get("/supervisor/workers", "workers", + [supervisor](fw::Request req) { + (void)req; + std::string body = "{\"workers\":["; + bool first = true; + for (const auto& w : supervisor->status()) { + if (!first) body += ","; + first = false; + body += "{\"name\":\"" + fw::json_escape(w.name) + + "\",\"running\":" + + (w.running ? "true" : "false") + ",\"restarts\":" + + std::to_string(w.restarts) + "}"; + } + return fw::JsonResponse(body + "]}", 200); + }); + + router.get("/restclient/self", "restclient", [](fw::Request req) { + (void)req; + fw::RestClient client([](fw::Request r) { + (void)r; + return fw::JsonResponse("{\"status\":\"ok\"}", 200); + }); + fw::Response upstream = client.get("/health"); + return fw::JsonResponse( + "{\"self_call\":" + std::to_string(upstream.status) + "}", 200); + }); + + openapi->add("/health", "get", "health probe"); + openapi->add("/orm/users", "get", "list users"); + openapi->add("/validate/user", "post", "validate user model"); + router.get("/openapi.json", "openapi", [openapi](fw::Request req) { + (void)req; + return fw::Response(200, openapi->to_json(), "application/json"); + }); + router.get("/docs", "docs", [env, openapi](fw::Request req) { + (void)req; + (void)openapi; + fw::TemplateCtx ctx; + ctx["title"] = std::string("API docs"); + fw::TemplateList rows; + rows.push_back({{"method", "GET"}, + {"path", "/health"}, + {"summary", "health probe"}}); + rows.push_back({{"method", "GET"}, + {"path", "/orm/users"}, + {"summary", "list users"}}); + rows.push_back({{"method", "POST"}, + {"path", "/validate/user"}, + {"summary", "validate user model"}}); + ctx["routes"] = rows; + return fw::Response(200, env->render("docs.html", ctx), + "text/html; charset=utf-8"); + }); + + router.get("/cli/ai", "ai", [](fw::Request req) { + std::string prompt = req.query_or("prompt", "add a route"); + fw::AiSample sample = fw::ai_harness_sample(prompt); + std::string body = "{\"base_url\":\"" + sample.base_url + + "\",\"model\":\"" + sample.model + + "\",\"prompt\":\"" + + fw::json_escape(sample.redacted_prompt) + + "\",\"stream_events\":" + + std::to_string(sample.stream_chunks.size()) + "}"; + return fw::JsonResponse(body, 200); + }); + + router.post("/workflow/release", "workflow", [](fw::Request req) { + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + std::string subjects = input.count("commits") ? input["commits"] : ""; + std::vector commits; + std::size_t start = 0; + while (start < subjects.size()) { + std::size_t bar = subjects.find('|', start); + std::string one = + subjects.substr(start, bar == std::string::npos ? bar : bar - start); + fw::CommitInfo info; + std::string error; + if (!fw::parse_commit(one, "", info, error)) + return fw::JsonResponse( + "{\"error\":\"bad commit: " + fw::json_escape(error) + "\"}", + 400); + commits.push_back(info); + if (bar == std::string::npos) break; + start = bar + 1; + } + fw::ReleasePlanner planner; + fw::ChangelogWriter writer; + fw::ReleaseVerifier verifier; + verifier.add_check("suite", true, "example suite green"); + verifier.add_check("live", true, "health probe ok"); + std::string version = planner.next_version("0.1.0", commits); + std::string body = + "{\"bump\":\"" + fw::bump_name(planner.decide(commits)) + + "\",\"version\":\"" + version + "\",\"green\":" + + (verifier.green() ? "true" : "false") + ",\"changelog\":\"" + + fw::json_escape(writer.entry(version, "2026-10-23", commits)) + "\"}"; + return fw::JsonResponse(body, 200); + }); +} + +} // namespace molo::example::app diff --git a/example/app/RealtimeDemo.cpp b/example/app/RealtimeDemo.cpp new file mode 100644 index 0000000..1da5d93 --- /dev/null +++ b/example/app/RealtimeDemo.cpp @@ -0,0 +1,93 @@ +#include "App.hpp" + +#include "../fw/Data.hpp" +#include "../fw/Router.hpp" +#include "../fw/Validate.hpp" + +namespace molo::example::app { + +// Realtime demos answer over plain HTTP in the skeleton: the WebSocket and +// Socket.IO endpoints describe the upgrade contract and echo posted frames, +// SSE streams real event text. Gap: real framing, namespaces, rooms, and +// long-polling fallback in 06-realtime.md plus matching clients. + +void register_realtime_demo(fw::Router& router, Services& services) { + fw::WebdavStore* webdav = services.webdav.get(); + + router.get("/streaming/feed", "stream", [](fw::Request req) { + (void)req; + return fw::Response( + fw::StreamingResponse({"chunk-1\n", "chunk-2\n", "chunk-3\n"})); + }); + router.get("/streaming/passthrough", "passthrough", [](fw::Request req) { + std::string target = req.query_or("target", "/health"); + return fw::Response(fw::StreamingResponse( + {"passthrough from " + target + "\n", "{\"status\":\"ok\"}\n"})); + }); + + router.get("/ws/chat", "ws", [](fw::Request req) { + std::string upgrade = req.headers.get("Upgrade"); + if (upgrade == "websocket") + return fw::JsonResponse( + "{\"upgraded\":false,\"note\":\"skeleton: echo over POST\"}", 200); + return fw::JsonResponse( + "{\"protocol\":\"websocket\",\"path\":\"/ws/chat\",\"upgrade\":" + "\"send Upgrade: websocket\"}", + 200); + }); + router.post("/ws/chat", "ws-echo", [](fw::Request req) { + return fw::JsonResponse( + "{\"echo\":\"" + fw::json_escape(req.body) + "\"}", 200); + }); + + router.get("/socketio/rooms", "socketio", [](fw::Request req) { + (void)req; + return fw::JsonResponse( + "{\"protocol\":\"socket.io\",\"namespaces\":[\"/\"],\"rooms\":[" + "\"lobby\"],\"fallback\":\"long-polling\"}", + 200); + }); + router.post("/socketio/rooms", "socketio-emit", [](fw::Request req) { + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + std::string room = input.count("room") ? input["room"] : "lobby"; + std::string event = input.count("event") ? input["event"] : "message"; + return fw::JsonResponse("{\"emitted\":\"" + fw::json_escape(event) + + "\",\"room\":\"" + fw::json_escape(room) + + "\",\"ack\":true}", + 200); + }); + + router.get("/sse/events", "sse", [](fw::Request req) { + (void)req; + return fw::Response( + fw::SseResponse({"hello", "tick", "goodbye"})); + }); + + router.any({fw::Method::Propfind}, "/webdav/files/", "webdav-propfind", + [webdav](fw::Request req) { + (void)req; + return fw::Response(207, webdav->propfind_xml("/files/"), + "application/xml"); + }); + router.any({fw::Method::Get}, "/webdav/files/{name}", "webdav-get", + [webdav](fw::Request req) -> fw::Response { + std::string content; + if (!webdav->get("/files/" + req.param_or("name", ""), content)) + return fw::JsonResponse("{\"error\":\"not found\"}", 404); + return fw::TextResponse(content, 200); + }); + router.any({fw::Method::Put}, "/webdav/files/{name}", "webdav-put", + [webdav](fw::Request req) { + webdav->put("/files/" + req.param_or("name", ""), req.body); + return fw::JsonResponse("{\"stored\":true}", 201); + }); + router.any({fw::Method::Delete}, "/webdav/files/{name}", "webdav-delete", + [webdav](fw::Request req) { + if (!webdav->remove("/files/" + req.param_or("name", ""))) + return fw::JsonResponse("{\"error\":\"not found\"}", 404); + return fw::JsonResponse("{\"deleted\":true}", 200); + }); +} + +} // namespace molo::example::app diff --git a/example/app/RoutingDemo.cpp b/example/app/RoutingDemo.cpp new file mode 100644 index 0000000..6c6a7ab --- /dev/null +++ b/example/app/RoutingDemo.cpp @@ -0,0 +1,80 @@ +#include "App.hpp" + +#include "../fw/Router.hpp" +#include "../fw/View.hpp" + +namespace molo::example::app { + +namespace { + +class GreetView : public fw::View { + public: + using fw::View::View; + + fw::Response get() override { + return fw::JsonResponse( + "{\"view\":\"GreetView\",\"method\":\"GET\",\"name\":\"" + + fw::json_escape(req_.query_or("name", "stranger")) + "\"}", + 200); + } + + fw::Response post() override { + auto form = fw::parse_form(req_.body); + auto it = form.find("name"); + std::string name = it == form.end() ? "stranger" : it->second; + return fw::JsonResponse( + "{\"view\":\"GreetView\",\"method\":\"POST\",\"name\":\"" + + fw::json_escape(name) + "\"}", + 200); + } +}; + +} // namespace + +void register_routing_demo(fw::Router& router, Services& services) { + (void)services; + router.get("/hello/{name}", "hello", [](fw::Request req) { + return fw::JsonResponse( + "{\"hello\":\"" + fw::json_escape(req.param_or("name", "")) + "\"}", + 200); + }); + + fw::Router views; + views.get("/greet", "greet", [](fw::Request req) { + GreetView view(std::move(req)); + return view.dispatch(); + }); + views.post("/greet", "greet-post", [](fw::Request req) { + GreetView view(std::move(req)); + return view.dispatch(); + }); + router.include_router("/views", views, "views."); + + fw::Router nested; + nested.get("/deep", "deep", [](fw::Request req) { + (void)req; + return fw::JsonResponse("{\"nested\":true}", 200); + }); + fw::Router middle; + middle.include_router("/nested", nested, "nested."); + router.include_router("/routers", middle, "routers."); +} + +void register_chain_demo(std::shared_ptr app, + Services& services) { + (void)services; + auto leaf = std::make_shared(); + fw::Router r; + r.get("/whoami", "whoami", [](fw::Request req) { + (void)req; + return fw::JsonResponse("{\"depth\":3,\"path\":\"/app1/app1/app1\"}", 200); + }); + leaf->include_router("", r); + auto mid = std::make_shared(); + mid->mount("/app1", leaf); + auto top = std::make_shared(); + top->mount("/app1", mid); + app->mount("/app1", top); +} + +} // namespace molo::example::app diff --git a/example/app/SecurityDemo.cpp b/example/app/SecurityDemo.cpp new file mode 100644 index 0000000..865f834 --- /dev/null +++ b/example/app/SecurityDemo.cpp @@ -0,0 +1,93 @@ +#include "App.hpp" + +#include "../fw/Cookies.hpp" +#include "../fw/Data.hpp" +#include "../fw/Router.hpp" +#include "../fw/Validate.hpp" + +namespace molo::example::app { + +void register_security_demo(fw::Router& router, Services& services) { + fw::Auth* auth = services.auth.get(); + fw::SessionStore* sessions = services.sessions.get(); + fw::UploadStore* uploads = services.uploads.get(); + std::string secret = services.cookie_secret; + + router.get("/auth/me", "auth-me", [auth](fw::Request req) { + std::string user, password; + if (!fw::parse_basic_auth(req.headers.get("Authorization"), user, + password)) + return fw::JsonResponse("{\"error\":\"basic auth required\"}", 401); + if (!auth->check(user, password)) + return fw::JsonResponse("{\"error\":\"bad credentials\"}", 403); + return fw::JsonResponse( + "{\"user\":\"" + fw::json_escape(user) + "\"}", 200); + }); + router.post("/auth/login", "auth-login", [auth](fw::Request req) { + auto input = fw::parse_json_object_flat(req.body); + if (input.empty()) input = fw::parse_form(req.body); + if (!input.count("username") || !input.count("password")) + return fw::JsonResponse("{\"error\":\"username and password required\"}", + 400); + if (!auth->check(input["username"], input["password"])) + return fw::JsonResponse("{\"error\":\"bad credentials\"}", 403); + return fw::JsonResponse( + "{\"user\":\"" + fw::json_escape(input["username"]) + "\"}", 200); + }); + + router.get("/sessions/counter", "session-counter", + [sessions, secret](fw::Request req) { + std::string raw = req.cookie_or("session", ""); + std::string id; + std::map data; + bool fresh = false; + if (raw.empty() || !fw::cookie_verify(raw, secret, id) || + !sessions->get(id, data)) { + id = sessions->create(); + fresh = true; + data.clear(); + } + long count = 0; + if (data.count("count")) { + try { + count = std::stol(data["count"]); + } catch (...) { + count = 0; + } + } + ++count; + sessions->put(id, "count", std::to_string(count)); + fw::Response res(200, + "{\"count\":" + std::to_string(count) + + ",\"fresh\":" + + (fresh ? "true" : "false") + "}", + "application/json"); + res.headers.set( + "Set-Cookie", + fw::make_set_cookie("session", fw::cookie_sign(id, secret), + "/", true, false, "Lax", 3600)); + return res; + }); + + router.post("/uploads/file", "upload", [uploads](fw::Request req) { + std::string filename = req.query_or("filename", "upload.txt"); + std::string content_type = + req.headers.get("Content-Type"); + if (content_type.empty()) content_type = "text/plain"; + fw::UploadInfo info; + std::string error; + if (!uploads->store(filename, content_type, req.body, info, error)) { + if (error == "file too large") + return fw::JsonResponse("{\"error\":\"" + error + "\"}", 413); + if (error == "content type not allowed") + return fw::JsonResponse("{\"error\":\"" + error + "\"}", 415); + return fw::JsonResponse("{\"error\":\"" + error + "\"}", 400); + } + return fw::JsonResponse("{\"file\":\"" + fw::json_escape(info.filename) + + "\",\"size\":" + std::to_string(info.size) + + "}", + 201); + }); +} + +} // namespace molo::example::app diff --git a/example/app/Seed.cpp b/example/app/Seed.cpp new file mode 100644 index 0000000..a2c0780 --- /dev/null +++ b/example/app/Seed.cpp @@ -0,0 +1,14 @@ +#include "App.hpp" + +namespace molo::example::app { + +void seed_services(Services& services) { + services.auth->add_user("ada", "correct-horse"); + services.auth->add_user("grace", "compiler-rocks"); + services.supervisor->add_worker("mailer"); + services.supervisor->add_worker("indexer"); + services.supervisor->start("mailer"); + services.webdav->put("/files/welcome.txt", "hello from webdav\n"); +} + +} // namespace molo::example::app diff --git a/example/app/Templates.cpp b/example/app/Templates.cpp new file mode 100644 index 0000000..b1c32c3 --- /dev/null +++ b/example/app/Templates.cpp @@ -0,0 +1,136 @@ +#include "App.hpp" + +#include "../fw/Templates.hpp" + +namespace molo::example::app { + +namespace { + +const char kBase[] = R"BASE( + + + + +{{ title }} + + + +
+{{ site }} + +
+Menu + +
+
+
+{% block content %}{% endblock %} +
+

{{ site }}: living example, {{ backend }}

+ + +)BASE"; + +const char kIndex[] = R"INDEX({% extends "base.html" %} +{% block content %} +

Living example

+

Every framework feature, one click away. Mobile-first: narrow the window +and the top nav collapses into a dropdown menu.

+{% if show_matrix %} + + + +{% for row in matrix %} + +{% endfor %} + +
FeatureRoute
{{ row.feature }}{{ row.route }}
+{% else %} +

Matrix hidden.

+{% endif %} +{% include "nav_hint.html" %} +{% endblock %} +)INDEX"; + +const char kPage[] = R"PAGE({% extends "base.html" %} +{% block content %} +

{{ heading }}

+

Author: {{ author|upper }}

+

Raw (escaped): {{ snippet }}

+

Trusted (safe): {{ snippet|safe }}

+
    +{% for item in items %} +
  • {{ item.name }} ({{ item.kind }})
    • +{% endfor %} +
+{% endblock %} +)PAGE"; + +const char kContact[] = R"CONTACT({% extends "base.html" %} +{% block content %} +

Contact

+{% if posted %} +

Thanks {{ name }}, message received.

+{% else %} +
+ + + + +
+{% endif %} +{% if error %} +

{{ error }}

+{% endif %} +{% endblock %} +)CONTACT"; + +const char kDocs[] = R"DOCS({% extends "base.html" %} +{% block content %} +

API docs

+

OpenAPI JSON: /openapi.json

+
    +{% for row in routes %} +
  • {{ row.method }} {{ row.path }} : {{ row.summary }}
    • +{% endfor %} +
+{% endblock %} +)DOCS"; + +const char kError[] = R"ERROR({% extends "base.html" %} +{% block content %} +

Error {{ status }}

+

{{ message }}

+{% endblock %} +)ERROR"; + +const char kNavHint[] = + "

Tip: this page works on a 360px phone.

\n"; + +} // namespace + +void load_templates(Services& services) { + services.templates->add_template("base.html", kBase); + services.templates->add_template("index.html", kIndex); + services.templates->add_template("page.html", kPage); + services.templates->add_template("contact.html", kContact); + services.templates->add_template("docs.html", kDocs); + services.templates->add_template("error.html", kError); + services.templates->add_template("nav_hint.html", kNavHint); + services.templates->add_global("site", "MoloApp living example"); + services.templates->add_global("backend", "skeleton"); +} + +} // namespace molo::example::app diff --git a/example/example.mk b/example/example.mk new file mode 100644 index 0000000..f1b5a16 --- /dev/null +++ b/example/example.mk @@ -0,0 +1,117 @@ +# example/example.mk: living example part (plan 10-workflow-example.md). +# Included by the root Makefile through per-part subdir .mk includes, or run +# standalone from the repo root: +# make -f example/example.mk example # build lib + server +# make -f example/example.mk example-test # build + run example suite +# make -f example/example.mk example-clean # remove build/example +# Portable make: variables plus sh loops only, no GNU-only functions. + +CXX ?= g++ +EXAMPLE_CXXFLAGS ?= -std=c++23 -Wall -Wextra -O1 +EXAMPLE_DIR ?= example +EXAMPLE_BUILD ?= build/example +EXAMPLE_TESTDIR ?= tests/example + +EXAMPLE_SRCS = \ + $(EXAMPLE_DIR)/fw/Http.cpp \ + $(EXAMPLE_DIR)/fw/Router.cpp \ + $(EXAMPLE_DIR)/fw/App.cpp \ + $(EXAMPLE_DIR)/fw/View.cpp \ + $(EXAMPLE_DIR)/fw/Validate.cpp \ + $(EXAMPLE_DIR)/fw/Templates.cpp \ + $(EXAMPLE_DIR)/fw/Negotiate.cpp \ + $(EXAMPLE_DIR)/fw/Cookies.cpp \ + $(EXAMPLE_DIR)/fw/StaticFiles.cpp \ + $(EXAMPLE_DIR)/fw/Tls.cpp \ + $(EXAMPLE_DIR)/fw/Kv.cpp \ + $(EXAMPLE_DIR)/fw/Observ.cpp \ + $(EXAMPLE_DIR)/fw/Data.cpp \ + $(EXAMPLE_DIR)/fw/Jobs.cpp \ + $(EXAMPLE_DIR)/fw/Net.cpp \ + $(EXAMPLE_DIR)/fw/Workflow.cpp \ + $(EXAMPLE_DIR)/fw/TestClient.cpp \ + $(EXAMPLE_DIR)/fw/Server.cpp \ + $(EXAMPLE_DIR)/app/App.cpp \ + $(EXAMPLE_DIR)/app/Seed.cpp \ + $(EXAMPLE_DIR)/app/Templates.cpp \ + $(EXAMPLE_DIR)/app/Embedded.cpp \ + $(EXAMPLE_DIR)/app/Index.cpp \ + $(EXAMPLE_DIR)/app/RoutingDemo.cpp \ + $(EXAMPLE_DIR)/app/DataDemo.cpp \ + $(EXAMPLE_DIR)/app/SecurityDemo.cpp \ + $(EXAMPLE_DIR)/app/FormatsDemo.cpp \ + $(EXAMPLE_DIR)/app/RealtimeDemo.cpp \ + $(EXAMPLE_DIR)/app/OpsDemo.cpp + +EXAMPLE_TESTS = \ + $(EXAMPLE_TESTDIR)/index_test.cpp \ + $(EXAMPLE_TESTDIR)/health_test.cpp \ + $(EXAMPLE_TESTDIR)/routing/hello_test.cpp \ + $(EXAMPLE_TESTDIR)/routing/views_test.cpp \ + $(EXAMPLE_TESTDIR)/routing/routers_test.cpp \ + $(EXAMPLE_TESTDIR)/chain/deep_test.cpp \ + $(EXAMPLE_TESTDIR)/orm/users_test.cpp \ + $(EXAMPLE_TESTDIR)/kv/items_test.cpp \ + $(EXAMPLE_TESTDIR)/cache/lookup_test.cpp \ + $(EXAMPLE_TESTDIR)/pubsub/publish_test.cpp \ + $(EXAMPLE_TESTDIR)/auth/login_test.cpp \ + $(EXAMPLE_TESTDIR)/auth/me_test.cpp \ + $(EXAMPLE_TESTDIR)/sessions/counter_test.cpp \ + $(EXAMPLE_TESTDIR)/uploads/file_test.cpp \ + $(EXAMPLE_TESTDIR)/forms/contact_test.cpp \ + $(EXAMPLE_TESTDIR)/validate/user_test.cpp \ + $(EXAMPLE_TESTDIR)/templates/page_test.cpp \ + $(EXAMPLE_TESTDIR)/formats/data_test.cpp \ + $(EXAMPLE_TESTDIR)/static/files_test.cpp \ + $(EXAMPLE_TESTDIR)/embedded/assets_test.cpp \ + $(EXAMPLE_TESTDIR)/streaming/feed_test.cpp \ + $(EXAMPLE_TESTDIR)/streaming/passthrough_test.cpp \ + $(EXAMPLE_TESTDIR)/ws/chat_test.cpp \ + $(EXAMPLE_TESTDIR)/socketio/rooms_test.cpp \ + $(EXAMPLE_TESTDIR)/sse/events_test.cpp \ + $(EXAMPLE_TESTDIR)/webdav/files_test.cpp \ + $(EXAMPLE_TESTDIR)/events/fire_test.cpp \ + $(EXAMPLE_TESTDIR)/metrics/counters_test.cpp \ + $(EXAMPLE_TESTDIR)/telemetry/trace_test.cpp \ + $(EXAMPLE_TESTDIR)/supervisor/tasks_test.cpp \ + $(EXAMPLE_TESTDIR)/supervisor/workers_test.cpp \ + $(EXAMPLE_TESTDIR)/replication/state_test.cpp \ + $(EXAMPLE_TESTDIR)/restclient/self_test.cpp \ + $(EXAMPLE_TESTDIR)/openapi/docs_test.cpp \ + $(EXAMPLE_TESTDIR)/cli/ai_test.cpp \ + $(EXAMPLE_TESTDIR)/workflow/release_test.cpp \ + $(EXAMPLE_TESTDIR)/redirect/here_test.cpp + +.PHONY: example example-test example-clean + +example: + mkdir -p $(EXAMPLE_BUILD)/obj + for s in $(EXAMPLE_SRCS); do \ + o=$(EXAMPLE_BUILD)/obj/`echo $$s | tr '/' '_'`.o; \ + $(CXX) $(EXAMPLE_CXXFLAGS) -I$(EXAMPLE_DIR) -c $$s -o $$o || exit 1; \ + done + ar rcs $(EXAMPLE_BUILD)/libexample.a $(EXAMPLE_BUILD)/obj/*.o + $(CXX) $(EXAMPLE_CXXFLAGS) -I$(EXAMPLE_DIR) $(EXAMPLE_DIR)/main.cpp \ + $(EXAMPLE_BUILD)/libexample.a -o $(EXAMPLE_BUILD)/living-server + +example-test: example + mkdir -p $(EXAMPLE_BUILD)/tests + pass=0; fail=0; \ + for t in $(EXAMPLE_TESTS); do \ + n=`echo $$t | sed 's|$(EXAMPLE_TESTDIR)/||; s|/|_|g; s|\.cpp$$||'`; \ + $(CXX) $(EXAMPLE_CXXFLAGS) -I$(EXAMPLE_DIR) -I$(EXAMPLE_TESTDIR) \ + $$t $(EXAMPLE_BUILD)/libexample.a \ + -o $(EXAMPLE_BUILD)/tests/$$n || exit 1; \ + if $(EXAMPLE_BUILD)/tests/$$n; then pass=$$((pass+1)); \ + else fail=$$((fail+1)); echo "FAIL: $$t"; fi; \ + done; \ + echo "example suite: $$pass passed, $$fail failed"; \ + test $$fail -eq 0 + +example-clean: + rm -rf $(EXAMPLE_BUILD) + +# Root Makefile aggregation points (double-colon: safe with other parts). +all:: example +test:: example-test +clean:: example-clean diff --git a/example/fw/App.cpp b/example/fw/App.cpp new file mode 100644 index 0000000..c1f0f8c --- /dev/null +++ b/example/fw/App.cpp @@ -0,0 +1,123 @@ +#include "App.hpp" + +#include + +#include "Caps.hpp" + +namespace molo::example::fw { + +HttpError::HttpError(int status, std::string code, std::string message) + : status_(status), code_(std::move(code)), message_(std::move(message)) {} + +int HttpError::status() const { return status_; } +const std::string& HttpError::code() const { return code_; } +const std::string& HttpError::message() const { return message_; } + +void Application::use(MiddlewareSlot slot, Middleware middleware) { + if (has_middleware_ && slot < last_slot_) + throw std::runtime_error("middleware category order violated"); + last_slot_ = slot; + has_middleware_ = true; + middlewares_.emplace_back(slot, std::move(middleware)); +} + +void Application::include_router(const std::string& prefix, + const Router& router, + const std::string& name_prefix) { + router_.include_router(prefix, router, name_prefix); +} + +void Application::mount(const std::string& path, + std::shared_ptr sub) { + mounts_.emplace_back(path, std::move(sub)); +} + +Response Application::dispatch(Request req) { + if (req.body.size() > caps::kMaxRequestBody) + return TextResponse("body too large", 413); + try { + return run_chain(std::move(req), 0); + } catch (const HttpError& e) { + std::string body = "{\"error\":\"" + json_escape(e.code()) + + "\",\"message\":\"" + json_escape(e.message()) + "\"}"; + return JsonResponse(body, e.status()); + } catch (const std::exception& e) { + return JsonResponse("{\"error\":\"internal\",\"message\":\"" + + json_escape(e.what()) + "\"}", + 500); + } +} + +Response Application::run_chain(Request req, int depth) { + Next next = [this, depth](Request r) { return dispatch_inner(r, depth); }; + for (std::size_t i = middlewares_.size(); i > 0; --i) { + Middleware mw = middlewares_[i - 1].second; + Next inner = next; + next = [mw, inner](Request r) { return mw(std::move(r), inner); }; + } + return next(std::move(req)); +} + +Response Application::dispatch_inner(Request req, int depth) { + if (depth > caps::kMaxMountDepth) + throw HttpError(500, "mount_depth", "sub-application nesting too deep"); + std::string best; + std::shared_ptr target; + for (auto& [prefix, sub] : mounts_) { + if (req.path.size() >= prefix.size() && + req.path.compare(0, prefix.size(), prefix) == 0 && + prefix.size() > best.size()) { + best = prefix; + target = sub; + } + } + if (target) { + req.path = req.path.substr(best.size()); + if (req.path.empty() || req.path.front() != '/') req.path = "/" + req.path; + return target->run_chain(std::move(req), depth + 1); + } + Handler handler; + if (router_.match(req, handler)) return handler(std::move(req)); + bool path_exists = false; + for (const auto& r : router_.routes()) { + std::map params; + if (match_pattern(r.pattern, req.path, params)) { + path_exists = true; + break; + } + } + if (path_exists) + throw HttpError(405, "method_not_allowed", "method not allowed"); + throw HttpError(404, "not_found", "no route for " + req.path); +} + +const std::vector& Application::routes() const { + return router_.routes(); +} + +Middleware errors_middleware() { + return [](Request req, Next next) -> Response { + try { + return next(std::move(req)); + } catch (const HttpError& e) { + std::string body = "{\"error\":\"" + json_escape(e.code()) + + "\",\"message\":\"" + json_escape(e.message()) + + "\"}"; + return JsonResponse(body, e.status()); + } + }; +} + +Middleware request_id_middleware() { + return [](Request req, Next next) -> Response { + if (req.correlation_id.empty()) { + static unsigned long counter = 1; + req.correlation_id = "req-" + std::to_string(counter++); + } + Response res = next(req); + res.headers.set("X-Request-ID", req.correlation_id); + return res; + }; +} + +} // namespace molo::example::fw diff --git a/example/fw/App.hpp b/example/fw/App.hpp new file mode 100644 index 0000000..fcdf372 --- /dev/null +++ b/example/fw/App.hpp @@ -0,0 +1,67 @@ +#pragma once + +// Application: composition root. Owns routers, middleware chain, sub-app +// mounts, and the typed error mapping. Gap: real molo::app::Application in +// 05-request-routing.md owns lifespan hooks, plugin registry, and the full DI +// container; this skeleton passes a Services struct to route factories. + +#include +#include +#include +#include + +#include "Http.hpp" +#include "Router.hpp" + +namespace molo::example::fw { + +enum class MiddlewareSlot { + Errors = 0, + Observability = 1, + Security = 2, + SessionAuth = 3, + User = 4, +}; + +using Next = std::function; +using Middleware = std::function; + +class HttpError { + public: + HttpError(int status, std::string code, std::string message); + int status() const; + const std::string& code() const; + const std::string& message() const; + + private: + int status_; + std::string code_; + std::string message_; +}; + +class Application : public std::enable_shared_from_this { + public: + void use(MiddlewareSlot slot, Middleware middleware); + void include_router(const std::string& prefix, const Router& router, + const std::string& name_prefix = ""); + void mount(const std::string& path, std::shared_ptr sub); + + Response dispatch(Request req); + + const std::vector& routes() const; + + private: + Response dispatch_inner(Request req, int depth); + Response run_chain(Request req, int depth); + + Router router_; + std::vector> middlewares_; + std::vector>> mounts_; + MiddlewareSlot last_slot_ = MiddlewareSlot::Errors; + bool has_middleware_ = false; +}; + +Middleware errors_middleware(); +Middleware request_id_middleware(); + +} // namespace molo::example::fw diff --git a/example/fw/Caps.hpp b/example/fw/Caps.hpp new file mode 100644 index 0000000..440eb2b --- /dev/null +++ b/example/fw/Caps.hpp @@ -0,0 +1,48 @@ +#pragma once + +// Static ceilings for the living example. Every knob has a cap; no autotuning. +// Gap: real framework caps live in 01-runtime.md through 09-cli-ops.md specs; +// these example caps mirror them until the framework lands. + +#include + +namespace molo::example::fw::caps { + +inline constexpr std::size_t kMaxRequestBody = 1 << 20; // 1 MiB +inline constexpr std::size_t kMaxHeaderCount = 100; +inline constexpr std::size_t kMaxHeaderBytes = 65536; +inline constexpr std::size_t kMaxFormFields = 200; +inline constexpr int kMaxJsonDepth = 32; +inline constexpr std::size_t kMaxUploadBytes = 8 << 20; // 8 MiB +inline constexpr std::size_t kMaxTemplateBytes = 256 << 10; +inline constexpr int kMaxTemplateDepth = 16; +inline constexpr std::size_t kMaxRouteCount = 4096; +inline constexpr int kMaxMountDepth = 8; +inline constexpr std::size_t kMaxKvEntries = 100000; +inline constexpr std::size_t kMaxCacheEntries = 10000; +inline constexpr std::size_t kMaxPubsubTopics = 1000; +inline constexpr std::size_t kMaxPubsubHistory = 100; +inline constexpr std::size_t kMaxMetrics = 1000; +inline constexpr std::size_t kMaxSpans = 10000; +inline constexpr std::size_t kMaxSessions = 100000; +inline constexpr std::size_t kMaxOrmRows = 100000; +inline constexpr std::size_t kMaxOrmTables = 500; +inline constexpr std::size_t kMaxTasks = 10000; +inline constexpr std::size_t kMaxWorkers = 64; +inline constexpr std::size_t kMaxReplEntries = 10000; +inline constexpr std::size_t kMaxStreamChunks = 1000; +inline constexpr std::size_t kMaxChunkBytes = 65536; +inline constexpr int kServerBacklog = 64; +inline constexpr int kServerTimeoutSec = 30; +inline constexpr int kMaxConnections = 512; +inline constexpr std::size_t kMaxTestBody = 4 << 20; +inline constexpr int kSuiteTimeoutSec = 600; +inline constexpr int kSingleTestTimeoutSec = 30; +inline constexpr int kMaxReleasePerRun = 1; +inline constexpr int kMaxChangelogLines = 20; +inline constexpr int kMaxCommitSubject = 100; +inline constexpr int kPipelineTimeoutMin = 30; +inline constexpr std::size_t kMaxOpenApiRoutes = 4096; +inline constexpr std::size_t kMaxWebdavEntries = 10000; + +} // namespace molo::example::fw::caps diff --git a/example/fw/Cookies.cpp b/example/fw/Cookies.cpp new file mode 100644 index 0000000..3d050ae --- /dev/null +++ b/example/fw/Cookies.cpp @@ -0,0 +1,49 @@ +#include "Cookies.hpp" + +#include + +namespace molo::example::fw { + +namespace { + +unsigned long long fnv1a(const std::string& s) { + unsigned long long h = 1469598103934665603ULL; + for (unsigned char c : s) { + h ^= c; + h *= 1099511628211ULL; + } + return h; +} + +} // namespace + +std::string cookie_sign(const std::string& value, const std::string& secret) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "%016llx", + (unsigned long long)fnv1a(secret + "|" + value)); + return value + "." + buf; +} + +bool cookie_verify(const std::string& signed_value, const std::string& secret, + std::string& out_value) { + std::size_t dot = signed_value.rfind('.'); + if (dot == std::string::npos) return false; + std::string value = signed_value.substr(0, dot); + return cookie_sign(value, secret) == signed_value + ? (out_value = value, true) + : false; +} + +std::string make_set_cookie(const std::string& name, const std::string& value, + const std::string& path, bool http_only, + bool secure, const std::string& same_site, + long max_age) { + std::string out = name + "=" + value + "; Path=" + path; + if (max_age >= 0) out += "; Max-Age=" + std::to_string(max_age); + if (http_only) out += "; HttpOnly"; + if (secure) out += "; Secure"; + out += "; SameSite=" + same_site; + return out; +} + +} // namespace molo::example::fw diff --git a/example/fw/Cookies.hpp b/example/fw/Cookies.hpp new file mode 100644 index 0000000..2dedb66 --- /dev/null +++ b/example/fw/Cookies.hpp @@ -0,0 +1,21 @@ +#pragma once + +// First-class cookies: sign and verify values for session cookies. +// Signing here is a keyed FNV-1a demonstration, NOT production crypto. +// Gap: real signing and encryption live in 03-security.md (AES-GCM or +// ChaCha20-Poly1305); this stub exists so the example compiles and runs. + +#include + +namespace molo::example::fw { + +std::string cookie_sign(const std::string& value, const std::string& secret); +bool cookie_verify(const std::string& signed_value, const std::string& secret, + std::string& out_value); + +std::string make_set_cookie(const std::string& name, const std::string& value, + const std::string& path, bool http_only, + bool secure, const std::string& same_site, + long max_age); + +} // namespace molo::example::fw diff --git a/example/fw/Data.cpp b/example/fw/Data.cpp new file mode 100644 index 0000000..3af3880 --- /dev/null +++ b/example/fw/Data.cpp @@ -0,0 +1,272 @@ +#include "Data.hpp" + +#include + +#include "Caps.hpp" + +namespace molo::example::fw { + +std::string SessionStore::create() { + std::lock_guard lock(mu_); + std::string id = "sess-" + std::to_string(next_id_++); + sessions_[id] = {}; + return id; +} + +bool SessionStore::get(const std::string& id, + std::map& out) { + std::lock_guard lock(mu_); + auto it = sessions_.find(id); + if (it == sessions_.end()) return false; + out = it->second; + return true; +} + +void SessionStore::put(const std::string& id, const std::string& key, + const std::string& value) { + std::lock_guard lock(mu_); + if (!sessions_.count(id) && sessions_.size() >= caps::kMaxSessions) return; + sessions_[id][key] = value; +} + +void SessionStore::rotate(const std::string& old_id, std::string& new_id) { + std::lock_guard lock(mu_); + new_id = "sess-" + std::to_string(next_id_++); + auto it = sessions_.find(old_id); + if (it != sessions_.end()) { + sessions_[new_id] = it->second; + sessions_.erase(it); + } else { + sessions_[new_id] = {}; + } +} + +std::string demo_hash_password(const std::string& password) { + unsigned long long h = 1469598103934665603ULL; + std::string salted = "demo-salt|" + password; + for (unsigned char c : salted) { + h ^= c; + h *= 1099511628211ULL; + } + char buf[32]; + std::snprintf(buf, sizeof(buf), "demo$%016llx", (unsigned long long)h); + return buf; +} + +namespace { + +int b64val(char c) { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+') return 62; + if (c == '/') return 63; + return -1; +} + +std::string b64decode(const std::string& in) { + std::string out; + int acc = 0, bits = 0; + for (char c : in) { + if (c == '=') break; + int v = b64val(c); + if (v < 0) continue; + acc = (acc << 6) | v; + bits += 6; + if (bits >= 8) { + bits -= 8; + out.push_back((char)((acc >> bits) & 0xFF)); + } + } + return out; +} + +} // namespace + +bool parse_basic_auth(const std::string& header, std::string& user, + std::string& password) { + const std::string prefix = "Basic "; + if (header.compare(0, prefix.size(), prefix) != 0) return false; + std::string decoded = b64decode(header.substr(prefix.size())); + std::size_t colon = decoded.find(':'); + if (colon == std::string::npos) return false; + user = decoded.substr(0, colon); + password = decoded.substr(colon + 1); + return true; +} + +void Auth::add_user(const std::string& username, const std::string& password) { + std::lock_guard lock(mu_); + users_[username] = demo_hash_password(password); +} + +bool Auth::check(const std::string& username, + const std::string& password) const { + std::lock_guard lock(mu_); + auto it = users_.find(username); + if (it == users_.end()) return false; + return it->second == demo_hash_password(password); +} + +std::string backend_name(OrmBackend backend) { + return backend == OrmBackend::Sqlite ? "sqlite" : "postgres"; +} + +OrmBackend parse_backend(const std::string& name) { + return name == "postgres" ? OrmBackend::Postgres : OrmBackend::Sqlite; +} + +OrmDb::OrmDb(OrmBackend backend) : backend_(backend) {} + +OrmBackend OrmDb::backend() const { return backend_; } + +OrmResult OrmDb::insert(const std::string& table, + const std::map& values) { + std::lock_guard lock(mu_); + OrmResult res; + auto& cols = schema_[table]; + if (cols.empty()) { + if (schema_.size() > caps::kMaxOrmTables) { + res.ok = false; + return res; + } + for (const auto& [k, v] : values) cols.push_back(k); + res.healed = true; + res.healed_note = "auto-created table " + table; + } else { + for (const auto& [k, v] : values) { + bool known = false; + for (const auto& c : cols) { + if (c == k) { + known = true; + break; + } + } + if (!known) { + cols.push_back(k); + res.healed = true; + res.healed_note = "auto-added column " + k + " to " + table; + } + } + } + auto& vec = rows_[table]; + if (vec.size() >= caps::kMaxOrmRows) { + res.ok = false; + return res; + } + OrmRow row; + row.columns = values; + vec.push_back(row); + res.rows = {row}; + return res; +} + +OrmResult OrmDb::select_all(const std::string& table) { + std::lock_guard lock(mu_); + OrmResult res; + auto it = rows_.find(table); + if (it == rows_.end()) { + if (!schema_.count(table)) { + schema_[table] = {"id"}; + res.healed = true; + res.healed_note = "auto-created table " + table; + } + return res; + } + res.rows = it->second; + return res; +} + +OrmResult OrmDb::ensure_column(const std::string& table, + const std::string& column) { + std::lock_guard lock(mu_); + OrmResult res; + auto& cols = schema_[table]; + for (const auto& c : cols) { + if (c == column) return res; + } + cols.push_back(column); + res.healed = true; + res.healed_note = "auto-added column " + column + " to " + table; + return res; +} + +std::vector OrmDb::tables() const { + std::lock_guard lock(mu_); + std::vector out; + for (const auto& [k, v] : schema_) out.push_back(k); + return out; +} + +std::vector OrmDb::columns(const std::string& table) const { + std::lock_guard lock(mu_); + auto it = schema_.find(table); + if (it == schema_.end()) return {}; + return it->second; +} + +bool UploadStore::store(const std::string& filename, + const std::string& content_type, + const std::string& data, UploadInfo& out, + std::string& error) { + if (data.size() > caps::kMaxUploadBytes) { + error = "file too large"; + return false; + } + bool allowed = content_type == "text/plain" || content_type == "image/png" || + content_type == "application/json"; + if (!allowed) { + error = "content type not allowed"; + return false; + } + if (filename.find("..") != std::string::npos || + filename.find('/') != std::string::npos) { + error = "unsafe filename"; + return false; + } + std::lock_guard lock(mu_); + out = UploadInfo{filename, content_type, data.size()}; + uploads_.push_back(out); + return true; +} + +std::vector UploadStore::list() const { + std::lock_guard lock(mu_); + return uploads_; +} + +void WebdavStore::put(const std::string& path, const std::string& content) { + std::lock_guard lock(mu_); + if (!files_.count(path) && files_.size() >= caps::kMaxWebdavEntries) return; + files_[path] = content; +} + +bool WebdavStore::get(const std::string& path, std::string& out) const { + std::lock_guard lock(mu_); + auto it = files_.find(path); + if (it == files_.end()) return false; + out = it->second; + return true; +} + +bool WebdavStore::remove(const std::string& path) { + std::lock_guard lock(mu_); + return files_.erase(path) > 0; +} + +std::string WebdavStore::propfind_xml(const std::string& root) const { + std::lock_guard lock(mu_); + std::string out = + ""; + for (const auto& [k, v] : files_) { + if (k.compare(0, root.size(), root) != 0) continue; + out += "" + k + + "" + + std::to_string(v.size()) + + "HTTP/1.1 200 OK" + ""; + } + return out + ""; +} + +} // namespace molo::example::fw diff --git a/example/fw/Data.hpp b/example/fw/Data.hpp new file mode 100644 index 0000000..868da99 --- /dev/null +++ b/example/fw/Data.hpp @@ -0,0 +1,113 @@ +#pragma once + +// Data layer skeletons: sessions, basic auth, dual-backend ORM stub with +// self-healing demo, upload store, and WebDAV store. +// Gap: real 04-data.md owns the Postgres wire client, SQLite handling, +// pooling, transactions, and production self-healing; real password hashing +// (Argon2id) and encryption live in 03-security.md. + +#include +#include +#include +#include + +namespace molo::example::fw { + +class SessionStore { + public: + std::string create(); + bool get(const std::string& id, std::map& out); + void put(const std::string& id, const std::string& key, + const std::string& value); + void rotate(const std::string& old_id, std::string& new_id); + + private: + mutable std::mutex mu_; + std::map> sessions_; + unsigned long next_id_ = 1; +}; + +struct BasicUser { + std::string username; + std::string password_hash; +}; + +std::string demo_hash_password(const std::string& password); +bool parse_basic_auth(const std::string& header, std::string& user, + std::string& password); + +class Auth { + public: + void add_user(const std::string& username, const std::string& password); + bool check(const std::string& username, const std::string& password) const; + + private: + mutable std::mutex mu_; + std::map users_; +}; + +enum class OrmBackend { Sqlite, Postgres }; + +std::string backend_name(OrmBackend backend); +OrmBackend parse_backend(const std::string& name); + +struct OrmRow { + std::map columns; +}; + +struct OrmResult { + bool ok = true; + bool healed = false; + std::string healed_note; + std::vector rows; +}; + +class OrmDb { + public: + explicit OrmDb(OrmBackend backend); + + OrmBackend backend() const; + OrmResult insert(const std::string& table, + const std::map& values); + OrmResult select_all(const std::string& table); + OrmResult ensure_column(const std::string& table, const std::string& column); + std::vector tables() const; + std::vector columns(const std::string& table) const; + + private: + OrmBackend backend_; + mutable std::mutex mu_; + std::map> schema_; + std::map> rows_; +}; + +struct UploadInfo { + std::string filename; + std::string content_type; + std::size_t size = 0; +}; + +class UploadStore { + public: + bool store(const std::string& filename, const std::string& content_type, + const std::string& data, UploadInfo& out, std::string& error); + std::vector list() const; + + private: + mutable std::mutex mu_; + std::vector uploads_; +}; + +class WebdavStore { + public: + void put(const std::string& path, const std::string& content); + bool get(const std::string& path, std::string& out) const; + bool remove(const std::string& path); + std::string propfind_xml(const std::string& root) const; + + private: + mutable std::mutex mu_; + std::map files_; +}; + +} // namespace molo::example::fw diff --git a/example/fw/Http.cpp b/example/fw/Http.cpp new file mode 100644 index 0000000..a146e5c --- /dev/null +++ b/example/fw/Http.cpp @@ -0,0 +1,266 @@ +#include "Http.hpp" + +#include + +#include "Caps.hpp" + +namespace molo::example::fw { + +Method parse_method(std::string_view name) { + if (name == "GET") return Method::Get; + if (name == "POST") return Method::Post; + if (name == "PUT") return Method::Put; + if (name == "PATCH") return Method::Patch; + if (name == "DELETE") return Method::Delete; + if (name == "HEAD") return Method::Head; + if (name == "OPTIONS") return Method::Options; + if (name == "PROPFIND") return Method::Propfind; + if (name == "PROPPATCH") return Method::Proppatch; + if (name == "MKCOL") return Method::Mkcol; + if (name == "COPY") return Method::Copy; + if (name == "MOVE") return Method::Move; + if (name == "LOCK") return Method::Lock; + if (name == "UNLOCK") return Method::Unlock; + return Method::Unknown; +} + +std::string method_name(Method method) { + switch (method) { + case Method::Get: return "GET"; + case Method::Post: return "POST"; + case Method::Put: return "PUT"; + case Method::Patch: return "PATCH"; + case Method::Delete: return "DELETE"; + case Method::Head: return "HEAD"; + case Method::Options: return "OPTIONS"; + case Method::Propfind: return "PROPFIND"; + case Method::Proppatch: return "PROPPATCH"; + case Method::Mkcol: return "MKCOL"; + case Method::Copy: return "COPY"; + case Method::Move: return "MOVE"; + case Method::Lock: return "LOCK"; + case Method::Unlock: return "UNLOCK"; + case Method::Unknown: return "UNKNOWN"; + } + return "UNKNOWN"; +} + +namespace { + +bool name_eq(std::string_view a, std::string_view b) { + if (a.size() != b.size()) return false; + for (std::size_t i = 0; i < a.size(); ++i) { + if (std::tolower((unsigned char)a[i]) != std::tolower((unsigned char)b[i])) + return false; + } + return true; +} + +} // namespace + +void Headers::set(std::string name, std::string value) { + for (auto& e : entries_) { + if (name_eq(e.first, name)) { + e.second = value; + return; + } + } + entries_.emplace_back(std::move(name), std::move(value)); +} + +std::string Headers::get(std::string_view name) const { + for (const auto& e : entries_) { + if (name_eq(e.first, name)) return e.second; + } + return ""; +} + +bool Headers::has(std::string_view name) const { + for (const auto& e : entries_) { + if (name_eq(e.first, name)) return true; + } + return false; +} + +const std::vector>& Headers::entries() + const { + return entries_; +} + +std::string Request::query_or(std::string_view key, + std::string_view fallback) const { + auto it = query.find(std::string(key)); + if (it == query.end()) return std::string(fallback); + return it->second; +} + +std::string Request::param_or(std::string_view key, + std::string_view fallback) const { + auto it = path_params.find(std::string(key)); + if (it == path_params.end()) return std::string(fallback); + return it->second; +} + +std::string Request::cookie_or(std::string_view key, + std::string_view fallback) const { + auto it = cookies.find(std::string(key)); + if (it == cookies.end()) return std::string(fallback); + return it->second; +} + +Response::Response(int status, std::string body, std::string content_type) + : status(status), body(std::move(body)) { + headers.set("Content-Type", std::move(content_type)); +} + +std::string Response::content_type() const { + return headers.get("Content-Type"); +} + +TextResponse::TextResponse(std::string body, int status) + : Response(status, std::move(body), "text/plain; charset=utf-8") {} + +HtmlResponse::HtmlResponse(std::string body, int status) + : Response(status, std::move(body), "text/html; charset=utf-8") {} + +JsonResponse::JsonResponse(std::string body, int status) + : Response(status, std::move(body), "application/json") {} + +XmlResponse::XmlResponse(std::string body, int status) + : Response(status, std::move(body), "application/xml") {} + +RedirectResponse::RedirectResponse(std::string location, bool permanent) + : Response(permanent ? 301 : 302, "", "text/plain; charset=utf-8") { + headers.set("Location", std::move(location)); +} + +FileResponse::FileResponse(std::string body, std::string content_type, + std::string filename) + : Response(200, std::move(body), std::move(content_type)) { + headers.set("Content-Disposition", + "attachment; filename=\"" + filename + "\""); +} + +StreamingResponse::StreamingResponse(std::vector chunks) + : Response(200, "", "text/plain; charset=utf-8"), chunks(std::move(chunks)) { + for (const auto& c : this->chunks) body += c; + headers.set("Transfer-Encoding", "chunked"); + if (this->chunks.size() > caps::kMaxStreamChunks) { + status = 500; + body = "chunk cap exceeded"; + } +} + +SseResponse::SseResponse(std::vector events) + : Response(200, "", "text/event-stream"), events(std::move(events)) { + for (const auto& e : this->events) body += "data: " + e + "\n\n"; +} + +std::string url_decode(std::string_view in) { + std::string out; + out.reserve(in.size()); + for (std::size_t i = 0; i < in.size(); ++i) { + if (in[i] == '%' && i + 2 < in.size() && + std::isxdigit((unsigned char)in[i + 1]) && + std::isxdigit((unsigned char)in[i + 2])) { + int hi = std::isdigit((unsigned char)in[i + 1]) + ? in[i + 1] - '0' + : std::tolower((unsigned char)in[i + 1]) - 'a' + 10; + int lo = std::isdigit((unsigned char)in[i + 2]) + ? in[i + 2] - '0' + : std::tolower((unsigned char)in[i + 2]) - 'a' + 10; + out.push_back((char)(hi * 16 + lo)); + i += 2; + } else if (in[i] == '+') { + out.push_back(' '); + } else { + out.push_back(in[i]); + } + } + return out; +} + +std::map parse_query(std::string_view qs) { + std::map out; + std::size_t start = 0; + std::size_t fields = 0; + while (start <= qs.size() && fields < caps::kMaxFormFields) { + std::size_t amp = qs.find('&', start); + if (amp == std::string_view::npos) amp = qs.size(); + std::string_view pair = qs.substr(start, amp - start); + std::size_t eq = pair.find('='); + if (eq == std::string_view::npos) { + if (!pair.empty()) out[url_decode(pair)] = ""; + } else { + out[url_decode(pair.substr(0, eq))] = url_decode(pair.substr(eq + 1)); + } + ++fields; + start = amp + 1; + } + return out; +} + +std::map parse_cookies(std::string_view header) { + std::map out; + std::size_t start = 0; + while (start < header.size()) { + std::size_t semi = header.find(';', start); + if (semi == std::string_view::npos) semi = header.size(); + std::string_view pair = header.substr(start, semi - start); + std::size_t eq = pair.find('='); + if (eq != std::string_view::npos) { + std::string k(pair.substr(0, eq)); + std::string v(pair.substr(eq + 1)); + while (!k.empty() && k.front() == ' ') k.erase(k.begin()); + while (!v.empty() && v.front() == ' ') v.erase(v.begin()); + if (!k.empty()) out[k] = v; + } + start = semi + 1; + } + return out; +} + +std::map parse_form(std::string_view body) { + return parse_query(body); +} + +std::string status_text(int status) { + switch (status) { + case 200: return "OK"; + case 201: return "Created"; + case 207: return "Multi-Status"; + case 301: return "Moved Permanently"; + case 302: return "Found"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 406: return "Not Acceptable"; + case 409: return "Conflict"; + case 413: return "Content Too Large"; + case 422: return "Unprocessable Entity"; + case 429: return "Too Many Requests"; + case 500: return "Internal Server Error"; + case 501: return "Not Implemented"; + default: return "Unknown"; + } +} + +std::string json_escape(std::string_view in) { + std::string out; + out.reserve(in.size() + 2); + for (char c : in) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out.push_back(c); break; + } + } + return out; +} + +} // namespace molo::example::fw diff --git a/example/fw/Http.hpp b/example/fw/Http.hpp new file mode 100644 index 0000000..539607d --- /dev/null +++ b/example/fw/Http.hpp @@ -0,0 +1,114 @@ +#pragma once + +// HTTP message types: one Request, one Response hierarchy with a subclass per +// format. Subclasses are constructor conveniences over the base value type so +// handlers can return by value without slicing loss. Gap: real molo::http_msg +// in 05-request-routing.md owns the final API; this skeleton mirrors it. + +#include +#include +#include +#include + +namespace molo::example::fw { + +enum class Method { + Get, + Post, + Put, + Patch, + Delete, + Head, + Options, + Propfind, + Proppatch, + Mkcol, + Copy, + Move, + Lock, + Unlock, + Unknown, +}; + +Method parse_method(std::string_view name); +std::string method_name(Method method); + +class Headers { + public: + void set(std::string name, std::string value); + std::string get(std::string_view name) const; + bool has(std::string_view name) const; + const std::vector>& entries() const; + + private: + std::vector> entries_; +}; + +struct Request { + Method method = Method::Get; + std::string path = "/"; + std::map query; + Headers headers; + std::map cookies; + std::map path_params; + std::string body; + std::string correlation_id; + + std::string query_or(std::string_view key, std::string_view fallback) const; + std::string param_or(std::string_view key, std::string_view fallback) const; + std::string cookie_or(std::string_view key, std::string_view fallback) const; +}; + +struct Response { + int status = 200; + Headers headers; + std::string body; + + Response() = default; + Response(int status, std::string body, std::string content_type); + + std::string content_type() const; +}; + +struct TextResponse : Response { + explicit TextResponse(std::string body, int status = 200); +}; + +struct HtmlResponse : Response { + explicit HtmlResponse(std::string body, int status = 200); +}; + +struct JsonResponse : Response { + explicit JsonResponse(std::string body, int status = 200); +}; + +struct XmlResponse : Response { + explicit XmlResponse(std::string body, int status = 200); +}; + +struct RedirectResponse : Response { + RedirectResponse(std::string location, bool permanent); +}; + +struct FileResponse : Response { + FileResponse(std::string body, std::string content_type, std::string filename); +}; + +struct StreamingResponse : Response { + explicit StreamingResponse(std::vector chunks); + std::vector chunks; +}; + +struct SseResponse : Response { + explicit SseResponse(std::vector events); + std::vector events; +}; + +std::string url_decode(std::string_view in); +std::map parse_query(std::string_view qs); +std::map parse_cookies(std::string_view header); +std::map parse_form(std::string_view body); +std::string status_text(int status); +std::string json_escape(std::string_view in); + +} // namespace molo::example::fw diff --git a/example/fw/Jobs.cpp b/example/fw/Jobs.cpp new file mode 100644 index 0000000..ba41bca --- /dev/null +++ b/example/fw/Jobs.cpp @@ -0,0 +1,113 @@ +#include "Jobs.hpp" + +#include "Caps.hpp" + +namespace molo::example::fw { + +void Supervisor::add_worker(const std::string& name) { + std::lock_guard lock(mu_); + if (!workers_.count(name) && workers_.size() >= caps::kMaxWorkers) return; + workers_[name] = WorkerInfo{name, false, 0}; +} + +bool Supervisor::start(const std::string& name) { + std::lock_guard lock(mu_); + auto it = workers_.find(name); + if (it == workers_.end()) return false; + it->second.running = true; + return true; +} + +bool Supervisor::stop(const std::string& name) { + std::lock_guard lock(mu_); + auto it = workers_.find(name); + if (it == workers_.end()) return false; + it->second.running = false; + return true; +} + +bool Supervisor::restart(const std::string& name) { + std::lock_guard lock(mu_); + auto it = workers_.find(name); + if (it == workers_.end()) return false; + it->second.running = true; + ++it->second.restarts; + return true; +} + +std::vector Supervisor::status() const { + std::lock_guard lock(mu_); + std::vector out; + for (const auto& [k, v] : workers_) out.push_back(v); + return out; +} + +std::string TaskQueue::enqueue(const std::string& kind, + const std::string& payload, long delay_sec) { + std::lock_guard lock(mu_); + if (tasks_.size() >= caps::kMaxTasks) return ""; + std::string id = "task-" + std::to_string(next_id_++); + TaskInfo t{id, kind, payload, + delay_sec > 0 ? "delayed" : "queued", 0}; + tasks_[id] = t; + return id; +} + +bool TaskQueue::cancel(const std::string& id) { + std::lock_guard lock(mu_); + auto it = tasks_.find(id); + if (it == tasks_.end()) return false; + it->second.state = "cancelled"; + return true; +} + +bool TaskQueue::retry(const std::string& id) { + std::lock_guard lock(mu_); + auto it = tasks_.find(id); + if (it == tasks_.end()) return false; + ++it->second.attempts; + it->second.state = "queued"; + return true; +} + +void TaskQueue::run_inline(TaskInfo& task) { + task.state = "done"; + std::lock_guard lock(mu_); + auto it = tasks_.find(task.id); + if (it != tasks_.end()) it->second.state = "done"; +} + +std::vector TaskQueue::list() const { + std::lock_guard lock(mu_); + std::vector out; + for (const auto& [k, v] : tasks_) out.push_back(v); + return out; +} + +ReplEntry ReplicationLog::append(const std::string& object, + const std::string& data) { + std::lock_guard lock(mu_); + ReplEntry e; + e.seq = entries_.size() + 1; + e.object = object; + e.data = data; + e.resolution = "last-writer-wins"; + if (entries_.size() < caps::kMaxReplEntries) entries_.push_back(e); + return e; +} + +std::vector ReplicationLog::since(unsigned long seq) const { + std::lock_guard lock(mu_); + std::vector out; + for (const auto& e : entries_) { + if (e.seq > seq) out.push_back(e); + } + return out; +} + +unsigned long ReplicationLog::latest() const { + std::lock_guard lock(mu_); + return entries_.size(); +} + +} // namespace molo::example::fw diff --git a/example/fw/Jobs.hpp b/example/fw/Jobs.hpp new file mode 100644 index 0000000..3e51288 --- /dev/null +++ b/example/fw/Jobs.hpp @@ -0,0 +1,75 @@ +#pragma once + +// Supervisor plus background tasks plus replication log skeletons. +// Gap: real 07-supervisor-replication.md owns worker processes, restarts, +// durable streams, conflict handling, and multi-instance deployment. + +#include +#include +#include +#include +#include + +namespace molo::example::fw { + +struct WorkerInfo { + std::string name; + bool running = false; + int restarts = 0; +}; + +class Supervisor { + public: + void add_worker(const std::string& name); + bool start(const std::string& name); + bool stop(const std::string& name); + bool restart(const std::string& name); + std::vector status() const; + + private: + mutable std::mutex mu_; + std::map workers_; +}; + +struct TaskInfo { + std::string id; + std::string kind; + std::string payload; + std::string state; + int attempts = 0; +}; + +class TaskQueue { + public: + std::string enqueue(const std::string& kind, const std::string& payload, + long delay_sec = 0); + bool cancel(const std::string& id); + bool retry(const std::string& id); + void run_inline(TaskInfo& task); + std::vector list() const; + + private: + mutable std::mutex mu_; + std::map tasks_; + unsigned long next_id_ = 1; +}; + +struct ReplEntry { + unsigned long seq = 0; + std::string object; + std::string data; + std::string resolution; +}; + +class ReplicationLog { + public: + ReplEntry append(const std::string& object, const std::string& data); + std::vector since(unsigned long seq) const; + unsigned long latest() const; + + private: + mutable std::mutex mu_; + std::vector entries_; +}; + +} // namespace molo::example::fw diff --git a/example/fw/Kv.cpp b/example/fw/Kv.cpp new file mode 100644 index 0000000..211c8f1 --- /dev/null +++ b/example/fw/Kv.cpp @@ -0,0 +1,95 @@ +#include "Kv.hpp" + +#include "Caps.hpp" + +namespace molo::example::fw { + +bool KvStore::set(const std::string& key, const std::string& value) { + std::lock_guard lock(mu_); + if (!data_.count(key) && data_.size() >= caps::kMaxKvEntries) return false; + data_[key] = value; + return true; +} + +bool KvStore::get(const std::string& key, std::string& out) const { + std::lock_guard lock(mu_); + auto it = data_.find(key); + if (it == data_.end()) return false; + out = it->second; + return true; +} + +bool KvStore::del(const std::string& key) { + std::lock_guard lock(mu_); + return data_.erase(key) > 0; +} + +std::size_t KvStore::size() const { + std::lock_guard lock(mu_); + return data_.size(); +} + +void Cache::put(const std::string& key, const std::string& value, long ttl_sec, + long now) { + std::lock_guard lock(mu_); + if (!data_.count(key) && data_.size() >= caps::kMaxCacheEntries) return; + data_[key] = CacheEntry{value, now + ttl_sec}; +} + +bool Cache::lookup(const std::string& key, std::string& out, long now) { + std::lock_guard lock(mu_); + auto it = data_.find(key); + if (it == data_.end()) return false; + if (it->second.expires_at <= now) { + data_.erase(it); + return false; + } + out = it->second.value; + return true; +} + +void Cache::invalidate(const std::string& key) { + std::lock_guard lock(mu_); + data_.erase(key); +} + +std::size_t Cache::size() const { + std::lock_guard lock(mu_); + return data_.size(); +} + +bool Pubsub::subscribe(const std::string& topic, PubsubHandler handler) { + std::lock_guard lock(mu_); + if (!subs_.count(topic) && subs_.size() >= caps::kMaxPubsubTopics) + return false; + subs_[topic].push_back(std::move(handler)); + return true; +} + +std::size_t Pubsub::publish(const std::string& topic, + const std::string& message) { + std::lock_guard lock(mu_); + auto& hist = history_[topic]; + hist.push_back(message); + if (hist.size() > caps::kMaxPubsubHistory) hist.erase(hist.begin()); + auto it = subs_.find(topic); + if (it == subs_.end()) return 0; + for (auto& h : it->second) h(message); + return it->second.size(); +} + +std::vector Pubsub::history(const std::string& topic) const { + std::lock_guard lock(mu_); + auto it = history_.find(topic); + if (it == history_.end()) return {}; + return it->second; +} + +std::vector Pubsub::topics() const { + std::lock_guard lock(mu_); + std::vector out; + for (const auto& [k, v] : subs_) out.push_back(k); + return out; +} + +} // namespace molo::example::fw diff --git a/example/fw/Kv.hpp b/example/fw/Kv.hpp new file mode 100644 index 0000000..e9ce60f --- /dev/null +++ b/example/fw/Kv.hpp @@ -0,0 +1,61 @@ +#pragma once + +// In-memory key-value store, TTL cache, and pubsub broker skeletons. +// Gap: real implementations in 04-data.md own persistence, replication, +// durability guarantees, and multi-instance sharing. + +#include +#include +#include +#include +#include +#include + +namespace molo::example::fw { + +class KvStore { + public: + bool set(const std::string& key, const std::string& value); + bool get(const std::string& key, std::string& out) const; + bool del(const std::string& key); + std::size_t size() const; + + private: + mutable std::mutex mu_; + std::map data_; +}; + +struct CacheEntry { + std::string value; + long expires_at = 0; +}; + +class Cache { + public: + void put(const std::string& key, const std::string& value, long ttl_sec, + long now); + bool lookup(const std::string& key, std::string& out, long now); + void invalidate(const std::string& key); + std::size_t size() const; + + private: + mutable std::mutex mu_; + std::map data_; +}; + +using PubsubHandler = std::function; + +class Pubsub { + public: + bool subscribe(const std::string& topic, PubsubHandler handler); + std::size_t publish(const std::string& topic, const std::string& message); + std::vector history(const std::string& topic) const; + std::vector topics() const; + + private: + mutable std::mutex mu_; + std::map> subs_; + std::map> history_; +}; + +} // namespace molo::example::fw diff --git a/example/fw/Negotiate.cpp b/example/fw/Negotiate.cpp new file mode 100644 index 0000000..6144e2e --- /dev/null +++ b/example/fw/Negotiate.cpp @@ -0,0 +1,70 @@ +#include "Negotiate.hpp" + +#include + +namespace molo::example::fw { + +namespace { + +std::string trim(std::string s) { + while (!s.empty() && (s.front() == ' ' || s.front() == '\t')) s.erase(s.begin()); + while (!s.empty() && (s.back() == ' ' || s.back() == '\t')) s.pop_back(); + return s; +} + +} // namespace + +std::string negotiate(const std::string& accept, + const std::vector& available) { + if (available.empty()) throw HttpError(500, "no_offers", "no offers"); + if (accept.empty()) return available.front(); + struct Range { + std::string type; + double q = 1.0; + }; + std::vector ranges; + std::size_t start = 0; + while (start < accept.size()) { + std::size_t comma = accept.find(',', start); + if (comma == std::string::npos) comma = accept.size(); + std::string part = trim(accept.substr(start, comma - start)); + std::size_t semi = part.find(';'); + Range r{trim(part.substr(0, semi)), 1.0}; + if (semi != std::string::npos) { + std::string params = part.substr(semi + 1); + std::size_t q = params.find("q="); + if (q != std::string::npos) { + try { + r.q = std::stod(params.substr(q + 2)); + } catch (...) { + r.q = 1.0; + } + } + } + ranges.push_back(r); + start = comma + 1; + } + std::string best; + double best_q = -1.0; + for (const auto& a : available) { + for (const auto& r : ranges) { + bool hit = r.type == "*/*" || r.type == a; + if (!hit) { + std::size_t slash = a.find('/'); + std::string wild = slash == std::string::npos + ? "*/*" + : a.substr(0, slash) + "/*"; + hit = r.type == wild; + } + if (hit && r.q > best_q) { + best_q = r.q; + best = a; + } + } + } + if (best.empty() || best_q <= 0.0) + throw HttpError(406, "not_acceptable", "no acceptable format"); + return best; +} + +} // namespace molo::example::fw diff --git a/example/fw/Negotiate.hpp b/example/fw/Negotiate.hpp new file mode 100644 index 0000000..2c92e9b --- /dev/null +++ b/example/fw/Negotiate.hpp @@ -0,0 +1,22 @@ +#pragma once + +// Server-driven content negotiation: parses Accept with q weights, picks one +// handler-declared format, answers 406 when nothing matches. +// Gap: real molo::negotiate in 05-request-routing.md owns suffix and +// format-override rules. + +#include +#include + +#include "App.hpp" + +namespace molo::example::fw { + +struct Offer { + std::string media_type; +}; + +std::string negotiate(const std::string& accept, + const std::vector& available); + +} // namespace molo::example::fw diff --git a/example/fw/Net.cpp b/example/fw/Net.cpp new file mode 100644 index 0000000..4b4e8b5 --- /dev/null +++ b/example/fw/Net.cpp @@ -0,0 +1,82 @@ +#include "Net.hpp" + +#include "Caps.hpp" + +namespace molo::example::fw { + +RestClient::RestClient(RestTransport transport) + : transport_(std::move(transport)) {} + +Response RestClient::get(const std::string& path) { + Request req; + req.method = Method::Get; + req.path = path; + return transport_(std::move(req)); +} + +Response RestClient::post_json(const std::string& path, + const std::string& json) { + Request req; + req.method = Method::Post; + req.path = path; + req.body = json; + req.headers.set("Content-Type", "application/json"); + return transport_(std::move(req)); +} + +void OpenApi::add(const std::string& path, const std::string& method, + const std::string& summary) { + if (routes_.size() >= caps::kMaxOpenApiRoutes) return; + routes_.push_back({path, method, summary}); +} + +std::string OpenApi::to_json() const { + std::string out = + "{\"openapi\":\"3.0.0\",\"info\":{\"title\":\"living-example\"," + "\"version\":\"0.1.0\"},\"paths\":{"; + bool first = true; + for (const auto& r : routes_) { + if (!first) out += ","; + first = false; + out += "\"" + json_escape(r.path) + "\":{\"" + json_escape(r.method) + + "\":{\"summary\":\"" + json_escape(r.summary) + "\"}}"; + } + return out + "}}"; +} + +std::size_t OpenApi::size() const { return routes_.size(); } + +std::string redact_secrets(const std::string& text) { + std::string out = text; + const char* keys[] = {"DEVPLACE_API_KEY", "API_KEY", "SECRET", "TOKEN", + "PASSWORD"}; + for (const char* k : keys) { + std::string key(k); + std::size_t pos = 0; + while ((pos = out.find(key, pos)) != std::string::npos) { + std::size_t eol = out.find('\n', pos); + if (eol == std::string::npos) eol = out.size(); + std::size_t eq = out.find('=', pos); + if (eq != std::string::npos && eq < eol) { + out.replace(eq + 1, eol - eq - 1, "[REDACTED]"); + pos = eq + 1; + } else { + pos += key.size(); + } + } + } + return out; +} + +AiSample ai_harness_sample(const std::string& prompt) { + AiSample s; + s.base_url = "https://devplace.net/openai/v1"; + s.model = "molodetz"; + s.redacted_prompt = redact_secrets(prompt); + s.stream_chunks = {"data: {\"delta\":\"plan: add route\"}\n\n", + "data: {\"delta\":\"apply: 1 file\"}\n\n", + "data: [DONE]\n\n"}; + return s; +} + +} // namespace molo::example::fw diff --git a/example/fw/Net.hpp b/example/fw/Net.hpp new file mode 100644 index 0000000..ba7fec4 --- /dev/null +++ b/example/fw/Net.hpp @@ -0,0 +1,57 @@ +#pragma once + +// Client parity skeletons: REST client (self-call transport), realtime +// client notes, OpenAPI registry, and the AI harness sample. +// Gap: real HTTP/1.1 plus HTTP/2 client in 08-clients-openapi.md, realtime +// clients in 06-realtime.md, and the AI harness CLI in 09-cli-ops.md. + +#include +#include +#include +#include + +#include "Http.hpp" + +namespace molo::example::fw { + +using RestTransport = std::function; + +class RestClient { + public: + explicit RestClient(RestTransport transport); + + Response get(const std::string& path); + Response post_json(const std::string& path, const std::string& json); + + private: + RestTransport transport_; +}; + +struct OpenApiRoute { + std::string path; + std::string method; + std::string summary; +}; + +class OpenApi { + public: + void add(const std::string& path, const std::string& method, + const std::string& summary); + std::string to_json() const; + std::size_t size() const; + + private: + std::vector routes_; +}; + +struct AiSample { + std::string base_url; + std::string model; + std::string redacted_prompt; + std::vector stream_chunks; +}; + +std::string redact_secrets(const std::string& text); +AiSample ai_harness_sample(const std::string& prompt); + +} // namespace molo::example::fw diff --git a/example/fw/Observ.cpp b/example/fw/Observ.cpp new file mode 100644 index 0000000..2602ef8 --- /dev/null +++ b/example/fw/Observ.cpp @@ -0,0 +1,86 @@ +#include "Observ.hpp" + +#include + +#include "Caps.hpp" + +namespace molo::example::fw { + +void EventBus::on(const std::string& type, + std::function listener) { + std::lock_guard lock(mu_); + if (!kEnabled) return; + listeners_[type].push_back(std::move(listener)); +} + +void EventBus::emit(const Event& event) { + std::lock_guard lock(mu_); + if (!kEnabled) return; + delivered_.push_back(event); + auto it = listeners_.find(event.type); + if (it == listeners_.end()) return; + for (auto& l : it->second) l(event); +} + +std::vector EventBus::delivered() const { + std::lock_guard lock(mu_); + return delivered_; +} + +void Metrics::inc(const std::string& name, double value) { + std::lock_guard lock(mu_); + if (!kEnabled) return; + if (!values_.count(name) && values_.size() >= caps::kMaxMetrics) return; + values_[name] += value; +} + +void Metrics::gauge(const std::string& name, double value) { + std::lock_guard lock(mu_); + if (!kEnabled) return; + if (!values_.count(name) && values_.size() >= caps::kMaxMetrics) return; + values_[name] = value; +} + +double Metrics::get(const std::string& name) const { + std::lock_guard lock(mu_); + auto it = values_.find(name); + return it == values_.end() ? 0.0 : it->second; +} + +std::string Metrics::render_prometheus() const { + std::lock_guard lock(mu_); + std::string out; + for (const auto& [k, v] : values_) { + char buf[64]; + std::snprintf(buf, sizeof(buf), "%g", v); + out += k + " " + buf + "\n"; + } + return out; +} + +Span Telemetry::start_span(const std::string& name, + const std::string& correlation_id) { + std::lock_guard lock(mu_); + Span s; + if (!kEnabled) return s; + s.trace_id = "trace-" + std::to_string(next_id_); + s.span_id = "span-" + std::to_string(next_id_); + ++next_id_; + s.name = name; + s.correlation_id = correlation_id; + return s; +} + +void Telemetry::finish_span(const Span& span) { + std::lock_guard lock(mu_); + if (!kEnabled) return; + if (spans_.size() >= caps::kMaxSpans) return; + spans_.push_back(span); +} + +std::vector Telemetry::spans() const { + std::lock_guard lock(mu_); + return spans_; +} + +} // namespace molo::example::fw diff --git a/example/fw/Observ.hpp b/example/fw/Observ.hpp new file mode 100644 index 0000000..1d4810a --- /dev/null +++ b/example/fw/Observ.hpp @@ -0,0 +1,72 @@ +#pragma once + +// Observability skeletons: typed event bus, metrics registry, and tracer. +// All three compile to near-zero overhead when kEnabled is false. +// Gap: real 02-observability.md owns the log format, Prometheus and OTel +// export, and the zero-cost macro layer. + +#include +#include +#include +#include +#include + +namespace molo::example::fw { + +struct Event { + std::string type; + std::string payload; + std::string correlation_id; +}; + +class EventBus { + public: + static constexpr bool kEnabled = true; + + void on(const std::string& type, std::function listener); + void emit(const Event& event); + std::vector delivered() const; + + private: + mutable std::mutex mu_; + std::map>> + listeners_; + std::vector delivered_; +}; + +class Metrics { + public: + static constexpr bool kEnabled = true; + + void inc(const std::string& name, double value = 1.0); + void gauge(const std::string& name, double value); + double get(const std::string& name) const; + std::string render_prometheus() const; + + private: + mutable std::mutex mu_; + std::map values_; +}; + +struct Span { + std::string trace_id; + std::string span_id; + std::string name; + std::string correlation_id; +}; + +class Telemetry { + public: + static constexpr bool kEnabled = true; + + Span start_span(const std::string& name, const std::string& correlation_id); + void finish_span(const Span& span); + std::vector spans() const; + + private: + mutable std::mutex mu_; + std::vector spans_; + unsigned long next_id_ = 1; +}; + +} // namespace molo::example::fw diff --git a/example/fw/Router.cpp b/example/fw/Router.cpp new file mode 100644 index 0000000..ba24d10 --- /dev/null +++ b/example/fw/Router.cpp @@ -0,0 +1,129 @@ +#include "Router.hpp" + +#include + +#include "Caps.hpp" + +namespace molo::example::fw { + +namespace { + +std::vector split_path(std::string_view path) { + std::vector segs; + std::string cur; + for (char c : path) { + if (c == '/') { + if (!cur.empty()) { + segs.push_back(cur); + cur.clear(); + } + } else { + cur.push_back(c); + } + } + if (!cur.empty()) segs.push_back(cur); + return segs; +} + +} // namespace + +bool match_pattern(std::string_view pattern, std::string_view path, + std::map& params) { + auto want = split_path(pattern); + auto got = split_path(path); + if (want.size() != got.size()) return false; + for (std::size_t i = 0; i < want.size(); ++i) { + const std::string& w = want[i]; + if (w.size() >= 2 && w.front() == '{' && w.back() == '}') { + params[w.substr(1, w.size() - 2)] = url_decode(got[i]); + } else if (w != got[i]) { + return false; + } + } + return true; +} + +void Router::add(std::vector methods, std::string pattern, + std::string name, Handler handler) { + if (routes_.size() >= caps::kMaxRouteCount) + throw std::length_error("route cap exceeded"); + for (const auto& r : routes_) { + if (r.name == name) + throw std::runtime_error("duplicate route name: " + name); + } + routes_.push_back( + Route{std::move(methods), std::move(pattern), std::move(name), + std::move(handler)}); +} + +void Router::get(std::string pattern, std::string name, Handler handler) { + add({Method::Get}, std::move(pattern), std::move(name), std::move(handler)); +} + +void Router::post(std::string pattern, std::string name, Handler handler) { + add({Method::Post}, std::move(pattern), std::move(name), std::move(handler)); +} + +void Router::put(std::string pattern, std::string name, Handler handler) { + add({Method::Put}, std::move(pattern), std::move(name), std::move(handler)); +} + +void Router::patch(std::string pattern, std::string name, Handler handler) { + add({Method::Patch}, std::move(pattern), std::move(name), std::move(handler)); +} + +void Router::del(std::string pattern, std::string name, Handler handler) { + add({Method::Delete}, std::move(pattern), std::move(name), + std::move(handler)); +} + +void Router::options(std::string pattern, std::string name, Handler handler) { + add({Method::Options}, std::move(pattern), std::move(name), + std::move(handler)); +} + +void Router::any(std::vector methods, std::string pattern, + std::string name, Handler handler) { + add(std::move(methods), std::move(pattern), std::move(name), + std::move(handler)); +} + +void Router::include_router(const std::string& prefix, const Router& other, + const std::string& name_prefix) { + for (const auto& r : other.routes_) { + std::string pattern = prefix; + if (!r.pattern.empty() && r.pattern != "/") { + if (pattern.empty() || pattern.back() != '/') pattern += "/"; + std::string sub = r.pattern; + if (!sub.empty() && sub.front() == '/') sub.erase(sub.begin()); + pattern += sub; + } + if (pattern.empty()) pattern = "/"; + std::string name = name_prefix.empty() ? r.name : name_prefix + r.name; + add(r.methods, pattern, name, r.handler); + } +} + +bool Router::match(Request& req, Handler& out) const { + for (const auto& r : routes_) { + bool method_ok = false; + for (Method m : r.methods) { + if (m == req.method) { + method_ok = true; + break; + } + } + if (!method_ok) continue; + std::map params; + if (match_pattern(r.pattern, req.path, params)) { + for (auto& [k, v] : params) req.path_params[k] = v; + out = r.handler; + return true; + } + } + return false; +} + +const std::vector& Router::routes() const { return routes_; } + +} // namespace molo::example::fw diff --git a/example/fw/Router.hpp b/example/fw/Router.hpp new file mode 100644 index 0000000..a69900d --- /dev/null +++ b/example/fw/Router.hpp @@ -0,0 +1,52 @@ +#pragma once + +// Router: FastAPI-style route groups mounted on the app with a shared prefix. +// Gap: real molo::routing in 05-request-routing.md owns path compilation, +// url_for and converters; this skeleton supports {param} segments only. + +#include +#include +#include + +#include "Http.hpp" + +namespace molo::example::fw { + +using Handler = std::function; + +struct Route { + std::vector methods; + std::string pattern; + std::string name; + Handler handler; +}; + +class Router { + public: + void add(std::vector methods, std::string pattern, std::string name, + Handler handler); + + void get(std::string pattern, std::string name, Handler handler); + void post(std::string pattern, std::string name, Handler handler); + void put(std::string pattern, std::string name, Handler handler); + void patch(std::string pattern, std::string name, Handler handler); + void del(std::string pattern, std::string name, Handler handler); + void options(std::string pattern, std::string name, Handler handler); + void any(std::vector methods, std::string pattern, std::string name, + Handler handler); + + void include_router(const std::string& prefix, const Router& other, + const std::string& name_prefix = ""); + + bool match(Request& req, Handler& out) const; + + const std::vector& routes() const; + + private: + std::vector routes_; +}; + +bool match_pattern(std::string_view pattern, std::string_view path, + std::map& params); + +} // namespace molo::example::fw diff --git a/example/fw/Server.cpp b/example/fw/Server.cpp new file mode 100644 index 0000000..1e2b995 --- /dev/null +++ b/example/fw/Server.cpp @@ -0,0 +1,205 @@ +#include "Server.hpp" + +#include +#include +#include +#include + +#include +#include + +#include "Caps.hpp" + +namespace molo::example::fw { + +Server::Server(std::shared_ptr app) : app_(std::move(app)) { + stop(); +} + +Server::~Server() { stop(); } + +bool Server::start(const std::string& host, int port) { + if (running_) return false; + listen_fd_ = ::socket(AF_INET, SOCK_STREAM, 0); + if (listen_fd_ < 0) return false; + int one = 1; + ::setsockopt(listen_fd_, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_port = htons((uint16_t)port); + if (::inet_pton(AF_INET, host.c_str(), &addr.sin_addr) != 1) { + ::close(listen_fd_); + listen_fd_ = -1; + return false; + } + if (::bind(listen_fd_, (sockaddr*)&addr, sizeof(addr)) != 0) { + ::close(listen_fd_); + listen_fd_ = -1; + return false; + } + if (port == 0) { + sockaddr_in bound{}; + socklen_t len = sizeof(bound); + if (::getsockname(listen_fd_, (sockaddr*)&bound, &len) == 0) + port_ = ntohs(bound.sin_port); + } else { + port_ = port; + } + if (::listen(listen_fd_, caps::kServerBacklog) != 0) { + ::close(listen_fd_); + listen_fd_ = -1; + return false; + } + running_ = true; + worker_ = std::thread([this]() { loop(); }); + return true; +} + +void Server::stop() { + bool was = running_.exchange(false); + if (listen_fd_ >= 0) { + ::shutdown(listen_fd_, SHUT_RDWR); + ::close(listen_fd_); + listen_fd_ = -1; + } + if (was && worker_.joinable() && + worker_.get_id() != std::this_thread::get_id()) + worker_.join(); + else if (worker_.joinable() && + worker_.get_id() == std::this_thread::get_id()) + worker_.detach(); +} + +int Server::port() const { return port_; } + +bool Server::running() const { return running_; } + +void Server::loop() { + int served = 0; + while (running_ && served < caps::kMaxConnections) { + sockaddr_in peer{}; + socklen_t len = sizeof(peer); + int fd = ::accept(listen_fd_, (sockaddr*)&peer, &len); + if (fd < 0) { + if (!running_) break; + continue; + } + struct timeval tv{}; + tv.tv_sec = caps::kServerTimeoutSec; + ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + handle_one(fd); + ::close(fd); + ++served; + } +} + +void Server::handle_one(int fd) { + std::string raw; + char buf[4096]; + while (raw.size() < caps::kMaxHeaderBytes) { + ssize_t n = ::recv(fd, buf, sizeof(buf), 0); + if (n <= 0) break; + raw.append(buf, (std::size_t)n); + if (raw.find("\r\n\r\n") != std::string::npos) break; + } + std::size_t hdr_end = raw.find("\r\n\r\n"); + if (hdr_end == std::string::npos) return; + std::string head = raw.substr(0, hdr_end); + std::string rest = raw.substr(hdr_end + 4); + std::istringstream lines(head); + std::string line; + if (!std::getline(lines, line)) return; + if (!line.empty() && line.back() == '\r') line.pop_back(); + std::istringstream rl(line); + std::string method_s, target, version; + rl >> method_s >> target >> version; + Request req; + req.method = parse_method(method_s); + std::size_t q = target.find('?'); + if (q == std::string::npos) { + req.path = target.empty() ? "/" : target; + } else { + req.path = target.substr(0, q); + req.query = parse_query(target.substr(q + 1)); + } + std::size_t header_count = 0; + std::size_t content_length = 0; + while (std::getline(lines, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.empty()) continue; + if (++header_count > caps::kMaxHeaderCount) break; + std::size_t colon = line.find(':'); + if (colon == std::string::npos) continue; + std::string name = line.substr(0, colon); + std::string value = line.substr(colon + 1); + while (!value.empty() && value.front() == ' ') value.erase(value.begin()); + req.headers.set(name, value); + if (name == "Content-Length" || name == "content-length") { + try { + content_length = (std::size_t)std::stoul(value); + } catch (...) { + content_length = 0; + } + } + } + std::string cookie_h = req.headers.get("Cookie"); + if (!cookie_h.empty()) req.cookies = parse_cookies(cookie_h); + req.body = rest; + while (req.body.size() < content_length && + req.body.size() < caps::kMaxRequestBody) { + ssize_t n = ::recv(fd, buf, sizeof(buf), 0); + if (n <= 0) break; + req.body.append(buf, (std::size_t)n); + } + Response res = app_->dispatch(std::move(req)); + if (res.body.size() > caps::kMaxTestBody) { + res.status = 500; + res.body = "response over cap"; + } + std::ostringstream out; + out << "HTTP/1.1 " << res.status << " " << status_text(res.status) << "\r\n"; + out << "Content-Length: " << res.body.size() << "\r\n"; + out << "Connection: close\r\n"; + for (const auto& [k, v] : res.headers.entries()) { + if (k == "Content-Length" || k == "content-length") continue; + if (k == "Transfer-Encoding" || k == "transfer-encoding") continue; + out << k << ": " << v << "\r\n"; + } + out << "\r\n" << res.body; + std::string wire = out.str(); + std::size_t sent = 0; + while (sent < wire.size()) { + ssize_t n = ::send(fd, wire.data() + sent, wire.size() - sent, 0); + if (n <= 0) break; + sent += (std::size_t)n; + } +} + +std::string http_roundtrip(const std::string& host, int port, + const std::string& raw_request) { + int fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return ""; + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_port = htons((uint16_t)port); + if (::inet_pton(AF_INET, host.c_str(), &addr.sin_addr) != 1) { + ::close(fd); + return ""; + } + if (::connect(fd, (sockaddr*)&addr, sizeof(addr)) != 0) { + ::close(fd); + return ""; + } + ::send(fd, raw_request.data(), raw_request.size(), 0); + std::string out; + char buf[4096]; + while (out.size() < caps::kMaxTestBody) { + ssize_t n = ::recv(fd, buf, sizeof(buf), 0); + if (n <= 0) break; + out.append(buf, (std::size_t)n); + } + ::close(fd); + return out; +} + +} // namespace molo::example::fw diff --git a/example/fw/Server.hpp b/example/fw/Server.hpp new file mode 100644 index 0000000..aa3ad63 --- /dev/null +++ b/example/fw/Server.hpp @@ -0,0 +1,44 @@ +#pragma once + +// Minimal loopback HTTP/1.1 server for manual verification and the real-HTTP +// round-trip test. Sequential accept loop, static caps, no TLS, no HTTP/2. +// Gap: real event loop, timeouts, keep-alive, H2, and compression live in +// 01-runtime.md; this server exists only to prove the stack end to end. + +#include +#include +#include +#include + +#include "App.hpp" + +namespace molo::example::fw { + +class Server { + public: + explicit Server(std::shared_ptr app); + ~Server(); + + Server(const Server&) = delete; + Server& operator=(const Server&) = delete; + + bool start(const std::string& host, int port); + void stop(); + int port() const; + bool running() const; + + private: + void loop(); + void handle_one(int fd); + + std::shared_ptr app_; + int listen_fd_ = -1; + int port_ = 0; + std::atomic running_{false}; + std::thread worker_; +}; + +std::string http_roundtrip(const std::string& host, int port, + const std::string& raw_request); + +} // namespace molo::example::fw diff --git a/example/fw/StaticFiles.cpp b/example/fw/StaticFiles.cpp new file mode 100644 index 0000000..a6b1be7 --- /dev/null +++ b/example/fw/StaticFiles.cpp @@ -0,0 +1,74 @@ +#include "StaticFiles.hpp" + +#include +#include +#include + +namespace molo::example::fw { + +bool path_is_safe(const std::string& request_path) { + if (request_path.find("..") != std::string::npos) return false; + if (request_path.find('\\') != std::string::npos) return false; + if (request_path.empty() || request_path.front() != '/') return false; + return true; +} + +namespace { + +std::string content_type_for(const std::string& name) { + auto ends = [&](const char* ext) { + std::string e(ext); + return name.size() >= e.size() && + name.compare(name.size() - e.size(), e.size(), e) == 0; + }; + if (ends(".html")) return "text/html; charset=utf-8"; + if (ends(".css")) return "text/css; charset=utf-8"; + if (ends(".js")) return "text/javascript; charset=utf-8"; + if (ends(".json")) return "application/json"; + if (ends(".svg")) return "image/svg+xml"; + if (ends(".png")) return "image/png"; + return "application/octet-stream"; +} + +} // namespace + +std::string etag_for(const std::string& body) { + unsigned long long h = 1469598103934665603ULL; + for (unsigned char c : body) { + h ^= c; + h *= 1099511628211ULL; + } + char buf[32]; + std::snprintf(buf, sizeof(buf), "\"%llx-%zu\"", + (unsigned long long)h, body.size()); + return buf; +} + +Response serve_disk_file(const std::string& root, + const std::string& subpath) { + std::string full = "/" + subpath; + if (!path_is_safe(full)) return TextResponse("unsafe path", 403); + std::ifstream in(root + full, std::ios::binary); + if (!in) return TextResponse("not found", 404); + std::ostringstream ss; + ss << in.rdbuf(); + Response res(200, ss.str(), content_type_for(full)); + res.headers.set("ETag", etag_for(res.body)); + res.headers.set("Cache-Control", "public, max-age=3600"); + return res; +} + +Response serve_embedded(const std::map& assets, + const std::string& subpath) { + std::string full = "/" + subpath; + if (!path_is_safe(full)) return TextResponse("unsafe path", 403); + auto it = assets.find(full); + if (it == assets.end()) return TextResponse("not found", 404); + Response res(200, it->second, content_type_for(full)); + res.headers.set("ETag", etag_for(res.body)); + res.headers.set("Cache-Control", "public, max-age=3600"); + res.headers.set("X-Asset-Source", "embedded"); + return res; +} + +} // namespace molo::example::fw diff --git a/example/fw/StaticFiles.hpp b/example/fw/StaticFiles.hpp new file mode 100644 index 0000000..01de0ad --- /dev/null +++ b/example/fw/StaticFiles.hpp @@ -0,0 +1,24 @@ +#pragma once + +// Static file serving from disk or from embedded C++ assets, with path +// traversal protection, ETags, and cache-control headers. +// Gap: real static serving in 01-runtime.md owns ranges, conditional +// requests, and directory indexing; the embedder CLI lives in 09-cli-ops.md. + +#include +#include + +#include "Http.hpp" + +namespace molo::example::fw { + +bool path_is_safe(const std::string& request_path); + +Response serve_disk_file(const std::string& root, const std::string& subpath); + +Response serve_embedded(const std::map& assets, + const std::string& subpath); + +std::string etag_for(const std::string& body); + +} // namespace molo::example::fw diff --git a/example/fw/Templates.cpp b/example/fw/Templates.cpp new file mode 100644 index 0000000..dc1d4b9 --- /dev/null +++ b/example/fw/Templates.cpp @@ -0,0 +1,246 @@ +#include "Templates.hpp" + +#include + +#include "Caps.hpp" + +namespace molo::example::fw { + +std::string html_escape(std::string_view in) { + std::string out; + out.reserve(in.size()); + for (char c : in) { + switch (c) { + case '&': out += "&"; break; + case '<': out += "<"; break; + case '>': out += ">"; break; + case '"': out += """; break; + default: out.push_back(c); break; + } + } + return out; +} + +namespace { + +std::string trim(std::string s) { + while (!s.empty() && (s.front() == ' ' || s.front() == '\t' || + s.front() == '\n' || s.front() == '\r')) + s.erase(s.begin()); + while (!s.empty() && (s.back() == ' ' || s.back() == '\t' || + s.back() == '\n' || s.back() == '\r')) + s.pop_back(); + return s; +} + +std::string apply_filter(std::string value, const std::string& filter) { + if (filter == "upper") { + for (char& c : value) c = (char)std::toupper((unsigned char)c); + } else if (filter == "lower") { + for (char& c : value) c = (char)std::tolower((unsigned char)c); + } else if (filter == "escape") { + value = html_escape(value); + } + return value; +} + +std::string ctx_string(const TemplateCtx& ctx, const std::string& key) { + auto it = ctx.find(key); + if (it == ctx.end()) return ""; + if (auto* s = std::get_if(&it->second)) return *s; + return ""; +} + +bool ctx_truthy(const TemplateCtx& ctx, const std::string& key) { + auto it = ctx.find(key); + if (it == ctx.end()) return false; + if (auto* s = std::get_if(&it->second)) + return !s->empty() && *s != "false" && *s != "0"; + if (auto* l = std::get_if(&it->second)) return !l->empty(); + return false; +} + +} // namespace + +void Environment::add_template(std::string name, std::string source) { + templates_[std::move(name)] = std::move(source); +} + +void Environment::add_global(std::string key, std::string value) { + globals_[std::move(key)] = std::move(value); +} + +bool Environment::has_template(const std::string& name) const { + return templates_.count(name) > 0; +} + +std::string Environment::render(const std::string& name, TemplateCtx ctx) const { + auto it = templates_.find(name); + if (it == templates_.end()) throw std::runtime_error("template not found"); + for (const auto& [k, v] : globals_) { + if (!ctx.count(k)) ctx[k] = v; + } + return render_source(name, it->second, ctx, 0); +} + +std::string Environment::render_source(const std::string& name, + const std::string& src, TemplateCtx& ctx, + int depth) const { + (void)name; + if (depth > caps::kMaxTemplateDepth) + throw std::runtime_error("template depth exceeded"); + std::string extends; + std::size_t pos = src.find("{% extends"); + if (pos != std::string::npos) { + std::size_t q1 = src.find('"', pos); + std::size_t q2 = q1 == std::string::npos + ? std::string::npos + : src.find('"', q1 + 1); + if (q1 != std::string::npos && q2 != std::string::npos) + extends = src.substr(q1 + 1, q2 - q1 - 1); + } + if (!extends.empty()) { + auto it = templates_.find(extends); + if (it == templates_.end()) throw std::runtime_error("base not found"); + std::map blocks; + std::size_t b = 0; + while (true) { + std::size_t bs = src.find("{% block", b); + if (bs == std::string::npos) break; + std::size_t be = src.find("%}", bs); + std::size_t cs = src.find("{% endblock %}", be); + if (be == std::string::npos || cs == std::string::npos) break; + std::string head = trim(src.substr(bs + 9, be - bs - 9)); + blocks[head] = src.substr(be + 2, cs - be - 2); + b = cs + 14; + } + std::string base = it->second; + for (const auto& [k, v] : blocks) { + std::string open = "{% block " + k + " %}"; + std::size_t s = base.find(open); + std::size_t e = s == std::string::npos + ? std::string::npos + : base.find("{% endblock %}", s); + if (s != std::string::npos && e != std::string::npos) + base = base.substr(0, s) + v + base.substr(e + 14); + } + return render_blocks(base, ctx, depth); + } + return render_blocks(src, ctx, depth); +} + +std::string Environment::render_blocks(const std::string& src, TemplateCtx& ctx, + int depth) const { + std::string out; + std::size_t i = 0; + while (i < src.size()) { + std::size_t v = src.find("{{", i); + std::size_t t = src.find("{%", i); + if (v == std::string::npos && t == std::string::npos) { + out += src.substr(i); + break; + } + if (v != std::string::npos && (t == std::string::npos || v < t)) { + out += src.substr(i, v - i); + std::size_t e = src.find("}}", v); + if (e == std::string::npos) break; + std::string expr = trim(src.substr(v + 2, e - v - 2)); + bool safe = false; + std::string var = expr; + std::vector filters; + std::size_t p = expr.find('|'); + if (p != std::string::npos) { + var = trim(expr.substr(0, p)); + std::string rest = expr.substr(p + 1); + std::size_t s = 0; + while (s < rest.size()) { + std::size_t q = rest.find('|', s); + std::string f = trim(rest.substr(s, q == std::string::npos + ? q + : q - s)); + if (f == "safe") { + safe = true; + } else if (!f.empty()) { + filters.push_back(f); + } + if (q == std::string::npos) break; + s = q + 1; + } + } + std::string value; + std::size_t dot = var.find('.'); + if (dot != std::string::npos) { + value = ctx_string(ctx, var); + } else { + value = ctx_string(ctx, var); + } + for (const auto& f : filters) value = apply_filter(value, f); + if (!safe) value = html_escape(value); + out += value; + i = e + 2; + } else { + out += src.substr(i, t - i); + std::size_t e = src.find("%}", t); + if (e == std::string::npos) break; + std::string tag = trim(src.substr(t + 2, e - t - 2)); + if (tag.rfind("if ", 0) == 0) { + std::string cond = trim(tag.substr(3)); + std::size_t end = src.find("{% endif %}", e); + if (end == std::string::npos) break; + std::string inner = src.substr(e + 2, end - e - 2); + std::size_t els = inner.find("{% else %}"); + std::string yes = els == std::string::npos ? inner + : inner.substr(0, els); + std::string no = + els == std::string::npos ? "" : inner.substr(els + 10); + out += render_blocks(ctx_truthy(ctx, cond) ? yes : no, ctx, depth + 1); + i = end + 11; + } else if (tag.rfind("for ", 0) == 0) { + std::string rest = trim(tag.substr(4)); + std::size_t in = rest.find(" in "); + std::string item = in == std::string::npos + ? "" + : trim(rest.substr(0, in)); + std::string list = in == std::string::npos + ? "" + : trim(rest.substr(in + 4)); + std::size_t end = src.find("{% endfor %}", e); + if (end == std::string::npos) break; + std::string inner = src.substr(e + 2, end - e - 2); + auto it = ctx.find(list); + if (it != ctx.end()) { + if (auto* l = std::get_if(&it->second)) { + for (const auto& row : *l) { + TemplateCtx sub = ctx; + for (const auto& [k, val] : row) { + sub[item + "." + k] = val; + sub[k] = val; + } + out += render_blocks(inner, sub, depth + 1); + } + } + } + i = end + 12; + } else if (tag.rfind("include ", 0) == 0) { + std::string rest = trim(tag.substr(8)); + if (rest.size() >= 2 && rest.front() == '"') { + std::string inc = rest.substr(1, rest.size() - 2); + auto it = templates_.find(inc); + if (it != templates_.end()) + out += render_blocks(it->second, ctx, depth + 1); + } + i = e + 2; + } else if (tag.rfind("block ", 0) == 0) { + std::size_t end = src.find("{% endblock %}", e); + if (end == std::string::npos) break; + out += render_blocks(src.substr(e + 2, end - e - 2), ctx, depth + 1); + i = end + 14; + } else { + i = e + 2; + } + } + } + return out; +} + +} // namespace molo::example::fw diff --git a/example/fw/Templates.hpp b/example/fw/Templates.hpp new file mode 100644 index 0000000..ab3cd10 --- /dev/null +++ b/example/fw/Templates.hpp @@ -0,0 +1,39 @@ +#pragma once + +// Jinja-like template environment: extends plus blocks, includes, if, for, +// filters, autoescape for .html/.htm/.xml with explicit safe opt-out. +// Gap: real molo::templates in 05-request-routing.md owns the full lexer, +// parser, sandbox, and streamed rendering. + +#include +#include +#include +#include + +namespace molo::example::fw { + +using TemplateList = std::vector>; +using TemplateValue = std::variant; +using TemplateCtx = std::map; + +class Environment { + public: + void add_template(std::string name, std::string source); + void add_global(std::string key, std::string value); + bool has_template(const std::string& name) const; + + std::string render(const std::string& name, TemplateCtx ctx) const; + + private: + std::string render_source(const std::string& name, const std::string& src, + TemplateCtx& ctx, int depth) const; + std::string render_blocks(const std::string& src, TemplateCtx& ctx, + int depth) const; + + std::map templates_; + std::map globals_; +}; + +std::string html_escape(std::string_view in); + +} // namespace molo::example::fw diff --git a/example/fw/TestClient.cpp b/example/fw/TestClient.cpp new file mode 100644 index 0000000..bfa8031 --- /dev/null +++ b/example/fw/TestClient.cpp @@ -0,0 +1,65 @@ +#include "TestClient.hpp" + +#include "Caps.hpp" +#include "Http.hpp" + +namespace molo::example::fw { + +TestClient::TestClient(std::shared_ptr app) + : app_(std::move(app)) {} + +Response TestClient::get(const std::string& path) { + return request(Method::Get, path); +} + +Response TestClient::post(const std::string& path, const std::string& body, + const std::string& content_type) { + return request(Method::Post, path, body, {{"Content-Type", content_type}}); +} + +Response TestClient::request( + Method method, const std::string& path, const std::string& body, + const std::map& headers) { + if (body.size() > caps::kMaxTestBody) + return TextResponse("test body over cap", 413); + Request req; + req.method = method; + std::size_t q = path.find('?'); + if (q == std::string::npos) { + req.path = path; + } else { + req.path = path.substr(0, q); + req.query = parse_query(path.substr(q + 1)); + } + req.headers.set("User-Agent", "living-example-test/1.0"); + for (const auto& [k, v] : headers) req.headers.set(k, v); + if (!jar_.empty()) { + std::string cookie; + for (const auto& [k, v] : jar_) { + if (!cookie.empty()) cookie += "; "; + cookie += k + "=" + v; + } + req.cookies = jar_; + req.headers.set("Cookie", cookie); + } + if (!frozen_time_.empty()) req.headers.set("X-Test-Time", frozen_time_); + req.body = body; + Response res = app_->dispatch(std::move(req)); + std::string set_cookie = res.headers.get("Set-Cookie"); + if (!set_cookie.empty()) { + std::size_t semi = set_cookie.find(';'); + std::string pair = set_cookie.substr(0, semi); + std::size_t eq = pair.find('='); + if (eq != std::string::npos) + jar_[pair.substr(0, eq)] = pair.substr(eq + 1); + } + return res; +} + +void TestClient::freeze_time(const std::string& iso) { frozen_time_ = iso; } + +const std::string& TestClient::frozen_time() const { return frozen_time_; } + +void TestClient::clear_cookies() { jar_.clear(); } + +} // namespace molo::example::fw diff --git a/example/fw/TestClient.hpp b/example/fw/TestClient.hpp new file mode 100644 index 0000000..dbbfe36 --- /dev/null +++ b/example/fw/TestClient.hpp @@ -0,0 +1,38 @@ +#pragma once + +// Test client: drives the real Application (routing, middleware, views, DI, +// error mapping, in-memory DB) without sockets, plus a cookie jar and frozen +// time hook. Gap: real test client in 08-clients-openapi.md owns fixtures, +// DB resets, and time freezing. + +#include +#include +#include + +#include "App.hpp" + +namespace molo::example::fw { + +class TestClient { + public: + explicit TestClient(std::shared_ptr app); + + Response get(const std::string& path); + Response post(const std::string& path, const std::string& body = "", + const std::string& content_type = "text/plain"); + Response request(Method method, const std::string& path, + const std::string& body = "", + const std::map& headers = + {}); + + void freeze_time(const std::string& iso); + const std::string& frozen_time() const; + void clear_cookies(); + + private: + std::shared_ptr app_; + std::map jar_; + std::string frozen_time_; +}; + +} // namespace molo::example::fw diff --git a/example/fw/Tls.cpp b/example/fw/Tls.cpp new file mode 100644 index 0000000..55d873e --- /dev/null +++ b/example/fw/Tls.cpp @@ -0,0 +1,9 @@ +#include "Tls.hpp" + +namespace molo::example::fw { + +std::string StubTlsContext::name() const { return "tls-stub-no-wolfssl"; } + +bool StubTlsContext::available() const { return false; } + +} // namespace molo::example::fw diff --git a/example/fw/Tls.hpp b/example/fw/Tls.hpp new file mode 100644 index 0000000..7d6ffdc --- /dev/null +++ b/example/fw/Tls.hpp @@ -0,0 +1,25 @@ +#pragma once + +// TLS posture: framework code depends only on ITlsContext. wolfSSL is NOT +// installed in this environment, so only StubTlsContext exists. It refuses +// handshakes loudly instead of silently serving plaintext as TLS. +// Gap: real wolfSSL-backed context from 03-security.md (TLS 1.3). + +#include + +namespace molo::example::fw { + +class ITlsContext { + public: + virtual ~ITlsContext() = default; + virtual std::string name() const = 0; + virtual bool available() const = 0; +}; + +class StubTlsContext : public ITlsContext { + public: + std::string name() const override; + bool available() const override; +}; + +} // namespace molo::example::fw diff --git a/example/fw/Validate.cpp b/example/fw/Validate.cpp new file mode 100644 index 0000000..852f3da --- /dev/null +++ b/example/fw/Validate.cpp @@ -0,0 +1,189 @@ +#include "Validate.hpp" + +#include + +#include "Caps.hpp" +#include "Http.hpp" + +namespace molo::example::fw { + +ValidationError::ValidationError(std::vector failures) + : HttpError(422, "validation_error", "validation failed"), + failures_(std::move(failures)) {} + +const std::vector& ValidationError::failures() const { + return failures_; +} + +std::string ValidationError::to_json() const { + std::string out = "{\"error\":\"validation_error\",\"failures\":["; + bool first = true; + for (const auto& f : failures_) { + if (!first) out += ","; + first = false; + out += "{\"field\":\"" + json_escape(f.field) + "\",\"message\":\"" + + json_escape(f.message) + "\"}"; + } + return out + "]}"; +} + +Model::Model(std::vector fields) : fields_(std::move(fields)) {} + +namespace { + +bool looks_like_email(const std::string& v) { + auto at = v.find('@'); + if (at == std::string::npos || at == 0) return false; + return v.find('.', at) != std::string::npos; +} + +bool parse_long(const std::string& v, long& out) { + if (v.empty()) return false; + std::size_t i = 0; + bool neg = false; + if (v[0] == '-' || v[0] == '+') { + neg = v[0] == '-'; + i = 1; + } + if (i >= v.size()) return false; + long acc = 0; + for (; i < v.size(); ++i) { + if (!std::isdigit((unsigned char)v[i])) return false; + acc = acc * 10 + (v[i] - '0'); + } + out = neg ? -acc : acc; + return true; +} + +} // namespace + +std::map Model::validate( + const std::map& input) const { + std::vector failures; + std::map clean; + for (const auto& f : fields_) { + auto it = input.find(f.name); + std::string value = it == input.end() ? "" : it->second; + bool missing = it == input.end() || value.empty(); + if (missing) { + if (!f.required) { + if (!f.default_value.empty()) clean[f.name] = f.default_value; + continue; + } + failures.push_back({f.name, "field is required"}); + continue; + } + if (f.type != FieldType::Integer && value.size() < f.min_len) + failures.push_back({f.name, "too short"}); + else if (f.type != FieldType::Integer && value.size() > f.max_len) + failures.push_back({f.name, "too long"}); + else if (f.type == FieldType::Integer) { + long n = 0; + if (!parse_long(value, n)) { + failures.push_back({f.name, "not an integer"}); + } else if (f.has_range && (n < f.min_value || n > f.max_value)) { + failures.push_back({f.name, "out of range"}); + } else { + clean[f.name] = value; + } + continue; + } else if (f.type == FieldType::Email && !looks_like_email(value)) { + failures.push_back({f.name, "not an email address"}); + continue; + } else if (f.type == FieldType::Boolean && + value != "true" && value != "false" && value != "1" && + value != "0") { + failures.push_back({f.name, "not a boolean"}); + continue; + } else { + clean[f.name] = value; + } + } + if (!failures.empty()) throw ValidationError(std::move(failures)); + return clean; +} + +const std::vector& Model::fields() const { return fields_; } + +std::map parse_json_object_flat( + const std::string& body) { + std::map out; + std::size_t i = 0; + int depth = 0; + auto skip_ws = [&]() { + while (i < body.size() && + std::isspace((unsigned char)body[i])) + ++i; + }; + skip_ws(); + if (i >= body.size() || body[i] != '{') return out; + ++i; + ++depth; + std::size_t fields = 0; + while (i < body.size() && fields < caps::kMaxFormFields) { + skip_ws(); + if (i < body.size() && body[i] == '}') break; + if (i >= body.size() || body[i] != '"') break; + ++i; + std::string key; + while (i < body.size() && body[i] != '"') { + if (body[i] == '\\' && i + 1 < body.size()) { + key.push_back(body[i + 1]); + i += 2; + } else { + key.push_back(body[i++]); + } + } + if (i < body.size()) ++i; + skip_ws(); + if (i >= body.size() || body[i] != ':') break; + ++i; + skip_ws(); + std::string value; + if (i < body.size() && body[i] == '"') { + ++i; + while (i < body.size() && body[i] != '"') { + if (body[i] == '\\' && i + 1 < body.size()) { + value.push_back(body[i + 1]); + i += 2; + } else { + value.push_back(body[i++]); + } + } + if (i < body.size()) ++i; + } else { + while (i < body.size() && body[i] != ',' && body[i] != '}') { + if (body[i] == '{' || body[i] == '[') { + if (++depth > caps::kMaxJsonDepth) return out; + } + value.push_back(body[i++]); + } + while (!value.empty() && + std::isspace((unsigned char)value.back())) + value.pop_back(); + } + out[key] = value; + ++fields; + skip_ws(); + if (i < body.size() && body[i] == ',') ++i; + } + return out; +} + +Form::Form(Model model) : model_(std::move(model)) {} + +std::map Form::bind(const Request& req) const { + std::string ct = req.headers.get("Content-Type"); + std::map input; + if (ct.find("application/json") != std::string::npos) { + input = parse_json_object_flat(req.body); + } else { + input = parse_form(req.body); + for (const auto& [k, v] : req.query) { + if (!input.count(k)) input[k] = v; + } + } + return model_.validate(input); +} + +} // namespace molo::example::fw diff --git a/example/fw/Validate.hpp b/example/fw/Validate.hpp new file mode 100644 index 0000000..8ac94ca --- /dev/null +++ b/example/fw/Validate.hpp @@ -0,0 +1,72 @@ +#pragma once + +// Single validation framework: field descriptors plus whole-model validation +// that reports ALL failures in one error. Forms bind JSON and urlencoded +// bodies into the same models. Gap: real molo::validate and molo::forms in +// 05-request-routing.md own coercions, nested models, and OpenAPI schemas. + +#include +#include +#include +#include + +#include "App.hpp" + +namespace molo::example::fw { + +enum class FieldType { String, Integer, Boolean, Email }; + +struct Field { + std::string name; + FieldType type = FieldType::String; + bool required = true; + std::size_t min_len = 0; + std::size_t max_len = 1024; + long min_value = 0; + long max_value = 0; + bool has_range = false; + std::string default_value; +}; + +struct FieldError { + std::string field; + std::string message; +}; + +class ValidationError : public HttpError { + public: + explicit ValidationError(std::vector failures); + const std::vector& failures() const; + std::string to_json() const; + + private: + std::vector failures_; +}; + +class Model { + public: + explicit Model(std::vector fields); + + std::map validate( + const std::map& input) const; + + const std::vector& fields() const; + + private: + std::vector fields_; +}; + +std::map parse_json_object_flat( + const std::string& body); + +class Form { + public: + explicit Form(Model model); + + std::map bind(const Request& req) const; + + private: + Model model_; +}; + +} // namespace molo::example::fw diff --git a/example/fw/View.cpp b/example/fw/View.cpp new file mode 100644 index 0000000..b79b469 --- /dev/null +++ b/example/fw/View.cpp @@ -0,0 +1,26 @@ +#include "View.hpp" + +namespace molo::example::fw { + +View::View(Request req) : req_(std::move(req)) {} + +Response View::dispatch() { + switch (req_.method) { + case Method::Get: return get(); + case Method::Post: return post(); + case Method::Put: return put(); + case Method::Patch: return patch(); + case Method::Delete: return del(); + case Method::Options: return options(); + default: return TextResponse("method not allowed", 405); + } +} + +Response View::get() { return TextResponse("method not allowed", 405); } +Response View::post() { return TextResponse("method not allowed", 405); } +Response View::put() { return TextResponse("method not allowed", 405); } +Response View::patch() { return TextResponse("method not allowed", 405); } +Response View::del() { return TextResponse("method not allowed", 405); } +Response View::options() { return TextResponse("method not allowed", 405); } + +} // namespace molo::example::fw diff --git a/example/fw/View.hpp b/example/fw/View.hpp new file mode 100644 index 0000000..ab3c279 --- /dev/null +++ b/example/fw/View.hpp @@ -0,0 +1,29 @@ +#pragma once + +// Class-based views, aiohttp style: constructed with the Request, one virtual +// coroutine-equivalent per HTTP method, dispatch selects by method. +// Gap: real molo::views::View in 05-request-routing.md is coroutine based. + +#include "Http.hpp" + +namespace molo::example::fw { + +class View { + public: + explicit View(Request req); + virtual ~View() = default; + + Response dispatch(); + + virtual Response get(); + virtual Response post(); + virtual Response put(); + virtual Response patch(); + virtual Response del(); + virtual Response options(); + + protected: + Request req_; +}; + +} // namespace molo::example::fw diff --git a/example/fw/Workflow.cpp b/example/fw/Workflow.cpp new file mode 100644 index 0000000..7bc1641 --- /dev/null +++ b/example/fw/Workflow.cpp @@ -0,0 +1,143 @@ +#include "Workflow.hpp" + +#include + +#include "Caps.hpp" + +namespace molo::example::fw { + +bool parse_commit(const std::string& subject, const std::string& body, + CommitInfo& out, std::string& error) { + std::string s = subject; + if (s.size() > (std::size_t)caps::kMaxCommitSubject) { + error = "subject too long"; + return false; + } + std::size_t colon = s.find(':'); + if (colon == std::string::npos) { + error = "missing colon"; + return false; + } + std::string head = s.substr(0, colon); + out.description = s.substr(colon + 1); + while (!out.description.empty() && out.description.front() == ' ') + out.description.erase(out.description.begin()); + if (!head.empty() && head.back() == '!') { + out.breaking = true; + head.pop_back(); + } + std::size_t paren = head.find('('); + if (paren == std::string::npos) { + out.type = head; + } else { + out.type = head.substr(0, paren); + std::size_t close = head.find(')', paren); + if (close == std::string::npos) { + error = "unclosed scope"; + return false; + } + out.scope = head.substr(paren + 1, close - paren - 1); + } + const char* allowed[] = {"feat", "fix", "docs", "style", "refactor", + "perf", "test", "build", "ci", "chore"}; + bool ok = false; + for (const char* t : allowed) { + if (out.type == t) { + ok = true; + break; + } + } + if (!ok) { + error = "unknown type: " + out.type; + return false; + } + if (body.find("BREAKING CHANGE") != std::string::npos) out.breaking = true; + return true; +} + +std::string bump_name(Bump bump) { + switch (bump) { + case Bump::None: return "none"; + case Bump::Patch: return "patch"; + case Bump::Minor: return "minor"; + case Bump::Major: return "major"; + } + return "none"; +} + +Bump ReleasePlanner::decide(const std::vector& commits) const { + bool feat = false, fix = false; + for (const auto& c : commits) { + if (c.breaking) return Bump::Major; + if (c.type == "feat") feat = true; + if (c.type == "fix") fix = true; + } + if (feat) return Bump::Minor; + if (fix) return Bump::Patch; + return Bump::None; +} + +std::string ReleasePlanner::next_version( + const std::string& current, + const std::vector& commits) const { + int major = 0, minor = 0, patch = 0; + std::sscanf(current.c_str(), "%d.%d.%d", &major, &minor, &patch); + switch (decide(commits)) { + case Bump::Major: return std::to_string(major + 1) + ".0.0"; + case Bump::Minor: + return std::to_string(major) + "." + std::to_string(minor + 1) + ".0"; + case Bump::Patch: + return std::to_string(major) + "." + std::to_string(minor) + "." + + std::to_string(patch + 1); + case Bump::None: return current; + } + return current; +} + +std::string ChangelogWriter::entry( + const std::string& version, const std::string& date, + const std::vector& commits) const { + std::string added, changed, fixed, removed; + for (const auto& c : commits) { + std::string line = + "- " + (c.scope.empty() ? "" : "(" + c.scope + ") ") + c.description + + "\n"; + if (c.breaking) + changed += line; + else if (c.type == "feat") + added += line; + else if (c.type == "fix") + fixed += line; + else + changed += line; + } + std::string out = "## [" + version + "] - " + date + "\n"; + if (!added.empty()) out += "### Added\n" + added; + if (!changed.empty()) out += "### Changed\n" + changed; + if (!fixed.empty()) out += "### Fixed\n" + fixed; + if (!removed.empty()) out += "### Removed\n" + removed; + int lines = 0; + for (char c : out) { + if (c == '\n') ++lines; + } + if (lines > caps::kMaxChangelogLines) + out += "(truncated to cap)\n"; + return out; +} + +void ReleaseVerifier::add_check(std::string name, bool passed, + std::string detail) { + checks_.push_back({std::move(name), passed, std::move(detail)}); +} + +bool ReleaseVerifier::green() const { + if (checks_.empty()) return false; + for (const auto& c : checks_) { + if (!c.passed) return false; + } + return true; +} + +std::vector ReleaseVerifier::checks() const { return checks_; } + +} // namespace molo::example::fw diff --git a/example/fw/Workflow.hpp b/example/fw/Workflow.hpp new file mode 100644 index 0000000..dc646f6 --- /dev/null +++ b/example/fw/Workflow.hpp @@ -0,0 +1,57 @@ +#pragma once + +// Autonomous release pipeline: commits since tag to bump decision, changelog +// writing, and pre-tag verification. Dry-run mode never writes. +// Final home is tools/release/ under namespace molo::workflow; this example +// copy proves the rules and is marked as a demo, not the pipeline itself. +// Gap: real molo::workflow in tools/release/ plus CLI surface in 09-cli-ops.md. + +#include +#include + +namespace molo::example::fw { + +struct CommitInfo { + std::string type; + std::string scope; + std::string description; + bool breaking = false; +}; + +bool parse_commit(const std::string& subject, const std::string& body, + CommitInfo& out, std::string& error); + +enum class Bump { None, Patch, Minor, Major }; + +std::string bump_name(Bump bump); + +class ReleasePlanner { + public: + Bump decide(const std::vector& commits) const; + std::string next_version(const std::string& current, + const std::vector& commits) const; +}; + +class ChangelogWriter { + public: + std::string entry(const std::string& version, const std::string& date, + const std::vector& commits) const; +}; + +struct VerifyCheck { + std::string name; + bool passed = false; + std::string detail; +}; + +class ReleaseVerifier { + public: + void add_check(std::string name, bool passed, std::string detail = ""); + bool green() const; + std::vector checks() const; + + private: + std::vector checks_; +}; + +} // namespace molo::example::fw diff --git a/example/main.cpp b/example/main.cpp new file mode 100644 index 0000000..27fa3db --- /dev/null +++ b/example/main.cpp @@ -0,0 +1,35 @@ +// Living example server entry: serves the demo app on loopback. +// Usage: living-server [port] (default 8080) + +#include +#include +#include +#include + +#include "app/App.hpp" +#include "fw/Server.hpp" + +int main(int argc, char** argv) { + int port = 8080; + if (argc > 1) { + try { + port = std::stoi(argv[1]); + } catch (...) { + std::cerr << "bad port\n"; + return 1; + } + } + molo::example::app::Services services; + auto app = molo::example::app::build_application(services); + molo::example::fw::Server server(app); + if (!server.start("127.0.0.1", port)) { + std::cerr << "bind failed\n"; + return 1; + } + std::cout << "living example on http://127.0.0.1:" << server.port() << "\n"; + std::cout << "press Ctrl-C to stop\n"; + while (server.running()) { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + return 0; +} diff --git a/example/static/app.css b/example/static/app.css new file mode 100644 index 0000000..f03d2f4 --- /dev/null +++ b/example/static/app.css @@ -0,0 +1,25 @@ +/* Living example stylesheet: mobile-first, responsive by default. */ +* { box-sizing: border-box; } +body { margin: 0; font-family: system-ui, sans-serif; line-height: 1.5; } +.topbar { display: flex; align-items: center; gap: 1rem; padding: 0.5rem 1rem; + background: #102030; color: #fff; } +.brand { color: #fff; font-weight: bold; text-decoration: none; } +.nav-wide { display: none; gap: 0.75rem; } +.nav-wide a { color: #cde; text-decoration: none; } +.nav-narrow { margin-left: auto; } +.nav-narrow nav { display: flex; flex-direction: column; gap: 0.25rem; + background: #fff; padding: 0.5rem; border: 1px solid #ccc; } +main { padding: 1rem; max-width: 60rem; margin: 0 auto; } +table { border-collapse: collapse; width: 100%; } +th, td { border: 1px solid #ccc; padding: 0.25rem 0.5rem; text-align: left; } +form { display: flex; flex-direction: column; gap: 0.5rem; max-width: 28rem; } +input, textarea, button { font-size: 1rem; padding: 0.4rem; } +.ok { color: #0a0; } +.error { color: #a00; } +.hint { color: #555; } +/* Big button group becomes a dropdown on small phones: the details element + above is the dropdown; on wide screens show the plain link row. */ +@media (min-width: 48rem) { + .nav-wide { display: flex; } + .nav-narrow { display: none; } +} diff --git a/example/static/app.js b/example/static/app.js new file mode 100644 index 0000000..d87a653 --- /dev/null +++ b/example/static/app.js @@ -0,0 +1,6 @@ +// Living example script: tiny health badge, no framework needed. +fetch("/health").then(function (r) { return r.json(); }).then(function (j) { + var el = document.createElement("p"); + el.textContent = "health: " + j.status + " tls: " + j.tls; + document.body.appendChild(el); +}); diff --git a/plan.md b/plan.md index 433895a..404a7a4 100644 --- a/plan.md +++ b/plan.md @@ -53,6 +53,11 @@ feature. C++26 stays excluded until published and stably supported. - Response formats: HTML, JSON, XML, plain text and others, with content negotiation - Streaming responses and passthrough streaming (proxy or generate bodies without full buffering) - Modern async HTTP/API/REST client with HTTP/1.1 and HTTP/2 support (pooling, retries, timeouts, JSON/XML/form handling, file upload, streaming, test double) +- Multi-tenancy: first-class tenants with silo/bridge/pool isolation, per-tenant config, quotas, rate limits, feature flags and metrics +- Audit and compliance: immutable hash-chained audit log, GDPR export/erasure cascade, retention with legal hold, PII marking, log redaction, non-prod masking +- Operability at scale: rolling/blue-green deploys, drains, distributed locks/election, breakers, bulkheads, load shedding, deadlines, SLO budgets, chaos hooks, debug endpoints +- API governance: API keys with scopes/quotas/expiry, router versioning with deprecation policy, OpenAPI breaking-change gate, rate-limit tiers, surface linter +- Supply chain and trust: SBOM (SPDX + CycloneDX), signed SLSA provenance, wolfSSL pin policy, vuln-scan gate, Sigstore-signed release artifacts ### 3. Core runtime and HTTP fundamentals @@ -80,6 +85,7 @@ folk knowledge: - Secrets management: loading and rotation story for passwords, API keys, encryption keys - XSS-safe templating defaults (auto-escaping), path traversal protection - TLS via wolfSSL (sole dependency exception, commercial license for closed-source use), natively compiled, TLS 1.3 +- Immutable audit log: append-only hash-chained record of security-relevant events (auth, grants, key use, config change, export/erasure), auditor-role reads, verification command ### 5. Data layer and ORM @@ -152,3 +158,6 @@ An AI agent keeps the tree consistent following `AGENTS.md`. 5. OpenAPI generation, test client, modern HTTP/API/REST client, dev server / CLI experience. 6. Replication/sync semantics and conflict handling. 7. Autonomous git/versioning/changelog pipeline + `AGENTS.md` + docs templates + big living example project updated with every feature. +8. Multi-tenancy jointly with the data layer: tenant context in place before ORM/pooling harden, then quotas, flags and per-tenant metrics. +9. Audit/compliance (PII marks, redaction, audit chain, retention/holds, rights engine, masking) plus operability at scale (coordination, drains, resilience, deploys, SLO, chaos, debug). +10. API governance (keys, versioning, diff gate, tiers, linter) plus supply chain and trust (SBOM, provenance, signing, vuln gate) wired into the release pipeline. diff --git a/plans/11-tenancy.md b/plans/11-tenancy.md new file mode 100644 index 0000000..bcffca3 --- /dev/null +++ b/plans/11-tenancy.md @@ -0,0 +1,187 @@ +# 11 - Multi-Tenancy + +Title: Multi-Tenancy +Scope: plan.md sections 2, 5, 7 partial. First-class tenants: isolated data paths, per-tenant config, quotas, rate limits, feature flags, per-tenant metrics; tenant-aware ORM, auth, routing, replication. Must land before the data layer hardens. + +## 1. Purpose and philosophy + +Every request, query, log record, metric, task, and replication entry belongs to exactly one tenant or carries an explicit system exemption. Tenancy is resolved once at the edge, attached to the request and data contexts, and enforced at the lowest layer that can do it (the database), never trusted to caller discipline. + +One API over a strategy spectrum: Silo (dedicated database instance or cluster per tenant), Bridge (database-per-tenant on a shared Postgres cluster, file-per-tenant on SQLite), Pool (shared tables with tenant_id plus Row-Level Security on Postgres). Strategy is chosen per tenant at provisioning time, because switching models later is costly (see Best practices). + +This file must land before 04-data.md hardens: the data Context shape, pool ownership, sync and healer behavior, and the migration journal all branch on isolation strategy. Retrofitting tenancy after those freeze would fork the ORM. + +## 2. Best practices (from research) + +1. Three partitioning models: "In the silo model, each tenant has their own unique physical resource. The bridge model logically separates tenants on a physical resource, for example, a database-per-tenant on a shared physical database. In the pool model, all tenants share a single logical construct on a physical resource, for example, a single schema shared by all tenants." (Source 1) +2. Silo gives the strongest isolation ("separate database instance and separate compute instance per tenant"), motivated by noisy-neighbour, compliance, or encryption requirements; bridge exists to cut cost by sharing the instance between tenants. (Source 1) +3. On Postgres bridge, AWS recommends database-per-tenant over schema-per-tenant: schemas are logical constructs that do not separate data on disk and share one directory, which risks system limits with many tenants. (Source 1, paraphrased) +4. A Postgres client connection "can only access data in a single database, the one specified in the connection request" (Source 4), so bridge needs one pool per tenant database while pool strategy keeps a single shared pool. +5. Pool enforcement belongs in the database: "When row security is enabled on a table ... all normal access to the table for selecting rows or modifying rows must be allowed by a row security policy", and "If no policy exists for the table, a default-deny policy is used" (Source 2). Policies can target ALL commands or SELECT, INSERT, UPDATE, DELETE, specific roles, or both. +6. The tenancy model "determines how each tenant's data is mapped to storage" and "Switching to a different model later is sometimes costly" (Source 3). Single-tenancy stores one tenant per database; multi-tenancy stores several tenants per database "with mechanisms to protect data privacy" (Source 3). Hence strategy-per-tenant at provisioning plus explicit migration procedures. +7. SQLite file-per-tenant rests on ATTACH: "The ATTACH DATABASE statement adds another database file to the current database connection" (Source 5), with a static limit on attached databases per connection. Under WAL, multi-file transactions stay atomic per file but a crash mid-COMMIT can split them across files (Source 5), so bridge on SQLite uses one connection per tenant file and never spans a transaction across files. +8. Tenant isolation "is fundamental to the design and development of software as a service (SaaS) systems" and crossing a tenant boundary "would represent a significant and potentially un-recoverable event for a SaaS business" (Source 6). Hence default-deny everywhere in this file. +9. Triangulation: AWS and Microsoft agree on the silo to pool spectrum with cost versus isolation tradeoffs (Sources 1, 3). AWS and the Postgres docs agree that bridge means database-per-tenant guarded by native access controls (Sources 1, 4). Pool safety is agreed from two sides: RLS default-deny at the engine (Source 2) plus app-level tenant scoping as defense in depth (this file, Rules R1 and R2). + +## 3. Structure + +Root: `src/molo/tenancy/`, namespace `molo::tenancy`. One class or concern per file pair. + +- `tenant_id.h`: `TenantId` strong type (uuid7 plus validated slug). +- `tenant.h/.cpp`: `Tenant` record (id, slug, strategy, status, timestamps). +- `strategy.h`: `IsolationStrategy { Silo, Bridge, Pool }`, `strategy_from_string`. +- `catalog.h/.cpp`: `TenantCatalog` (create, lookup, suspend, tombstone, purge); `store/catalog_store.h` interface plus `store/memory_catalog.h`, `store/db_catalog.h`. +- `resolver.h/.cpp`: `TenantResolver` (ordered chain: subdomain, header, path prefix, auth claim). +- `context.h`: `TenantContext` (id, strategy, connection selector), embedded in the 04-data.md data Context and the request scope. +- `middleware.h/.cpp`: `TenantMiddleware` (resolve, attach, reject unknown or suspended). +- `provisioner.h/.cpp`: `TenantProvisioner` (provision, strategy migration, suspend, delete with floor). +- `config_overlay.h/.cpp`: `TenantConfig` (layered typed overlay over global config, cache, invalidation). +- `quota.h/.cpp`: `QuotaManager` (usage counters, pre-allocation checks). +- `tenant_buckets.h/.cpp`: per-tenant rate-limit buckets built on the 03-security.md limiter core. +- `feature_flags.h/.cpp`: `FeatureFlags` (per-tenant typed flags, cache, change events). +- `metrics.h/.cpp`: `TenantMetrics` (tenant-labeled recording with allowlist cap). +- `orm_scope.h/.cpp`: `TenantScope` (automatic query filter injection, privileged escape hatch). + +## 4. Rules (hard, numbered) + +- R1. Every ORM query in pool mode carries tenant scope. Unscoped data access fails closed unless it uses the explicit privileged API, which logs tenant pair plus reason. +- R2. Pool on Postgres always enables Row-Level Security with default-deny policies on every tenant table. The app-level filter is defense in depth, never the only enforcement. +- R3. Tenant database state is set per transaction or per checkout (SET LOCAL semantics) and reset on release. Shared pooled connections never retain a tenant between checkouts. +- R4. Bridge means database-per-tenant on Postgres and file-per-tenant on SQLite. Schema-per-tenant is not offered. +- R5. Cross-tenant reads and writes are forbidden except through explicit system APIs that record both tenant ids and the reason in logs and audit. +- R6. Tenant resolution runs before auth in the middleware chain. Unknown tenant yields 404 `tenant_unknown`; suspended tenant yields 403 `tenant_suspended`. +- R7. Quotas are checked before allocation. Over-quota fails closed: 429 with Retry-After for rate quotas, 507 for storage quotas. +- R8. The `tenant` metric label is allowed only for tenants on the metrics allowlist (static cap); all others record as `tenant="other"` and count an overflow metric. +- R9. Tenant deletion is a two-phase tombstone plus purge with a retention window. Purge (data destruction) requires explicit human confirmation and is never autonomous. +- R10. Every task envelope, oplog entry, cache key namespace, pubsub channel, log record, and trace span attribute set carries the tenant id. Replay and sync filter by tenant. +- R11. Per-tenant config overlays validate against the same typed schema as global config. An invalid overlay fails the update atomically; partial application is forbidden. +- R12. Isolation strategy is fixed per tenant at provisioning. Strategy migration is an explicit procedure with pre-checks, backfill, and post-verification, never silent. + +## 5. Development guidelines + +- Build in section 7 order: resolver plus context plus catalog first, then pool scoping, then bridge pools, then quotas, flags, and metrics. Each step ends green through the real stack. +- Keep the strategy matrix in exactly one place (`TenantProvisioner` plus the pool factory). Callers take a `TenantContext` and never branch on backend or strategy. +- Fixtures always provision at least two tenants plus system scope. Every tenant-visible route test asserts the neighbor tenant sees nothing (404 or empty, never a leak). +- RLS policies are generated from the model registry, reviewed as SQL text, and applied by the same migrator as schema DDL. No hand-written policy drift. +- Quota checks sit at admission points (middleware, pool checkout, upload start, task enqueue), not sprinkled in business logic. +- Feature flags default from global config; a tenant overlay only overrides named flags. Unknown flag names fail closed at update time. +- Deletion and purge paths are dry-run first with an itemized report (databases, files, keys, streams), then confirmed execution. + +## 6. Specifications + +### 6.1 Identity and resolution + +- `TenantId`: internal uuid7 plus public slug matching `[a-z0-9-]{3,63}`, starting and ending alphanumeric. Slug is unique per deployment; uuid is the stored key. +- Resolvers, in default order (configurable): subdomain (`.example.com`), header `X-Tenant-ID` (slug or uuid), path prefix `/t//` (stripped before routing), auth claim `tid` (verified token only). +- First match wins; conflicting signals (header says A, path says B) yield 400 `tenant_conflict`. No signal yields the configured default tenant in single-tenant dev mode, else 404 `tenant_unknown`. +- Statuses: `provisioning`, `active`, `suspended`, `tombstoned`. Only `active` serves traffic; `suspended` serves 403 with a fixed body; `tombstoned` behaves as unknown. + +### 6.2 Catalog record (JSON, also the on-disk shape) + +`{"id":"uuid7","slug":"acme","strategy":"pool","status":"active","pg_database":"molo_t_acme","sqlite_file":"tenants/acme.db","quotas":{...},"flags":{...},"created_at":"iso8601","updated_at":"iso8601"}`. Slug to id cached in memory (cap 65536 entries, LRU); catalog reads bypass tenant scope via the privileged API and are audit-logged. + +### 6.3 Pool mode (shared tables) + +- Every tenant model gains `tenant_id UUID NOT NULL`; composite indexes lead with `tenant_id`. The ORM injects `tenant_id = current` into every generated query; raw fragments must bind it explicitly. +- Postgres: `ALTER TABLE ENABLE ROW LEVEL SECURITY` plus generated policy `CREATE POLICY tenant_isolation ON USING (tenant_id = current_setting('app.current_tenant')::uuid) WITH CHECK (...)` for ALL commands on the app role. The tenant GUC is set with `SET LOCAL` inside each transaction; pooled checkouts assert the GUC is empty before use. +- SQLite pool: shared file with `tenant_id` column plus app filter; RLS does not exist, so integration tests additionally run a hostile-shape suite where hand-written SQL without scope must fail at the driver guard. +- Max tenants per pool deployment: 100000 (soft, operations warning at 80 percent). Beyond that, split pools or move tenants to bridge. + +### 6.4 Bridge mode + +- Postgres: one database per tenant on the shared cluster, name `molo_t_` (truncated to 63 chars with uuid suffix on collision). Per-tenant pool: min 0, max 8, acquire timeout 5 s. Connection strings derive from the catalog, never from request input. +- SQLite: one file per tenant at `tenants/.db`, same pragmas as 04-data.md (WAL, foreign keys on, busy timeout 5000). One writer gate per file; no transaction ever spans two tenant files (see Best practices 7). +- Bridge caps: max 1024 tenant databases per Postgres cluster, max 4096 tenant files per deployment (soft; shard beyond). + +### 6.5 Silo mode + +- Dedicated Postgres cluster (or instance) per tenant, or dedicated file root per tenant on SQLite. The catalog stores a connection profile name; credentials come from 03-security.md secrets per profile. All pool defaults from 04-data.md apply unchanged inside the silo. Silo tenants skip RLS (single tenant per database) but keep `tenant_id` columns for portability. + +### 6.6 Sync, healer, migrations per tenant + +- Sync and self-heal run per tenant target: pool runs once on shared tables (policies included); bridge and silo run once per tenant database or file. Heal DDL in pool mode never touches `tenant_id` columns except widening the type. +- The migration journal is per target (shared journal for pool, per-database or per-file journal otherwise). `molo migrate` (09-cli-ops.md) iterates tenant targets with a per-tenant report and continues past single-tenant failure, exiting nonzero with a summary. + +### 6.7 Quotas and rate limits + +- Defaults per tenant (all overridable via overlay): requests 600/min with burst 2x for 10 s; concurrent connections 256; realtime connections 1024; storage 10 GB; uploads 1 GB/day; queued tasks 10000; export rows per request 100000. +- Rate overage: 429 with `Retry-After` and `X-Tenant-ID` echo. Storage overage: 507 with a problem body naming the quota. Usage counters live in memory plus periodic catalog persist (every 60 s or 1000 deltas); restart rebuilds from stores, never trusts memory alone. +- Noisy-neighbour guard: a tenant exceeding 50 percent of shared pool checkouts over a 60 s window is throttled to its fair share and emits a warning event; sustained abuse triggers an automatic suspension proposal that a human confirms. + +### 6.8 Per-tenant config and feature flags + +- Overlay keys mirror global config paths (`http.body_max`, `auth.session_idle`, ...); unknown paths rejected. Overlay cache TTL 30 s, tag-invalidated on update; every read path takes the merged view, never global alone when a tenant is active. +- Flags: typed map (bool, int64, string; max 256 flags per tenant, name `[a-z0-9_.]{1,128}`, string value max 4 KB). Reads are synchronous from cache; updates emit a `tenant.flags_changed` event that cache invalidation and long-lived connections honor within 30 s. + +### 6.9 Per-tenant metrics, logs, traces + +- Metrics: shared registry with `tenant` label gated by the metrics allowlist (default cap 512 tenants, LRU admission, overflow to `tenant="other"` plus `molo_tenant_label_overflows_total`). Framework per-tenant series: requests total, request duration, db queries total, quota denials total, realtime connections gauge. +- Logs: every record in tenant scope carries `tenant_id`; the redaction rules of 02-observability.md apply unchanged. Traces: `tenant.id` span attribute on server spans; sampling stays head-based and tenant-blind. +- Cardinality audit: CI fails if any new metric uses an unbounded tenant-derived label outside this section. + +### 6.10 Tenant-aware auth, routing, realtime, replication + +- Auth (with 03-security.md): users belong to one or more tenants; sessions bind to exactly one tenant at login; switching tenant requires re-resolution plus a membership check and rotates the session id. The login URL carries the tenant signal; token `tid` must equal the resolved tenant. +- Routing (with 05-request-routing.md): `TenantMiddleware` runs immediately after error mapping and observability, before security guards that need tenant quotas. Route helpers accept a tenant parameter for URL building in path-prefix mode. +- Realtime (with 06-realtime.md): rooms and channels are namespaced `tenant::`; cross-tenant join is rejected; connection caps enforced per tenant at upgrade; broadcasts fan out only within the tenant unless a system channel explicitly allows wider. +- Replication (with 07-supervisor-replication.md): oplog entries gain `"tenant"`; sync sessions declare a tenant filter (single tenant, tenant list cap 128, or privileged all); conflict policies apply per tenant key; tenant provision, suspend, and purge replicate as control entries with human-confirm gates at apply time on peers. + +### 6.11 Provisioning and deletion + +- Provision: validate slug and strategy, allocate target (database, file, or pool row), run sync plus RLS policies, seed flags and quotas, warm caches, emit `tenant.provisioned`. Provision timeout 120 s; partial state rolls back with an itemized cleanup log. +- Suspend: flip status, drain tenant connections within 30 s, reject new work with 403, keep data. Unsuspend reverses with a health pre-check. +- Delete: mark `tombstoned`, revoke access, retain data for the retention window (default 30 days), then require human-confirmed purge. Purge emits an itemized destruction report and is blocked while replicas lag on that tenant. + +## 7. Execution steps (in build order) + +- E11-1: `TenantId`, `Tenant`, `IsolationStrategy`, in-memory catalog, `TenantContext` type shared with 04-data.md. Joint milestone: data Context carries tenant before drivers harden. +- E11-2: `TenantResolver` (all four signals, conflict rule) plus `TenantMiddleware` ordering test with 05-request-routing.md. +- E11-3: Pool scoping: `TenantScope` filter injection, `tenant_id` column convention, Postgres RLS policy generator, SET LOCAL checkout discipline. +- E11-4: Bridge pools: per-tenant Postgres databases plus pools, per-tenant SQLite files plus writer gates, catalog-driven connection factory. +- E11-5: `TenantProvisioner`: provision, sync integration, suspend and unsuspend, tombstone plus human-gated purge. +- E11-6: Quotas plus per-tenant rate buckets at admission points; usage persistence; 429 and 507 paths. +- E11-7: Config overlay plus feature flags with cache, validation, and change events. +- E11-8: `TenantMetrics` allowlist plus per-tenant series; log and trace tenant fields; replication tenant filter and control entries; living-example tenant demo. + +## 8. Validation (how to verify each step) + +- E11-1: context propagation test: tenant set at edge reaches ORM, log record, and task envelope unchanged; missing tenant fails closed in pool mode. +- E11-2: resolution matrix: subdomain, header, path, claim, conflict (400), unknown (404), suspended (403), dev default; order reconfiguration honored. +- E11-3: two-tenant CRUD suite on both backends: each tenant sees only its rows; direct SQL without scope returns zero rows on Postgres (RLS proof) and is rejected by the driver guard on SQLite; pooled checkout leak test (100k checkouts, GUC never bleeds). +- E11-4: bridge suite: per-tenant databases and files created, isolated pools saturate independently, cross-file transaction attempt refused, connection factory rejects request-derived names. +- E11-5: provision plus suspend plus purge drill: partial-provision rollback leaves no residue; tombstone retains data; purge without human confirmation refuses; purge with lagging replica blocks. +- E11-6: quota matrix: each quota trips at its limit with the documented status; burst and fair-share throttle behave within 20 percent; restart preserves usage within one persist window. +- E11-7: overlay validation: unknown path rejected atomically; flag change propagates to cached readers within 30 s; event observed on the bus. +- E11-8: metrics allowlist: 513th tenant records as "other" with overflow counter; per-tenant series render in `/metrics`; replication filter syncs one tenant without touching others; full suite green with the living-example tenant demo. + +## 9. References + +- Depends on 01-runtime.md: connection lifecycle, checkout discipline, static caps pattern. +- Depends on 02-observability.md: log schema, metric registry and label caps, trace attributes, redaction. +- Depends on 03-security.md: auth and session binding, rate limiter core, secrets for silo profiles, upload quotas. +- Depends on 04-data.md: ORM query builder, pooling, transactions, sync and healer, migration journal, KV and cache namespaces, pubsub channels. Joint hardening milestone before either file freezes. +- Depends on 05-request-routing.md: middleware ordering, Request scope, DI providers, error shapes. +- Depends on 06-realtime.md: room and channel namespacing, upgrade-time caps, broadcast fan-out. +- Depends on 07-supervisor-replication.md: task envelope tenant field, scheduler fairness, oplog shape, sync filters, multi-instance shared stores. +- Depends on 08-clients-openapi.md: test client tenant fixtures, per-tenant OpenAPI examples. +- Depends on 09-cli-ops.md: `molo tenant` commands, `/metrics` exposition, health aggregation per tenant. +- Follows 10-workflow-example.md: spec template, integration-only tests, living example coverage. +- Feeds 12-audit-compliance.md: tenant id on every auditable record, purge reports, suspension trail. +- Feeds 14-api-governance.md: per-tenant quota and versioning surface. + +## 10. Open questions + +1. RLS plus prepared plans under transaction pooling: which pool mode stays default for pool tenants, and does SET LOCAL survive the pooler path? Needs a joint test with 04-data.md. +2. Schema-per-tenant is excluded per AWS guidance; if a reviewer demands it, which cap (max schemas) makes it safe? +3. Should tenant slug renames be supported (pointer plus async rewrite) or forbidden (new tenant plus migration)? Leaning forbidden for v1. +4. Fair-share throttle constants (50 percent, 60 s window) are placeholders pending load measurements. +5. Cross-tenant analytics for operators: privileged read API only, or a dedicated export pipeline with its own audit? Proposal: export pipeline with audit owned alongside 12-audit-compliance.md. + +## 11. Sources + +1. AWS Database Blog, Choose the right PostgreSQL data access pattern for your SaaS application (silo, bridge, pool; database-per-tenant recommendation; noisy neighbours): https://aws.amazon.com/blogs/database/choose-the-right-postgresql-data-access-pattern-for-your-saas-application/ (retrieved 2026-10-23). +2. PostgreSQL 18 documentation, 5.9 Row Security Policies (ENABLE ROW LEVEL SECURITY, per-command and per-role policies, default-deny): https://www.postgresql.org/docs/current/ddl-rowsecurity.html (retrieved 2026-10-23). +3. Microsoft Learn, Multitenant SaaS Patterns, Azure SQL Database (tenancy model definitions, switching cost): https://learn.microsoft.com/en-us/azure/azure-sql/database/saas-tenancy-app-design-patterns?view=azuresql (retrieved 2026-10-23). +4. PostgreSQL 18 documentation, 5.10 Schemas (single-database connections, CREATE SCHEMA, search path): https://www.postgresql.org/docs/current/ddl-schemas.html (retrieved 2026-10-23). +5. SQLite documentation, ATTACH DATABASE (multi-file connections, per-file atomicity under WAL, attach limit): https://www.sqlite.org/lang_attach.html (retrieved 2026-10-23). +6. AWS whitepaper, SaaS Tenant Isolation Strategies (isolation as foundational SaaS concern): https://docs.aws.amazon.com/whitepapers/latest/saas-tenant-isolation-strategies/saas-tenant-isolation-strategies.html (retrieved 2026-10-23). \ No newline at end of file diff --git a/plans/12-audit-compliance.md b/plans/12-audit-compliance.md new file mode 100644 index 0000000..61a679f --- /dev/null +++ b/plans/12-audit-compliance.md @@ -0,0 +1,159 @@ +# 12 - Audit and Compliance + +File: plans/12-audit-compliance.md +Title: Audit and Compliance +Scope: Immutable audit log (who/what/when/where, before/after), GDPR export and erasure cascade, retention policies with legal hold, field-level PII marking, log redaction, non-prod data masking. + +## 1. Purpose and philosophy + +This file owns proof: what happened, who did it, when, where, and what changed. It gives every MoloApp deployment an append-only audit log with tamper-evident chaining, a GDPR rights engine (export and erasure with cascade), retention policies with legal hold override, field-level PII marking on ORM models, centralized log redaction, and deterministic masking for non-prod data. + +Philosophy: + +- Accountability is a feature, not paperwork. GDPR Art. 5(2) makes the controller responsible for demonstrating compliance, so the framework generates the evidence by default. +- Append-only with teeth. Audit records are hash-chained and sealed; verification is a command, not a hope. Detection of gaps follows the signed-syslog model (sequencing plus missing-message detection). +- Redact before persistence. Sensitive values never reach a log, audit record, export, or non-prod copy. Scrubbing copies later is not a strategy. +- Erasure is a cascade with named exceptions. Art. 17 grounds drive deletion across ORM, KV, cache, audit metadata, and backups policy; Art. 17(3) exceptions (legal claims, legal obligation) plus legal holds are the only brakes, and each brake is itself audited. +- Minimize by mark. Every ORM field declares a PII class once; redaction, export, erasure, retention, and masking all read that mark. No second inventory anywhere. + +## 2. Best practices (from research) + +1. Audit records capture when, where, who and what for each event: timestamps plus interaction identifier (when), app identity plus entry point plus code location (where), source address plus user identity (who), event type plus severity plus action/object/result/reason (what). (Source: https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html, retrieved 2026-10-23) +2. Log security events always: auth success and failure, authorization failure, session failures, admin actions, sensitive-data access, key use and rotation, data import and export, config change, legal opt-ins such as consent. Data changes and config modifications are at minimum considered. (Source: https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html, retrieved 2026-10-23) +3. Never record directly, instead remove, mask, sanitize, hash, or encrypt: session IDs, access tokens, passwords, DB connection strings, encryption keys, bank or card data, sensitive PII (health, government identifiers, vulnerable people), data above the logging system's classification, opted-out data. Non-sensitive PII (names, phones, emails) gets special treatment; prefer de-identification (deletion, scrambling, pseudonymization). (Source: https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html, retrieved 2026-10-23) +4. GDPR principles: lawfulness/fairness/transparency, purpose limitation, data minimisation, accuracy, storage limitation (identifiable form no longer than necessary), integrity and confidentiality, plus accountability (the controller must demonstrate compliance). (Source: https://gdpr-info.eu/art-5-gdpr/, retrieved 2026-10-23) +5. Erasure without undue delay when a ground applies: data no longer necessary, consent withdrawn with no other ground, successful objection, unlawful processing, legal-obligation erasure, child consent services data. Public-data controllers take reasonable steps to inform downstream processors. Exceptions: expression, legal obligation or public task, public health, archiving/research/statistics, legal claims. (Source: https://gdpr-info.eu/art-17-gdpr/, retrieved 2026-10-23) +6. Portability: the subject receives provided data in a structured, commonly used and machine-readable format and may transmit it to another controller; direct controller-to-controller transfer applies where technically feasible; the right must not adversely affect others' rights. It is without prejudice to erasure. (Source: https://gdpr-info.eu/art-20-gdpr/, retrieved 2026-10-23) +7. Rights requests: information on action taken without undue delay and within one month of receipt, extendable by two months for complex or numerous requests with reasons given inside the first month; electronic requests get electronic answers where possible; identity may be re-confirmed on reasonable doubt; manifestly unfounded or excessive requests may be charged or refused with the burden of proof on the controller. (Source: https://gdpr-info.eu/art-12-gdpr/, retrieved 2026-10-23) +8. Tamper evidence for logs: signed syslog adds origin authentication, message integrity, replay resistance, message sequencing, and detection of missing messages on top of RFC 5424. (Source: https://www.rfc-editor.org/rfc/rfc5848.txt, retrieved 2026-10-23) +9. Triangulated decisions: (a) OWASP when/where/who/what plus Art. 5 accountability define the audit record schema in section 6.1, nothing else may define it. (b) Art. 17 grounds plus Art. 17(3) exceptions plus OWASP consent logging define the erasure cascade with legal-hold brake. (c) Art. 20 machine-readable plus Art. 12 electronic-means define the JSON export format and delivery rule. (d) OWASP remove/mask/hash plus Art. 5 minimisation define redact-before-persist and PII marking. (e) RFC 5848 sequencing plus integrity define the hash chain and gap detection. + +## 3. Structure + +Root: `src/molo/audit/`, headers mirror under `include/molo/audit/`. One class or concern per file. Namespace root `molo::audit`, sub-namespaces `log`, `rights`, `retention`, `pii`, `redact`, `mask`. + +- `log/Record.h`: `struct Record` (schema in 6.1, fixed JSON keys, no free text PII). +- `log/Appender.h`: `class Appender` (append-only write, monotonic seq, prev-hash link). +- `log/Chain.h`: `class Chain` (hash computation, seal block, verify range, gap report). +- `log/Store.h`: `class Store` interface (append, read range, seal); `log/SqlStore.h` (dual-backend impl over 04-data.md drivers). +- `log/Query.h`: `class Query` (filter by subject, actor, object, time range, paged). +- `rights/Subject.h`: `struct SubjectRef` (subject id type, tenant id from 11-tenancy.md). +- `rights/Exporter.h`: `class Exporter` (collect marked fields, emit JSON bundle). +- `rights/Eraser.h`: `class Eraser` (plan cascade, apply, report per-store outcome). +- `rights/Request.h`: `class RightsRequest` (id, type export/erase, state machine, due date). +- `retention/Policy.h`: `struct Policy` (per-model TTL, legal basis tag, action erase or anonymize). +- `retention/Enforcer.h`: `class Enforcer` (periodic scan driver, executed via 07-supervisor-replication.md scheduler). +- `retention/Hold.h`: `class LegalHold` (hold id, scope, reason, placed-by, release rule). +- `pii/Mark.h`: `enum class PiiClass`, `struct FieldMark` (see 6.4). +- `pii/Registry.h`: `class Registry` (model to field-mark map, built from ORM descriptors). +- `redact/Redactor.h`: `class Redactor` (single entry `redact(record, context)`). +- `redact/Pattern.h`: `struct Pattern` (named matchers for tokens, keys, card data). +- `mask/Masker.h`: `class Masker` (deterministic per-class masking, seeded). +- `mask/Fixture.h`: `class FixtureScrubber` (DB dump and fixture rewrite for non-prod). + +## 4. Rules (hard, numbered) + +1. Audit writes are append-only. No update or delete API exists on the store; correction is a new compensating record linking the original seq. +2. Every audit record carries seq, prev-hash, timestamp, actor, subject, action, object, result, before/after digests or redacted diffs, and request/trace IDs. Missing fields fail the write loudly. +3. Before/after values pass through the Redactor before persistence. Raw PII, secrets, tokens, or passwords in an audit record are a build-breaking defect. +4. All data-subject rights flow through `RightsRequest` with identity confirmation and due-date tracking. Ad-hoc export or delete code paths are forbidden. +5. Erasure cascades to every store holding the subject: ORM rows, KV keys, cache entries and tags, sessions, uploads metadata, task payloads, audit subject index (pseudonymized, never content-deleted). Each store reports done, held, or exempted. +6. Legal hold and Art. 17(3) exemptions block erasure and retention deletion for held records only, with reason recorded. Holds never silently expand scope. +7. Retention deletion never touches held records and never runs without a dry-run report first. Enforcer applies only the policy version that was dry-run. +8. Every ORM field is marked with exactly one PII class at model declaration. Unmarked fields fail model registration at startup. +9. Non-prod environments never hold prod PII. Seed, fixture, dump-restore, and staging-sync paths run the Masker; unmasked prod copies outside prod are forbidden. +10. Redaction is centralized in `molo::audit::redact`. Sibling log, error, and export code calls it; local regex scrubbers are forbidden. +11. Audit log reads require an explicit auditor role and are themselves audited (who queried what range, when). Export bundles are signed and expire. +12. All caps in section 6 are static constants. No autotuning of retention, chain sealing, export size, or masking throughput. + +## 5. Development guidelines + +- Build order follows section 7: PII marks first (everything reads them), then redaction, then the audit chain, then retention and holds, then rights, then masking. +- Extremely object-oriented: one small class per file, deep namespaces, shared behavior in exactly one base. No cross-layer imports outside declared interfaces. +- Every audit emission is a one-line call at the mutation site: `AUDIT(model, action, before, after)`; the macro fills actor, subject, IDs, and chain link. Reviews reject manual record assembly. +- Structured over stringly: fixed JSON schemas for records, export bundles, erasure reports, and retention reports. Free-text reason fields are capped and redacted. +- Blocking work (chain sealing, export assembly, erasure cascade, retention scan, masking) runs off the event loop via 01-runtime.md blocking-work path or as 07-supervisor-replication.md tasks. +- Denials and holds emit structured logs plus metrics per 02-observability.md with reason codes, never the held content. +- Mobile-first and simplicity rules from plan.md apply to the auditor UI and rights self-service pages: responsive, plain language, one action per screen. + +## 6. Specifications (concrete behaviors, caps, formats) + +### 6.1 Audit record schema (JSON, fixed keys) + +`{"seq":n,"prev":"hex","hash":"hex","ts":"RFC3339-UTC","tenant":"id","actor":"user:42|system:scheduler|api:key-id","subject":"subject:uuid|none","action":"create|update|delete|read-sensitive|export|erase|login|grant|hold.place|hold.release|retention.run|config.change","object":"model:table:pk","result":"success|fail|defer","reason":"code","request_id":"...","trace_id":"...","before":"redacted-json|null","after":"redacted-json|null","policy":"retention-policy-version|null"}` + +Chain: `hash = SHA256(prev || canonical-json-without-hash)` using in-house hash per 03-security.md; genesis `prev` is 64 zeros. Seal block every 1000 records or 60 s: `{"seal_seq":n,"root":"hex-chain-tip","ts":"..."}` signed with the audit key epoch. Verify command replays hashes and reports first gap. + +### 6.2 Retention and legal hold + +Policy per model: `{"model":"...","ttl_days":n,"basis":"contract|consent|legal-obligation|legitimate-interest","on_expiry":"erase|anonymize","grace_days":30}`. Defaults: sessions 1 day, logs with PII 90 days, audit records 7 years (legal basis tag, jurisdiction-configurable), user content per-app config. Enforcer runs daily via 07-supervisor-replication.md scheduler, dry-run report first, then apply; report lists scanned, expired, held, deleted, anonymized counts. Hold: `{"hold_id":"...","scope":"subject:uuid|model:table:pk|query-hash","reason":"...","by":"...","ts":"...","release":"manual|date"}`; held records are skipped with counter `audit_retention_held_total`. Holds survive retention runs and erasure requests until released; release is audited. + +### 6.3 Rights engine (export, erasure) + +Request states: received, identity-pending, queued, running, done, refused, expired. Due date: received plus 30 days; one extension of 60 days with reason notice. Export bundle JSON: `{"subject":"...","exported_at":"...","sources":[{"store":"orm:users","rows":n}],"data":{...marked fields only...},"checksum":"sha256"}`; max bundle 256 MB, beyond that split into numbered parts; delivered electronically with signed URL expiry 7 days. Erasure cascade order: sessions, cache, KV, ORM rows (or anonymize where FK and audit law requires), uploads metadata, task payloads, audit subject index pseudonymized to `erased:`. Report per store: done, held (hold id), exempted (Art. 17(3) limb). Refusals cite the limb and log the reason code. + +### 6.4 PII marking + +`enum class PiiClass { None, Identifier, Contact, CredentialRef, Financial, Health, Location, Biometric, Child, Legal }`. `FieldMark { PiiClass cls; bool exportable; bool erasure; const char* basis; }`. Defaults: Identifier and Contact are exportable and erasure-bound; CredentialRef never exported in raw form (presence boolean only); Financial, Health, Biometric, Child default to non-exportable raw, exportable only as category-present flags unless policy opts in. Registry is built at ORM model registration; unknown model or unmarked field aborts boot with named error. + +### 6.5 Redaction and masking + +Redactor runs on: audit before/after, log records via 02-observability.md sink hook, error bodies via 05-request-routing.md error renderer, export bundles for non-exportable classes. Patterns: session/token/key headers, `*_KEY/*_SECRET/*_TOKEN`, card-shaped digit runs, DB connection strings; replacement is `[redacted:]`, stable per class for grepability. Masker per class: Identifier to `user-`, Contact to `user@example.invalid` and `+00-000-000`, Financial to zeros preserving length, others to class token; deterministic under seed so FK joins survive masking. FixtureScrubber rewrites dumps and seeds; max dump 2 GB, streams in 1 MB chunks, verifies zero unmasked marked values by rescan. + +### 6.6 Caps (static) + +Audit record max 64 KB (before/after each max 16 KB, overflow truncated with flag); chain seal every 1000 records or 60 s; log store retention of sealed tips forever, record bodies per policy; query page max 1000 rows; export bundle max 256 MB and max 1M rows per request; erasure cascade max 100k objects per run then continue-as-new-task; retention scan max 1M rows per run; redactor input max 1 MB per call; masker throughput floor 50 MB/s on reference host; holds max 100k active; rights requests max 10k open per tenant. + +## 7. Execution steps (in build order) + +1. PII marks: `PiiClass`, `FieldMark`, `Registry` wired into 04-data.md model registration with boot-time unmarked-field failure. +2. Redactor: patterns, class replacements, single `redact()` entry, hooks into 02-observability.md sinks and 05-request-routing.md errors. +3. Audit chain: `Record`, `Appender`, `Chain`, `SqlStore` on both backends, seal blocks, `AUDIT()` macro at ORM write hooks. +4. Audit query plus verify: `Query` with auditor-role guard, verify command with gap report, metrics per 02-observability.md. +5. Retention: `Policy` config, dry-run report, `Enforcer` as scheduled task via 07-supervisor-replication.md, per-model TTLs. +6. Legal hold: `LegalHold` CRUD with audit trail, hold check inside Enforcer and Eraser, release flow. +7. Rights requests: `RightsRequest` state machine with due dates, identity gate, export bundle builder with checksums. +8. Erasure cascade: per-store deleters (ORM, KV, cache, sessions, uploads, tasks), pseudonymization of audit subject index, report format. +9. Masking: `Masker` per class plus `FixtureScrubber` for dumps, seeds, and staging sync; rescan verifier. +10. Hardening: caps audit, auditor UI routes, CLI surface via 09-cli-ops.md, docs and living-example coverage. + +## 8. Validation (how to verify each step) + +- Step 1: model with an unmarked field refuses to boot naming the field; registry dump lists every model with its marks; integration test through a create URL. +- Step 2: planted secrets, tokens, card-shaped strings, and marked PII in logs, errors, and audit diffs all render as `[redacted:]`; no raw value in any sink under failure injection. +- Step 3: 10k writes produce an unbroken chain on both backends; tamper-one-byte fixture fails verify naming the exact seq; seal blocks present at the configured interval. +- Step 4: non-auditor query returns 403 and is audited; auditor paged query returns exact ranges; verify on a pruned-gap fixture reports the gap range. +- Step 5: dry-run report counts match a manual query; apply deletes only expired unheld rows; TTL boundary test (day 89 kept, day 91 plus grace handled). +- Step 6: held subject survives retention and erasure with reason cited; release then re-run completes; hold place and release each emit audit records. +- Step 7: export bundle validates against schema, contains only marked exportable fields, checksum matches, second generation for identical data is byte-identical except timestamps. +- Step 8: erased subject has zero rows, keys, cache entries, sessions, or task payloads on rescan; audit records keep chain validity with pseudonymized subject; exempted-limb fixture refuses with cited reason. +- Step 9: masked dump restores and passes app boot; rescan finds zero raw marked values; FK joins still resolve; determinism check (same seed gives same output). +- Step 10: full integration suite green in the same session; caps review signed; living example demonstrates audit view, export, erasure, hold, and masked staging per 10-workflow-example.md rule 1. + +## 9. References + +- Depends on 02-observability.md: log record format, sink hook for redaction, audit metrics and trace linkage. +- Depends on 03-security.md: hash and signing primitives, key epochs, auditor auth guards, secret handling. +- Depends on 04-data.md: ORM descriptors for PII marks, dual-backend stores, KV/cache invalidation, migration journal for audit tables. +- Depends on 05-request-routing.md: error renderer hook for redaction, auditor and self-service routes, validation for rights forms. +- Depends on 07-supervisor-replication.md: scheduler for retention runs, task queue for export/erasure cascades, durable streams for audit replication. +- Depends on 11-tenancy.md: tenant selector on every record, per-tenant retention and holds isolation. +- Feeds 09-cli-ops.md: verify, export, erase, hold, retention dry-run commands and health inputs. +- Feeds 10-workflow-example.md: compliance demo routes and integration test templates. +- Defers to 13-operability.md for cross-system audit exchange formats; defers API policy surface to 14-api-governance.md; defers SBOM and provenance attestation to 15-supply-chain.md. + +## 10. Open questions + +1. Audit store engine: reuse 04-data.md SQLite WAL path for the chain, or a dedicated append-only segment file shared with 07-supervisor-replication.md streams? +2. Backup erasure: crypto-shredding via per-subject keys, or documented backup-expiry window (restores re-run erasure)? Leaning to crypto-shredding for flagged classes. +3. Default audit retention is 7 years; should the default vary per deployment region template, or stay single-default with explicit override? +4. Should audit reads require quorum (two auditor roles) for bulk export, or is single auditor plus audit-of-access enough for v1? + +## 11. Sources + +- OWASP Logging Cheat Sheet (event attributes when/where/who/what, events to log, data to exclude, de-identification): https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html (retrieved 2026-10-23) +- GDPR Art. 5, Principles relating to processing of personal data: https://gdpr-info.eu/art-5-gdpr/ (retrieved 2026-10-23) +- GDPR Art. 12, rights modalities and one-month deadline: https://gdpr-info.eu/art-12-gdpr/ (retrieved 2026-10-23) +- GDPR Art. 17, Right to erasure: https://gdpr-info.eu/art-17-gdpr/ (retrieved 2026-10-23) +- GDPR Art. 20, Right to data portability: https://gdpr-info.eu/art-20-gdpr/ (retrieved 2026-10-23) +- RFC 5848, Signed Syslog Messages (origin authentication, integrity, sequencing, missing-message detection): https://www.rfc-editor.org/rfc/rfc5848.txt (retrieved 2026-10-23) diff --git a/plans/13-operability.md b/plans/13-operability.md new file mode 100644 index 0000000..a2510ed --- /dev/null +++ b/plans/13-operability.md @@ -0,0 +1,175 @@ +# 13 - Operability at Scale + +Title: Operability at Scale +Scope: Rolling and blue-green deploys, startup/shutdown drains, distributed locks, leader election, circuit breakers, bulkheads, load shedding, client deadlines, chaos hooks, SLO and error-budget metrics, debug endpoints. + +## 1. Purpose and philosophy + +This file owns what keeps MoloApp alive under real production pressure: safe deploys, clean startup and shutdown, single-owner coordination across instances, fast failure under overload, propagated deadlines, deliberate fault injection, SLO accounting, and debug introspection. It does not own the primitives it builds on: probes live in 09-cli-ops.md, metric and trace primitives live in 02-observability.md, retry and timeout phases live in 08-clients-openapi.md, scheduler leadership and replication live in 07-supervisor-replication.md, connection draining mechanics live in 01-runtime.md. + +Philosophy: + +- Fail fast, shed early, degrade loudly. It is nearly always better to fail quickly and apply back pressure downstream as soon as possible (Envoy circuit breaking docs). Overload is handled by policy, never by unbounded queues. +- Deploys are boring by construction. Health-gated, reversible, and version-tolerant: two versions serve side by side during every rollout, and shared state never assumes a single version. +- Coordination needs a lease and a fence. Every lock and leadership claim expires on its own and carries a fencing token, so a slow holder can never corrupt state after losing ownership. +- Deadlines flow downhill. Every inbound request carries a deadline, and every outbound call inherits what remains, converted to a remaining-time budget so clock skew cannot stretch it. +- Chaos is a first-class test input. Fault hooks are compiled in, disabled by default, authenticated always, and every experiment states its steady-state hypothesis up front. + +## 2. Best practices (from research) + +1. Error budgets make reliability negotiable. An error budget is 1 minus the SLO of the service: a 99.9% SLO service has a 0.1% error budget, so 1,000,000 requests in four weeks allow 1,000 errors (Source: https://sre.google/sre-book/embracing-risk, retrieved 2026-10-23). SLOs are the target, the budget is the spendable allowance for deploys and experiments. +2. Rolling updates gate on readiness. Kubernetes RollingUpdate strategy exposes maxUnavailable (pods allowed down) and maxSurge (extra pods allowed above desired), both defaulting to 25%, and with maxUnavailable 0 a new pod must pass its readiness probe before an old one is terminated (Source: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/, retrieved 2026-10-23; parameter table at https://kubernetes.io/docs/tasks/run-application/update-deployment-rolling/, retrieved 2026-10-23). MoloApp deploys mirror this: readiness gates traffic, startup gates protect slow boots. +3. Circuit breaking is distributed and fail-fast. Envoy enforces circuit breaking limits at the network level with fully distributed (not coordinated) breakers: cluster maximum connections, pending requests, active requests, and retries, each with overflow counters (Source: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/circuit_breaking, retrieved 2026-10-23). Triangulates with the SRE overload chapter: gracefully handling overload is fundamental to a reliable serving system, via degraded responses and client-side throttling (Source: https://sre.google/sre-book/handling-overload/, retrieved 2026-10-23). +4. Deadlines are absolute, propagation is relative. A deadline is a point in time past which a client is unwilling to wait; because clocks differ, gRPC converts the deadline to a remaining timeout with elapsed time already deducted before propagating it downstream, and expiry surfaces as DEADLINE_EXCEEDED (Source: https://grpc.io/docs/guides/deadlines/, retrieved 2026-10-23). MoloApp copies this: absolute on the wire, remaining-budget on forward. +5. Locks and elections ride on lease-bound sessions. etcd concurrency builds Mutex and Election on Sessions: multiple sessions may contend for one election prefix but only one can be leader at a time, and session loss releases everything it held (Source: https://pkg.go.dev/go.etcd.io/etcd/client/v3/concurrency, retrieved 2026-10-23). MoloApp implements the same shape over the shared KV from 04-data.md, which answers open question O7-4 in 07-supervisor-replication.md by adding fencing tokens in v1. +6. Chaos experiments follow four steps: define steady state as measurable output, hypothesize it holds for control and experimental groups, inject real-world variables (crashes, severed links, failing disks), and try to disprove the hypothesis by comparing the groups (Source: https://principlesofchaos.org/, retrieved 2026-10-23). MoloApp chaos hooks encode exactly this loop: steady-state metric, hypothesis record, fault action, comparison verdict. +7. Debug endpoints are separate, conventional, and cheap. Go exposes pprof under /debug/pprof/ with heap, CPU profile, trace, goroutine, block, and mutex views (Source: https://pkg.go.dev/net/http/pprof, retrieved 2026-10-23). MoloApp mirrors the convention under /debug/ with its own auth, caps, and opt-in compile flag, never on the public listener by default. + +## 3. Structure + +Root: `src/ops/` continues in 09-cli-ops.md; this file owns `src/resilience/` plus deploy and debug surface. One class or concern per file, deep over flat. + +``` +src/deploy/ + strategy.h molo::deploy::Strategy (rolling, blue_green) + rolling.h/.cpp molo::deploy::RollingUpdater (surge, unavailable, gates) + blue_green.h/.cpp molo::deploy::BlueGreenSwitcher (standby, cutover, rollback) + lifecycle.h/.cpp molo::deploy::Lifecycle (startup gate, drain phases, hooks) +src/coord/ + lease.h/.cpp molo::coord::Lease (TTL, renewal, expiry callback) + dist_lock.h/.cpp molo::coord::DistributedLock (acquire, fencing token) + election.h/.cpp molo::coord::LeaderElection (campaign, observe, resign) + fence.h molo::coord::FencingToken (monotonic, compare) +src/resilience/ + breaker.h/.cpp molo::resilience::CircuitBreaker (closed, open, half_open) + bulkhead.h/.cpp molo::resilience::Bulkhead (partition, semaphore, queue) + shedder.h/.cpp molo::resilience::LoadShedder (priority, overload policy) + deadline.h/.cpp molo::resilience::Deadline (absolute, remaining, propagate) + slo.h/.cpp molo::resilience::SloTracker (SLI windows, burn rate, budget) +src/chaos/ + hook.h molo::chaos::FaultHook (interface: fault points) + injector.h/.cpp molo::chaos::Injector (latency, error, kill, partition) + experiment.h/.cpp molo::chaos::Experiment (hypothesis, steady state, verdict) +src/debug/ + endpoint.h/.cpp molo::debug::DebugEndpoint (route table, auth gate) + snapshot.h/.cpp molo::debug::StateSnapshot (loops, pools, breakers, locks) +``` + +Namespaces: `molo::deploy`, `molo::coord`, `molo::resilience`, `molo::chaos`, `molo::debug`. No sibling namespace may implement its own breaker, lock, or deadline; they consume these. + +## 4. Rules (hard, numbered) + +1. Every deploy keeps the old version serving until the new version passes readiness; traffic never moves to an unreadied instance. +2. Mixed versions must interoperate: shared state formats (oplog, envelopes, sessions) stay backward compatible across one minor version, old readers ignore unknown fields. +3. Shutdown always drains: stop accepting, finish in-flight work within the deadline, then exit; dropped work lands in the dead-letter queue with a reason, never vanishes silently (mechanics in 01-runtime.md and 07-supervisor-replication.md, orchestration here). +4. Every lock acquisition and leadership claim has a TTL and returns a fencing token; writes without a current token are rejected by shared stores. +5. Only one leader per election name at a time; leadership loss is delivered as an event within one renewal interval, and the loser stops acting immediately. +6. Circuit breakers fail fast when open: no queuing behind an open breaker, no retries into it, every rejection counted and logged at debug. +7. Bulkheads partition by downstream dependency and by priority class; one saturated partition never starves another. +8. Load shedding drops lowest priority first by declared policy; shed requests return 503 with Retry-After, never hang. +9. Every request has an effective deadline; outbound calls inherit the remaining budget minus a safety margin, and expiry returns 504 (or DEADLINE_EXCEEDED equivalent on realtime paths). +10. Chaos hooks are disabled by default, require explicit enable plus auth, and every experiment records hypothesis, blast radius, and verdict. +11. Debug endpoints bind to the admin listener only, require auth, and are compiled out unless MOLO_ENABLE_DEBUG is set. +12. SLO burn-rate alerts fire on budget consumption rate, not on raw error counts; paging alerts require both fast-burn and slow-burn windows to agree. + +## 5. Development guidelines + +- Build coordination first (lease, lock, election over the 04-data.md shared KV), then drains, then breakers and bulkheads, then deadlines, then deploy strategies, then SLO tracking, then chaos and debug last. +- State machines everywhere: breaker (closed, open, half_open), election (candidate, leader, follower), lifecycle (booting, serving, draining, stopped). Explicit transitions, logged, with metrics per transition. +- Time comes from an injectable clock (see 08-clients-openapi.md frozen clock) so TTLs, backoffs, and burn windows are deterministic in tests. +- Fencing tokens are monotonic per resource (uint64 from the shared store); compare with greater-than, never equality, and persist the highest seen token alongside the data. +- Prefer middleware composition for resilience: breaker, bulkhead, shedder, and deadline middlewares wrap handlers in a fixed order (deadline outermost, then shedder, bulkhead, breaker innermost near the call). +- Chaos faults implement one interface with dry-run support: every fault can report what it would do without doing it, and dry-run is the default first step of every experiment. +- Debug output is snapshots, not live references: serialize state under a short lock, then render; a slow debugger must never stall the loop. + +## 6. Specifications (concrete behaviors, caps, formats) + +Deploy: + +- Rolling: max_unavailable default 0, max_surge default 1; per-instance steps are wait_ready (cap 120 s) then terminate_old; revision history kept for last 5 deploys; automatic rollback when new-version error rate exceeds 5x baseline over 2 minutes. +- Blue-green: standby pool warms to ready, cutover flips router weight in 10% steps every 30 s, instant rollback flips back in one step; both pools share the same shared-KV and durable bus (no split brain). +- Startup gate: /startupz (09-cli-ops.md) returns 503 until lifecycle reaches serving; slow boot budget 60 s, then process exits nonzero so the orchestrator retries. +- Drain phases on SIGTERM: phase 1 stop accept plus fail /readyz (immediate); phase 2 finish in-flight requests (cap 30 s); phase 3 close realtime connections with code 1001 (cap 10 s); phase 4 flush tasks and spans (cap 10 s); total cap 60 s, then hard exit with exit code 0 and a loud log of anything dropped. + +Coordination: + +- Lease: TTL default 10 s, renewal every 3 s, expiry callback fires within 1 renewal interval; max 1024 live leases per process. +- DistributedLock: acquire timeout default 5 s, hold TTL 10 s with auto-renew while held; returns FencingToken uint64 strictly increasing per lock name; unlock with stale token is a no-op counted in metrics. +- LeaderElection: one name, one leader; campaign blocks until elected or cancelled; heartbeat 3 s; followers get leader-change events; resign hands over within 1 s. Scheduler leadership in 07-supervisor-replication.md is the first consumer. +- Fencing enforcement: shared KV compare-and-set on (key, token) rejects writes with token older than stored; rejection error carries winner token. + +Resilience: + +- CircuitBreaker per downstream: opens after 20 failures or 50% failure rate over a 10 s sliding window (min 10 calls); open duration 30 s; half_open allows 3 probe calls, closes on 3 successes, reopens on 1 failure. States exported as gauge plus transition counter. +- Bulkhead: max 32 partitions per process; each partition has max_concurrent (default 64) plus max_queue (default 128); overflow sheds with 503; partitions named per downstream host or priority class. +- LoadShedder: priority classes critical, normal, best_effort; when CPU queue depth or memory pressure crosses thresholds, shed best_effort first; shed decision under 1 ms; every shed emits metric with class and reason. +- Deadline: header `X-Deadline-Ms` (absolute Unix millis) on inbound; outbound propagation sends remaining minus 50 ms floor-margin; default request budget 30 s, max 300 s; realtime paths carry deadline in the connection context from 06-realtime.md. + +SLO and error budget: + +- SLI windows: 30-day rolling for budget accounting, plus 1 h and 6 h burn-rate windows for alerting; availability SLI = good_events / total_events per route template (never raw path). +- Budget: budget = 1 - SLO; default SLO 99.9% per service; burn_rate = observed_error_ratio / budget_ratio; fast-burn page at 14x over 1 h, slow-burn ticket at 6x over 6 h; both must fire for a page. +- Metrics: `molo_slo_budget_remaining_ratio`, `molo_slo_burn_rate{window}`, `molo_slo_bad_events_total{route,reason}`; cardinality capped per 02-observability.md rules. + +Chaos and debug: + +- Fault hooks: latency_add_ms (cap 10000), error_rate (cap 1.0), conn_kill, partition_downstream, clock_skew_ms (cap 60000, test only); each hook scoped to route template or downstream name, max 64 active hooks. +- Experiment record: {id, hypothesis, steady_state_metric, threshold, duration_cap 30 min, blast_radius, verdict}; auto-abort when steady state breaches threshold; all actions audit-logged (feeds 12-audit-compliance.md). +- Debug endpoints (admin listener, auth required): GET /debug/snapshot (loops, pools, breakers, locks, elections), GET /debug/breakers, GET /debug/elections, GET /debug/slo; response cap 1 MB; pprof-style CPU and heap views where the platform allows, same path convention /debug/pprof/. + +## 7. Execution steps (in build order) + +1. Coordination core: Lease, DistributedLock with fencing tokens, LeaderElection over shared KV; migrate the 07-supervisor-replication.md scheduler lease onto it. +2. Lifecycle orchestration: startup gate, four-phase drain, signal wiring with 01-runtime.md, drop accounting into dead-letter queue. +3. Breaker plus bulkhead: per-downstream breaker state machines, partitions, middleware wrappers, overflow metrics. +4. Shedder plus deadlines: priority shedder, X-Deadline-Ms parse and propagate through 08-clients-openapi.md client and 06-realtime.md contexts. +5. Deploy strategies: rolling updater with readiness gates and auto-rollback, blue-green switcher with weighted cutover, revision history. +6. SLO tracking: SLI windows, budget math, burn-rate alerts, Prometheus series for budget and burn. +7. Chaos hooks: fault interface, injector actions, experiment runner with auto-abort and audit records. +8. Debug endpoints: admin-listener routes, snapshot serializers, auth gate, compile-flag removal check. + +## 8. Validation (how to verify each step) + +- Step 1: two processes campaign for one election, kill the leader, follower takes over within 5 s; stale-token write rejected with winner token; lease expiry releases lock within 4 s of holder death (frozen-clock plus real-clock runs). +- Step 2: SIGTERM during 1000 in-flight requests drains with zero silent drops; /readyz flips to 503 within 100 ms; realtime peers receive close 1001; hard cap enforced at 60 s with loud log. +- Step 3: failing downstream opens breaker within window, calls fail fast under 1 ms with rejection counter; bulkhead saturation on one host leaves other hosts at full speed; half-open probes close the breaker after recovery. +- Step 4: low-priority flood sheds best_effort first while critical stays green; propagated deadline expires downstream first and returns 504 upstream with matching request IDs; margin floor never sends negative budgets. +- Step 5: rolling deploy with max_unavailable 0 shows zero failed requests across cutover; injected 10% new-version errors trigger auto-rollback within 3 minutes; blue-green cutover and instant rollback both verified through the real router. +- Step 6: recorded error rates reproduce budget math within rounding; 14x fast-burn plus 6x slow-burn fixtures page, single-window fixtures do not; series cardinality stays under caps at 10k routes. +- Step 7: experiment with latency fault shows steady-state breach, auto-abort, and verdict record; dry-run changes nothing; unauthenticated chaos call rejected; disabled-by-default build contains no reachable fault path. +- Step 8: debug routes 404 on the public listener and 200 on admin with auth; snapshot under 1 MB renders in under 100 ms; MOLO_ENABLE_DEBUG=0 build has zero debug symbols in the binary. + +## 9. References (sibling files this depends on or feeds) + +- Depends on 01-runtime.md: signal handling, connection drain mechanics, H2 GOAWAY, loop introspection for snapshots. +- Depends on 02-observability.md: metric, event, and span primitives; label cardinality rules; zero-cost gating for all resilience hooks. +- Depends on 04-data.md: shared KV for leases, locks, elections, and fencing compare-and-set. +- Depends on 05-request-routing.md: middleware ordering slot for deadline, shedder, bulkhead, breaker layers. +- Depends on 06-realtime.md: connection context for deadlines, close codes for drain, broadcast behavior during deploys. +- Depends on 07-supervisor-replication.md: drain deadlines, dead-letter queue, scheduler leadership migration, multi-instance topology. +- Depends on 08-clients-openapi.md: timeout phases, retry policy interplay with breakers, frozen clock, deadline propagation in the HTTP client. +- Depends on 09-cli-ops.md: /healthz, /readyz, /startupz, /metrics endpoints consumed as deploy gates; admin listener shape. +- Feeds 09-cli-ops.md: deploy and chaos CLI surface (rollout status, rollback, experiment run/abort), debug endpoint hosting. +- Feeds 10-workflow-example.md: living example must demo rolling deploy, breaker trip, deadline expiry, and one chaos experiment. +- Feeds 12-audit-compliance.md: chaos action and debug access audit records (no duplication of audit formats). +- Boundary with 03-security.md: auth for debug and chaos endpoints is owned there; this file only names the gate. + +## 10. Open questions + +1. Should blue-green weight steps be time-based (10% per 30 s) or metric-gated per step (advance only when error rate holds)? Proposal: metric-gated with time cap. +2. Fencing token scope: per lock name or global per store? Proposal: per lock name for v1, global optional later. +3. Default SLO: single 99.9% for the whole service, or per-route SLOs from day one? Proposal: service default plus opt-in per-route overrides. +4. Chaos in production: allow guarded production experiments in v1, or restrict to staging by config? Proposal: staging-only until the audit trail in 12-audit-compliance.md lands. +5. Debug CPU and heap profiling: in-house sampler or platform hooks only? Proposal: platform hooks where available, documented gap elsewhere. + +## 11. Sources (exact URLs with retrieval dates) + +- Google SRE Book, Chapter 3 Embracing Risk (error budget equals 1 minus SLO): https://sre.google/sre-book/embracing-risk (retrieved 2026-10-23). +- Google SRE Book, Chapter 21 Handling Overload (graceful overload, degraded responses, client-side throttling): https://sre.google/sre-book/handling-overload/ (retrieved 2026-10-23). +- Kubernetes Deployments (RollingUpdate strategy, readiness gating): https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ (retrieved 2026-10-23). +- Kubernetes rolling update parameters (maxUnavailable, maxSurge defaults): https://kubernetes.io/docs/tasks/run-application/update-deployment-rolling/ (retrieved 2026-10-23). +- Envoy Circuit Breaking (fail fast, distributed breakers, overflow counters): https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/circuit_breaking (retrieved 2026-10-23). +- gRPC Deadlines (absolute deadline, remaining-timeout propagation, DEADLINE_EXCEEDED): https://grpc.io/docs/guides/deadlines/ (retrieved 2026-10-23). +- etcd clientv3 concurrency (Session, Mutex, Election, single leader): https://pkg.go.dev/go.etcd.io/etcd/client/v3/concurrency (retrieved 2026-10-23). +- Principles of Chaos Engineering (four-step experiment loop): https://principlesofchaos.org/ (retrieved 2026-10-23). +- Go net/http/pprof (debug endpoint convention): https://pkg.go.dev/net/http/pprof (retrieved 2026-10-23). diff --git a/plans/14-api-governance.md b/plans/14-api-governance.md new file mode 100644 index 0000000..6d9ff2f --- /dev/null +++ b/plans/14-api-governance.md @@ -0,0 +1,162 @@ +# 14 - API Governance + +Title: API Governance +Scope: plan.md sections 2, 6 partial. Topics: API keys with scopes/quotas/expiry, router versioning and deprecation policy, breaking-change detection from OpenAPI diff, rate-limit tiers, developer-facing API surface rules. + +## 1. Purpose and philosophy + +This file owns the contract between MoloApp applications and their API consumers: who may call (keys, scopes), how much they may call (quotas, tiers), which contract version they bind to (router versioning), how that contract retires (deprecation policy), how breakage is caught before release (OpenAPI diff gate), and what a conforming endpoint looks like (surface rules). + +Philosophy: + +- Explicit contracts, explicit versions. Every public route belongs to exactly one major API version. No unversioned public surface. +- Additive evolution inside a version, new version for breakage. Compatible changes ship in place; incompatible changes ship a new `/v{n}` prefix and retire the old one on a schedule. +- Machine-checkable governance. Keys, scopes, tiers, versions, deprecations and surface rules are declared data validated at startup and in CI, never tribal knowledge. +- Standards wire format, in-house enforcement. Deprecation/Sunset headers, OpenAPI flags and rate-limit headers follow their specs byte for byte; the checker, linter and gate are dependency-free C++23. +- This file declares policy and detection. Mechanism lives in siblings: crypto and base throttling in 03-security.md, routers and views in 05-request-routing.md, OpenAPI emission in 08-clients-openapi.md, audit trail in 12-audit-compliance.md. + +## 2. Best practices (from research) + +1. Explicit versioning is mandatory and Major.Minor shaped. Microsoft requires: "All APIs compliant with the Microsoft REST API Guidelines MUST support explicit versioning", versioned "using a Major.Minor versioning scheme", with version in path or `?api-version=` query; services that cannot keep URL paths stable across versions MUST embed the version in the URL path (Source: https://github.com/Microsoft/api-guidelines/blob/master/Guidelines.md, retrieved 2026-10-23). Triangulated with 08-clients-openapi.md (`/v{n}` path prefix rule) and OpenAPI `info.version` below: MoloApp uses path prefix `/v{n}` (major only) as the single mechanism. +2. OpenAPI separates spec version from API version and flags retirement per item. The `openapi` field "MUST be the version number of the OpenAPI Specification" and "is not related to the API `info.version` string"; Operation and Parameter objects carry `deprecated: boolean` where consumers "SHOULD refrain from usage" (Source: https://spec.openapis.org/oas/v3.1.0, retrieved 2026-10-23). MoloApp therefore versions the API in `info.version` plus the `/v{n}` prefix, and marks deprecation with `deprecated: true` derived from router declarations in 08-clients-openapi.md. +3. Runtime deprecation uses two headers plus a link. RFC 8594 "defines the Sunset HTTP response header field, which indicates that a URI is likely to become unresponsive at a specified point in the future" plus "a sunset link relation type" (Source: https://www.rfc-editor.org/rfc/rfc8594, retrieved 2026-10-23). RFC 9745 adds: "The Deprecation HTTP response header field is used to signal to consumers of a resource (identified by a URI) that the resource will be or has been deprecated", with a `deprecation` link relation; "The act of deprecation does not change any behavior of the resource"; `Deprecation` value MUST be a Structured Field Date; "The timestamp given in the `Sunset` HTTP header field MUST NOT be earlier than the one given in the `Deprecation` header field" (Source: https://www.rfc-editor.org/rfc/rfc9745, retrieved 2026-10-23). Triangulated with item 2: static flag in OpenAPI, runtime headers on responses, both from one router declaration. +4. Scopes are space-delimited least-privilege grants. RFC 6749 section 3.3: "The value of the scope parameter is expressed as a list of space-delimited, case-sensitive strings", issued scope may differ from requested scope per resource-owner policy, and violations use `invalid_scope` errors (Source: https://www.rfc-editor.org/rfc/rfc6749, retrieved 2026-10-23). MoloApp API keys carry the same grammar (`resource.action`, space-delimited sets) enforced per route; under-scoped calls fail 403 with a problem-details body. +5. Rate-limit signalling is converging on two fields plus 429/Retry-After. Draft revision 11 "defines the RateLimit-Policy and RateLimit HTTP header fields for servers to advertise their quota policies" with quota parameters, available-quota parameters, and problem types including "quota-exceeded" (Source: https://datatracker.ietf.org/doc/draft-ietf-httpapi-ratelimit-headers/, retrieved 2026-10-23). Until the draft becomes an RFC, MoloApp emits both the draft fields (revision pinned in config) and the de-facto `X-RateLimit-*` trio, always with `Retry-After` on 429. +6. Breaking-change detection is a three-mode OpenAPI diff with a rule engine. oasdiff "compare[s] and detect[s] breaking changes in OpenAPI specs": `breaking` shows "only the changes that break existing API clients", `changelog` the human-readable consumer-affecting list, `diff` the full machine-readable definition diff, backed by a customizable checks list (Source: https://github.com/oasdiff/oasdiff, retrieved 2026-10-23). Triangulated with Microsoft section 12.3 (breaking-change definition) and 10-workflow-example.md release gates: MoloApp ships an in-house checker with the same three outputs and fails CI on `breaking` unless the major version bumped. +7. Surface rules stay boring: plural noun collections, verbs in methods, standard codes, problem details, pagination in query. Microsoft guidelines plus 05-request-routing.md/08-clients-openapi.md agree on this shape; this file turns that agreement into a startup linter with numbered checks rather than prose advice. + +## 3. Structure + +Root: `src/molo/govern/`, headers mirrored under `include/molo/govern/`. One class or concern per file pair. Namespace root `molo::govern`. + +- `keys/ApiKey.h`: `struct ApiKey` (id, prefix, hash ref, owner, scopes, tier, quota overrides, issued_at, expires_at, revoked flag, last_used). +- `keys/Scope.h`: `class Scope` grammar (`resource.action`, `*` segment rules), parse, `allows(route_scope)` check. +- `keys/KeyStore.h`: `class IKeyStore` interface (lookup by prefix, record use); `keys/MemoryKeyStore.h`, `keys/DbKeyStore.h` (ORM-backed, see 04-data.md). +- `keys/KeyIssuer.h`: mint, rotate, revoke; `keys/KeyAuth.h`: middleware hook extracting `Authorization: Bearer` or `X-Api-Key`, constant-time verify via 03-security.md. +- `versions/ApiVersion.h`: `struct ApiVersion` (major, status: current/deprecated/sunset, deprecation_date, sunset_date, successor link). +- `versions/VersionedRouter.h`: binds a `Router` (05-request-routing.md) to `/v{n}`, declares status and dates, validates uniqueness. +- `versions/DeprecationMiddleware.h`: injects `Deprecation`, `Sunset`, `Link: rel="deprecation"` and `rel="successor-version"` per policy. +- `diff/SpecSnapshot.h`: stored canonical OpenAPI JSON per version (`specs/api/v{n}.json`). +- `diff/SpecDiffer.h`: path/method/schema diff engine over 08-clients-openapi.md documents. +- `diff/BreakingRules.h`: numbered check list (see 6.3), severity ERR/WARN/INFO. +- `diff/GovernGate.h`: CI verdict (pass/fail with reason codes), major-bump detector. +- `tiers/TierTable.h`: `struct Tier` (name, req/min, req/day, burst, concurrency, quota units); `tiers/TierPolicy.h`: key-to-tier resolution plus per-route overrides. +- `tiers/QuotaLedger.h`: usage counters interface over 04-data.md KV; `tiers/TierHeaders.h`: RateLimit-Policy/RateLimit plus X-RateLimit-* plus Retry-After emission. +- `surface/SurfaceRules.h`: numbered lint checks (see 6.5); `surface/RouteLinter.h`: runs checks over routers at startup and in CI. + +## 4. Rules (hard, numbered) + +1. Every public API route belongs to exactly one major version mounted at `/v{n}`. Unversioned public routes fail startup. +2. At most 3 live majors per app; a 4th requires retiring the oldest first. Deprecated majors keep serving until their Sunset date. +3. Breaking changes (section 6.3 ERR list) inside one major without a new `/v{n}` fail the CI gate. No override flag; ship a new major instead. +4. Every deprecated route/version emits `Deprecation` plus `Sunset` plus `Link rel="deprecation"` on every response; Sunset date MUST NOT precede Deprecation date. After Sunset, the route returns 410 Gone, never silent 404. +5. OpenAPI `deprecated: true` and runtime headers derive from the same router declaration. Hand-editing one without the other is a defect caught by the diff gate. +6. API keys authenticate the calling application only, never a user. User context comes from 03-security.md sessions or bearer user tokens; mixing the two in one credential is rejected at issue time. +7. Every key has scopes (space-delimited `resource.action`), an expiry, and a tier. Keys without expiry are rejected at issue. Expired or revoked keys fail 401; valid keys lacking scope fail 403. +8. Key secrets are shown once at mint, stored as Argon2id hashes (03-security.md), looked up by public prefix. Plaintext secrets never appear in logs, errors, metrics or audit bodies (see 12-audit-compliance.md). +9. Rate-limit tiers bind to the 03-security.md limiter; this file defines the tier table and headers, never a second counting algorithm. Every 429 carries `Retry-After` and the tier headers. +10. Surface rules in section 6.5 are startup errors for new routes and CI warnings for deprecated ones. New violations never merge. +11. All caps in section 6 are static constants with documented defaults. No autotuning of quotas, windows, versions or diff thresholds. +12. Governance data (keys metadata, tiers, versions, snapshots) is tenant-scoped where 11-tenancy.md applies; cross-tenant key reuse is impossible by construction. + +## 5. Development guidelines + +- Declare first: version, scopes, tier and deprecation dates live on the router/handler declaration; middleware, headers, OpenAPI flags and docs all render from that one declaration. +- Build in section 7 order; each step ends green through the real stack per 10-workflow-example.md (no unit-only key checks, real HTTP with real headers). +- Reuse sibling machinery: hashing and constant-time compare from 03-security.md, routers and error shapes from 05-request-routing.md, OpenAPI builder from 08-clients-openapi.md, KV counters from 04-data.md, metrics/log macros from 02-observability.md. +- Keep the diff deterministic: canonical JSON (sorted keys, fixed floats) before compare, so identical apps produce byte-identical snapshots. +- Errors are RFC 9457 problem details (`application/problem+json`) with `type` URIs per reason (expired_key, insufficient_scope, quota_exceeded, version_gone, version_deprecated is a warning header, not an error). +- Document every new check (diff or surface) with id, severity, example violation and fix, in the same change that adds it. + +## 6. Specifications (concrete behaviors, caps, formats) + +### 6.1 API keys, scopes, quotas, expiry + +- Key format: `molo__`; id 16 chars public prefix used for lookup and logs; secret 32 bytes from 03-security.md CSPRNG. Storage: `{id, prefix, argon2id_hash, owner, scopes[], tier, quota_override?, issued_at, expires_at, revoked_at?, last_used_at, last_used_ip?}`. +- Scope grammar: `resource.action` segments `[a-z0-9_]+`, `*` allowed as full segment only (`billing.*`, `*.read`); max 64 scopes per key, max 64 chars per scope. Route declares `required_scopes[]`; request passes when key scopes cover all required (AND) with `*` expansion. Empty route list means key-only auth, no scope check. +- Expiry: default 90 days, max 366 days, min 1 hour; rotation issues a successor with 24 h overlap (`supersedes` link), old key keeps serving 24 h then 401. Revocation is immediate and replicated per 07-supervisor-replication.md within 5 s p99. +- Quota binding: key resolves tier from TierTable; per-key `quota_override` may only lower limits, never raise above tier. Usage ledger keys `gov:quota:{key_id}:{window}` in 04-data.md KV, TTL = window length. +- Caps: 100k keys per tenant, 10k lookups/s per process (in-memory prefix index, overflow spills to DB store with metric), secret verify budget under 1 s via off-loop Argon2id path. + +### 6.2 Router versioning and deprecation policy + +- Mount: `VersionedRouter(version=2, status=current)` mounts all routes under `/v2/...`; `url_for` includes the version. Major-only integers 1..99. Minor evolution is additive inside the major and tracked in `info.version` (`2.3.0`) without URL change. +- Lifecycle: current -> deprecated (announce: headers on, OpenAPI flag on, changelog entry) -> sunset (date reached: 410 Gone with problem body + `Link rel="successor-version"`) -> removed (route gone, major slot freed). Minimum deprecation window 90 days; minimum announce-to-sunset 30 days even for pre-1.0 majors. +- Headers: `Deprecation: @` (Structured Field Date per RFC 9745), `Sunset: ` (RFC 8594), `Link: <{guide}>; rel="deprecation"`, `Link: <{successor}>; rel="successor-version"`. Startup validation rejects Sunset earlier than Deprecation. +- Version negotiation: path prefix only; no header or query negotiation in v1 (explicit non-goal, avoids the matrix Microsoft documents as optional). Wrong-major requests get 404 plus `Link rel="successor-version"` to the current major, not a silent redirect. +- Snapshots: one canonical `specs/api/v{n}.json` per live major, committed in-repo, regenerated by 08-clients-openapi.md builder, diffed by 6.3 gate. + +### 6.3 Breaking-change detection (OpenAPI diff) + +- Modes mirror oasdiff: `molo apidiff breaking ` (ERR only, exit 1 on any), `changelog` (ERR+WARN human list), `diff` (full JSON incl. INFO/doc-only). Same taxonomy: ERR breaks clients, WARN may break, INFO is safe. +- ERR (breaking): remove path/method; add required request param/property/header; Narrow type/format/enum (remove enum value, int32 to int, string to number, widen is WARN); raise required-ness; remove response status previously documented; change response property from optional to required; change auth scheme or add required security; reduce rate-limit tier defaults. WARN: add optional field, add enum value, add response status, widen type, deprecate flag flip, description-only edits are INFO. +- Gate: CI regenerates the live-major snapshot, runs `breaking` against the committed one. Any ERR fails unless the change also adds a new `/v{n+1}` snapshot carrying the change (major-bump path). Deprecated majors accept INFO/WARN only; any ERR fails unconditionally. +- Caps: diff input max 10 MB per document (matches 08-clients-openapi.md cap), max 5000 paths, rule run under 30 s on reference hardware, output max 1 MB (truncated with counter beyond). + +### 6.4 Rate-limit tiers + +- Default table: `free` 60 req/min, 10k req/day, burst 2x/10 s, concurrency 4; `pro` 600 req/min, 1M req/day, burst 2x/60 s, concurrency 32; `enterprise` 6000 req/min, 50M req/day, burst 1.5x/60 s, concurrency 256. All per key, all overridable per route downward only. +- Headers on every keyed response: `RateLimit-Policy: "tier";q=;w=` and `RateLimit: "tier";r=;t=` (draft-ietf-httpapi-ratelimit-headers rev 11 shape) plus `X-RateLimit-Limit/Remaining/Reset` for legacy clients. 429 adds `Retry-After: ` (min(reset, cap 300)) and `application/problem+json` body with type `quota-exceeded`. +- Counting owned by 03-security.md limiter buckets keyed `(tier, key_id, route)`; this file supplies limits and emits headers. Cross-instance counts converge via 07-supervisor-replication.md shared counters; documented skew bound 2x window edge under partition. +- Quota day is UTC midnight to midnight; minute windows are fixed 60 s; burst bucket refills linearly. Exceeding any one dimension returns 429 for that dimension (reason named in body). + +### 6.5 Developer-facing API surface rules (linter) + +- S1 nouns not verbs: collection paths are plural nouns (`/v1/users/{id}`), no verbs (`/getUsers`, `/users/create` fail). S2 method semantics: GET read, POST create, PUT full replace, PATCH partial, DELETE remove. S3 status codes from the fixed set 200/201/204/400/401/403/404/405/406/409/410/415/422/429/500/503 with 410 reserved for sunset routes. +- S4 errors are RFC 9457 bodies with stable `type` URIs; S5 pagination is `?page=&per_page=` (default 20, max 100) or cursor, envelope `{data, meta, links}` per 08-clients-openapi.md; S6 filtering/sorting in query only; S7 new error `code` values require a minor version note, new nested detail never breaks (Microsoft innererror pattern). +- S8 auth declared: every route declares `auth: none|key|key+scopes|session` plus scopes when applicable; undeclared auth fails startup. S9 idempotency: POST create endpoints accept `Idempotency-Key` (24 h result store per 08-clients-openapi.md). S10 docs: every route has summary + one example; missing items render `description: TBD` and fail the strict lint profile. + +## 7. Execution steps (in build order) + +1. Scopes + key model: Scope grammar, ApiKey struct, IKeyStore + memory backend, KeyIssuer mint/rotate/revoke with 03-security.md hashing. +2. Key auth path: KeyAuth middleware (Bearer + X-Api-Key), constant-time verify, 401/403 problem bodies, scope AND-check, per-request key context for tiers and audit. +3. Versioned routers: ApiVersion + VersionedRouter mounting `/v{n}`, uniqueness and max-3-majors validation, `url_for` with version, 410 Gone handler. +4. Deprecation runtime: DeprecationMiddleware headers (Deprecation/Sunset/Link), date validation (Sunset >= Deprecation), OpenAPI `deprecated: true` derivation via 08-clients-openapi.md builder hook. +5. Tier table + headers: TierTable defaults, key/tier resolution, QuotaLedger over 04-data.md KV, TierHeaders emission (draft + legacy + Retry-After), 429 mapping into 03-security.md limiter. +6. Spec snapshots + differ: canonical JSON writer, SpecSnapshot store per major, SpecDiffer path/method/schema engine, INFO/WARN/ERR taxonomy. +7. Breaking gate + CLI: BreakingRules full list, GovernGate verdict, `molo apidiff breaking|changelog|diff` commands (CLI surface follows 09-cli-ops.md conventions), CI wiring per 10-workflow-example.md. +8. Surface linter + docs: RouteLinter S1-S10, startup fail on new-route violations, strict CI profile, developer guide page in living example. + +## 8. Validation (how to verify each step) + +- Step 1: mint 1k keys, assert prefix lookup, hash-only storage (dump contains no secret), expiry/rotation windows (old serves 24 h then 401), revoke immediacy, scope parse/expand matrix incl. `*` edges and 65-scope rejection. +- Step 2: real-HTTP matrix: no key 401, expired 401, revoked 401, wrong scope 403 with `insufficient_scope`, correct 200; key context visible to audit probe; verify latency within 03-security.md budget. +- Step 3: mount v1+v2, assert prefix routing, triple-mount path merge still works under `/v{n}`, 4th major fails startup, sunset date reached returns 410 with successor link, `url_for` round-trips. +- Step 4: deprecated route returns all four headers with Sunset >= Deprecation; generated OpenAPI shows `deprecated: true` for the same routes; misconfigured dates refuse boot with named error. +- Step 5: tier matrix per key (free/pro/enterprise + override-down) enforced through real burst traffic; every response carries both header families; 429 carries Retry-After and `quota-exceeded` body; counters converge across two instances within the documented skew bound. +- Step 6: golden vector diffs (remove path, add required field, widen type, doc-only edit) classify ERR/WARN/INFO exactly; determinism check (snapshot twice, diff empty); 10 MB + 5000-path caps enforced with clear errors. +- Step 7: fixture PRs: breaking change without new major fails CI with reason codes; same change with new `/v{n+1}` snapshot passes; deprecated-major ERR always fails; CLI exit codes match 09-cli-ops.md mapping. +- Step 8: lint fixtures per S1-S10 pass/fail exactly; new-route violation blocks startup; full integration suite green same session; living example exposes versioned, keyed, tiered routes with deprecation demo page. + +## 9. References (sibling files this depends on or feeds) + +- Depends on 01-runtime.md: header emission path, 429/410 status plumbing, request lifecycle hooks. +- Depends on 02-observability.md: key-use metrics, version/deprecation counters, quota gauges, trace linkage. +- Depends on 03-security.md: Argon2id hashing, CSPRNG, constant-time compare, base rate-limiter buckets, session/user auth boundary. +- Depends on 04-data.md: ORM/DB key store backend, KV quota ledger, invalidation hooks. +- Depends on 05-request-routing.md: Router, Route, middleware chain, error-to-status mapping, problem-details rendering. +- Depends on 08-clients-openapi.md: OpenAPI builder input (routers + models), canonical document, `info.version`, security schemes. +- Depends on 10-workflow-example.md: CI gate placement, spec/test templates, living-example coverage rule. +- Depends on 11-tenancy.md: tenant scope for keys/tiers/versions; feeds it the per-tenant governance shapes. +- Feeds 09-cli-ops.md: `molo apidiff` command implementation target (UX owned there). +- Feeds 12-audit-compliance.md: key lifecycle events (mint/rotate/revoke/use), gate verdicts, deprecation audit records. +- Feeds 13-operability.md: tier/quota export formats shared with external gateways. +- Parallel with 07-supervisor-replication.md (key/quota replication transport) and 15-supply-chain.md (snapshot provenance signing, if adopted). + +## 10. Open questions + +1. Should v1 ship OAuth2 scopes beyond API-key scopes (user-delegated flows), or stay key-only with sessions covering users? Proposal: key scopes only, RFC 6749 flows deferred. +2. Draft header shape is pinned to rev 11; when the IETF publishes the RFC, do we dual-emit old and final shapes during a window? Proposal: yes, one minor version of dual emission. +3. Cross-instance quota exactness: is 2x edge skew acceptable for paid tiers, or do enterprise tiers need synchronous counting? Needs a joint decision with 07-supervisor-replication.md. +4. Should the surface linter auto-fix (rename suggestions) or only report? Proposal: report with fix hints, no auto-rename in v1. +5. Sunset 410 body: include machine-readable `successor` operation mapping (old path to new path table), or just the version link? Proposal: version link in v1, path map later. + +## 11. Sources (exact URLs with retrieval dates) + +- RFC 8594, The Sunset HTTP Header Field (E. Wilde, May 2019, Informational): https://www.rfc-editor.org/rfc/rfc8594 (retrieved 2026-10-23). +- RFC 9745, The Deprecation HTTP Response Header Field (S. Dalal, E. Wilde, Proposed Standard): https://www.rfc-editor.org/rfc/rfc9745 (retrieved 2026-10-23). +- OpenAPI Specification v3.1.0 (2021-02-15): https://spec.openapis.org/oas/v3.1.0 (retrieved 2026-10-23). +- IETF draft-ietf-httpapi-ratelimit-headers-11, RateLimit header fields for HTTP (May 2026, Internet-Draft): https://datatracker.ietf.org/doc/draft-ietf-httpapi-ratelimit-headers/ (retrieved 2026-10-23). +- oasdiff repository, OpenAPI Diff and Breaking Changes (breaking/changelog/diff, checker rules): https://github.com/oasdiff/oasdiff (retrieved 2026-10-23). +- Microsoft REST API Guidelines (explicit versioning, Major.Minor, breaking-change definition): https://github.com/Microsoft/api-guidelines/blob/master/Guidelines.md (retrieved 2026-10-23). +- RFC 6749, The OAuth 2.0 Authorization Framework, section 3.3 Access Token Scope (Oct 2012, Standards Track): https://www.rfc-editor.org/rfc/rfc6749 (retrieved 2026-10-23). diff --git a/plans/15-supply-chain.md b/plans/15-supply-chain.md new file mode 100644 index 0000000..5e24da0 --- /dev/null +++ b/plans/15-supply-chain.md @@ -0,0 +1,150 @@ +# 15 - Supply Chain and Trust + +File: plans/15-supply-chain.md +Scope: plan.md sections 4 (wolfSSL exception), 9 (autonomous releases). Topics: SBOM generation, signed release artifacts, dependency policy and wolfSSL exception handling, vulnerability-scan pipeline hook, provenance and release attestation. + +## 1. Purpose and philosophy + +MoloApp ships almost no supply chain: stdlib plus OS syscalls plus wolfSSL as the sole third-party dependency. This file makes that claim verifiable. Every release carries a machine-readable SBOM, a signed provenance statement describing where, when and how it was built, and detached signatures over all artifacts, so a consumer can answer "what is inside this binary and who built it" without trusting the download server. + +Philosophy: + +- Minimal surface, maximal proof. One allowed exception (wolfSSL) is pinned, hashed and attested; everything else in-house is versioned by git commit. +- Proof over promise. SBOM, provenance and signatures are generated by the release pipeline in 10-workflow-example.md, never hand-edited. +- Standards first. SPDX and CycloneDX for SBOM, in-toto plus SLSA for provenance, Sigstore/cosign for signing, NIST SSDF for practice grouping. +- Closed by default. A new dependency, an unsigned artifact, or a failing vuln gate blocks the release. Overrides are explicit, logged and expire. + +## 2. Best practices (from research) + +1. SLSA provenance is verifiable information about where, when and how an artifact was produced, defined as an in-toto attestation predicate with predicateType `https://slsa.dev/provenance/v1`, where the build platform attests it produced artifacts by executing a buildDefinition (Source: https://slsa.dev/spec/v1.0/provenance, retrieved 2026-10-23; triangulated by https://slsa.dev/spec/v1.0/about, retrieved 2026-10-23). +2. The in-toto attestation framework is the envelope: a Statement binds subjects (artifact digests) to a predicateType plus predicate, and SLSA provenance is one predicate type inside it (Source: https://github.com/in-toto/attestation, retrieved 2026-10-23; triangulated by SLSA provenance spec above). +3. SPDX (Linux Foundation, ISO/IEC 5962:2021 for SPDX 2.2.1) and CycloneDX (OWASP, ECMA-424) are the two interoperable machine-readable SBOM formats satisfying the NTIA minimum elements: supplier name, component name, version, other unique identifiers, dependency relationship, SBOM author, timestamp (Source: https://spdx.dev/learn/overview/, retrieved 2026-10-23; Source: https://cyclonedx.org/specification/overview/, retrieved 2026-10-23; NTIA element list triangulated by https://github.com/spdx/ntia-conformance-checker, retrieved 2026-10-23). MoloApp emits SPDX JSON as canonical and CycloneDX JSON as secondary. +4. Sigstore cosign keyless blob signing (`cosign sign-blob` without `--key`) obtains a short-lived certificate from Fulcio via OIDC identity and records the signature in the Rekor transparency log, producing a `.sigstore.json` bundle verified against the expected signer identity (`--certificate-identity`) and OIDC issuer (Source: https://docs.sigstore.dev/cosign/signing/signing_with_blobs/, retrieved 2026-10-23; triangulated by https://docs.sigstore.dev/about/overview/, retrieved 2026-10-23). +5. NIST SP 800-218 v1.1 (SSDF, Feb 2022) defines outcome-based secure development practices in four groups: Prepare the Organization (PO), Protect the Software (PS), Produce Well-Secured Software (PW), Respond to Vulnerabilities (RV); it mandates outcomes, not tools (Source: https://csrc.nist.gov/pubs/sp/800/218/final, retrieved 2026-10-23). +6. Triangulated release rule from (1)+(4)+(5): a trustworthy release needs all three of SBOM (PW.4.1 lineage), signed provenance from the build platform (PS.3, SLSA Build L2+), and a vulnerability response hook (RV) that blocks or discloses known CVEs before ship. Missing any one fails the gate. + +## 3. Structure + +Directories (one class or concern per file, deep over flat): + +``` +tools/supply/ release-time only, never linked into the framework + SbomGenerator.hpp/.cpp molo::supply::sbom::Generator (inventory assembly) + SpdxEmitter.hpp/.cpp molo::supply::sbom::SpdxEmitter (SPDX 2.3 JSON) + CycloneDxEmitter.hpp/.cpp molo::supply::sbom::CycloneDxEmitter (CycloneDX 1.6 JSON) + SbomCheck.hpp/.cpp molo::supply::sbom::ConformanceCheck (NTIA elements) +include/molo/supply/ +src/supply/ + policy/DependencyPolicy.hpp molo::supply::policy::DependencyPolicy (allow-list) + policy/WolfPin.hpp/.cpp molo::supply::policy::WolfPin (version, digest, license receipt) + attest/ProvenanceBuilder.hpp molo::supply::attest::ProvenanceBuilder (SLSA v1 predicate) + attest/Attestor.hpp/.cpp molo::supply::attest::Attestor (in-toto Statement envelope) + attest/Signer.hpp/.cpp molo::supply::attest::Signer (cosign wrapper, keyless first) + attest/Verify.hpp/.cpp molo::supply::attest::Verifier (bundle + digest verify) + scan/VulnHook.hpp/.cpp molo::supply::scan::VulnHook (SBOM in, verdict out) + scan/OsdvClient.hpp/.cpp molo::supply::scan::OsvClient (OSV API query, capped) +sbom/ generated per release, committed as tag asset only +specs/supply.md normative spec mirror (owned by 10-workflow-example.md template) +``` + +Namespaces: `molo::supply::sbom`, `::policy`, `::attest`, `::scan`. No other namespace signs, attests or scans. Signing keys never enter `src/`; only digests and bundle paths cross into `tools/release/` from 10-workflow-example.md. + +## 4. Rules (hard, numbered) + +R15-1: wolfSSL is the sole third-party dependency. Any second dependency blocks the build until this spec is amended and the SBOM schema covers it. +R15-2: Every tagged release ships four artifacts minimum: binary or tarball, `sbom.spdx.json`, `sbom.cdx.json`, `provenance.intoto.jsonl` plus one `.sigstore.json` bundle per signed file. Missing file blocks the tag. +R15-3: SPDX JSON is canonical and must pass NTIA minimum-element conformance; CycloneDX JSON must describe the same component set (same name/version/digest triples). +R15-4: Provenance predicateType is exactly `https://slsa.dev/provenance/v1` inside an in-toto Statement; subject digests are SHA-256 over released bytes. +R15-5: Signing is Sigstore keyless first (OIDC identity of the release workflow); long-lived keys are forbidden unless a human-approved exception with expiry exists. +R15-6: The vuln-scan hook runs on the final SBOM before signing; a CRITICAL with a fix available blocks release, others attach as disclosed advisories. +R15-7: wolfSSL is pinned by exact version plus SHA-256 of the vendored tarball plus license receipt path; any mismatch fails the build loudly. +R15-8: Reproducibility: same git commit plus same toolchain record plus same wolfSSL tarball yields byte-identical SBOM component section (timestamps only in metadata, never in digests). +R15-9: No network fetch during normal build. The only network allowed is the release pipeline calling OSV, Fulcio and Rekor, each behind timeout, retry cap and offline fail-closed behavior. +R15-10: Verification is public and offline-capable: `molo verify` checks digests, bundle chain and provenance subject match using only release assets plus Sigstore roots. +R15-11: All caps in section 6 are static constants. No silent growth of SBOM size, scan window or signature count. +R15-12: This file never duplicates sibling scope: no TLS config (03-security.md), no git/tag mechanics (10-workflow-example.md), no CLI UX (09-cli-ops.md), no audit log schema (12-audit-compliance.md). It provides SBOM, policy verdicts, provenance and bundles those siblings consume. + +## 5. Development guidelines + +- Build order follows section 7: policy pin first, then SBOM, then scan hook, then provenance, then signing, then verify. Each step lands with its spec delta and template output checked in as golden files. +- Generators are pure functions over (git commit, toolchain file, wolfSSL pin, file manifest). No timestamps inside hashed content; timestamps live only in SBOM metadata and provenance `buildMetadata.finishedOn`. +- Treat the SBOM as code: diff it in review, fail CI on unexpected component delta, keep golden SBOMs for the last three releases in `sbom/golden/`. +- Wrap external CLIs (cosign) behind `Signer` with a fake for tests; never shell out from framework code, only from `tools/supply/`. +- Log every policy decision with reason code (allow, pin-mismatch, vuln-block, exception-expired) through 02-observability.md; denials are loud, never silent skips. +- License hygiene: wolfSSL commercial license receipt path is recorded in the SBOM supplier field and provenance materials; receipt missing blocks release. + +## 6. Specifications (concrete behaviors, caps, formats) + +Dependency policy: + +- Allow-list file `supply.allow.json`: exactly one entry `{name: wolfssl, version: X.Y.Z, sha256: hex, source: vendored-tarball-url-or-path, license: commercial, receipt: path}`. Unknown entry or version drift fails `make supply-check`. +- Toolchain record `supply.toolchain.json`: compiler id plus version, linker, CMake or Make version, build flags hash. Max 100 entries, max 64 KB file. + +SBOM: + +- Formats: SPDX 2.3 JSON (`sbom.spdx.json`) canonical, CycloneDX 1.6 JSON (`sbom.cdx.json`) secondary. Both UTF-8, LF, sorted keys, 2-space indent. +- Components: `moloapp` (supplier MoloApp, version from VERSION, id git commit), `wolfssl` (supplier wolfSSL Inc, pinned version, externalRefs: url plus purl `pkg:generic/wolfssl@X.Y.Z`), `toolchain` (supplier recorded, version string). Each carries SHA-256 checksum, license declared plus concluded, downloadLocation or NOASSERTION with justification. +- NTIA fields mandatory on every component: supplier, name, version, unique id (SPDXID plus purl/CPE where known), relationship (DEPENDS_ON from moloapp), author (`MoloApp release pipeline`), timestamp (RFC 3339 UTC). +- Caps: max 64 components, max 256 relationships, SBOM JSON max 1 MB per format, component name max 256 chars, version max 64 chars. + +Vuln-scan hook: + +- Input: canonical SPDX JSON. Query OSV API `https://api.osv.dev/v1/query` per component (purl plus version), timeout 10 s per request, max 64 queries per release, 1 retry, offline yields `unknown` verdict which blocks signing until a human override with 7-day expiry. +- Verdicts: `clean`, `disclosed` (non-critical or no fix, attached as `advisories.json`), `blocked` (critical CVSS >= 9.0 with fix, or wolfSSL CVE with fix regardless of score). Blocked stops the pipeline before provenance generation. + +Provenance and signing: + +- Provenance: in-toto Statement v1, `predicateType: https://slsa.dev/provenance/v1`, subjects are `{name, digest: {sha256: hex}}` per released file, `buildDefinition.buildType` is `https://moloapp.dev/build/v1`, `externalParameters` holds git commit, toolchain hash, wolfSSL pin digest; `runDetails.builder.id` is the release workflow identity URI; `runDetails.metadata.invocationId` is the pipeline run id. +- Signing: `cosign sign-blob --yes --bundle .sigstore.json ` per artifact; verify with `cosign verify-blob --bundle --certificate-identity --certificate-oidc-issuer https://token.actions.githubusercontent.com `. SHA-256SUMS file signed as one blob additionally. +- Layout per tag `vX.Y.Z`: `molo-X.Y.Z.tar.gz`, `SHA256SUMS`, `sbom.spdx.json`, `sbom.cdx.json`, `advisories.json`, `provenance.intoto.jsonl`, plus `.sigstore.json` per file. Max 32 signed files per release. +- `molo verify `: recompute SHA-256, check bundle chain, check provenance subjects match digests, check SBOM component digests match wolfSSL pin; exit 0 verified, 1 mismatch (first mismatch quoted), 2 usage error. No network required. + +## 7. Execution steps (in build order) + +E15-1: DependencyPolicy plus WolfPin: allow-list schema, pin check in `make supply-check`, golden fixtures (good pin, drifted version, bad digest). +E15-2: SbomGenerator plus SpdxEmitter: inventory assembly, canonical JSON, sorted output, NTIA conformance self-check. +E15-3: CycloneDxEmitter plus equivalence test: same triples as SPDX, golden pair diff in CI. +E15-4: VulnHook plus OsvClient: query, verdict mapping, advisories.json writer, offline fail-closed path. +E15-5: ProvenanceBuilder plus Attestor: SLSA v1 predicate, in-toto Statement envelope, subject digest binding. +E15-6: Signer plus Verifier: cosign keyless wrapper, bundle layout, offline `molo verify` command core. +E15-7: Pipeline wiring into 10-workflow-example.md release order (suite green, SBOM, scan, provenance, sign, tag) and 09-cli-ops.md `molo release` surface. +E15-8: Hardening: golden SBOMs for three releases, tamper drills (bit-flip, pin drift, expired exception), docs and living-example supply page. + +## 8. Validation (how to verify each step) + +V15-1: Pin fixtures: correct tarball passes, one-byte change fails with digest quoted, version bump without allow-list edit fails, missing license receipt fails. +V15-2: SPDX golden test: regenerate twice, byte-identical except timestamp; NTIA checker passes; component count is exactly 3 (moloapp, wolfssl, toolchain) on a clean tree. +V15-3: Equivalence test: extract (name, version, sha256) triples from both SBOMs, sets equal; CycloneDX validates against its JSON schema. +V15-4: Scan matrix: clean SBOM yields clean; fixture with known-fixed critical yields blocked; fixture with no-fix advisory yields disclosed plus advisories.json entry; network cut yields unknown and signing refused. +V15-5: Provenance test: subjects match `sha256sum` of release files; predicateType exact; builder id equals workflow identity; tampered artifact fails subject match. +V15-6: Sign and verify round-trip against local Sigstore stub plus one manual live run: valid bundle verifies, wrong identity fails, bit-flipped file fails, offline verify works with cached roots. +V15-7: End-to-end dry run on fixture repo: full order executes, a blocked verdict stops before signing, a green run produces all files in the layout with caps respected. +V15-8: Drill suite: expired exception blocks, oversized SBOM (>1 MB) rejected, 33rd signed file rejected, living-example supply page shows SBOM plus verify instructions on a small-phone viewport. + +## 9. References (sibling files this depends on or feeds) + +- Depends on 03-security.md: wolfSSL version and TLS posture, hash and RNG primitives for digests, secret redaction in SBOM and provenance. +- Depends on 09-cli-ops.md: `molo release`, `molo verify`, `molo version` UX, embedder manifest hash reuse, exit-code mapping. +- Depends on 10-workflow-example.md: release pipeline order, tag gating, VERSION and CHANGELOG linkage, spec and test templates. +- Depends on 02-observability.md: structured logs and metrics for policy verdicts and scan outcomes. +- Feeds 12-audit-compliance.md: SBOM, provenance, bundles and advisories as audit evidence; retention pointers. +- Feeds 14-api-governance.md: SBOM and provenance as governed release contracts (formats versioned, breaking changes major). +- Feeds 11-tenancy.md and 13-operability.md: per-tenant build identity inputs and SBOM exchange notes (formats defined here, transport there). +- Never duplicates: 10-workflow-example.md owns git mechanics, 09-cli-ops.md owns CLI flags, 12-audit-compliance.md owns audit retention. + +## 10. Open questions + +Q1: SLSA Build L3 (hardened builder, non-falsifiable provenance) in v1 or L2 first with a documented L3 path? Proposal: L2 now, L3 tracked as gap. +Q2: OSV only, or OSV plus a second feed (wolfSSL security list, CISA KEV) for the vuln hook? Proposal: OSV plus wolfSSL advisory check, KEV later. +Q3: Rekor transparency proof required at verify time, or bundle signature plus identity sufficient for v1? Proposal: bundle sufficient, checkpoint proof optional flag. +Q4: SPDX 3.0 migration timeline? Proposal: stay on SPDX 2.3 JSON until tooling converges, track 3.0 as gap. + +## 11. Sources (exact URLs with retrieval dates) + +- SLSA Provenance specification v1.0 (predicateType, model, purpose): https://slsa.dev/spec/v1.0/provenance (retrieved 2026-10-23) +- SLSA About page (SSDF link, supply-chain rationale): https://slsa.dev/spec/v1.0/about (retrieved 2026-10-23) +- in-toto Attestation Framework repository (Statement, predicate model): https://github.com/in-toto/attestation (retrieved 2026-10-23) +- SPDX Project overview (formats, ISO standard): https://spdx.dev/learn/overview/ (retrieved 2026-10-23) +- CycloneDX Specification Overview (object model, ECMA-424): https://cyclonedx.org/specification/overview/ (retrieved 2026-10-23) +- Sigstore cosign Signing Blobs (keyless sign-blob, bundles): https://docs.sigstore.dev/cosign/signing/signing_with_blobs/ (retrieved 2026-10-23) +- NIST SP 800-218 SSDF v1.1 project page (PO, PS, PW, RV practice groups): https://csrc.nist.gov/pubs/sp/800/218/final (retrieved 2026-10-23) diff --git a/plans/README.md b/plans/README.md index 6a00084..488fe6b 100644 --- a/plans/README.md +++ b/plans/README.md @@ -1,6 +1,6 @@ # MoloApp Detailed Plans -Index for the ten detailed plan files. Each file is self-contained with purpose, best practices, structure, rules, guidelines, specifications, execution steps, validation, references, open questions, and sources. +Index for the fifteen detailed plan files. Each file is self-contained with purpose, best practices, structure, rules, guidelines, specifications, execution steps, validation, references, open questions, and sources. ## Index @@ -16,10 +16,15 @@ Index for the ten detailed plan files. Each file is self-contained with purpose, | 8 | 08-clients-openapi.md | Clients, OpenAPI, Test Client | HTTP/REST client, REST standards, OpenAPI generation, test client. | | 9 | 09-cli-ops.md | CLI Tooling, Dev Server, Operations | CLI, dev server, embedder, AI harness, health/metrics, i18n/CSP. | | 10 | 10-workflow-example.md | Workflow, Testing Discipline, Living Example | Specs, AGENTS.md, autonomous git, integration testing, living example. | +| 11 | 11-tenancy.md | Multi-Tenancy | First-class tenants: isolation strategies, quotas, flags, per-tenant metrics. | +| 12 | 12-audit-compliance.md | Audit and Compliance | Immutable audit log, GDPR export/erasure, retention, PII marking, masking. | +| 13 | 13-operability.md | Operability at Scale | Safe deploys, drains, locks/election, breakers, deadlines, SLO, chaos, debug. | +| 14 | 14-api-governance.md | API Governance | API keys, versioning/deprecation, OpenAPI diff gate, rate tiers, surface rules. | +| 15 | 15-supply-chain.md | Supply Chain and Trust | SBOM, signed provenance, dependency policy, vuln-scan hook, release attestation. | ## How files reference each other -- References always use exact sibling filenames (`01-runtime.md` through `10-workflow-example.md`). +- References always use exact sibling filenames (`01-runtime.md` through `15-supply-chain.md`). - Each file has a References section naming Depends on (interfaces it consumes) and Feeds (interfaces it provides). - Dependency direction (consumes, upstream first): - `01-runtime.md` depends on `02-observability.md` (hooks) and `03-security.md` (TLS handle); feeds `05-request-routing.md`, `06-realtime.md`, `08-clients-openapi.md`, `09-cli-ops.md`. @@ -32,6 +37,11 @@ Index for the ten detailed plan files. Each file is self-contained with purpose, - `08-clients-openapi.md` depends on `01-runtime.md`, `02-observability.md`, `03-security.md`, `04-data.md`, `05-request-routing.md`; defers realtime clients to `06-realtime.md`; feeds `09-cli-ops.md`, `10-workflow-example.md`. - `09-cli-ops.md` depends on `01-runtime.md` through `08-clients-openapi.md`; feeds `10-workflow-example.md`. - `10-workflow-example.md` depends on `02-observability.md`, `08-clients-openapi.md`, `09-cli-ops.md`; feeds `01-runtime.md` through `09-cli-ops.md` (spec template, test discipline, release gates). + - `11-tenancy.md` depends on `01-runtime.md` through `09-cli-ops.md` (joint hardening milestone with `04-data.md` before either file freezes); follows `10-workflow-example.md`; feeds `12-audit-compliance.md`, `14-api-governance.md`. + - `12-audit-compliance.md` depends on `02-observability.md`, `03-security.md`, `04-data.md`, `05-request-routing.md`, `07-supervisor-replication.md`, `11-tenancy.md`; feeds `09-cli-ops.md`, `10-workflow-example.md`; defers exchange formats to `13-operability.md`, API policy to `14-api-governance.md`, attestation to `15-supply-chain.md`. + - `13-operability.md` depends on `01-runtime.md`, `02-observability.md`, `04-data.md`, `05-request-routing.md`, `06-realtime.md`, `07-supervisor-replication.md`, `08-clients-openapi.md`, `09-cli-ops.md`; feeds `09-cli-ops.md`, `10-workflow-example.md`, `12-audit-compliance.md`. + - `14-api-governance.md` depends on `01-runtime.md`, `02-observability.md`, `03-security.md`, `04-data.md`, `05-request-routing.md`, `08-clients-openapi.md`, `10-workflow-example.md`, `11-tenancy.md`; feeds `09-cli-ops.md`, `12-audit-compliance.md`, `13-operability.md`; parallel with `07-supervisor-replication.md`, `15-supply-chain.md`. + - `15-supply-chain.md` depends on `02-observability.md`, `03-security.md`, `09-cli-ops.md`, `10-workflow-example.md`; feeds `11-tenancy.md`, `12-audit-compliance.md`, `13-operability.md`, `14-api-governance.md`. ## Overall execution order @@ -44,5 +54,8 @@ Build order follows plan.md section 10, with the file sequence matching dependen 5. `05-request-routing.md` steps E1-E12 plus `08-clients-openapi.md` steps 1-8 plus `09-cli-ops.md` steps 1-10 (plan step 5: app layer, clients, OpenAPI, CLI/ops). 6. `07-supervisor-replication.md` steps E7-5 through E7-8 (plan step 6: durable streams, replication, multi-instance). 7. `10-workflow-example.md` steps 1-8 throughout and finally (plan step 7: specs, AGENTS.md, pipeline, living example, autonomous end-to-end run). +8. `11-tenancy.md` steps E11-1 through E11-4 jointly with `04-data.md` (tenant context in the data Context before the data layer hardens), then E11-5 through E11-8 with the app layer (plan step 8: multi-tenancy). +9. `12-audit-compliance.md` steps 1-10 plus `13-operability.md` steps 1-8 (plan step 9: audit/compliance plus operability at scale). +10. `14-api-governance.md` steps 1-8 plus `15-supply-chain.md` steps E15-1 through E15-8 (plan step 10: API governance plus supply chain and trust). Rule: no file is done until its validation section passes through the real stack and `10-workflow-example.md` rule 1 holds (spec updated, docs updated, living example demonstrates it, full suite green, same session). diff --git a/src/app/Makefile b/src/app/Makefile new file mode 100644 index 0000000..52fdbbb --- /dev/null +++ b/src/app/Makefile @@ -0,0 +1,12 @@ +# Standalone entry for the request/routing part. Root Makefile includes +# app.mk instead; this wrapper only fixes ROOT for isolated builds. +ROOT := $(abspath ../..) +include app.mk + +.PHONY: all test clean + +all: app-build + +test: app-test + +clean: app-clean diff --git a/src/app/SPEC.md b/src/app/SPEC.md new file mode 100644 index 0000000..6fbd8ec --- /dev/null +++ b/src/app/SPEC.md @@ -0,0 +1,96 @@ +# Spec: request/routing part (plan 05) + +Implementation of `plans/05-request-routing.md`, execution steps E1 through +E12, rules R1 through R12. All code lives under `src/app/` in namespaces +`molo::app::*` (namespaces mirror directories). Tests live under +`tests/src/app/` mirroring URL paths. Build outputs go under `build/` only. +C++23, standard library plus OS syscalls only. No TLS code in this part, so +no wolfSSL use and no stub needed here. + +## Module map + +| Directory | Contents | +|---|---| +| `async/` | `Task`: minimal eager coroutine task (handler return type) | +| `seams/` | Sibling-owned seams: request IDs, session store (in-memory default) | +| `http_msg/` | Method, Headers, Query, Cookies, Request, Response hierarchy, HTML/XML escape | +| `routing/` | Path compiler, Route, Router, match, url_for | +| `app/` | Application, Mount, PluginRegistry, PluginContext | +| `validate/` | Value, JSON codec, Field, Model, ValidationError | +| `forms/` | Form binding: JSON, urlencoded, multipart | +| `errors/` | HttpError hierarchy, ErrorHandler with JSON/HTML renderers | +| `negotiate/` | Accept parser, Negotiator, variant rendering | +| `views/` | View base, handler adapters, Flash helpers | +| `middleware/` | Middleware base, Next guard, Chain, built-ins | +| `di/` | Container with app and request scopes, cycle detection | +| `templates/` | Lexer, Parser, AST, Renderer, Environment, Loaders, Cache, Filters, Tests, TemplateResponse | + +Caps: `caps.h` holds every static ceiling (R11). No autotuning anywhere. + +## Key behaviors (plan section 6) + +- Matching order: static segments first, converters in registration order, + path catch-alls last. Precedence: full match, 405 with `Allow`, 422 on + converter failure with `loc: ["path", name]`, else 404. +- Trailing slash is strict; `slash_insensitive` registers the declared form + plus a companion 308 redirect route (query string preserved). +- Mounts merge paths (`/app1/app1/app1` for triple nesting) and prefix + route names with the mount alias (`app1:app1:deep`) so R10 uniqueness + holds after merge. Mounted apps keep their own DI container and run + their own middleware chain inside the parent chain. +- One 422 aggregates failures across query, headers, cookies and body + (R3). Shape: `{"detail":[{"loc":[...],"msg":"...","type":"..."}]}` + (lowercase `detail`). +- 406 shape: `{"Detail":{"available":[...]}}` (capital `D`, per plan). +- Other errors: `{"error":{"code":...,"message":...,"request_id":"..."}}`, + plus a responsive HTML page when the client prefers HTML. +- `?format=` override applies only when the route sets + `allow_format_override` (default off, Q2). +- Redirects are 307 temporary / 308 permanent (method-preserving). +- Flash messages ride the session store and render exactly once. +- Middleware categories are enforced at registration; `Next` twice is a + loud error. ExceptionMapping is outermost by construction; + Negotiation wraps the terminal dispatch so it stays innermost. +- DI has app and request scopes; request scope is created per `handle()` + call and cleared before returning. Declared dependency cycles fail + startup with the cycle quoted; resolution carries a dynamic guard too. +- Templates: `{{ }}`, `{% %}`, `{# #}` with extends/blocks, include, + if/elif/else, for/else with `loop`, set, macros with defaults and + kwargs, filters, tests, autoescape blocks. Autoescape is on for + `.html`/`.htm`/`.xml` with no global disable (R5). Undefined policy + defaults to Strict and Chain renders empty plus a stats counter (Q3). + Documented Jinja gaps: no ternary, no `**`, no string `%` formatting, + no `super()`, no block-level set, macro defaults are literal-only, + case conversion is ASCII-only, Accept range params are ignored. + +## Sibling seams (R12) + +Consumed through seams with standalone defaults, owned elsewhere: + +- 01-runtime: byte caps, transport, production task executor. Local + `async::Task` is the eager stand-in with the same `co_await` shape. +- 02-observability: correlation IDs (`seams::request_id`), log sink hook + on `ErrorHandler`, per-route metrics (not wired here). +- 03-security: session store (`Application::set_session_store`), upload + quotas and scanning, auth guards consumed via DI and middleware. +- 04-data: ORM/KV/cache handles provided through DI by the host app. + +Provided to siblings: `Request`/`Response` shells, router mounting, +middleware and DI participation points, validation models for OpenAPI, +template loaders (embedded assets plug into `EmbeddedLoader`), request +scope teardown at end of `handle()` for background handoff. + +## Static assertions and diagnostics + +- R4 is enforced by explicit constructors plus `static_assert` in + `tests/src/app/00_compile_all_test.cpp`. +- Route names unique after merge (quoted paths), unknown converters name + the route, DI cycles quote the cycle, middleware reorder names layers, + plugin capability denial names plugin and capability. + +## ID counters note (R8) + +`generate_request_id` and `generate_session_id` use process-wide atomic +counters. This is ID uniqueness, not service state: no service singleton +or mutable file-static state is reachable from handlers. All services +flow through `Application` and request extensions. diff --git a/src/app/app.mk b/src/app/app.mk new file mode 100644 index 0000000..f3ee34a --- /dev/null +++ b/src/app/app.mk @@ -0,0 +1,29 @@ +# Per-part makefile for the request/routing part (plan 05). Includable by +# the root Makefile: set ROOT to the repo root, then `include src/app/app.mk`. +# Provides: app-build, app-test, app-clean. All outputs go under build/. +ROOT ?= . +APP_CXX ?= g++ +APP_CXXFLAGS ?= -std=c++23 -Wall -Wextra -Isrc +APP_BUILD ?= $(ROOT)/build/app +APP_TEST_SRCS := $(shell find $(ROOT)/tests/src/app -name '*_test.cpp' | sort) +APP_HEADERS := $(shell find $(ROOT)/src/app -name '*.h' | sort) +APP_TEST_BINS := $(patsubst $(ROOT)/tests/src/app/%.cpp,$(APP_BUILD)/%,$(APP_TEST_SRCS)) + +$(APP_BUILD)/%: $(ROOT)/tests/src/app/%.cpp $(APP_HEADERS) + @mkdir -p $(dir $@) + $(APP_CXX) $(APP_CXXFLAGS) -I$(ROOT)/src -I$(ROOT)/tests/src/app $< -o $@ + +.PHONY: app-build app-test app-clean + +app-build: $(APP_TEST_BINS) + +app-test: $(APP_TEST_BINS) + @pass=0; fail=0; \ + for t in $(APP_TEST_BINS); do \ + if (cd $(ROOT) && "$$t"); then pass=$$((pass+1)); else fail=$$((fail+1)); fi; \ + done; \ + echo "app tests: $$pass passed, $$fail failed"; \ + test $$fail -eq 0 + +app-clean: + rm -rf $(APP_BUILD) $(ROOT)/build/tmp_templates diff --git a/src/app/app/application.h b/src/app/app/application.h new file mode 100644 index 0000000..9ea35a2 --- /dev/null +++ b/src/app/app/application.h @@ -0,0 +1,528 @@ +#pragma once +// Spec: molo::app::app::Application. Composition root (plan 05, E3): owns +// the router, middleware chain, DI container, template environment, error +// handler, session store seam and plugin registry. include_router merges a +// router under a prefix; mount nests a sub-application with automatic path +// merge (triple nesting yields /app1/app1/app1). startup() validates names +// (unique after merge; mounted names gain the mount alias prefix), +// converters (fail fast at add), DI cycles and caps, then runs lifespan +// hooks and plugin hooks. handle() matches the route before the chain runs +// so all middleware sees route info, forks a request DI scope, runs parent +// chain outer plus mount chains inner, validates bindings, and invokes the +// handler. 404/405/422 precedence follows Router::match. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../governance/problem.h" +#include "../../governance/versions/ApiVersion.h" +#include "../async/task.h" +#include "../caps.h" +#include "../di/container.h" +#include "../errors/handler.h" +#include "../errors/http_error.h" +#include "../forms/form.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" +#include "../middleware/builtins.h" +#include "../middleware/chain.h" +#include "../routing/router.h" +#include "../seams/request_id.h" +#include "../seams/session.h" +#include "../templates/environment.h" +#include "../validate/errors.h" +#include "../validate/model.h" +#include "../validate/value.h" +#include "../views/flash.h" +#include "plugin.h" + +namespace molo::app::app { + +class StartupError : public std::runtime_error { + public: + explicit StartupError(const std::string& msg) : std::runtime_error(msg) {} +}; + +class Application; +struct Mount { + std::string prefix; + std::string name_prefix; + std::shared_ptr sub; +}; + +struct Resolved { + Application* owner = nullptr; + const routing::Route* route = nullptr; + std::map params; + std::vector allow; + std::vector path_errors; + std::vector levels; +}; + +class Application { + public: + Application() + : error_handler_(std::make_shared()), + sessions_(seams::default_session_store()) { + // ExceptionMapping is outermost by construction. Negotiation runs around + // the terminal dispatch (see run_mount_levels), not in the chain, so it + // stays innermost no matter when user layers register. + chain_.use(std::make_shared(error_handler_)); + error_handler_->set_render_hook([this](int status, const std::string& code) { + last_error_code_ = code; + plugins_.fire_error_render(this, status, code); + }); + } + + routing::Router& router() { return router_; } + middleware::Chain& chain() { return chain_; } + di::Container& di() { return di_; } + templates::Environment& templates_env() { return templates_; } + errors::ErrorHandler& error_handler() { return *error_handler_; } + PluginRegistry& plugins() { return plugins_; } + + std::shared_ptr sessions() { return sessions_; } + void set_session_store(std::shared_ptr store) { + sessions_ = std::move(store); + } + const std::string& last_error_code() const { return last_error_code_; } + + void include_router(const std::string& prefix, const routing::Router& router, + const std::vector& tags = {}) { + if (started_) throw StartupError("cannot include_router after startup"); + router_.include_router(prefix, router, tags); + } + + void mount(std::string prefix, std::shared_ptr sub, std::string alias = "") { + if (started_) throw StartupError("cannot mount after startup"); + if (prefix.empty() || prefix[0] != '/') { + throw StartupError("mount prefix must start with /"); + } + while (prefix.size() > 1 && prefix.back() == '/') prefix.pop_back(); + if (sub->depth() + 1 > caps::kMaxSubAppNestingDepth) { + throw StartupError("sub-app nesting exceeds depth cap at '" + prefix + "'"); + } + Mount m; + m.prefix = prefix; + if (!alias.empty()) { + m.name_prefix = alias; + } else if (prefix == "/") { + m.name_prefix = "root"; + } else { + m.name_prefix = prefix.substr(1); + for (char& c : m.name_prefix) { + if (c == '/') c = '_'; + } + } + m.sub = std::move(sub); + mounts_.push_back(std::move(m)); + } + + void on_startup(std::function hook) { + startup_hooks_.push_back(std::move(hook)); + } + void on_shutdown(std::function hook) { + shutdown_hooks_.push_back(std::move(hook)); + } + + void startup() { + if (started_) return; + for (const Mount& m : mounts_) m.sub->startup(); + if (total_routes() > caps::kMaxRoutesPerApp) { + throw StartupError("too many routes in application"); + } + std::vector> names; + collect_names("", names); + std::map> by_name; + for (const auto& kv : names) by_name[kv.first].push_back(kv.second); + for (const auto& kv : by_name) { + if (kv.second.size() > 1) { + std::string msg = "duplicate route name '" + kv.first + "' at paths:"; + for (const auto& p : kv.second) msg += " '" + p + "'"; + throw StartupError(msg); + } + } + di_.validate_no_cycles(); + try { + router_.validate_governance(); + } catch (const std::runtime_error& e) { + throw StartupError(e.what()); + } + gov_sunsets_ = CollectGovSunsets(router_); + for (const routing::Route& route : router_.routes()) { + if (!route.is_slash_redirect() && !route.options().name.empty()) { + plugins_.fire_route_added(this, route); + } + } + for (auto& hook : startup_hooks_) hook(*this); + plugins_.fire_app_started(this); + router_.freeze(); + started_ = true; + } + + void shutdown() { + if (!started_) return; + for (auto& hook : shutdown_hooks_) hook(*this); + for (const Mount& m : mounts_) m.sub->shutdown(); + started_ = false; + } + + bool started() const { return started_; } + + async::Task handle(http_msg::Request& req) { + if (!started_) startup(); + // Governance: a sunset-version match answers 410 instead of 404. + if (IsSunsetVersion(req.path())) { + const auto problem = ::molo::govern::Problem::VersionGone(); + http_msg::Response res(410); + res.set_content_type(std::string(::molo::govern::Problem::kContentType)); + res.set_body(problem.to_json_string()); + co_return res; + } + if (req.request_id().empty()) { + std::string rid = req.headers().get(seams::kRequestIdHeader); + req.set_request_id(rid.empty() ? seams::generate_request_id() : rid); + } + // Resolve runs before the chain so every middleware sees route info. + // Resolve-time failures (414 on overlong paths) render here so they + // still get the exact error shape, request ID echo and plugin hooks. + Resolved rr; + try { + rr = resolve_with_method(req.method(), req.path()); + } catch (const errors::HttpError& e) { + req.set_app(this); + http_msg::Response early = error_handler_->render(e, req, req.request_id()); + early.headers().set(seams::kRequestIdHeader, req.request_id()); + co_return early; + } catch (const std::exception& e) { + req.set_app(this); + http_msg::Response early = error_handler_->render_unknown(req, req.request_id(), e.what()); + early.headers().set(seams::kRequestIdHeader, req.request_id()); + co_return early; + } + if (rr.route != nullptr) { + req.set_route(rr.route); + for (const auto& kv : rr.params) req.set_path_param(kv.first, kv.second); + req.set_app(rr.owner); + } else { + req.set_app(this); + } + // Request DI scope: lives for this call, cleared before returning so + // the scope ends at response completion (R8). Background handoff must + // copy values out; the scope pointer is not valid after handle(). + auto scope = std::make_shared( + rr.owner != nullptr ? &rr.owner->di_ : &di_); + req.set_ext("di", scope); + std::function(http_msg::Request&)> terminal = + [this, rr](http_msg::Request& r) -> async::Task { + co_return co_await run_mount_levels(r, rr, 0); + }; + http_msg::Response res = co_await chain_.run(req, terminal); + req.clear_ext("di"); + co_return res; + } + + std::string url_for(const std::string& name, + const std::map& params = {}, + const std::string& query = "") const { + if (router_.has_route_name(name)) return router_.url_for(name, params, query); + for (const Mount& m : mounts_) { + std::string head = m.name_prefix + ":"; + if (name.size() > head.size() && name.substr(0, head.size()) == head) { + std::string rest = name.substr(head.size()); + std::string sub_url = m.sub->url_for(rest, params, query); + if (m.prefix == "/") return sub_url; + return m.prefix + sub_url; + } + } + throw errors::NotFound("url_for: unknown route '" + name + "'"); + } + + std::size_t depth() const { + std::size_t deep = 0; + for (const Mount& m : mounts_) { + std::size_t d = m.sub->depth(); + if (d > deep) deep = d; + } + return 1 + deep; + } + + std::size_t total_routes() const { + std::size_t n = router_.size(); + for (const Mount& m : mounts_) n += m.sub->total_routes(); + return n; + } + + void collect_names(const std::string& prefix, + std::vector>& out) const { + for (const routing::Route& route : router_.routes()) { + if (route.is_slash_redirect() || route.options().name.empty()) continue; + out.emplace_back(prefix + route.options().name, route.pattern()); + } + for (const Mount& m : mounts_) { + std::vector> sub; + m.sub->collect_names(m.name_prefix + ":", sub); + for (const auto& kv : sub) out.emplace_back(prefix + kv.first, m.prefix + kv.second); + } + } + + static di::RequestScope* di_scope(http_msg::Request& req) { + auto* holder = req.get_ext>("di"); + if (holder == nullptr || holder->get() == nullptr) { + throw errors::InternalError("no DI scope bound to request"); + } + return holder->get(); + } + + // Method-aware resolve used by handle(). Public for tests. + Resolved resolve_with_method(http_msg::Method method, const std::string& path) const { + Resolved out; + routing::MatchResult own = router_.match(method, path); + if (own.route != nullptr) { + out.owner = const_cast(this); + out.route = own.route; + out.params = std::move(own.params); + return out; + } + bool have_mount_405 = false; + bool have_mount_422 = false; + Resolved mount_best; + for (const Mount& m : mounts_) { + bool hit = (path == m.prefix) || (m.prefix == "/" && !path.empty()) || + (m.prefix != "/" && path.size() > m.prefix.size() && + path.substr(0, m.prefix.size()) == m.prefix && path[m.prefix.size()] == '/'); + if (!hit) continue; + std::string sub_path = + (m.prefix == "/") ? path : path.substr(m.prefix.size()); + if (sub_path.empty()) sub_path = "/"; + Resolved sub = m.sub->resolve_with_method(method, sub_path); + if (sub.route != nullptr) { + sub.levels.insert(sub.levels.begin(), &m); + return sub; + } + if (!have_mount_405 && !sub.allow.empty()) { + mount_best = sub; + have_mount_405 = true; + } else if (!have_mount_405 && !have_mount_422 && !sub.path_errors.empty()) { + mount_best = sub; + have_mount_422 = true; + } + } + if (!own.allow.empty()) { + out.allow = std::move(own.allow); + return out; + } + if (!own.path_errors.empty()) { + out.path_errors = std::move(own.path_errors); + return out; + } + return mount_best; + } + + private: + // Governance: earliest sunset per /v{n} major, collected at startup so + // handle() answers sunset versions in O(1). + static std::map CollectGovSunsets(const routing::Router& router) { + std::map out; + for (const routing::Route& r : router.routes()) { + if (r.is_slash_redirect()) continue; + const routing::RouteOptions& o = r.options(); + if (o.gov_sunset_httpdate.empty()) continue; + const int major = + ::molo::govern::versions::ApiVersion::ParseMajor(r.pattern()); + if (major == 0) continue; + const long sunset = + routing::detail::ParseImfFixdate(o.gov_sunset_httpdate); + if (sunset < 0) continue; + auto it = out.find(major); + if (it == out.end() || sunset < it->second) out[major] = sunset; + } + return out; + } + + bool IsSunsetVersion(const std::string& path) const { + if (gov_sunsets_.empty()) return false; + const int major = ::molo::govern::versions::ApiVersion::ParseMajor(path); + if (major == 0) return false; + auto it = gov_sunsets_.find(major); + if (it == gov_sunsets_.end()) return false; + return std::time(nullptr) >= it->second; + } + + // Two-step resolve keeps handle() readable; this section holds dispatch. + async::Task run_mount_levels(http_msg::Request& req, Resolved rr, + std::size_t level) const { + if (level >= rr.levels.size()) { + middleware::NegotiationMiddleware negotiation; + middleware::Next guard( + [this, rr](http_msg::Request& r) -> async::Task { + co_return co_await dispatch_final(r, rr); + }); + co_return co_await negotiation.handle(req, guard); + } + const Mount* m = rr.levels[level]; + std::function(http_msg::Request&)> inner = + [this, rr, level](http_msg::Request& r) -> async::Task { + co_return co_await run_mount_levels(r, rr, level + 1); + }; + co_return co_await m->sub->chain_.run(req, inner); + } + + async::Task dispatch_final(http_msg::Request& req, + const Resolved& rr) const { + if (rr.route == nullptr) { + if (!rr.allow.empty()) throw errors::MethodNotAllowed(rr.allow); + if (!rr.path_errors.empty()) { + throw errors::ValidationFailed(validate::ValidationError(rr.path_errors)); + } + throw errors::NotFound("no route for " + req.path()); + } + const routing::Route& route = *rr.route; + const routing::RouteOptions& opts = route.options(); + validate::Object path_obj; + for (const auto& kv : rr.params) path_obj[kv.first] = validate::Value(kv.second); + req.set_validated("path", std::move(path_obj)); + // R3: one 422 aggregates failures across every bound source. + std::vector all_failures; + auto run_model = [&](const std::shared_ptr& model, + const validate::Object& input, const std::string& source, + const std::string& slot) { + try { + req.set_validated(slot, model->validate(input, source)); + } catch (const validate::ValidationError& e) { + for (const auto& item : e.failures()) all_failures.push_back(item); + } + }; + if (opts.query_model) run_model(opts.query_model, query_object(req), "query", "query"); + if (opts.header_model) { + validate::Object input; + for (const auto& kv : req.headers().items()) { + if (!input.count(kv.first)) input[kv.first] = validate::Value(kv.second); + } + run_model(opts.header_model, input, "header", "headers"); + } + if (opts.cookie_model) { + validate::Object input; + for (const auto& kv : req.cookies().items()) input[kv.first] = validate::Value(kv.second); + run_model(opts.cookie_model, input, "cookie", "cookies"); + } + if (opts.body_model) { + validate::Object fields; + if (req.body().empty() && req.content_type().empty()) { + fields = validate::Object{}; + } else { + if (opts.max_body == 0 && !req.body().empty()) { + throw errors::PayloadTooLarge("route rejects bodies"); + } + forms::FormData data = forms::Form::parse_body(req, opts.max_body); + fields = std::move(data.fields); + req.set_ext("form_files", data.files); + } + run_model(opts.body_model, fields, "body", "body"); + } + if (!all_failures.empty()) { + throw errors::ValidationFailed(validate::ValidationError(std::move(all_failures))); + } + co_return co_await route.handler()(req); + } + + static validate::Object query_object(const http_msg::Request& req) { + validate::Object out; + std::map> grouped; + for (const auto& kv : req.query().items()) grouped[kv.first].push_back(kv.second); + for (const auto& kv : grouped) { + if (kv.second.size() == 1) { + out[kv.first] = validate::Value(kv.second[0]); + } else { + validate::Array arr; + for (const auto& v : kv.second) arr.emplace_back(v); + out[kv.first] = validate::Value(std::move(arr)); + } + } + return out; + } + + routing::Router router_; + middleware::Chain chain_; + di::Container di_; + templates::Environment templates_; + std::shared_ptr error_handler_; + std::shared_ptr sessions_; + PluginRegistry plugins_; + std::vector mounts_; + std::vector> startup_hooks_; + std::vector> shutdown_hooks_; + std::string last_error_code_; + bool started_ = false; + std::map gov_sunsets_; +}; + +// PluginContext definitions (need the complete Application type). + +inline PluginContext::PluginContext(Application* app, const std::set& capabilities, + std::string plugin_name) + : app_(app), capabilities_(capabilities), plugin_name_(std::move(plugin_name)) {} + +inline bool PluginContext::has(const std::string& capability) const { + return capabilities_.count(capability) > 0; +} + +inline void PluginContext::require(const std::string& capability) const { + if (!has(capability)) { + throw PluginAccessError("plugin '" + plugin_name_ + "' lacks capability '" + capability + + "'"); + } +} + +inline std::size_t PluginContext::route_count() const { + require("routes"); + return app_->total_routes(); +} + +inline bool PluginContext::di_has(const std::string& provider) const { + require("di"); + return app_->di().has(provider); +} + +inline std::string PluginContext::last_error_code() const { + require("errors"); + return app_->last_error_code(); +} + +} // namespace molo::app::app + +namespace molo::app::views { + +// Flash definitions (need the complete Application type). + +inline void Flash::flash(http_msg::Request& req, http_msg::Response& res, std::string level, + std::string text) { + app::Application* app = req.app(); + if (app == nullptr) throw errors::InternalError("flash needs a bound application"); + std::string sid = req.cookie(kSessionCookie); + if (sid.empty()) { + sid = generate_session_id(); + http_msg::SetCookie cookie; + cookie.name = kSessionCookie; + cookie.value = sid; + res.headers().add("set-cookie", cookie.render()); + } + app->sessions()->add_flash(sid, seams::FlashMessage{std::move(level), std::move(text)}); +} + +inline std::vector Flash::take(http_msg::Request& req) { + app::Application* app = req.app(); + if (app == nullptr) throw errors::InternalError("flash needs a bound application"); + std::string sid = req.cookie(kSessionCookie); + if (sid.empty()) return {}; + return app->sessions()->take_flashes(sid); +} + +} // namespace molo::app::views diff --git a/src/app/app/plugin.h b/src/app/app/plugin.h new file mode 100644 index 0000000..565339f --- /dev/null +++ b/src/app/app/plugin.h @@ -0,0 +1,109 @@ +#pragma once +// Spec: molo::app::app plugin registry (plan 05, E12). Named hooks: +// route_added, app_started, error_render. Plugins declare capabilities +// ("routes", "di", "errors"); PluginContext getters throw PluginAccessError +// unless the plugin declared the matching capability, so plugins cannot +// reach undeclared internals. Hook order per event is registration order. + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::app::routing { +class Route; +} + +namespace molo::app::app { + +class Application; + +class PluginAccessError : public std::runtime_error { + public: + explicit PluginAccessError(const std::string& msg) : std::runtime_error(msg) {} +}; + +class PluginContext { + public: + PluginContext(Application* app, const std::set& capabilities, + std::string plugin_name); + + const std::string& plugin_name() const { return plugin_name_; } + bool has(const std::string& capability) const; + + // Capability "routes". + std::size_t route_count() const; + // Capability "di". + bool di_has(const std::string& provider) const; + // Capability "errors". + std::string last_error_code() const; + + private: + void require(const std::string& capability) const; + + Application* app_; + std::set capabilities_; + std::string plugin_name_; +}; + +struct PluginHooks { + std::function route_added; + std::function app_started; + std::function error_render; +}; + +struct Plugin { + std::string name; + std::set capabilities; + PluginHooks hooks; +}; + +class PluginRegistry { + public: + void register_plugin(Plugin plugin) { + for (const auto& cap : plugin.capabilities) { + if (cap != "routes" && cap != "di" && cap != "errors") { + throw PluginAccessError("plugin '" + plugin.name + "' declares unknown capability '" + + cap + "'"); + } + } + plugins_.push_back(std::move(plugin)); + } + + const std::vector& plugins() const { return plugins_; } + + void fire_route_added(Application* app, const routing::Route& route) const { + for (const Plugin& p : plugins_) { + if (p.hooks.route_added) { + PluginContext ctx(app, p.capabilities, p.name); + p.hooks.route_added(ctx, route); + } + } + } + + void fire_app_started(Application* app) const { + for (const Plugin& p : plugins_) { + if (p.hooks.app_started) { + PluginContext ctx(app, p.capabilities, p.name); + p.hooks.app_started(ctx); + } + } + } + + void fire_error_render(Application* app, int status, const std::string& code) const { + for (const Plugin& p : plugins_) { + if (p.hooks.error_render) { + PluginContext ctx(app, p.capabilities, p.name); + p.hooks.error_render(ctx, status, code); + } + } + } + + private: + std::vector plugins_; +}; + +} // namespace molo::app::app diff --git a/src/app/async/task.h b/src/app/async/task.h new file mode 100644 index 0000000..1461c07 --- /dev/null +++ b/src/app/async/task.h @@ -0,0 +1,123 @@ +#pragma once +// Spec: molo::app::async::Task. Minimal eager coroutine task used as the +// handler return type (plan 05, R1 and section 6 Middleware). Seam note: the +// production executor is owned by 01-runtime; this local type keeps the app +// layer compilable standalone and has the same co_await shape for ready +// values. It runs eagerly and synchronously: every co_await inside app code +// awaits an already complete Task. Integration swaps this header for the +// runtime task without touching call sites. + +#include +#include +#include +#include + +namespace molo::app::async { + +template +class Task { + public: + struct promise_type { + std::optional value; + std::exception_ptr error; + + Task get_return_object() { + return Task{std::coroutine_handle::from_promise(*this)}; + } + std::suspend_never initial_suspend() noexcept { return {}; } + std::suspend_always final_suspend() noexcept { return {}; } + void return_value(T v) { value.emplace(std::move(v)); } + void unhandled_exception() { error = std::current_exception(); } + }; + + using Handle = std::coroutine_handle; + + Task() = default; + explicit Task(Handle h) : handle_(h) {} + Task(Task&& other) noexcept : handle_(other.handle_) { other.handle_ = nullptr; } + Task& operator=(Task&& other) noexcept { + if (this != &other) { + destroy(); + handle_ = other.handle_; + other.handle_ = nullptr; + } + return *this; + } + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + ~Task() { destroy(); } + + bool await_ready() const noexcept { return true; } + void await_suspend(std::coroutine_handle<>) const noexcept {} + T await_resume() { return value(); } + + T value() && { + if (!handle_) throw std::logic_error("molo::app::async::Task: empty task"); + if (handle_.promise().error) std::rethrow_exception(handle_.promise().error); + return std::move(*handle_.promise().value); + } + T value() & { + if (!handle_) throw std::logic_error("molo::app::async::Task: empty task"); + if (handle_.promise().error) std::rethrow_exception(handle_.promise().error); + T out = *handle_.promise().value; + return out; + } + + private: + void destroy() { + if (handle_) handle_.destroy(); + handle_ = nullptr; + } + Handle handle_{nullptr}; +}; + +template <> +class Task { + public: + struct promise_type { + std::exception_ptr error; + + Task get_return_object() { + return Task{std::coroutine_handle::from_promise(*this)}; + } + std::suspend_never initial_suspend() noexcept { return {}; } + std::suspend_always final_suspend() noexcept { return {}; } + void return_void() {} + void unhandled_exception() { error = std::current_exception(); } + }; + + using Handle = std::coroutine_handle; + + Task() = default; + explicit Task(Handle h) : handle_(h) {} + Task(Task&& other) noexcept : handle_(other.handle_) { other.handle_ = nullptr; } + Task& operator=(Task&&) = delete; + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + ~Task() { + if (handle_) handle_.destroy(); + } + + bool await_ready() const noexcept { return true; } + void await_suspend(std::coroutine_handle<>) const noexcept {} + void await_resume() { value(); } + + void value() { + if (!handle_) throw std::logic_error("molo::app::async::Task: empty task"); + if (handle_.promise().error) std::rethrow_exception(handle_.promise().error); + } + + private: + Handle handle_{nullptr}; +}; + +// Block on an eager task. Valid only because every awaited task completes +// synchronously; the runtime executor replaces this on integration. +template +T run(Task&& task) { + return std::move(task).value(); +} + +inline void run(Task&& task) { task.value(); } + +} // namespace molo::app::async diff --git a/src/app/caps.h b/src/app/caps.h new file mode 100644 index 0000000..d211461 --- /dev/null +++ b/src/app/caps.h @@ -0,0 +1,52 @@ +#pragma once +// Spec: molo::app::caps. Single home of every static ceiling owned by this +// part (plan 05, section 6 plus R11). All values are compile-time constants. +// No autotuning, no silent growth. Sibling-owned byte caps (body, headers) +// are enforced by 01-runtime before this layer; the values below bind the +// app-layer structures only. + +#include + +namespace molo::app::caps { + +// Routing and application (section 6, Global caps). +inline constexpr std::size_t kMaxRoutesPerApp = 100000; +inline constexpr std::size_t kMaxPathLength = 2048; +inline constexpr std::size_t kMaxPathParamsPerRoute = 32; +inline constexpr std::size_t kMaxMiddlewareLayers = 64; +inline constexpr std::size_t kMaxSubAppNestingDepth = 8; +inline constexpr std::size_t kMaxDiProviders = 4096; + +// HTTP message views (standalone parsing; runtime owns wire caps). +inline constexpr std::size_t kMaxHeaders = 100; +inline constexpr std::size_t kMaxHeaderName = 256; +inline constexpr std::size_t kMaxHeaderValue = 8192; +inline constexpr std::size_t kMaxQueryParams = 1000; +inline constexpr std::size_t kMaxCookies = 200; + +// Validation and forms (section 6, Validation and forms). +inline constexpr std::size_t kMaxFormFields = 1000; +inline constexpr std::size_t kMaxFieldName = 256; +inline constexpr std::size_t kMaxNonFileValue = 1048576; // 1 MiB +inline constexpr std::size_t kMaxJsonDepth = 32; +inline constexpr std::size_t kMaxJsonDocument = 10485760; // 10 MiB +inline constexpr std::size_t kMaxMultipartFiles = 100; +inline constexpr std::size_t kMaxFileBytes = 16777216; // 16 MiB default; 03-security owns policy +inline constexpr std::size_t kMaxMultipartTotal = 67108864; // 64 MiB + +// Templates (section 6, Templates). +inline constexpr std::size_t kMaxTemplateSize = 1048576; // 1 MiB +inline constexpr std::size_t kMaxIncludeDepth = 16; +inline constexpr std::size_t kMaxExtendsChain = 8; +inline constexpr std::size_t kMaxLoopIterations = 100000; +inline constexpr std::size_t kMaxRenderOutput = 33554432; // 32 MiB +inline constexpr std::size_t kMaxCompiledTemplates = 512; +inline constexpr std::size_t kMaxMacroArgs = 32; +inline constexpr std::size_t kMaxExprDepth = 128; + +// Errors and misc. +inline constexpr std::size_t kMaxErrorDetails = 64; +inline constexpr std::size_t kMaxRouteName = 256; +inline constexpr std::size_t kMaxFlashMessages = 100; + +} // namespace molo::app::caps diff --git a/src/app/di/container.h b/src/app/di/container.h new file mode 100644 index 0000000..66cb4ef --- /dev/null +++ b/src/app/di/container.h @@ -0,0 +1,208 @@ +#pragma once +// Spec: molo::app::di::Container (plan 05, R8, E9). Dependency injection +// without globals: named providers with app scope (one per Application) or +// request scope (one per request, destroyed after the response). Providers +// are factories or values. Declared dependencies feed startup cycle +// detection (cycle quoted on failure); resolution carries a dynamic guard +// with the same quoting. No service singletons, no file-static mutable +// state reachable from handlers. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../caps.h" + +namespace molo::app::di { + +enum class Scope { App, Request }; + +class DiError : public std::runtime_error { + public: + explicit DiError(const std::string& msg) : std::runtime_error(msg) {} +}; + +class CycleError : public DiError { + public: + explicit CycleError(const std::string& msg) : DiError(msg) {} +}; + +class Container; +class RequestScope; + +class Resolver { + public: + std::any get(const std::string& name); + template + T get_as(const std::string& name) { + return std::any_cast(get(name)); + } + + private: + friend class Container; + friend class RequestScope; + Resolver(Container* owner, std::map* request_cache, + std::vector* stack) + : owner_(owner), request_cache_(request_cache), stack_(stack) {} + Container* owner_; + std::map* request_cache_; + std::vector* stack_; +}; + +struct Provider { + std::function factory; + Scope scope = Scope::App; + std::vector deps; + std::any cached; + bool resolved = false; +}; + +class Container { + public: + void provide(std::string name, std::function factory, + Scope scope = Scope::App, std::vector deps = {}) { + if (providers_.count(name)) throw DiError("duplicate DI provider '" + name + "'"); + if (providers_.size() >= caps::kMaxDiProviders) { + throw DiError("too many DI providers"); + } + Provider p; + p.factory = std::move(factory); + p.scope = scope; + p.deps = std::move(deps); + providers_[std::move(name)] = std::move(p); + } + + template + void provide_typed(std::string name, std::function factory, + Scope scope = Scope::App, std::vector deps = {}) { + provide(std::move(name), + [factory = std::move(factory)](Resolver& r) -> std::any { + return std::any(factory(r)); + }, + scope, std::move(deps)); + } + + template + void provide_value(std::string name, T value) { + provide(std::move(name), + [v = std::move(value)](Resolver&) -> std::any { return std::any(v); }, + Scope::App, {}); + } + + bool has(const std::string& name) const { return providers_.count(name) > 0; } + std::size_t size() const { return providers_.size(); } + + void validate_no_cycles() const { + std::set done; + for (const auto& kv : providers_) { + std::vector path; + visit(kv.first, path, done); + } + } + + RequestScope fork_request(); + + private: + friend class Resolver; + friend class RequestScope; + + std::any resolve_into(const std::string& name, std::map* request_cache, + std::vector* stack) { + auto it = providers_.find(name); + if (it == providers_.end()) throw DiError("unknown DI provider '" + name + "'"); + for (const auto& frame : *stack) { + if (frame == name) { + std::string cycle; + bool started = false; + for (const auto& f : *stack) { + if (f == name) started = true; + if (started) { + if (!cycle.empty()) cycle += " -> "; + cycle += "'" + f + "'"; + } + } + cycle += " -> '" + name + "'"; + throw CycleError("dependency cycle: " + cycle); + } + } + Provider& p = it->second; + if (p.scope == Scope::App && p.resolved) return p.cached; + if (p.scope == Scope::Request && request_cache != nullptr) { + auto hit = request_cache->find(name); + if (hit != request_cache->end()) return hit->second; + } + stack->push_back(name); + Resolver resolver(this, request_cache, stack); + std::any value = p.factory(resolver); + stack->pop_back(); + if (p.scope == Scope::App) { + p.cached = value; + p.resolved = true; + } else if (request_cache != nullptr) { + (*request_cache)[name] = value; + } + return value; + } + + void visit(const std::string& name, std::vector& path, + std::set& done) const { + if (done.count(name)) return; + for (const auto& frame : path) { + if (frame == name) { + std::string cycle; + bool started = false; + for (const auto& f : path) { + if (f == name) started = true; + if (started) { + if (!cycle.empty()) cycle += " -> "; + cycle += "'" + f + "'"; + } + } + cycle += " -> '" + name + "'"; + throw CycleError("dependency cycle: " + cycle); + } + } + auto it = providers_.find(name); + if (it == providers_.end()) return; + path.push_back(name); + for (const auto& dep : it->second.deps) visit(dep, path, done); + path.pop_back(); + done.insert(name); + } + + std::map providers_; +}; + +class RequestScope { + public: + explicit RequestScope(Container* owner) : owner_(owner) {} + + std::any get(const std::string& name) { + return owner_->resolve_into(name, &cache_, &stack_); + } + + template + T get_as(const std::string& name) { + return std::any_cast(get(name)); + } + + Resolver resolver() { return Resolver(owner_, &cache_, &stack_); } + + private: + Container* owner_; + std::map cache_; + std::vector stack_; +}; + +inline std::any Resolver::get(const std::string& name) { + return owner_->resolve_into(name, request_cache_, stack_); +} + +inline RequestScope Container::fork_request() { return RequestScope(this); } + +} // namespace molo::app::di diff --git a/src/app/errors/handler.h b/src/app/errors/handler.h new file mode 100644 index 0000000..5290c8c --- /dev/null +++ b/src/app/errors/handler.h @@ -0,0 +1,209 @@ +#pragma once +// Spec: molo::app::errors::ErrorHandler (plan 05, R9, E5). Maps exceptions +// to status plus negotiated body. JSON shape: +// {"error":{"code":"...","message":"...","request_id":"..."}}. +// HTML shape: responsive error page with the same fields, no internals. +// ValidationFailed renders the 422 detail shape instead: +// {"detail":[{"loc":[...],"msg":"...","type":"..."}]}. +// Unknown exceptions become 500; client-visible text is fixed, internals go +// to the log sink hook (owned by 02-observability on integration). + +#include +#include +#include +#include + +#include "../../audit/redact/redactor.h" +#include "../http_msg/html.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" +#include "../seams/request_id.h" +#include "../validate/json.h" +#include "../validate/value.h" +#include "http_error.h" + +namespace molo::app::errors { + +inline bool wants_html(const http_msg::Request& req) { + std::string accept = req.headers().get("accept"); + std::string low; + for (char c : accept) low += (char)std::tolower((unsigned char)c); + bool html = low.find("text/html") != std::string::npos; + bool json = low.find("application/json") != std::string::npos; + if (html && !json) return true; + if (html && json) return low.find("text/html") < low.find("application/json"); + return false; +} + +inline std::string error_page_html(int status, const std::string& code, const std::string& message, + const std::string& request_id) { + std::string html = + "" + "" + "" + + std::to_string(status) + " " + http_msg::escape_html(code) + + "

" + + std::to_string(status) + " " + http_msg::escape_html(message) + "

" + + http_msg::escape_html(code) + " request " + http_msg::escape_html(request_id) + + "

"; + return html; +} + +inline std::string validation_page_html(const validate::ValidationError& cause, + const std::string& request_id) { + std::string rows; + for (const auto& f : cause.failures()) { + std::string loc; + for (std::size_t i = 0; i < f.loc.size(); ++i) { + if (i) loc += "."; + loc += f.loc[i]; + } + rows += "
  • " + http_msg::escape_html(loc) + " " + + http_msg::escape_html(f.msg) + " " + http_msg::escape_html(f.type) + + "
    • "; + } + std::string html = + "" + "" + "422 validation failed

    422 validation failed

    " + "
      " + + rows + "

    request " + http_msg::escape_html(request_id) + + "

    "; + return html; +} + +class ErrorHandler { + public: + using LogSink = std::function; + + void set_log_sink(LogSink sink) { sink_ = std::move(sink); } + void set_render_hook(std::function hook) { + render_hook_ = std::move(hook); + } + + http_msg::Response render(const HttpError& err, const http_msg::Request& req, + std::string request_id) const { + // Audit patch: user-visible detail and diagnostics pass through the + // redactor before they reach the response body and the log sink. + const std::string safe_detail = + ::molo::audit::redact::Redactor::RedactDetail(err.detail_message()); + log(err.status(), err.code(), request_id, safe_detail); + if (render_hook_) render_hook_(err.status(), err.code()); + if (const auto* vf = dynamic_cast(&err)) { + if (wants_html(req)) { + return http_msg::HtmlResponse(validation_page_html(vf->cause(), request_id), 422); + } + return http_msg::JsonResponse(vf->cause().to_value(), 422); + } + if (const auto* na = dynamic_cast(&err)) { + // Spec 406 shape: {"Detail":{"available":[...]}} (capital D). + validate::Array avail; + for (const auto& a : na->available()) avail.emplace_back(a); + validate::Object inner; + inner["available"] = validate::Value(std::move(avail)); + validate::Object root; + root["Detail"] = validate::Value(std::move(inner)); + if (wants_html(req)) { + std::string items; + for (const auto& a : na->available()) { + items += "
  • " + http_msg::escape_html(a) + "
    • "; + } + std::string page = + "" + "" + "406 not acceptable
    " + "

    406 not acceptable

    Available types:

      " + + items + "

    request " + http_msg::escape_html(request_id) + + "

    "; + return http_msg::HtmlResponse(page, 406); + } + return http_msg::JsonResponse(validate::Value(std::move(root)), 406); + } + http_msg::Response res = + wants_html(req) + ? http_msg::Response(http_msg::HtmlResponse( + error_page_html(err.status(), err.code(), safe_detail, request_id), + err.status())) + : http_msg::Response(http_msg::JsonResponse( + validate::Value([&] { + validate::Object inner; + inner["code"] = validate::Value(err.code()); + inner["message"] = validate::Value(safe_detail); + inner["request_id"] = validate::Value(request_id); + validate::Object root; + root["error"] = validate::Value(std::move(inner)); + return root; + }()), + err.status())); + if (const auto* mna = dynamic_cast(&err)) { + res.headers().set("allow", mna->allow_header()); + } + return res; + } + + // Governance: RFC 9457 renderer for problem bodies. Copies Retry-After + // and Link headers from the source headers map when present. + http_msg::Response render_problem(int status, std::string_view type, std::string_view title, + std::string_view detail, + const http_msg::Headers& extra) const { + validate::Object root; + root["type"] = validate::Value(std::string(type)); + root["title"] = validate::Value(std::string(title)); + root["status"] = validate::Value(static_cast(status)); + if (!detail.empty()) root["detail"] = validate::Value(std::string(detail)); + http_msg::Response res = http_msg::JsonResponse(validate::Value(std::move(root)), status); + res.set_content_type("application/problem+json"); + for (const char* name : {"retry-after", "link"}) { + std::string v = extra.get(name); + if (!v.empty()) res.headers().set(name, v); + } + return res; + } + + http_msg::Response render_unknown(const http_msg::Request& req, std::string request_id, + const std::string& internal) const { + const std::string safe_internal = + ::molo::audit::redact::Redactor::RedactDetail(internal); + log(500, "internal_error", request_id, safe_internal); + InternalError err; + if (wants_html(req)) { + return http_msg::HtmlResponse( + error_page_html(500, err.code(), err.detail_message(), request_id), 500); + } + validate::Object inner; + inner["code"] = validate::Value(err.code()); + inner["message"] = validate::Value(err.detail_message()); + inner["request_id"] = validate::Value(request_id); + validate::Object root; + root["error"] = validate::Value(std::move(inner)); + return http_msg::JsonResponse(validate::Value(std::move(root)), 500); + } + + private: + void log(int status, const std::string& code, const std::string& request_id, + const std::string& internal) const { + if (sink_) sink_(status, code, request_id, internal); + } + LogSink sink_; + std::function render_hook_; +}; + +} // namespace molo::app::errors diff --git a/src/app/errors/http_error.h b/src/app/errors/http_error.h new file mode 100644 index 0000000..80303e0 --- /dev/null +++ b/src/app/errors/http_error.h @@ -0,0 +1,178 @@ +#pragma once +// Spec: molo::app::errors HttpError hierarchy (plan 05, R9, section 6 +// Errors). Every framework exception maps to exactly one HTTP status via +// this table. Unmapped exceptions become 500 in ErrorHandler. + +#include +#include +#include +#include + +#include "../validate/errors.h" + +namespace molo::app::errors { + +class HttpError : public std::exception { + public: + HttpError(int status, std::string code, std::string message) + : status_(status), code_(std::move(code)), message_(std::move(message)) {} + + const char* what() const noexcept override { return message_.c_str(); } + + int status() const { return status_; } + const std::string& code() const { return code_; } + const std::string& detail_message() const { return message_; } + const std::string& request_id() const { return request_id_; } + void set_request_id(std::string id) { request_id_ = std::move(id); } + + private: + int status_; + std::string code_; + std::string message_; + std::string request_id_; +}; + +class BadRequest : public HttpError { + public: + explicit BadRequest(std::string message = "bad request") + : HttpError(400, "bad_request", std::move(message)) {} +}; + +class Unauthorized : public HttpError { + public: + explicit Unauthorized(std::string message = "unauthorized") + : HttpError(401, "unauthorized", std::move(message)) {} +}; + +class Forbidden : public HttpError { + public: + explicit Forbidden(std::string message = "forbidden") + : HttpError(403, "forbidden", std::move(message)) {} +}; + +class NotFound : public HttpError { + public: + explicit NotFound(std::string message = "not found") + : HttpError(404, "not_found", std::move(message)) {} +}; + +class MethodNotAllowed : public HttpError { + public: + explicit MethodNotAllowed(std::vector allow, + std::string message = "method not allowed") + : HttpError(405, "method_not_allowed", std::move(message)), allow_(std::move(allow)) {} + const std::vector& allow() const { return allow_; } + std::string allow_header() const { + std::string out; + for (std::size_t i = 0; i < allow_.size(); ++i) { + if (i) out += ", "; + out += allow_[i]; + } + return out; + } + + private: + std::vector allow_; +}; + +class NotAcceptable : public HttpError { + public: + explicit NotAcceptable(std::vector available = {}, + std::string message = "not acceptable") + : HttpError(406, "not_acceptable", std::move(message)), available_(std::move(available)) {} + const std::vector& available() const { return available_; } + + private: + std::vector available_; +}; + +class RequestTimeout : public HttpError { + public: + explicit RequestTimeout(std::string message = "request timeout") + : HttpError(408, "request_timeout", std::move(message)) {} +}; + +class PayloadTooLarge : public HttpError { + public: + explicit PayloadTooLarge(std::string message = "payload too large") + : HttpError(413, "payload_too_large", std::move(message)) {} +}; + +class UriTooLong : public HttpError { + public: + explicit UriTooLong(std::string message = "uri too long") + : HttpError(414, "uri_too_long", std::move(message)) {} +}; + +class HeaderFieldsTooLarge : public HttpError { + public: + explicit HeaderFieldsTooLarge(std::string message = "header fields too large") + : HttpError(431, "header_fields_too_large", std::move(message)) {} +}; + +class UnsupportedMediaType : public HttpError { + public: + explicit UnsupportedMediaType(std::string message = "unsupported media type") + : HttpError(415, "unsupported_media_type", std::move(message)) {} +}; + +class ValidationFailed : public HttpError { + public: + explicit ValidationFailed(validate::ValidationError cause) + : HttpError(422, "validation_failed", "validation failed"), cause_(std::move(cause)) {} + const validate::ValidationError& cause() const { return cause_; } + + private: + validate::ValidationError cause_; +}; + +class TooManyRequests : public HttpError { + public: + explicit TooManyRequests(std::string message = "too many requests") + : HttpError(429, "too_many_requests", std::move(message)) {} +}; + +class Gone : public HttpError { + public: + explicit Gone(std::string message = "gone") + : HttpError(410, "gone", std::move(message)) {} +}; + +// Tenant errors (PATCHES 11.13), real subclass style. +class TenantUnknown : public HttpError { + public: + explicit TenantUnknown(std::string message = "tenant unknown") + : HttpError(404, "tenant_unknown", std::move(message)) {} +}; + +class TenantSuspended : public HttpError { + public: + explicit TenantSuspended(std::string message = "tenant suspended") + : HttpError(403, "tenant_suspended", std::move(message)) {} +}; + +class TenantConflict : public HttpError { + public: + explicit TenantConflict(std::string message = "tenant conflict") + : HttpError(400, "tenant_conflict", std::move(message)) {} +}; + +class InsufficientStorage : public HttpError { + public: + explicit InsufficientStorage(std::string message = "insufficient storage") + : HttpError(507, "insufficient_storage", std::move(message)) {} +}; + +class InternalError : public HttpError { + public: + explicit InternalError(std::string message = "internal server error") + : HttpError(500, "internal_error", std::move(message)) {} +}; + +class Unavailable : public HttpError { + public: + explicit Unavailable(std::string message = "service unavailable") + : HttpError(503, "unavailable", std::move(message)) {} +}; + +} // namespace molo::app::errors diff --git a/src/app/forms/form.h b/src/app/forms/form.h new file mode 100644 index 0000000..9a23a08 --- /dev/null +++ b/src/app/forms/form.h @@ -0,0 +1,99 @@ +#pragma once +// Spec: molo::app::forms::Form. Binds application/json, +// application/x-www-form-urlencoded and multipart/form-data into a +// validate::Model (plan 05, R2, E10). One entry point, content-type +// dispatch inside. Status policy: malformed shape, bad JSON and depth +// overrun yield 400; size and count overruns (document, value, file, +// total, field count) yield 413. + +#include +#include + +#include "../errors/http_error.h" +#include "../http_msg/request.h" +#include "../validate/errors.h" +#include "../validate/json.h" +#include "../validate/model.h" +#include "../validate/value.h" +#include "multipart.h" +#include "urlencoded.h" + +namespace molo::app::forms { + +struct FormData { + validate::Object fields; + std::vector files; +}; + +class Form { + public: + static FormData parse_body(const http_msg::Request& req, + std::size_t max_json_document = caps::kMaxJsonDocument) { + std::string ctype = req.content_type(); + if (ctype == "application/json") return parse_json_body(req.body(), max_json_document); + if (ctype == "application/x-www-form-urlencoded") { + FormData out; + try { + out.fields = parse_urlencoded(req.body()); + } catch (const std::length_error& e) { + throw errors::PayloadTooLarge(e.what()); + } catch (const std::exception& e) { + throw errors::BadRequest(e.what()); + } + return out; + } + if (ctype == "multipart/form-data") { + FormData out; + try { + MultipartBody body = + parse_multipart(req.headers().get("content-type"), req.body()); + out.fields = std::move(body.fields); + out.files = std::move(body.files); + } catch (const std::length_error& e) { + throw errors::PayloadTooLarge(e.what()); + } catch (const errors::HttpError&) { + throw; + } catch (const std::exception& e) { + throw errors::BadRequest(e.what()); + } + return out; + } + throw errors::UnsupportedMediaType("unsupported form content type: " + ctype); + } + + // Parse then validate into a clean object. Throws ValidationFailed (422) + // carrying every field failure (R3). + static validate::Object bind(const http_msg::Request& req, const validate::Model& model, + const std::string& source = "body", + std::size_t max_json_document = caps::kMaxJsonDocument) { + FormData data = parse_body(req, max_json_document); + try { + return model.validate(data.fields, source); + } catch (const validate::ValidationError& e) { + throw errors::ValidationFailed(e); + } + } + + private: + static FormData parse_json_body(const std::string& body, std::size_t max_doc) { + if (max_doc == 0 && !body.empty()) { + throw errors::PayloadTooLarge("route rejects bodies"); + } + if (body.size() > max_doc) throw errors::PayloadTooLarge("JSON document too large"); + FormData out; + validate::Value v; + try { + v = validate::parse_json(body); + } catch (const validate::JsonError& e) { + throw errors::BadRequest(e.what()); + } + if (!v.is_object()) throw errors::BadRequest("JSON body must be an object"); + out.fields = *std::get_if(&v.storage()); + if (out.fields.size() > caps::kMaxFormFields) { + throw errors::BadRequest("too many JSON fields"); + } + return out; + } +}; + +} // namespace molo::app::forms diff --git a/src/app/forms/multipart.h b/src/app/forms/multipart.h new file mode 100644 index 0000000..255f8a3 --- /dev/null +++ b/src/app/forms/multipart.h @@ -0,0 +1,127 @@ +#pragma once +// Spec: molo::app::forms multipart binding. Parses multipart/form-data field +// parts into an Object; file parts become FileUpload records (bytes held in +// memory under the standalone caps; quota policy and scanning belong to +// 03-security, wired through Application on integration). + +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "../validate/value.h" + +namespace molo::app::forms { + +struct FileUpload { + std::string field; + std::string filename; + std::string content_type = "application/octet-stream"; + std::string data; +}; + +struct MultipartBody { + validate::Object fields; + std::vector files; +}; + +inline std::string multipart_boundary(std::string_view content_type_header) { + std::string lower(content_type_header); + for (char& c : lower) c = (char)std::tolower((unsigned char)c); + std::string::size_type pos = lower.find("boundary="); + if (pos == std::string::npos) return ""; + std::string b(content_type_header.substr(pos + 9)); + while (!b.empty() && (b.back() == ' ' || b.back() == '\t' || b.back() == '"')) b.pop_back(); + while (!b.empty() && (b.front() == ' ' || b.front() == '\t' || b.front() == '"')) + b.erase(b.begin()); + return b; +} + +inline std::string part_header_attr(std::string_view headers, std::string_view attr) { + std::string lower(headers); + for (char& c : lower) c = (char)std::tolower((unsigned char)c); + std::string key(attr); + for (char& c : key) c = (char)std::tolower((unsigned char)c); + key += "=\""; + std::string::size_type pos = lower.find(key); + if (pos == std::string::npos) { + key = std::string(attr) + "="; + for (char& c : key) c = (char)std::tolower((unsigned char)c); + pos = lower.find(key); + if (pos == std::string::npos) return ""; + std::string::size_type start = pos + key.size(); + std::string::size_type end = lower.find_first_of("; \t\r\n", start); + return std::string(headers.substr(start, end - start)); + } + std::string::size_type start = pos + key.size(); + std::string::size_type end = headers.find('"', start); + if (end == std::string_view::npos) return ""; + return std::string(headers.substr(start, end - start)); +} + +inline MultipartBody parse_multipart(std::string_view content_type_header, std::string_view body) { + std::string boundary = multipart_boundary(content_type_header); + if (boundary.empty()) throw std::runtime_error("multipart body without boundary"); + if (body.size() > caps::kMaxMultipartTotal) throw std::length_error("multipart body too large"); + MultipartBody out; + std::string delim = "--" + boundary; + std::string end_delim = delim + "--"; + std::size_t pos = 0; + std::size_t field_count = 0; + while (true) { + std::size_t start = body.find(delim, pos); + if (start == std::string_view::npos) break; + start += delim.size(); + if (body.substr(start, 2) == "--") break; + if (body.substr(start, 2) == "\r\n") start += 2; + std::size_t head_end = body.find("\r\n\r\n", start); + if (head_end == std::string_view::npos) break; + std::string_view headers = body.substr(start, head_end - start); + std::size_t data_start = head_end + 4; + std::size_t next = body.find(delim, data_start); + if (next == std::string_view::npos) break; + std::size_t data_end = next; + if (data_end >= 2 && body.substr(data_end - 2, 2) == "\r\n") data_end -= 2; + std::string data(body.substr(data_start, data_end - data_start)); + std::string name = part_header_attr(headers, "name"); + std::string filename = part_header_attr(headers, "filename"); + if (name.size() > caps::kMaxFieldName) throw std::length_error("form field name too long"); + if (filename.empty()) { + if (field_count >= caps::kMaxFormFields) throw std::length_error("too many form fields"); + if (data.size() > caps::kMaxNonFileValue) throw std::length_error("form value too large"); + out.fields[std::move(name)] = validate::Value(std::move(data)); + ++field_count; + } else { + if (out.files.size() >= caps::kMaxMultipartFiles) { + throw std::length_error("too many uploaded files"); + } + if (data.size() > caps::kMaxFileBytes) throw std::length_error("uploaded file too large"); + FileUpload upload; + upload.field = std::move(name); + upload.filename = std::move(filename); + std::string ctype = part_header_attr(headers, "content-type"); + // Content-Type of a part is a full header line, not an attr; parse it. + std::string lower(headers); + for (char& c : lower) c = (char)std::tolower((unsigned char)c); + std::string::size_type cpos = lower.find("content-type:"); + if (cpos != std::string::npos) { + std::string::size_type vstart = cpos + 13; + while (vstart < headers.size() && (headers[vstart] == ' ' || headers[vstart] == '\t')) + ++vstart; + std::string::size_type vend = headers.find("\r", vstart); + ctype = std::string(headers.substr(vstart, vend - vstart)); + std::string::size_type semi = ctype.find(';'); + if (semi != std::string::npos) ctype = ctype.substr(0, semi); + } + if (!ctype.empty()) upload.content_type = ctype; + upload.data = std::move(data); + out.files.push_back(std::move(upload)); + } + pos = next; + } + return out; +} + +} // namespace molo::app::forms diff --git a/src/app/forms/urlencoded.h b/src/app/forms/urlencoded.h new file mode 100644 index 0000000..abed457 --- /dev/null +++ b/src/app/forms/urlencoded.h @@ -0,0 +1,65 @@ +#pragma once +// Spec: molo::app::forms urlencoded binding. Parses +// application/x-www-form-urlencoded into an Object (last value wins per key, +// full multimap available). Caps from caps.h. + +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "../http_msg/query.h" +#include "../validate/value.h" + +namespace molo::app::forms { + +inline validate::Object parse_urlencoded(std::string_view body) { + validate::Object out; + std::size_t count = 0; + std::size_t i = 0; + while (i <= body.size()) { + std::size_t amp = body.find('&', i); + if (amp == std::string_view::npos) amp = body.size(); + std::string_view pair = body.substr(i, amp - i); + if (!pair.empty()) { + if (count >= caps::kMaxFormFields) throw std::length_error("too many form fields"); + std::size_t eq = pair.find('='); + std::string_view k = (eq == std::string_view::npos) ? pair : pair.substr(0, eq); + std::string_view v = (eq == std::string_view::npos) ? "" : pair.substr(eq + 1); + std::string key = http_msg::percent_decode(k, true); + std::string val = http_msg::percent_decode(v, true); + if (key.size() > caps::kMaxFieldName) throw std::length_error("form field name too long"); + if (val.size() > caps::kMaxNonFileValue) throw std::length_error("form value too large"); + out[std::move(key)] = validate::Value(std::move(val)); + ++count; + } + if (amp == body.size()) break; + i = amp + 1; + } + return out; +} + +inline std::vector> parse_urlencoded_multi( + std::string_view body) { + std::vector> out; + std::size_t i = 0; + while (i <= body.size()) { + std::size_t amp = body.find('&', i); + if (amp == std::string_view::npos) amp = body.size(); + std::string_view pair = body.substr(i, amp - i); + if (!pair.empty()) { + if (out.size() >= caps::kMaxFormFields) throw std::length_error("too many form fields"); + std::size_t eq = pair.find('='); + std::string_view k = (eq == std::string_view::npos) ? pair : pair.substr(0, eq); + std::string_view v = (eq == std::string_view::npos) ? "" : pair.substr(eq + 1); + out.emplace_back(http_msg::percent_decode(k, true), http_msg::percent_decode(v, true)); + } + if (amp == body.size()) break; + i = amp + 1; + } + return out; +} + +} // namespace molo::app::forms diff --git a/src/app/http_msg/cookies.h b/src/app/http_msg/cookies.h new file mode 100644 index 0000000..7f4ac51 --- /dev/null +++ b/src/app/http_msg/cookies.h @@ -0,0 +1,82 @@ +#pragma once +// Spec: molo::app::http_msg::Cookies. Read-only cookie view parsed from the +// Cookie header. Signing and encryption live in 03-security; this is the +// plain view plus a Set-Cookie builder for simple values. + +#include +#include +#include +#include + +#include "../caps.h" +#include "query.h" + +namespace molo::app::http_msg { + +class Cookies { + public: + static Cookies parse(std::string_view header) { + Cookies c; + std::size_t i = 0; + while (i < header.size()) { + while (i < header.size() && (header[i] == ' ' || header[i] == ';')) ++i; + if (i >= header.size()) break; + std::size_t semi = header.find(';', i); + if (semi == std::string_view::npos) semi = header.size(); + std::string_view pair = header.substr(i, semi - i); + std::size_t eq = pair.find('='); + std::string name = eq == std::string_view::npos + ? std::string(pair) + : std::string(pair.substr(0, eq)); + std::string value = + eq == std::string_view::npos ? "" : percent_decode(pair.substr(eq + 1), false); + while (!name.empty() && name.back() == ' ') name.pop_back(); + if (!name.empty()) { + if (c.items_.size() >= caps::kMaxCookies) break; + c.items_.emplace_back(name, value); + } + i = semi + 1; + } + return c; + } + + std::string get(std::string_view name) const { + for (const auto& kv : items_) { + if (kv.first == name) return kv.second; + } + return ""; + } + + bool has(std::string_view name) const { + for (const auto& kv : items_) { + if (kv.first == name) return true; + } + return false; + } + + const std::vector>& items() const { return items_; } + + private: + std::vector> items_; +}; + +struct SetCookie { + std::string name; + std::string value; + std::string path = "/"; + std::string same_site = "Lax"; + bool http_only = true; + bool secure = false; + long max_age = -1; + + std::string render() const { + std::string out = name + "=" + percent_encode(value) + "; Path=" + path; + if (max_age >= 0) out += "; Max-Age=" + std::to_string(max_age); + if (http_only) out += "; HttpOnly"; + if (secure) out += "; Secure"; + out += "; SameSite=" + same_site; + return out; + } +}; + +} // namespace molo::app::http_msg diff --git a/src/app/http_msg/headers.h b/src/app/http_msg/headers.h new file mode 100644 index 0000000..7a3ce40 --- /dev/null +++ b/src/app/http_msg/headers.h @@ -0,0 +1,95 @@ +#pragma once +// Spec: molo::app::http_msg::Headers. Case-insensitive header multimap. +// Lookup lowercases the name. Count and size caps from caps.h guard +// standalone parsing; the runtime enforces wire caps first. + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" + +namespace molo::app::http_msg { + +inline std::string lower_header_name(std::string_view name) { + std::string out(name); + for (char& c : out) c = (char)std::tolower((unsigned char)c); + return out; +} + +class Headers { + public: + void set(std::string name, std::string value) { + if (name.size() > caps::kMaxHeaderName) throw std::length_error("header name too long"); + if (value.size() > caps::kMaxHeaderValue) throw std::length_error("header value too long"); + std::string key = lower_header_name(name); + for (auto& kv : items_) { + if (kv.first == key) { + kv.second = value; + return; + } + } + if (items_.size() >= caps::kMaxHeaders) throw std::length_error("too many headers"); + items_.emplace_back(key, value); + } + + void add(std::string name, std::string value) { + if (name.size() > caps::kMaxHeaderName) throw std::length_error("header name too long"); + if (value.size() > caps::kMaxHeaderValue) throw std::length_error("header value too long"); + if (items_.size() >= caps::kMaxHeaders) throw std::length_error("too many headers"); + items_.emplace_back(lower_header_name(name), value); + } + + std::string get(std::string_view name) const { + std::string key = lower_header_name(name); + for (const auto& kv : items_) { + if (kv.first == key) return kv.second; + } + return ""; + } + + std::string get_or(std::string_view name, std::string fallback) const { + std::string key = lower_header_name(name); + for (const auto& kv : items_) { + if (kv.first == key) return kv.second; + } + return fallback; + } + + bool contains(std::string_view name) const { + std::string key = lower_header_name(name); + for (const auto& kv : items_) { + if (kv.first == key) return true; + } + return false; + } + + std::vector get_all(std::string_view name) const { + std::string key = lower_header_name(name); + std::vector out; + for (const auto& kv : items_) { + if (kv.first == key) out.push_back(kv.second); + } + return out; + } + + void remove(std::string_view name) { + std::string key = lower_header_name(name); + std::vector> kept; + for (const auto& kv : items_) { + if (kv.first != key) kept.push_back(kv); + } + items_.swap(kept); + } + + const std::vector>& items() const { return items_; } + std::size_t size() const { return items_.size(); } + + private: + std::vector> items_; +}; + +} // namespace molo::app::http_msg diff --git a/src/app/http_msg/html.h b/src/app/http_msg/html.h new file mode 100644 index 0000000..5cce2d2 --- /dev/null +++ b/src/app/http_msg/html.h @@ -0,0 +1,42 @@ +#pragma once +// Spec: molo::app::http_msg::escape_html. Shared HTML escaper for error +// pages and template filters (one implementation, plan 05 section 9). + +#include +#include + +namespace molo::app::http_msg { + +inline std::string escape_html(std::string_view s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + switch (c) { + case '&': out += "&"; break; + case '<': out += "<"; break; + case '>': out += ">"; break; + case '"': out += """; break; + case '\'': out += "'"; break; + default: out += c; + } + } + return out; +} + +inline std::string escape_xml(std::string_view s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + switch (c) { + case '&': out += "&"; break; + case '<': out += "<"; break; + case '>': out += ">"; break; + case '"': out += """; break; + case '\'': out += "'"; break; + default: out += c; + } + } + return out; +} + +} // namespace molo::app::http_msg diff --git a/src/app/http_msg/method.h b/src/app/http_msg/method.h new file mode 100644 index 0000000..98b419b --- /dev/null +++ b/src/app/http_msg/method.h @@ -0,0 +1,37 @@ +#pragma once +// Spec: molo::app::http_msg::Method. HTTP methods supported on routes. +// Covers server methods; WebDAV extras arrive with 06-realtime. + +#include +#include +#include + +namespace molo::app::http_msg { + +enum class Method { GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS }; + +inline std::string to_string(Method m) { + switch (m) { + case Method::GET: return "GET"; + case Method::POST: return "POST"; + case Method::PUT: return "PUT"; + case Method::PATCH: return "PATCH"; + case Method::DELETE: return "DELETE"; + case Method::HEAD: return "HEAD"; + case Method::OPTIONS: return "OPTIONS"; + } + return "GET"; +} + +inline std::optional parse_method(std::string_view s) { + if (s == "GET") return Method::GET; + if (s == "POST") return Method::POST; + if (s == "PUT") return Method::PUT; + if (s == "PATCH") return Method::PATCH; + if (s == "DELETE") return Method::DELETE; + if (s == "HEAD") return Method::HEAD; + if (s == "OPTIONS") return Method::OPTIONS; + return std::nullopt; +} + +} // namespace molo::app::http_msg diff --git a/src/app/http_msg/query.h b/src/app/http_msg/query.h new file mode 100644 index 0000000..e30368e --- /dev/null +++ b/src/app/http_msg/query.h @@ -0,0 +1,122 @@ +#pragma once +// Spec: molo::app::http_msg percent-decoding and Query multimap. +// Query holds repeated keys in arrival order. Parsing caps come from caps.h. + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" + +namespace molo::app::http_msg { + +inline int hex_val(char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +inline std::string percent_decode(std::string_view s, bool plus_is_space) { + std::string out; + out.reserve(s.size()); + for (std::size_t i = 0; i < s.size(); ++i) { + char c = s[i]; + if (c == '+' && plus_is_space) { + out += ' '; + } else if (c == '%' && i + 2 < s.size()) { + int hi = hex_val(s[i + 1]); + int lo = hex_val(s[i + 2]); + if (hi >= 0 && lo >= 0) { + out += (char)((hi << 4) | lo); + i += 2; + } else { + out += c; + } + } else { + out += c; + } + } + return out; +} + +inline std::string percent_encode(std::string_view s) { + static const char* kHex = "0123456789ABCDEF"; + std::string out; + for (unsigned char c : s) { + bool plain = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~'; + if (plain) { + out += (char)c; + } else { + out += '%'; + out += kHex[c >> 4]; + out += kHex[c & 15]; + } + } + return out; +} + +class Query { + public: + static Query parse(std::string_view raw) { + Query q; + std::size_t i = 0; + if (!raw.empty() && raw[0] == '?') i = 1; + while (i <= raw.size()) { + std::size_t amp = raw.find('&', i); + if (amp == std::string_view::npos) amp = raw.size(); + std::string_view pair = raw.substr(i, amp - i); + if (!pair.empty()) { + if (q.params_.size() >= caps::kMaxQueryParams) { + throw std::length_error("too many query params"); + } + std::size_t eq = pair.find('='); + std::string_view k = (eq == std::string_view::npos) ? pair : pair.substr(0, eq); + std::string_view v = (eq == std::string_view::npos) ? "" : pair.substr(eq + 1); + q.params_.emplace_back(percent_decode(k, true), percent_decode(v, true)); + } + if (amp == raw.size()) break; + i = amp + 1; + } + return q; + } + + void add(std::string key, std::string value) { + if (params_.size() >= caps::kMaxQueryParams) throw std::length_error("too many query params"); + params_.emplace_back(std::move(key), std::move(value)); + } + + std::string get_first(std::string_view key) const { + for (const auto& kv : params_) { + if (kv.first == key) return kv.second; + } + return ""; + } + + bool has(std::string_view key) const { + for (const auto& kv : params_) { + if (kv.first == key) return true; + } + return false; + } + + std::vector get_all(std::string_view key) const { + std::vector out; + for (const auto& kv : params_) { + if (kv.first == key) out.push_back(kv.second); + } + return out; + } + + const std::vector>& items() const { return params_; } + std::size_t size() const { return params_.size(); } + + private: + std::vector> params_; +}; + +} // namespace molo::app::http_msg diff --git a/src/app/http_msg/request.h b/src/app/http_msg/request.h new file mode 100644 index 0000000..196dcbd --- /dev/null +++ b/src/app/http_msg/request.h @@ -0,0 +1,138 @@ +#pragma once +// Spec: molo::app::http_msg::Request. One request object per handler call +// (plan 05, R1). Carries method, path, parsed query, headers, cookie view, +// body, matched route plus path params, owning app pointer, request ID, and +// a typed extension map for middleware, DI and validation results. +// Body and header byte caps are enforced by 01-runtime before this layer. + +#include +#include +#include +#include + +#include "../validate/value.h" +#include "cookies.h" +#include "headers.h" +#include "method.h" +#include "query.h" + +namespace molo::app::routing { +class Route; +} + +namespace molo::app::app { +class Application; +} + +namespace molo::app::http_msg { + +class Request { + public: + Request() = default; + Request(Method method, std::string path, Query query = Query(), + Headers headers = Headers(), std::string body = "") + : method_(method), + path_(std::move(path)), + query_(std::move(query)), + headers_(std::move(headers)), + body_(std::move(body)), + cookies_(Cookies::parse(headers_.get("cookie"))) {} + + Method method() const { return method_; } + std::string method_string() const { return to_string(method_); } + const std::string& path() const { return path_; } + void set_path(std::string p) { path_ = std::move(p); } + const Query& query() const { return query_; } + Query& query() { return query_; } + const std::string& query_raw() const { return query_raw_; } + void set_query_raw(std::string q) { query_raw_ = std::move(q); } + const Headers& headers() const { return headers_; } + Headers& headers() { return headers_; } + const Cookies& cookies() const { return cookies_; } + const std::string& body() const { return body_; } + void set_body(std::string b) { body_ = std::move(b); } + + // Query shortcut: first value or fallback. + std::string get(std::string_view key, std::string fallback = "") const { + if (query_.has(key)) return query_.get_first(key); + return fallback; + } + std::string header(std::string_view name) const { return headers_.get(name); } + std::string cookie(std::string_view name) const { return cookies_.get(name); } + + // Content-Type without parameters, lowercased. + std::string content_type() const { + std::string ct = headers_.get("content-type"); + std::size_t semi = ct.find(';'); + if (semi != std::string::npos) ct = ct.substr(0, semi); + while (!ct.empty() && (ct.back() == ' ' || ct.back() == '\t')) ct.pop_back(); + std::size_t start = 0; + while (start < ct.size() && (ct[start] == ' ' || ct[start] == '\t')) ++start; + std::string out = ct.substr(start); + for (char& c : out) c = (char)std::tolower((unsigned char)c); + return out; + } + + const std::map& path_params() const { return path_params_; } + std::string path_param(std::string_view name) const { + auto it = path_params_.find(std::string(name)); + return it == path_params_.end() ? "" : it->second; + } + void set_path_param(std::string name, std::string value) { + path_params_[std::move(name)] = std::move(value); + } + + const routing::Route* route() const { return route_; } + void set_route(const routing::Route* r) { route_ = r; } + + app::Application* app() const { return app_; } + void set_app(app::Application* a) { app_ = a; } + + const std::string& request_id() const { return request_id_; } + void set_request_id(std::string id) { request_id_ = std::move(id); } + + // Validated binding results, keyed by source ("query", "body", ...). + void set_validated(std::string source, validate::Object obj) { + validated_[std::move(source)] = std::move(obj); + } + const validate::Object* validated(std::string_view source) const { + auto it = validated_.find(std::string(source)); + return it == validated_.end() ? nullptr : &it->second; + } + + // Typed extension slot for middleware and DI. + template + void set_ext(std::string key, T value) { + ext_[std::move(key)] = std::any(std::move(value)); + } + template + T* get_ext(const std::string& key) { + auto it = ext_.find(key); + if (it == ext_.end()) return nullptr; + return std::any_cast(&it->second); + } + template + const T* get_ext(const std::string& key) const { + auto it = ext_.find(key); + if (it == ext_.end()) return nullptr; + return std::any_cast(&it->second); + } + void clear_ext(const std::string& key) { ext_.erase(key); } + + private: + Method method_ = Method::GET; + std::string path_ = "/"; + std::string query_raw_; + Query query_; + Headers headers_; + std::string body_; + Cookies cookies_; + std::map path_params_; + const routing::Route* route_ = nullptr; + app::Application* app_ = nullptr; + std::string request_id_; + std::map validated_; + std::map ext_; +}; + +} // namespace molo::app::http_msg diff --git a/src/app/http_msg/response.h b/src/app/http_msg/response.h new file mode 100644 index 0000000..7fc2cd9 --- /dev/null +++ b/src/app/http_msg/response.h @@ -0,0 +1,155 @@ +#pragma once +// Spec: molo::app::http_msg::Response hierarchy (plan 05, R1, R4, E1). +// Response is a value type: status, headers, body, plus an extension payload +// for streaming and negotiated variants, so returning by value never slices. +// Subclasses only preset fields. All string-taking constructors are explicit +// (R4: no implicit conversion of a raw string to a 200 text response). + +#include +#include +#include +#include +#include +#include + +#include "../validate/json.h" +#include "../validate/value.h" +#include "headers.h" + +namespace molo::app::http_msg { + +class Response { + public: + explicit Response(int status = 200, std::string body = "", + std::string content_type = "text/plain; charset=utf-8") + : status_(status), body_(std::move(body)) { + headers_.set("content-type", std::move(content_type)); + } + + int status() const { return status_; } + void set_status(int s) { status_ = s; } + const std::string& body() const { return body_; } + void set_body(std::string b) { body_ = std::move(b); } + const Headers& headers() const { return headers_; } + Headers& headers() { return headers_; } + std::string content_type() const { return headers_.get("content-type"); } + void set_content_type(std::string ct) { headers_.set("content-type", std::move(ct)); } + + template + void set_payload(T value) { + payload_ = std::any(std::move(value)); + } + template + T* get_payload() { + return std::any_cast(&payload_); + } + template + const T* get_payload() const { + return std::any_cast(&payload_); + } + bool has_payload() const { return payload_.has_value(); } + + protected: + int status_ = 200; + Headers headers_; + std::string body_; + std::any payload_; +}; + +class TextResponse : public Response { + public: + explicit TextResponse(std::string body, int status = 200) + : Response(status, std::move(body), "text/plain; charset=utf-8") {} +}; + +class HtmlResponse : public Response { + public: + explicit HtmlResponse(std::string body, int status = 200) + : Response(status, std::move(body), "text/html; charset=utf-8") {} +}; + +class XmlResponse : public Response { + public: + explicit XmlResponse(std::string body, int status = 200) + : Response(status, std::move(body), "application/xml; charset=utf-8") {} +}; + +class JsonResponse : public Response { + public: + explicit JsonResponse(validate::Value value, int status = 200) + : Response(status, validate::to_json(value), "application/json") {} + explicit JsonResponse(std::string raw_json, int status = 200) + : Response(status, std::move(raw_json), "application/json") {} +}; + +class RedirectResponse : public Response { + public: + // Method-preserving redirects: 308 permanent, 307 temporary. + explicit RedirectResponse(std::string url, bool permanent = false) + : Response(permanent ? 308 : 307, "") { + headers_.set("location", std::move(url)); + } + static RedirectResponse temporary(std::string url) { return RedirectResponse(std::move(url), false); } + static RedirectResponse permanent(std::string url) { return RedirectResponse(std::move(url), true); } +}; + +class FileResponse : public Response { + public: + explicit FileResponse(std::string content, std::string content_type, + std::string filename = "", int status = 200) + : Response(status, std::move(content), std::move(content_type)) { + if (!filename.empty()) { + headers_.set("content-disposition", "attachment; filename=\"" + filename + "\""); + } + headers_.set("accept-ranges", "bytes"); + } +}; + +// Streaming shell: chunk mechanics live in 06-realtime; the class shell +// lives here (plan 05, section 3). The producer returns the next chunk or +// nullopt when done. collect() materializes with a byte cap. +struct StreamPayload { + std::function()> producer; +}; + +class StreamingResponse : public Response { + public: + explicit StreamingResponse(std::function()> producer, + std::string content_type = "text/plain; charset=utf-8", + int status = 200) + : Response(status, "", std::move(content_type)) { + set_payload(StreamPayload{std::move(producer)}); + } + + std::string collect(std::size_t max_bytes) { + auto* p = get_payload(); + if (p == nullptr) return body_; + std::string out; + while (true) { + std::optional chunk = p->producer(); + if (!chunk.has_value()) break; + if (out.size() + chunk->size() > max_bytes) break; + out += *chunk; + } + return out; + } +}; + +// Negotiated variant: holds one data value plus the media types the route +// can render it as. NegotiationMiddleware (molo::app::middleware) replaces +// the body and content type with the negotiated representation, or answers +// 406 when nothing matches. Concrete responses pass through untouched. +struct VariantPayload { + validate::Value data; + std::vector available; +}; + +class VariantResponse : public Response { + public: + explicit VariantResponse(validate::Value data, std::vector available) + : Response(200, "") { + set_payload(VariantPayload{std::move(data), std::move(available)}); + } +}; + +} // namespace molo::app::http_msg diff --git a/src/app/middleware/builtins.h b/src/app/middleware/builtins.h new file mode 100644 index 0000000..efbb5b2 --- /dev/null +++ b/src/app/middleware/builtins.h @@ -0,0 +1,92 @@ +#pragma once +// Spec: molo::app::middleware built-ins (plan 05, section 6 Middleware). +// ExceptionMappingMiddleware (outermost, category Errors) converts thrown +// errors to rendered responses. NegotiationMiddleware (RouterAdjacent) +// enforces consumes (415) and renders VariantResponse in the negotiated +// format (406 when nothing matches). CORS policy belongs to 03-security; +// these layers only preserve headers they do not own. + +#include +#include +#include +#include + +#include "../async/task.h" +#include "../errors/handler.h" +#include "../errors/http_error.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" +#include "../negotiate/negotiator.h" +#include "../routing/route.h" +#include "../seams/request_id.h" +#include "middleware.h" + +namespace molo::app::middleware { + +class ExceptionMappingMiddleware : public Middleware { + public: + explicit ExceptionMappingMiddleware( + std::shared_ptr handler = std::make_shared()) + : handler_(std::move(handler)) {} + + Category category() const override { return Category::Errors; } + std::string name() const override { return "exception_mapping"; } + + async::Task handle(http_msg::Request& req, Next next) override { + std::string rid = req.request_id().empty() ? seams::generate_request_id() : req.request_id(); + try { + http_msg::Response res = co_await next(req); + res.headers().set(seams::kRequestIdHeader, rid); + co_return res; + } catch (const errors::HttpError& e) { + http_msg::Response res = handler_->render(e, req, rid); + res.headers().set(seams::kRequestIdHeader, rid); + co_return res; + } catch (const validate::ValidationError& e) { + errors::ValidationFailed wrapped(e); + http_msg::Response res = handler_->render(wrapped, req, rid); + res.headers().set(seams::kRequestIdHeader, rid); + co_return res; + } catch (const std::exception& e) { + http_msg::Response res = handler_->render_unknown(req, rid, e.what()); + res.headers().set(seams::kRequestIdHeader, rid); + co_return res; + } catch (...) { + http_msg::Response res = handler_->render_unknown(req, rid, "non-standard exception"); + res.headers().set(seams::kRequestIdHeader, rid); + co_return res; + } + } + + private: + std::shared_ptr handler_; +}; + +class NegotiationMiddleware : public Middleware { + public: + Category category() const override { return Category::RouterAdjacent; } + std::string name() const override { return "negotiation"; } + + async::Task handle(http_msg::Request& req, Next next) override { + const routing::Route* route = req.route(); + if (route == nullptr) { + co_return co_await next(req); + } + negotiate::Negotiator::check_consumes(req.content_type(), route->options().consumes); + http_msg::Response res = co_await next(req); + auto* variant = res.get_payload(); + if (variant == nullptr) co_return res; + std::vector produces = route->options().produces; + if (produces.empty()) produces = variant->available; + if (produces.empty()) produces = {"application/json"}; + std::string chosen = negotiate::Negotiator::select( + req.headers().get("accept"), produces, route->options().allow_format_override, + req.query().get_first("format")); + auto rendered = negotiate::render_variant(variant->data, chosen); + res.set_body(std::move(rendered.first)); + res.set_content_type(std::move(rendered.second)); + co_return res; + } +}; + +} // namespace molo::app::middleware diff --git a/src/app/middleware/chain.h b/src/app/middleware/chain.h new file mode 100644 index 0000000..96f490b --- /dev/null +++ b/src/app/middleware/chain.h @@ -0,0 +1,63 @@ +#pragma once +// Spec: molo::app::middleware::Chain. Composes middleware in registration +// order around a terminal handler. Category order is enforced at use(): +// adding a layer with a lower category than the previous one throws +// ChainOrderError (startup error, R7). Layer count capped at 64. + +#include +#include +#include +#include +#include + +#include "../async/task.h" +#include "../caps.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" +#include "middleware.h" + +namespace molo::app::middleware { + +class ChainOrderError : public std::runtime_error { + public: + explicit ChainOrderError(const std::string& msg) : std::runtime_error(msg) {} +}; + +class Chain { + public: + void use(std::shared_ptr layer) { + if (layers_.size() >= caps::kMaxMiddlewareLayers) { + throw ChainOrderError("too many middleware layers"); + } + if (!layers_.empty() && layer->category() < layers_.back()->category()) { + throw ChainOrderError( + "middleware order violation: '" + layer->name() + "' (" + + category_name(layer->category()) + ") registered after '" + + layers_.back()->name() + "' (" + category_name(layers_.back()->category()) + ")"); + } + layers_.push_back(std::move(layer)); + } + + async::Task run( + http_msg::Request& req, + std::function(http_msg::Request&)> terminal) { + std::function(http_msg::Request&)> next = terminal; + for (std::size_t i = layers_.size(); i-- > 0;) { + std::shared_ptr layer = layers_[i]; + auto inner = next; + next = [layer, inner](http_msg::Request& r) -> async::Task { + Next guard(inner); + co_return co_await layer->handle(r, guard); + }; + } + co_return co_await next(req); + } + + const std::vector>& layers() const { return layers_; } + std::size_t size() const { return layers_.size(); } + + private: + std::vector> layers_; +}; + +} // namespace molo::app::middleware diff --git a/src/app/middleware/middleware.h b/src/app/middleware/middleware.h new file mode 100644 index 0000000..1713068 --- /dev/null +++ b/src/app/middleware/middleware.h @@ -0,0 +1,69 @@ +#pragma once +// Spec: molo::app::middleware base (plan 05, R7, E8). Signature: +// Task handle(Request, Next). Next invokes the remainder exactly +// once; calling twice throws MiddlewareError. Category order is fixed: +// Errors, Observability, Security, SessionAuth, User, RouterAdjacent. +// Middleware must preserve the correlation/request ID. + +#include +#include +#include +#include + +#include "../async/task.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" + +namespace molo::app::middleware { + +enum class Category { + Errors = 0, + Observability = 1, + Security = 2, + SessionAuth = 3, + User = 4, + RouterAdjacent = 5 +}; + +inline std::string category_name(Category c) { + switch (c) { + case Category::Errors: return "errors"; + case Category::Observability: return "observability"; + case Category::Security: return "security"; + case Category::SessionAuth: return "session_auth"; + case Category::User: return "user"; + case Category::RouterAdjacent: return "router_adjacent"; + } + return "user"; +} + +class MiddlewareError : public std::runtime_error { + public: + explicit MiddlewareError(const std::string& msg) : std::runtime_error(msg) {} +}; + +class Next { + public: + explicit Next(std::function(http_msg::Request&)> fn) + : fn_(std::move(fn)), called_(std::make_shared(false)) {} + + async::Task operator()(http_msg::Request& req) const { + if (*called_) throw MiddlewareError("Next invoked twice in one middleware"); + *called_ = true; + co_return co_await fn_(req); + } + + private: + std::function(http_msg::Request&)> fn_; + std::shared_ptr called_; +}; + +class Middleware { + public: + virtual ~Middleware() = default; + virtual Category category() const = 0; + virtual std::string name() const = 0; + virtual async::Task handle(http_msg::Request& req, Next next) = 0; +}; + +} // namespace molo::app::middleware diff --git a/src/app/negotiate/accept.h b/src/app/negotiate/accept.h new file mode 100644 index 0000000..52c0d9f --- /dev/null +++ b/src/app/negotiate/accept.h @@ -0,0 +1,98 @@ +#pragma once +// Spec: molo::app::negotiate Accept parser. Parses an Accept header into +// media ranges (type, subtype, q, params). Missing or empty header means +// */*. Invalid q values are treated as weight 0 (entry ignored), documented +// leniency choice. Range params other than q are parsed and ignored for +// matching (documented gap vs RFC 9110 section 12). + +#include +#include +#include +#include +#include + +namespace molo::app::negotiate { + +struct MediaRange { + std::string type; + std::string subtype; + double q = 1.0; + std::vector> params; + std::size_t order = 0; +}; + +inline std::string trim_ws(std::string_view s) { + std::size_t a = 0; + while (a < s.size() && (s[a] == ' ' || s[a] == '\t')) ++a; + std::size_t b = s.size(); + while (b > a && (s[b - 1] == ' ' || s[b - 1] == '\t')) --b; + std::string out(s.substr(a, b - a)); + for (char& c : out) c = (char)std::tolower((unsigned char)c); + return out; +} + +inline double parse_q(std::string_view s) { + if (s.empty() || s.size() > 5) return 0.0; + try { + std::size_t used = 0; + double v = std::stod(std::string(s), &used); + if (used != s.size() || v < 0.0 || v > 1.0) return 0.0; + return v; + } catch (...) { + return 0.0; + } +} + +inline std::vector parse_accept(std::string_view header) { + std::vector out; + if (trim_ws(header).empty()) { + out.push_back(MediaRange{"*", "*", 1.0, {}, 0}); + return out; + } + std::size_t i = 0; + std::size_t order = 0; + while (i <= header.size()) { + std::size_t comma = header.find(',', i); + if (comma == std::string_view::npos) comma = header.size(); + std::string_view entry = header.substr(i, comma - i); + std::size_t semi = entry.find(';'); + std::string_view full = (semi == std::string_view::npos) ? entry : entry.substr(0, semi); + std::size_t slash = full.find('/'); + if (slash != std::string_view::npos) { + MediaRange range; + range.type = trim_ws(full.substr(0, slash)); + range.subtype = trim_ws(full.substr(slash + 1)); + range.order = order++; + if (!range.type.empty() && !range.subtype.empty()) { + if (semi != std::string_view::npos) { + std::string_view rest = entry.substr(semi + 1); + std::size_t j = 0; + while (j <= rest.size()) { + std::size_t s2 = rest.find(';', j); + if (s2 == std::string_view::npos) s2 = rest.size(); + std::string_view kv = rest.substr(j, s2 - j); + std::size_t eq = kv.find('='); + if (eq != std::string_view::npos) { + std::string k = trim_ws(kv.substr(0, eq)); + std::string v = trim_ws(kv.substr(eq + 1)); + if (k == "q") { + range.q = parse_q(v); + } else { + range.params.emplace_back(k, v); + } + } + if (s2 == rest.size()) break; + j = s2 + 1; + } + } + out.push_back(std::move(range)); + } + } + if (comma == header.size()) break; + i = comma + 1; + } + if (out.empty()) out.push_back(MediaRange{"*", "*", 1.0, {}, 0}); + return out; +} + +} // namespace molo::app::negotiate diff --git a/src/app/negotiate/negotiator.h b/src/app/negotiate/negotiator.h new file mode 100644 index 0000000..522fb91 --- /dev/null +++ b/src/app/negotiate/negotiator.h @@ -0,0 +1,137 @@ +#pragma once +// Spec: molo::app::negotiate::Negotiator (plan 05, R6, E6). Server-driven +// content negotiation: sorts Accept ranges by q desc, then specificity +// (exact beats type star beats star star), then Accept order; matches +// against handler-declared produces in declaration order. Explicit ?format= +// or suffix override applies only when the route sets allow_format_override +// (Q2 answer: off by default). No match throws NotAcceptable carrying the +// available list. Consumes check throws UnsupportedMediaType (415). + +#include +#include +#include +#include +#include + +#include "../errors/http_error.h" +#include "../http_msg/html.h" +#include "../validate/json.h" +#include "../validate/value.h" +#include "accept.h" + +namespace molo::app::negotiate { + +inline int range_specificity(const MediaRange& r) { + if (r.type == "*" && r.subtype == "*") return 0; + if (r.subtype == "*") return 1; + return 2; +} + +inline bool range_matches(const MediaRange& r, std::string_view candidate) { + std::size_t slash = candidate.find('/'); + if (slash == std::string_view::npos) return false; + std::string_view ct = candidate.substr(0, slash); + std::string_view cs = candidate.substr(slash + 1); + if (r.type != "*" && r.type != ct) return false; + if (r.subtype != "*" && r.subtype != cs) return false; + return true; +} + +inline std::string format_name_to_media(std::string_view name) { + std::string low(name); + for (char& c : low) c = (char)std::tolower((unsigned char)c); + if (low == "json") return "application/json"; + if (low == "html") return "text/html"; + if (low == "xml") return "application/xml"; + if (low == "text" || low == "txt") return "text/plain"; + return ""; +} + +class Negotiator { + public: + static std::string select(std::string_view accept_header, + const std::vector& produces, + bool allow_format_override = false, + std::string_view format_param = "") { + if (produces.empty()) return ""; + if (allow_format_override && !format_param.empty()) { + std::string mapped = format_name_to_media(format_param); + for (const auto& p : produces) { + if (p == mapped) return p; + } + throw errors::NotAcceptable(produces); + } + std::vector ranges = parse_accept(accept_header); + std::sort(ranges.begin(), ranges.end(), [](const MediaRange& a, const MediaRange& b) { + if (a.q != b.q) return a.q > b.q; + int sa = range_specificity(a); + int sb = range_specificity(b); + if (sa != sb) return sa > sb; + return a.order < b.order; + }); + for (const MediaRange& r : ranges) { + if (r.q <= 0.0) continue; + for (const auto& p : produces) { + if (range_matches(r, p)) return p; + } + } + throw errors::NotAcceptable(produces); + } + + static void check_consumes(std::string_view content_type, + const std::vector& consumes) { + if (consumes.empty()) return; + for (const auto& c : consumes) { + if (c == content_type) return; + } + throw errors::UnsupportedMediaType("content type not consumed by this route"); + } +}; + +// Render one data value into a concrete (body, content type) pair for the +// negotiated media type. Used by NegotiationMiddleware for VariantResponse. +inline std::pair render_variant(const validate::Value& data, + std::string_view media) { + if (media == "application/json") { + return {validate::to_json(data), "application/json"}; + } + if (media == "application/xml") { + std::string body = ""; + if (const auto* o = std::get_if(&data.storage())) { + for (const auto& kv : *o) { + body += "<" + kv.first + ">" + http_msg::escape_xml(validate::to_json(kv.second)) + + ""; + } + } else { + body += http_msg::escape_xml(validate::to_json(data)); + } + body += ""; + return {body, "application/xml; charset=utf-8"}; + } + if (media == "text/html") { + std::string rows; + if (const auto* o = std::get_if(&data.storage())) { + for (const auto& kv : *o) { + rows += "" + http_msg::escape_html(kv.first) + "" + + http_msg::escape_html(validate::to_json(kv.second)) + ""; + } + } else { + rows = "" + http_msg::escape_html(validate::to_json(data)) + ""; + } + std::string body = + "" + "" + "response" + "" + + rows + "
    "; + return {body, "text/html; charset=utf-8"}; + } + std::string text = data.is_string() ? data.as_text() : validate::to_json(data); + return {text, "text/plain; charset=utf-8"}; +} + +} // namespace molo::app::negotiate diff --git a/src/app/routing/path_compiler.h b/src/app/routing/path_compiler.h new file mode 100644 index 0000000..574d8ad --- /dev/null +++ b/src/app/routing/path_compiler.h @@ -0,0 +1,227 @@ +#pragma once +// Spec: molo::app::routing path compiler (plan 05, section 6 Route matching). +// Path syntax: /users/{id:int} with converters int, float, str (single +// segment), path (remainder, must be last). Unknown converter is a startup +// error naming the route. Trailing slash is strict. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "../errors/http_error.h" +#include "../validate/errors.h" + +namespace molo::app::routing { + +class PathCompileError : public std::runtime_error { + public: + explicit PathCompileError(const std::string& msg) : std::runtime_error(msg) {} +}; + +enum class Converter { Int, Float, Str, Path }; + +struct Segment { + bool is_param = false; + std::string literal; + std::string name; + Converter converter = Converter::Str; +}; + +struct CompiledPath { + std::vector segments; + bool has_catchall = false; + std::size_t static_count = 0; +}; + +inline std::vector split_path(std::string_view path) { + std::vector out; + std::size_t i = 0; + if (!path.empty() && path[0] == '/') i = 1; + while (true) { + std::size_t slash = path.find('/', i); + if (slash == std::string_view::npos) { + out.emplace_back(path.substr(i)); + break; + } + out.emplace_back(path.substr(i, slash - i)); + i = slash + 1; + } + return out; +} + +inline CompiledPath compile_path(std::string_view pattern, std::string_view route_label) { + if (pattern.empty() || pattern[0] != '/') { + throw PathCompileError("route '" + std::string(route_label) + "': pattern must start with /"); + } + CompiledPath out; + std::vector parts = split_path(pattern); + std::size_t param_count = 0; + for (std::size_t idx = 0; idx < parts.size(); ++idx) { + const std::string& part = parts[idx]; + if (part.size() >= 2 && part.front() == '{' && part.back() == '}') { + std::string inner = part.substr(1, part.size() - 2); + std::string name = inner; + std::string conv = "str"; + std::size_t colon = inner.find(':'); + if (colon != std::string::npos) { + name = inner.substr(0, colon); + conv = inner.substr(colon + 1); + } + if (name.empty()) { + throw PathCompileError("route '" + std::string(route_label) + "': empty param name"); + } + Segment seg; + seg.is_param = true; + seg.name = name; + if (conv == "int") { + seg.converter = Converter::Int; + } else if (conv == "float") { + seg.converter = Converter::Float; + } else if (conv == "str") { + seg.converter = Converter::Str; + } else if (conv == "path") { + seg.converter = Converter::Path; + if (idx != parts.size() - 1) { + throw PathCompileError("route '" + std::string(route_label) + + "': path converter must be last"); + } + out.has_catchall = true; + } else { + throw PathCompileError("route '" + std::string(route_label) + "': unknown converter '" + + conv + "'"); + } + ++param_count; + if (param_count > caps::kMaxPathParamsPerRoute) { + throw PathCompileError("route '" + std::string(route_label) + "': too many path params"); + } + out.segments.push_back(std::move(seg)); + } else { + Segment seg; + seg.literal = part; + out.segments.push_back(std::move(seg)); + ++out.static_count; + } + } + return out; +} + +inline bool strict_int(std::string_view s) { + if (s.empty()) return false; + std::size_t i = 0; + if (s[0] == '+' || s[0] == '-') i = 1; + if (i == s.size()) return false; + for (std::size_t j = i; j < s.size(); ++j) { + if (!std::isdigit((unsigned char)s[j])) return false; + } + return true; +} + +inline bool strict_float(std::string_view s) { + if (s.empty()) return false; + try { + std::size_t used = 0; + std::stod(std::string(s), &used); + return used == s.size(); + } catch (...) { + return false; + } +} + +struct PathMatch { + bool full = false; + bool skeleton = false; + std::map params; + std::vector errors; +}; + +inline PathMatch match_compiled(const CompiledPath& cp, std::string_view path) { + PathMatch m; + if (path.empty() || path[0] != '/') return m; + std::vector parts = split_path(path); + std::size_t need = cp.segments.size(); + bool count_ok = cp.has_catchall ? (parts.size() >= need - 1) : (parts.size() == need); + // Skeleton pass: converters match any single segment, catch-all the rest. + bool skeleton = count_ok; + if (skeleton) { + for (std::size_t i = 0; i < cp.segments.size() && skeleton; ++i) { + const Segment& seg = cp.segments[i]; + if (seg.is_param) { + if (seg.converter == Converter::Path) break; + continue; + } + if (i >= parts.size() || parts[i] != seg.literal) skeleton = false; + } + } + m.skeleton = skeleton; + if (!skeleton) return m; + // Full pass with converter checks. + for (std::size_t i = 0; i < cp.segments.size(); ++i) { + const Segment& seg = cp.segments[i]; + if (!seg.is_param) continue; + if (seg.converter == Converter::Path) { + std::string rest; + for (std::size_t j = i; j < parts.size(); ++j) { + if (j > i) rest += '/'; + rest += parts[j]; + } + m.params[seg.name] = rest; + break; + } + const std::string& raw = parts[i]; + if (seg.converter == Converter::Str) { + if (raw.empty()) { + m.errors.push_back({{"path", seg.name}, "empty path segment", "string_type"}); + return m; + } + m.params[seg.name] = raw; + } else if (seg.converter == Converter::Int) { + if (!strict_int(raw)) { + m.errors.push_back({{"path", seg.name}, "invalid integer", "int_parsing"}); + return m; + } + m.params[seg.name] = raw; + } else if (seg.converter == Converter::Float) { + if (!strict_float(raw)) { + m.errors.push_back({{"path", seg.name}, "invalid number", "float_parsing"}); + return m; + } + m.params[seg.name] = raw; + } + } + m.full = true; + return m; +} + +inline std::string render_compiled(const CompiledPath& cp, + const std::map& params, + std::string_view route_name) { + std::string out; + for (const Segment& seg : cp.segments) { + out += '/'; + if (!seg.is_param) { + out += seg.literal; + } else { + auto it = params.find(seg.name); + if (it == params.end()) { + throw errors::NotFound("url_for: route '" + std::string(route_name) + + "' missing param '" + seg.name + "'"); + } + out += it->second; + } + } + if (out.empty()) out = "/"; + // Collapse the "/"+"" root case back to "/". + if (out.size() > 1 && out.back() == '/' && cp.segments.size() == 1 && + !cp.segments[0].is_param && cp.segments[0].literal.empty()) { + out = "/"; + } + return out; +} + +} // namespace molo::app::routing diff --git a/src/app/routing/route.h b/src/app/routing/route.h new file mode 100644 index 0000000..20ee2d5 --- /dev/null +++ b/src/app/routing/route.h @@ -0,0 +1,113 @@ +#pragma once +// Spec: molo::app::routing::Route. One route record: methods, compiled path, +// handler, name, tags, validation models, negotiation declarations, +// slash policy (plan 05, section 3). Handler takes Request and returns +// Task (R1). slash_insensitive registers the declared route plus a +// companion 308 redirect route for the alternate trailing-slash form. + +#include +#include +#include +#include +#include +#include + +#include "../async/task.h" +#include "../caps.h" +#include "../http_msg/method.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" +#include "../validate/model.h" +#include "path_compiler.h" + +namespace molo::app::routing { + +using Handler = std::function(http_msg::Request&)>; + +struct RouteOptions { + std::string name; + std::vector tags; + std::vector produces; + std::vector consumes; + bool allow_format_override = false; + bool slash_insensitive = false; + std::size_t max_body = caps::kMaxJsonDocument; + // Governance (molo::govern): auth and lifecycle for this route. + enum class GovAuth { kNone, kKey, kKeyScopes, kSession }; + GovAuth gov_auth = GovAuth::kNone; + std::vector gov_required_scopes; + int gov_api_version = 0; // 0 = derived from the /v{n} prefix + long gov_deprecation_unix = 0; + std::string gov_sunset_httpdate; + std::string gov_tier_override; // "", "free", "pro", "enterprise" + bool gov_has_summary = false; + bool gov_has_example = false; + std::shared_ptr query_model; + std::shared_ptr body_model; + std::shared_ptr header_model; + std::shared_ptr cookie_model; + // Internal: non-empty on slash-redirect companions (target declared path). + std::string redirect_target; +}; + +class Route { + public: + Route(std::set methods, std::string pattern, Handler handler, + RouteOptions options = {}, std::size_t index = 0) + : methods_(std::move(methods)), + pattern_(std::move(pattern)), + handler_(std::move(handler)), + options_(std::move(options)), + index_(index) { + if (options_.name.size() > caps::kMaxRouteName) { + throw PathCompileError("route name too long: " + options_.name); + } + compiled_ = compile_path(pattern_, options_.name.empty() ? pattern_ : options_.name); + } + + const std::set& methods() const { return methods_; } + const std::string& pattern() const { return pattern_; } + const CompiledPath& compiled() const { return compiled_; } + const Handler& handler() const { return handler_; } + const RouteOptions& options() const { return options_; } + std::size_t index() const { return index_; } + bool is_slash_redirect() const { return !options_.redirect_target.empty(); } + + bool allows(http_msg::Method m) const { return methods_.count(m) > 0; } + + std::vector allow_list() const { + std::vector out; + for (auto m : methods_) out.push_back(http_msg::to_string(m)); + return out; + } + + Route with_prefix(const std::string& prefix, + const std::vector& extra_tags) const { + std::string clean = prefix; + if (clean == "/") clean = ""; + RouteOptions opts = options_; + for (const auto& t : extra_tags) opts.tags.push_back(t); + std::string new_pattern = clean + pattern_; + Handler h = handler_; + if (!opts.redirect_target.empty()) { + opts.redirect_target = clean + opts.redirect_target; + std::string target = opts.redirect_target; + h = [target](http_msg::Request& req) -> async::Task { + std::string url = target; + if (!req.query_raw().empty()) url += "?" + req.query_raw(); + co_return http_msg::RedirectResponse::permanent(url); + }; + } + return Route(methods_, new_pattern, h, opts, index_); + } + + private: + std::set methods_; + std::string pattern_; + Handler handler_; + RouteOptions options_; + CompiledPath compiled_; + std::size_t index_ = 0; +}; + +} // namespace molo::app::routing diff --git a/src/app/routing/router.h b/src/app/routing/router.h new file mode 100644 index 0000000..a1de32f --- /dev/null +++ b/src/app/routing/router.h @@ -0,0 +1,319 @@ +#pragma once +// Spec: molo::app::routing::Router (plan 05, E2). FastAPI-style router: +// get/post/put/patch/delete/head/options, nested include_router with prefix +// and tags, match with 404/405/422 precedence, url_for reversal. +// Match order: static segments first (more static wins), then converters in +// registration order, then path catch-alls. Precedence: full match, then +// 405 (skeleton path exists, method missing), then 422 (path converter +// failure), then 404. Method mismatch sets the Allow header. + +#include +#include +#include +#include +#include +#include + +#include "../../governance/caps.h" +#include "../../governance/versions/ApiVersion.h" +#include "../../tenancy/resolver.h" +#include "../caps.h" +#include "../errors/http_error.h" +#include "../http_msg/method.h" +#include "../http_msg/response.h" +#include "../validate/errors.h" +#include "path_compiler.h" +#include "route.h" + +namespace molo::app::routing { + +namespace detail { + +// Strict IMF-fixdate ("Wed, 21 Oct 2015 07:28:00 GMT") to unix seconds. +// Returns -1 when unparseable. Self-contained so router.h gains no deps. +inline long ParseImfFixdate(const std::string& text) { + if (text.size() != 29) return -1; + if (text[3] != ',' || text[4] != ' ' || text[7] != ' ' || text[11] != ' ' || + text[16] != ' ' || text[19] != ':' || text[22] != ':' || text[25] != ' ') { + return -1; + } + if (text.compare(26, 3, "GMT") != 0) return -1; + auto digits = [&](std::size_t pos, std::size_t len, int& out) { + int v = 0; + for (std::size_t i = 0; i < len; ++i) { + char c = text[pos + i]; + if (c < '0' || c > '9') return false; + v = v * 10 + (c - '0'); + } + out = v; + return true; + }; + int day, year, hh, mm, ss; + if (!digits(5, 2, day) || !digits(12, 4, year) || !digits(17, 2, hh) || + !digits(20, 2, mm) || !digits(23, 2, ss)) { + return -1; + } + static const char* kMonths[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + int mon = 0; + for (int i = 0; i < 12; ++i) { + if (text.compare(8, 3, kMonths[i]) == 0) { + mon = i + 1; + break; + } + } + if (mon == 0 || day < 1 || day > 31 || hh > 23 || mm > 59 || ss > 60) return -1; + // Days from civil (Howard Hinnant), 1970-01-01 based. + int y = year; + unsigned m = static_cast(mon); + unsigned d = static_cast(day); + y -= (m <= 2); + const int era = (y >= 0 ? y : y - 399) / 400; + const unsigned yoe = static_cast(y - era * 400); + const unsigned doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + const long days = static_cast(era) * 146097 + static_cast(doe) - 719468; + return days * 86400L + hh * 3600L + mm * 60L + ss; +} + +} // namespace detail + +struct MatchResult { + const Route* route = nullptr; + std::map params; + std::vector allow; + std::vector path_errors; + bool skeleton_matched = false; +}; + +class Router { + public: + const Route& add(std::set methods, std::string pattern, Handler handler, + RouteOptions options = {}) { + if (frozen_) throw PathCompileError("router frozen: cannot add routes after startup"); + if (routes_.size() + 1 > caps::kMaxRoutesPerApp) { + throw PathCompileError("too many routes in router"); + } + std::size_t index = next_index_++; + std::size_t at = routes_.size(); + routes_.emplace_back(std::move(methods), pattern, std::move(handler), options, index); + const Route& added = routes_.at(at); + if (options.slash_insensitive) { + std::string alt; + if (pattern.size() > 1 && pattern.back() == '/') { + alt = pattern.substr(0, pattern.size() - 1); + } else if (pattern == "/") { + alt = ""; + } else { + alt = pattern + "/"; + } + if (!alt.empty()) { + if (routes_.size() + 1 > caps::kMaxRoutesPerApp) { + throw PathCompileError("too many routes in router"); + } + RouteOptions redirect_opts; + redirect_opts.redirect_target = pattern; + std::string target = pattern; + Handler redirect = [target](http_msg::Request& req) + -> async::Task { + std::string url = target; + if (!req.query_raw().empty()) url += "?" + req.query_raw(); + co_return http_msg::RedirectResponse::permanent(url); + }; + routes_.emplace_back(added.methods(), alt, std::move(redirect), redirect_opts, + next_index_++); + } + } + rebuild_order(); + return routes_.at(at); + } + + const Route& get(std::string pattern, Handler handler, RouteOptions options = {}) { + return add({http_msg::Method::GET}, std::move(pattern), std::move(handler), + std::move(options)); + } + const Route& post(std::string pattern, Handler handler, RouteOptions options = {}) { + return add({http_msg::Method::POST}, std::move(pattern), std::move(handler), + std::move(options)); + } + const Route& put(std::string pattern, Handler handler, RouteOptions options = {}) { + return add({http_msg::Method::PUT}, std::move(pattern), std::move(handler), + std::move(options)); + } + const Route& patch(std::string pattern, Handler handler, RouteOptions options = {}) { + return add({http_msg::Method::PATCH}, std::move(pattern), std::move(handler), + std::move(options)); + } + const Route& del(std::string pattern, Handler handler, RouteOptions options = {}) { + return add({http_msg::Method::DELETE}, std::move(pattern), std::move(handler), + std::move(options)); + } + const Route& head(std::string pattern, Handler handler, RouteOptions options = {}) { + return add({http_msg::Method::HEAD}, std::move(pattern), std::move(handler), + std::move(options)); + } + const Route& options(std::string pattern, Handler handler, RouteOptions opts = {}) { + return add({http_msg::Method::OPTIONS}, std::move(pattern), std::move(handler), + std::move(opts)); + } + + void include_router(std::string prefix, const Router& other, + std::vector extra_tags = {}) { + if (frozen_) throw PathCompileError("router frozen: cannot include after startup"); + if (prefix.empty() || prefix[0] != '/') { + throw PathCompileError("include_router prefix must start with /"); + } + for (const Route& r : other.routes_) { + if (routes_.size() + 1 > caps::kMaxRoutesPerApp) { + throw PathCompileError("too many routes in router"); + } + Route merged = r.with_prefix(prefix, extra_tags); + routes_.emplace_back(merged.methods(), merged.pattern(), merged.handler(), + merged.options(), next_index_++); + } + rebuild_order(); + } + + MatchResult match(http_msg::Method method, std::string_view path) const { + if (path.size() > caps::kMaxPathLength) { + throw errors::UriTooLong("path exceeds length cap"); + } + MatchResult out; + std::set allow_set; + for (const Route* route : order_) { + PathMatch pm = match_compiled(route->compiled(), path); + if (pm.full) { + if (route->allows(method)) { + out.route = route; + out.params = std::move(pm.params); + return out; + } + out.skeleton_matched = true; + for (const auto& a : route->allow_list()) allow_set.insert(a); + continue; + } + if (pm.skeleton) { + out.skeleton_matched = true; + if (!route->allows(method)) { + for (const auto& a : route->allow_list()) allow_set.insert(a); + } else if (out.path_errors.empty() && !pm.errors.empty()) { + out.path_errors = std::move(pm.errors); + } + } + } + out.allow.assign(allow_set.begin(), allow_set.end()); + return out; + } + + std::string url_for(std::string_view name, + const std::map& params = {}, + std::string_view query = "") const { + for (const Route& r : routes_) { + if (r.options().name == name && !r.is_slash_redirect()) { + std::string url = render_compiled(r.compiled(), params, name); + if (!query.empty()) { + url += '?'; + url += query; + } + return url; + } + } + throw errors::NotFound("url_for: unknown route '" + std::string(name) + "'"); + } + + bool has_route_name(std::string_view name) const { + for (const Route& r : routes_) { + if (!r.is_slash_redirect() && r.options().name == name) return true; + } + return false; + } + + // Tenant-aware helpers (PATCHES 11.14): strip the /t/ prefix and + // build URLs that carry the tenant. + static std::pair strip_tenant_prefix( + const std::string& path) { + return ::molo::tenancy::Resolver::StripTenantPrefix(path); + } + + std::string url_for_tenant(const std::string& tenant, std::string_view name, + const std::map& params = {}, + std::string_view query = "") const { + return "/t/" + tenant + url_for(name, params, query); + } + + const std::vector& routes() const { return routes_; } + std::size_t size() const { return routes_.size(); } + + // Application::startup freezes the router so match pointers stay stable. + void freeze() { frozen_ = true; } + bool frozen() const { return frozen_; } + + // Governance hook: validates /v{n} prefixes, at most 3 live majors, + // Sunset >= Deprecation, and declared scopes on scoped routes. Routes + // without governance fields set are skipped. Throws std::runtime_error + // (Application::startup rethrows as StartupError). + void validate_governance() const { + std::set live_majors; + for (const Route& r : routes_) { + if (r.is_slash_redirect()) continue; + const RouteOptions& o = r.options(); + const bool managed = o.gov_auth != RouteOptions::GovAuth::kNone || + o.gov_api_version != 0 || o.gov_deprecation_unix != 0 || + !o.gov_sunset_httpdate.empty() || !o.gov_tier_override.empty(); + if (!managed) continue; + const int major = + ::molo::govern::versions::ApiVersion::ParseMajor(r.pattern()); + if (major == 0) { + throw std::runtime_error("governance: route '" + r.pattern() + + "' is gov-managed but has no /v{n} prefix"); + } + if (o.gov_api_version != 0 && o.gov_api_version != major) { + throw std::runtime_error("governance: route '" + r.pattern() + + "' gov_api_version does not match its /v{n} prefix"); + } + live_majors.insert(major); + if (o.gov_auth == RouteOptions::GovAuth::kKeyScopes && + o.gov_required_scopes.empty()) { + throw std::runtime_error("governance: route '" + r.pattern() + + "' requires scopes but declares none"); + } + if (!o.gov_sunset_httpdate.empty()) { + const long sunset = detail::ParseImfFixdate(o.gov_sunset_httpdate); + if (sunset < 0) { + throw std::runtime_error("governance: route '" + r.pattern() + + "' has an unparseable gov_sunset_httpdate"); + } + if (o.gov_deprecation_unix != 0 && sunset < o.gov_deprecation_unix) { + throw std::runtime_error("governance: route '" + r.pattern() + + "' sunset predates deprecation"); + } + } + } + if (static_cast(live_majors.size()) > + ::molo::govern::Caps::kMaxLiveMajors) { + throw std::runtime_error("governance: more than 3 live API majors"); + } + } + + private: + void rebuild_order() { + order_.clear(); + for (const Route& r : routes_) order_.push_back(&r); + std::stable_sort(order_.begin(), order_.end(), [](const Route* a, const Route* b) { + int ca = a->compiled().has_catchall ? 1 : 0; + int cb = b->compiled().has_catchall ? 1 : 0; + if (ca != cb) return ca < cb; + if (a->compiled().static_count != b->compiled().static_count) { + return a->compiled().static_count > b->compiled().static_count; + } + return a->index() < b->index(); + }); + } + + std::vector routes_; + std::vector order_; + std::size_t next_index_ = 0; + bool frozen_ = false; +}; + +} // namespace molo::app::routing diff --git a/src/app/seams/request_id.h b/src/app/seams/request_id.h new file mode 100644 index 0000000..1ce8636 --- /dev/null +++ b/src/app/seams/request_id.h @@ -0,0 +1,23 @@ +#pragma once +// Spec: molo::app::seams::request_id. Correlation ID seam owned by +// 02-observability. Request ID is read from X-Request-ID when present, else +// generated. Middleware must preserve it (plan 05, section 6 Middleware). + +#include +#include +#include +#include + +namespace molo::app::seams { + +inline std::string generate_request_id() { + static std::atomic counter{1}; + std::uint64_t n = counter.fetch_add(1, std::memory_order_relaxed); + char buf[32]; + std::snprintf(buf, sizeof(buf), "req-%016llx", (unsigned long long)n); + return std::string(buf); +} + +inline constexpr const char* kRequestIdHeader = "X-Request-ID"; + +} // namespace molo::app::seams diff --git a/src/app/seams/session.h b/src/app/seams/session.h new file mode 100644 index 0000000..a608dcf --- /dev/null +++ b/src/app/seams/session.h @@ -0,0 +1,62 @@ +#pragma once +// Spec: molo::app::seams::session. Session storage seam owned by +// 03-security. The app layer (flash messages, views) programs against +// SessionStore only. InMemorySessionStore is the standalone default so this +// part compiles and tests without siblings; production wires the security +// store through Application::set_session_store. + +#include +#include +#include +#include +#include + +#include "../caps.h" + +namespace molo::app::seams { + +struct FlashMessage { + std::string level; + std::string text; +}; + +class SessionStore { + public: + virtual ~SessionStore() = default; + virtual void add_flash(const std::string& session_id, FlashMessage msg) = 0; + virtual std::vector take_flashes(const std::string& session_id) = 0; + virtual void clear(const std::string& session_id) = 0; +}; + +class InMemorySessionStore : public SessionStore { + public: + void add_flash(const std::string& session_id, FlashMessage msg) override { + std::lock_guard lock(mutex_); + auto& vec = flashes_[session_id]; + if (vec.size() < caps::kMaxFlashMessages) vec.push_back(std::move(msg)); + } + + std::vector take_flashes(const std::string& session_id) override { + std::lock_guard lock(mutex_); + auto it = flashes_.find(session_id); + if (it == flashes_.end()) return {}; + std::vector out = std::move(it->second); + flashes_.erase(it); + return out; + } + + void clear(const std::string& session_id) override { + std::lock_guard lock(mutex_); + flashes_.erase(session_id); + } + + private: + std::mutex mutex_; + std::map> flashes_; +}; + +inline std::shared_ptr default_session_store() { + return std::make_shared(); +} + +} // namespace molo::app::seams diff --git a/src/app/templates/ast.h b/src/app/templates/ast.h new file mode 100644 index 0000000..cb23db4 --- /dev/null +++ b/src/app/templates/ast.h @@ -0,0 +1,116 @@ +#pragma once +// Spec: molo::app::templates AST. Plain data nodes built by the parser, +// executed by the renderer. Expressions cover literals, names, attribute +// and index access, calls, arithmetic, comparisons, logic, membership, +// filters and tests. Gaps vs Jinja (documented): no ternary, no ** power, +// no string % formatting, no super(), no block-level set. + +#include +#include +#include +#include +#include +#include + +#include "../validate/value.h" + +namespace molo::app::templates { + +struct Expr; +using ExprPtr = std::shared_ptr; + +struct Expr { + enum class Kind { + Literal, + Name, + Attr, + Index, + Call, + Unary, + Binary, + Compare, + BoolOp, + Not, + In, + Filter, + Test, + List, + Dict + }; + Kind kind = Kind::Literal; + validate::Value literal; + std::string name; // Name, Attr attribute, Call callee, Filter/Test name + ExprPtr left; + ExprPtr right; + ExprPtr operand; + ExprPtr seq; // In: left in seq + std::string op; + bool negated = false; // In (not in), Test (is not) + std::vector args; + std::vector> kwargs; + std::vector> pairs; // Dict +}; + +struct Node; +using NodePtr = std::shared_ptr; + +struct IfBranch { + ExprPtr cond; // null cond means else + std::vector body; +}; + +struct Node { + enum class Kind { + Text, + Output, + If, + For, + Block, + Extends, + Include, + Set, + Macro, + Autoescape + }; + Kind kind = Kind::Text; + std::string text; // Text content, Block/Macro/Set name + ExprPtr expr; // Output, Extends, Include, Set value + std::vector branches; // If + std::vector targets; // For + ExprPtr seq; // For + std::vector body; // For, Block, Macro, Autoescape + std::vector else_body; // For + std::vector params; // Macro + std::vector> defaults; // Macro + bool flag = false; // Autoescape enabled +}; + +struct MacroDef { + std::string name; + std::vector params; + std::vector> defaults; + std::vector body; +}; + +inline ExprPtr lit(validate::Value v) { + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Literal; + e->literal = std::move(v); + return e; +} + +inline ExprPtr name_expr(std::string n) { + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Name; + e->name = std::move(n); + return e; +} + +inline NodePtr text_node(std::string t) { + NodePtr n = std::make_shared(); + n->kind = Node::Kind::Text; + n->text = std::move(t); + return n; +} + +} // namespace molo::app::templates diff --git a/src/app/templates/cache.h b/src/app/templates/cache.h new file mode 100644 index 0000000..eee49e6 --- /dev/null +++ b/src/app/templates/cache.h @@ -0,0 +1,62 @@ +#pragma once +// Spec: molo::app::templates compiled-template cache. LRU with max 512 +// entries (caps.h). Keyed by template name. + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "ast.h" + +namespace molo::app::templates { + +struct CompiledTemplate { + std::string name; + std::vector nodes; + bool autoescape = false; +}; + +class TemplateCache { + public: + std::shared_ptr get(const std::string& name) { + auto it = entries_.find(name); + if (it == entries_.end()) return nullptr; + order_.erase(it->second.second); + order_.push_front(name); + it->second.second = order_.begin(); + return it->second.first; + } + + void put(std::string name, std::shared_ptr compiled) { + auto it = entries_.find(name); + if (it != entries_.end()) { + order_.erase(it->second.second); + entries_.erase(it); + } + order_.push_front(name); + entries_[name] = {std::move(compiled), order_.begin()}; + while (entries_.size() > caps::kMaxCompiledTemplates) { + entries_.erase(order_.back()); + order_.pop_back(); + } + } + + void clear() { + entries_.clear(); + order_.clear(); + } + + std::size_t size() const { return entries_.size(); } + + private: + std::map, + std::list::iterator>> + entries_; + std::list order_; +}; + +} // namespace molo::app::templates diff --git a/src/app/templates/context.h b/src/app/templates/context.h new file mode 100644 index 0000000..70774b3 --- /dev/null +++ b/src/app/templates/context.h @@ -0,0 +1,66 @@ +#pragma once +// Spec: molo::app::templates::Context. Lexical scopes for rendering: a stack +// of variable scopes plus macro definitions. set writes the innermost +// scope; loops, includes and macro calls push and pop. + +#include +#include +#include +#include + +#include "../validate/value.h" +#include "ast.h" + +namespace molo::app::templates { + +struct Scope { + validate::Object vars; + std::map> macros; +}; + +class Context { + public: + Context() { scopes_.emplace_back(); } + + explicit Context(validate::Object globals) { + Scope root; + root.vars = std::move(globals); + scopes_.push_back(std::move(root)); + } + + void push() { scopes_.emplace_back(); } + void pop() { + if (scopes_.size() > 1) scopes_.pop_back(); + } + + void set(const std::string& name, validate::Value value) { + scopes_.back().vars[name] = std::move(value); + } + + void define_macro(std::shared_ptr macro) { + scopes_.back().macros[macro->name] = std::move(macro); + } + + validate::Value lookup(const std::string& name) const { + for (std::size_t i = scopes_.size(); i-- > 0;) { + auto it = scopes_[i].vars.find(name); + if (it != scopes_[i].vars.end()) return it->second; + } + return validate::Value::undefined(); + } + + std::shared_ptr lookup_macro(const std::string& name) const { + for (std::size_t i = scopes_.size(); i-- > 0;) { + auto it = scopes_[i].macros.find(name); + if (it != scopes_[i].macros.end()) return it->second; + } + return nullptr; + } + + const std::vector& scopes() const { return scopes_; } + + private: + std::vector scopes_; +}; + +} // namespace molo::app::templates diff --git a/src/app/templates/environment.h b/src/app/templates/environment.h new file mode 100644 index 0000000..3afe5f8 --- /dev/null +++ b/src/app/templates/environment.h @@ -0,0 +1,129 @@ +#pragma once +// Spec: molo::app::templates::Environment. Reusable template environment +// holding loaders, filters, tests and globals (plan 05, E11). Autoescape is +// ON for .html, .htm, .xml; there is deliberately no global disable flag +// (R5). Undefined policy defaults to Strict (dev default per spec); Chain +// renders empty plus a stats counter. dev_reload bypasses the compiled +// cache so disk edits show on the next render. + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "../validate/value.h" +#include "cache.h" +#include "filters.h" +#include "lexer.h" +#include "loaders.h" +#include "parser.h" +#include "predicates.h" + +namespace molo::app::templates { + +class Renderer; + +enum class UndefinedPolicy { Chain, Strict }; + +struct TemplateStats { + std::size_t renders = 0; + std::size_t cache_hits = 0; + std::size_t undefined_hits = 0; + std::size_t errors = 0; +}; + +class Environment { + public: + Environment() : filters_(builtin_filters()), tests_(builtin_tests()) {} + + void add_loader(std::shared_ptr loader) { loaders_.push_back(std::move(loader)); } + + void set_global(const std::string& name, validate::Value value) { + globals_[name] = std::move(value); + } + void add_filter(const std::string& name, FilterFn fn) { filters_[name] = std::move(fn); } + void add_test(const std::string& name, TestFn fn) { tests_[name] = std::move(fn); } + + void set_undefined_policy(UndefinedPolicy p) { undefined_policy_ = p; } + UndefinedPolicy undefined_policy() const { return undefined_policy_; } + void set_dev_reload(bool on) { dev_reload_ = on; } + bool dev_reload() const { return dev_reload_; } + + const validate::Object& globals() const { return globals_; } + const std::map& filters() const { return filters_; } + const std::map& tests() const { return tests_; } + TemplateStats& stats() { return stats_; } + const TemplateStats& stats() const { return stats_; } + TemplateCache& cache() { return cache_; } + + bool autoescape_for(const std::string& name) const { + std::size_t dot = name.rfind('.'); + if (dot == std::string::npos) return false; + std::string ext = name.substr(dot + 1); + for (char& c : ext) c = (char)std::tolower((unsigned char)c); + return ext == "html" || ext == "htm" || ext == "xml"; + } + + std::string load_source(const std::string& name) const { + for (const auto& loader : loaders_) { + std::optional hit = loader->load(name); + if (hit.has_value()) { + if (hit->size() > caps::kMaxTemplateSize) { + throw TemplateError("template too large: " + name); + } + return *hit; + } + } + throw TemplateError("template not found: " + name); + } + + std::shared_ptr compile_source(const std::string& source, + const std::string& name) const { + if (source.size() > caps::kMaxTemplateSize) { + throw TemplateError("template too large: " + name); + } + std::vector toks = lex_template(source, name); + std::vector nodes = parse_template(toks, name); + auto compiled = std::make_shared(); + compiled->name = name; + compiled->nodes = std::move(nodes); + compiled->autoescape = autoescape_for(name); + return compiled; + } + + std::shared_ptr get_compiled(const std::string& name) { + if (!dev_reload_) { + std::shared_ptr hit = cache_.get(name); + if (hit) { + ++stats_.cache_hits; + return hit; + } + } + std::string source = load_source(name); + std::shared_ptr compiled = compile_source(source, name); + if (!dev_reload_) cache_.put(name, compiled); + return compiled; + } + + // Defined in renderer.h (needs the full Renderer). + std::string render(const std::string& name, validate::Object context = {}); + std::string render_source(const std::string& source, const std::string& name, + validate::Object context = {}); + + private: + std::vector> loaders_; + validate::Object globals_; + std::map filters_; + std::map tests_; + TemplateCache cache_; + UndefinedPolicy undefined_policy_ = UndefinedPolicy::Strict; + bool dev_reload_ = false; + TemplateStats stats_; +}; + +} // namespace molo::app::templates + +#include "renderer.h" diff --git a/src/app/templates/filters.h b/src/app/templates/filters.h new file mode 100644 index 0000000..fab424b --- /dev/null +++ b/src/app/templates/filters.h @@ -0,0 +1,163 @@ +#pragma once +// Spec: molo::app::templates shipped filters (plan 05, section 6 Templates): +// escape, safe, upper, lower, trim, striptags, urlencode, length, default, +// join, date, tojson. Case conversion is ASCII-only (documented gap). + +#include +#include +#include +#include +#include +#include + +#include "../http_msg/html.h" +#include "../http_msg/query.h" +#include "../validate/json.h" +#include "../validate/value.h" +#include "lexer.h" + +namespace molo::app::templates { + +using FilterFn = + std::function&)>; + +inline std::string value_to_text(const validate::Value& v) { + if (v.is_string() || v.is_safe()) return v.as_text(); + if (v.is_null()) return "None"; + if (v.is_undefined()) return ""; + if (const auto* b = std::get_if(&v.storage())) return *b ? "True" : "False"; + if (const auto* i = std::get_if(&v.storage())) return std::to_string(*i); + if (const auto* d = std::get_if(&v.storage())) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "%.17g", *d); + return std::string(buf); + } + return validate::to_json(v); +} + +inline std::string strip_tags_impl(std::string_view s) { + std::string out; + bool in_tag = false; + bool in_comment = false; + for (std::size_t i = 0; i < s.size(); ++i) { + if (!in_tag && !in_comment && s.substr(i, 4) == "") { + in_comment = false; + i += 2; + } + continue; + } + if (!in_tag && s[i] == '<') { + in_tag = true; + continue; + } + if (in_tag && s[i] == '>') { + in_tag = false; + continue; + } + if (!in_tag) out += s[i]; + } + return out; +} + +inline std::map builtin_filters() { + std::map f; + f["escape"] = [](const validate::Value& v, const std::vector&) { + if (v.is_safe()) return v; + return validate::Value::safe(http_msg::escape_html(value_to_text(v))); + }; + f["e"] = f["escape"]; + f["safe"] = [](const validate::Value& v, const std::vector&) { + return validate::Value::safe(value_to_text(v)); + }; + f["upper"] = [](const validate::Value& v, const std::vector&) { + std::string s = value_to_text(v); + for (char& c : s) c = (char)std::toupper((unsigned char)c); + return validate::Value(s); + }; + f["lower"] = [](const validate::Value& v, const std::vector&) { + std::string s = value_to_text(v); + for (char& c : s) c = (char)std::tolower((unsigned char)c); + return validate::Value(s); + }; + f["trim"] = [](const validate::Value& v, const std::vector& args) { + std::string chars = " \t\r\n"; + if (!args.empty()) chars = value_to_text(args[0]); + std::string s = value_to_text(v); + std::size_t a = s.find_first_not_of(chars); + if (a == std::string::npos) return validate::Value(std::string("")); + std::size_t b = s.find_last_not_of(chars); + return validate::Value(s.substr(a, b - a + 1)); + }; + f["striptags"] = [](const validate::Value& v, const std::vector&) { + return validate::Value(strip_tags_impl(value_to_text(v))); + }; + f["urlencode"] = [](const validate::Value& v, const std::vector&) { + return validate::Value(http_msg::percent_encode(value_to_text(v))); + }; + f["length"] = [](const validate::Value& v, const std::vector&) { + if (v.is_string() || v.is_safe()) return validate::Value((long long)v.as_text().size()); + if (const auto* a = std::get_if(&v.storage())) { + return validate::Value((long long)a->size()); + } + if (const auto* o = std::get_if(&v.storage())) { + return validate::Value((long long)o->size()); + } + throw TemplateError("length filter needs string, sequence or mapping"); + }; + f["count"] = f["length"]; + f["default"] = [](const validate::Value& v, const std::vector& args) { + validate::Value fallback = args.empty() ? validate::Value("") : args[0]; + bool boolean = args.size() > 1 && args[1].truthy(); + if (v.is_undefined() || v.is_null()) return fallback; + if (boolean && !v.truthy()) return fallback; + return v; + }; + f["d"] = f["default"]; + f["join"] = [](const validate::Value& v, const std::vector& args) { + std::string delim = args.empty() ? "" : value_to_text(args[0]); + std::string attr; + if (args.size() > 1) attr = value_to_text(args[1]); + const auto* a = std::get_if(&v.storage()); + if (a == nullptr) throw TemplateError("join filter needs a sequence"); + std::string out; + bool first = true; + for (const auto& item : *a) { + if (!first) out += delim; + first = false; + if (!attr.empty() && item.is_object()) { + const auto* o = std::get_if(&item.storage()); + auto it = o->find(attr); + out += (it == o->end()) ? "" : value_to_text(it->second); + } else { + out += value_to_text(item); + } + } + return validate::Value(out); + }; + f["date"] = [](const validate::Value& v, const std::vector& args) { + if (!v.is_number()) throw TemplateError("date filter needs an epoch number"); + std::string fmt = args.empty() ? "%Y-%m-%d %H:%M:%S" : value_to_text(args[0]); + std::time_t t = (std::time_t)v.as_number(); + std::tm tmv{}; +#if defined(_WIN32) + gmtime_s(&tmv, &t); +#else + gmtime_r(&t, &tmv); +#endif + char buf[256]; + if (std::strftime(buf, sizeof(buf), fmt.c_str(), &tmv) == 0) return validate::Value(""); + return validate::Value(std::string(buf)); + }; + f["tojson"] = [](const validate::Value& v, const std::vector&) { + return validate::Value::safe(validate::to_json(v)); + }; + return f; +} + +} // namespace molo::app::templates diff --git a/src/app/templates/lexer.h b/src/app/templates/lexer.h new file mode 100644 index 0000000..c48c04d --- /dev/null +++ b/src/app/templates/lexer.h @@ -0,0 +1,139 @@ +#pragma once +// Spec: molo::app::templates lexer. Splits source into Text, Var ({{ }}), +// Block ({% %}) tokens; comments ({# #}) are dropped. Whitespace control +// dashes ({{- , -}}) strip adjacent whitespace runs. Tag close search +// respects single and double quoted strings. + +#include +#include +#include +#include + +namespace molo::app::templates { + +class TemplateError : public std::runtime_error { + public: + explicit TemplateError(const std::string& msg) : std::runtime_error(msg) {} +}; + +class UndefinedError : public TemplateError { + public: + explicit UndefinedError(const std::string& msg) : TemplateError(msg) {} +}; + +enum class TokenKind { Text, Var, Block }; + +struct Token { + TokenKind kind = TokenKind::Text; + std::string text; + std::size_t line = 1; +}; + +inline bool is_space_char(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; +} + +inline void rstrip_token(std::string& s) { + while (!s.empty() && is_space_char(s.back())) s.pop_back(); +} + +inline std::size_t find_tag_close(std::string_view src, std::size_t from, char close_first, + char close_second) { + char quote = 0; + for (std::size_t i = from; i + 1 < src.size(); ++i) { + char c = src[i]; + if (quote) { + if (c == '\\') { + ++i; + continue; + } + if (c == quote) quote = 0; + continue; + } + if (c == '\'' || c == '"') { + quote = c; + continue; + } + if (c == close_first && src[i + 1] == close_second) return i; + } + return std::string_view::npos; +} + +inline std::vector lex_template(std::string_view src, std::string_view source_name) { + std::vector out; + std::size_t line = 1; + std::size_t i = 0; + auto push_text = [&](std::string_view t) { + if (t.empty()) return; + Token tok; + tok.kind = TokenKind::Text; + tok.text = std::string(t); + tok.line = line; + for (char c : t) { + if (c == '\n') ++line; + } + out.push_back(std::move(tok)); + }; + while (i < src.size()) { + std::size_t var_at = src.find("{{", i); + std::size_t blk_at = src.find("{%", i); + std::size_t com_at = src.find("{#", i); + std::size_t next = std::string_view::npos; + if (var_at != std::string_view::npos) next = var_at; + if (blk_at != std::string_view::npos && (next == std::string_view::npos || blk_at < next)) { + next = blk_at; + } + if (com_at != std::string_view::npos && (next == std::string_view::npos || com_at < next)) { + next = com_at; + } + if (next == std::string_view::npos) { + push_text(src.substr(i)); + break; + } + push_text(src.substr(i, next - i)); + if (next == com_at) { + std::size_t end = src.find("#}", next + 2); + if (end == std::string_view::npos) { + throw TemplateError("unterminated comment in " + std::string(source_name)); + } + for (std::size_t k = next; k < end + 2; ++k) { + if (src[k] == '\n') ++line; + } + i = end + 2; + continue; + } + bool is_var = (next == var_at); + char cf = is_var ? '}' : '%'; + std::size_t open_end = next + 2; + bool lstrip = open_end < src.size() && src[open_end] == '-'; + if (lstrip) ++open_end; + std::size_t close = find_tag_close(src, open_end, cf, '}'); + if (close == std::string_view::npos) { + throw TemplateError("unterminated tag in " + std::string(source_name)); + } + std::size_t inner_end = close; + bool rstrip = inner_end > open_end && src[inner_end - 1] == '-'; + if (rstrip) --inner_end; + if (lstrip && !out.empty() && out.back().kind == TokenKind::Text) { + rstrip_token(out.back().text); + } + Token tok; + tok.kind = is_var ? TokenKind::Var : TokenKind::Block; + tok.text = std::string(src.substr(open_end, inner_end - open_end)); + tok.line = line; + for (std::size_t k = next; k < close + 2; ++k) { + if (src[k] == '\n') ++line; + } + out.push_back(std::move(tok)); + i = close + 2; + if (rstrip) { + while (i < src.size() && is_space_char(src[i])) { + if (src[i] == '\n') ++line; + ++i; + } + } + } + return out; +} + +} // namespace molo::app::templates diff --git a/src/app/templates/loaders.h b/src/app/templates/loaders.h new file mode 100644 index 0000000..f7de349 --- /dev/null +++ b/src/app/templates/loaders.h @@ -0,0 +1,115 @@ +#pragma once +// Spec: molo::app::templates loaders. DictLoader (in-memory map), +// EmbeddedLoader (populated by the 09-cli-ops asset embedder) and +// FileLoader (directory root with path traversal protection: rejects +// absolute paths, drive letters, backslashes, null bytes and .. segments). + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lexer.h" + +namespace molo::app::templates { + +class Loader { + public: + virtual ~Loader() = default; + virtual std::optional load(const std::string& name) = 0; + virtual std::string kind() const = 0; +}; + +class DictLoader : public Loader { + public: + explicit DictLoader(std::map templates = {}) + : templates_(std::move(templates)) {} + + void set(const std::string& name, std::string source) { templates_[name] = std::move(source); } + + std::optional load(const std::string& name) override { + auto it = templates_.find(name); + if (it == templates_.end()) return std::nullopt; + return it->second; + } + std::string kind() const override { return "dict"; } + + private: + std::map templates_; +}; + +class EmbeddedLoader : public Loader { + public: + void embed(const std::string& name, std::string source) { + assets_[name] = std::move(source); + } + + std::optional load(const std::string& name) override { + auto it = assets_.find(name); + if (it == assets_.end()) return std::nullopt; + return it->second; + } + std::string kind() const override { return "embedded"; } + + private: + std::map assets_; +}; + +class FileLoader : public Loader { + public: + explicit FileLoader(std::string root) : root_(std::move(root)) {} + + std::optional load(const std::string& name) override { + std::string safe = sanitize(name); + if (safe.empty()) throw TemplateError("unsafe template path: " + name); + std::string full = root_ + "/" + safe; + std::ifstream in(full, std::ios::binary); + if (!in) return std::nullopt; + std::ostringstream ss; + ss << in.rdbuf(); + return ss.str(); + } + std::string kind() const override { return "file"; } + + // Exposed for tests: sanitized relative path or "" when rejected. + static std::string sanitize(std::string_view name) { + if (name.empty() || name.size() > 1024) return ""; + if (name[0] == '/' || name[0] == '\\') return ""; + if (name.size() >= 2 && name[1] == ':') return ""; + std::vector parts; + std::size_t i = 0; + while (i <= name.size()) { + std::size_t slash = name.find('/', i); + if (slash == std::string_view::npos) slash = name.size(); + std::string_view seg = name.substr(i, slash - i); + if (seg.empty() || seg == ".") { + // Skip. + } else if (seg == "..") { + return ""; + } else { + for (char c : seg) { + if (c == '\\' || c == '\0') return ""; + } + parts.emplace_back(seg); + } + if (slash == name.size()) break; + i = slash + 1; + } + if (parts.empty()) return ""; + std::string out; + for (std::size_t k = 0; k < parts.size(); ++k) { + if (k) out += '/'; + out += parts[k]; + } + return out; + } + + private: + std::string root_; +}; + +} // namespace molo::app::templates diff --git a/src/app/templates/parser.h b/src/app/templates/parser.h new file mode 100644 index 0000000..bd94045 --- /dev/null +++ b/src/app/templates/parser.h @@ -0,0 +1,823 @@ +#pragma once +// Spec: molo::app::templates parser. Builds the AST from lexer tokens: +// extends, block/endblock, include, if/elif/else/endif, for/else/endfor, +// set, macro/endmacro, autoescape/endautoescape, plus {{ }} output. +// Expression grammar with precedence: or, and, not, comparison (== != < > +// <= >= in not-in is is-not), additive (+ - ~), multiplicative (* / // %), +// unary, postfix (.attr [index] (call) |filter), primary. + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "ast.h" +#include "lexer.h" + +namespace molo::app::templates { + +struct ExprToken { + enum class Kind { End, Number, String, Name, Op, Punct }; + Kind kind = Kind::End; + std::string text; +}; + +inline std::vector lex_expr(std::string_view s, std::string_view source_name) { + std::vector out; + std::size_t i = 0; + while (i < s.size()) { + char c = s[i]; + if (c == ' ' || c == '\t' || c == '\r' || c == '\n') { + ++i; + continue; + } + if (std::isdigit((unsigned char)c) || (c == '.' && i + 1 < s.size() && + std::isdigit((unsigned char)s[i + 1]))) { + std::size_t start = i; + while (i < s.size() && std::isdigit((unsigned char)s[i])) ++i; + if (i < s.size() && s[i] == '.') { + ++i; + while (i < s.size() && std::isdigit((unsigned char)s[i])) ++i; + } + if (i < s.size() && (s[i] == 'e' || s[i] == 'E')) { + std::size_t save = i; + ++i; + if (i < s.size() && (s[i] == '+' || s[i] == '-')) ++i; + if (i < s.size() && std::isdigit((unsigned char)s[i])) { + while (i < s.size() && std::isdigit((unsigned char)s[i])) ++i; + } else { + i = save; + } + } + out.push_back({ExprToken::Kind::Number, std::string(s.substr(start, i - start))}); + continue; + } + if (c == '\'' || c == '"') { + char quote = c; + ++i; + std::string val; + bool closed = false; + while (i < s.size()) { + char d = s[i++]; + if (d == '\\' && i < s.size()) { + char e = s[i++]; + switch (e) { + case 'n': val += '\n'; break; + case 't': val += '\t'; break; + case 'r': val += '\r'; break; + case '\\': val += '\\'; break; + case '\'': val += '\''; break; + case '"': val += '"'; break; + default: val += e; break; + } + continue; + } + if (d == quote) { + closed = true; + break; + } + val += d; + } + if (!closed) throw TemplateError("unterminated string in " + std::string(source_name)); + out.push_back({ExprToken::Kind::String, val}); + continue; + } + if (std::isalpha((unsigned char)c) || c == '_') { + std::size_t start = i; + while (i < s.size() && (std::isalnum((unsigned char)s[i]) || s[i] == '_')) ++i; + out.push_back({ExprToken::Kind::Name, std::string(s.substr(start, i - start))}); + continue; + } + if (c == '=' && i + 1 < s.size() && s[i + 1] == '=') { + out.push_back({ExprToken::Kind::Op, "=="}); + i += 2; + continue; + } + if (c == '!' && i + 1 < s.size() && s[i + 1] == '=') { + out.push_back({ExprToken::Kind::Op, "!="}); + i += 2; + continue; + } + if (c == '<' && i + 1 < s.size() && s[i + 1] == '=') { + out.push_back({ExprToken::Kind::Op, "<="}); + i += 2; + continue; + } + if (c == '>' && i + 1 < s.size() && s[i + 1] == '=') { + out.push_back({ExprToken::Kind::Op, ">="}); + i += 2; + continue; + } + if (c == '/' && i + 1 < s.size() && s[i + 1] == '/') { + out.push_back({ExprToken::Kind::Op, "//"}); + i += 2; + continue; + } + if (c == '<' || c == '>' || c == '+' || c == '-' || c == '*' || c == '/' || c == '%' || + c == '~' || c == '=') { + out.push_back({ExprToken::Kind::Op, std::string(1, c)}); + ++i; + continue; + } + if (c == '(' || c == ')' || c == '[' || c == ']' || c == '{' || c == '}' || c == ',' || + c == '.' || c == '|' || c == ':') { + out.push_back({ExprToken::Kind::Punct, std::string(1, c)}); + ++i; + continue; + } + throw TemplateError("bad character in expression in " + std::string(source_name)); + } + out.push_back({ExprToken::Kind::End, ""}); + return out; +} + +class ExprParser { + public: + ExprParser(const std::vector& toks, std::string_view source_name) + : toks_(toks), source_(source_name) {} + + ExprPtr parse() { + ExprPtr e = parse_or(0); + if (peek().kind != ExprToken::Kind::End) { + throw TemplateError("trailing tokens in expression in " + std::string(source_)); + } + return e; + } + + private: + const ExprToken& peek() const { return toks_[pos_]; } + const ExprToken& peek_at(std::size_t ahead) const { + std::size_t p = pos_ + ahead; + return toks_[p < toks_.size() ? p : toks_.size() - 1]; + } + bool accept_name(const std::string& w) { + if (peek().kind == ExprToken::Kind::Name && peek().text == w) { + ++pos_; + return true; + } + return false; + } + bool accept_op(const std::string& o) { + if (peek().kind == ExprToken::Kind::Op && peek().text == o) { + ++pos_; + return true; + } + return false; + } + bool accept_punct(const std::string& p) { + if (peek().kind == ExprToken::Kind::Punct && peek().text == p) { + ++pos_; + return true; + } + return false; + } + void expect_punct(const std::string& p) { + if (!accept_punct(p)) { + throw TemplateError("expected '" + p + "' in " + std::string(source_)); + } + } + + void guard_depth(std::size_t depth) { + if (depth > caps::kMaxExprDepth) { + throw TemplateError("expression too deep in " + std::string(source_)); + } + } + + ExprPtr parse_or(std::size_t depth) { + guard_depth(depth); + ExprPtr left = parse_and(depth + 1); + while (accept_name("or")) { + ExprPtr right = parse_and(depth + 1); + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::BoolOp; + e->op = "or"; + e->left = left; + e->right = right; + left = e; + } + return left; + } + + ExprPtr parse_and(std::size_t depth) { + guard_depth(depth); + ExprPtr left = parse_not(depth + 1); + while (accept_name("and")) { + ExprPtr right = parse_not(depth + 1); + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::BoolOp; + e->op = "and"; + e->left = left; + e->right = right; + left = e; + } + return left; + } + + ExprPtr parse_not(std::size_t depth) { + guard_depth(depth); + if (accept_name("not")) { + // "not in" is handled in comparison; rewind if followed by "in". + ExprPtr inner = parse_not(depth + 1); + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Not; + e->operand = inner; + return e; + } + return parse_comparison(depth + 1); + } + + ExprPtr parse_comparison(std::size_t depth) { + guard_depth(depth); + ExprPtr left = parse_additive(depth + 1); + if (peek().kind == ExprToken::Kind::Op && + (peek().text == "==" || peek().text == "!=" || peek().text == "<" || + peek().text == ">" || peek().text == "<=" || peek().text == ">=")) { + std::string op = peek().text; + ++pos_; + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Compare; + e->op = op; + e->left = left; + e->right = parse_additive(depth + 1); + return e; + } + if (peek().kind == ExprToken::Kind::Name && peek().text == "in") { + ++pos_; + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::In; + e->left = left; + e->seq = parse_additive(depth + 1); + return e; + } + if (peek().kind == ExprToken::Kind::Name && peek().text == "not" && + peek_at(1).kind == ExprToken::Kind::Name && peek_at(1).text == "in") { + pos_ += 2; + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::In; + e->negated = true; + e->left = left; + e->seq = parse_additive(depth + 1); + return e; + } + if (peek().kind == ExprToken::Kind::Name && peek().text == "is") { + ++pos_; + bool neg = accept_name("not"); + if (peek().kind != ExprToken::Kind::Name) { + throw TemplateError("expected test name after 'is' in " + std::string(source_)); + } + std::string test = peek().text; + ++pos_; + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Test; + e->name = test; + e->negated = neg; + e->operand = left; + if (accept_punct("(")) { + e->args = parse_args(); + expect_punct(")"); + } + return e; + } + return left; + } + + ExprPtr parse_additive(std::size_t depth) { + guard_depth(depth); + ExprPtr left = parse_multiplicative(depth + 1); + while (peek().kind == ExprToken::Kind::Op && + (peek().text == "+" || peek().text == "-" || peek().text == "~")) { + std::string op = peek().text; + ++pos_; + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Binary; + e->op = op; + e->left = left; + e->right = parse_multiplicative(depth + 1); + left = e; + } + return left; + } + + ExprPtr parse_multiplicative(std::size_t depth) { + guard_depth(depth); + ExprPtr left = parse_unary(depth + 1); + while (peek().kind == ExprToken::Kind::Op && + (peek().text == "*" || peek().text == "/" || peek().text == "//" || + peek().text == "%")) { + std::string op = peek().text; + ++pos_; + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Binary; + e->op = op; + e->left = left; + e->right = parse_unary(depth + 1); + left = e; + } + return left; + } + + ExprPtr parse_unary(std::size_t depth) { + guard_depth(depth); + if (accept_op("-")) { + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Unary; + e->op = "-"; + e->operand = parse_unary(depth + 1); + return e; + } + if (accept_op("+")) { + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Unary; + e->op = "+"; + e->operand = parse_unary(depth + 1); + return e; + } + return parse_postfix(depth + 1); + } + + ExprPtr parse_postfix(std::size_t depth) { + guard_depth(depth); + ExprPtr base = parse_primary(depth + 1); + while (true) { + if (accept_punct(".")) { + if (peek().kind != ExprToken::Kind::Name) { + throw TemplateError("expected attribute name in " + std::string(source_)); + } + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Attr; + e->operand = base; + e->name = peek().text; + ++pos_; + base = e; + continue; + } + if (accept_punct("[")) { + ExprPtr idx = parse_or(depth + 1); + expect_punct("]"); + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Index; + e->operand = base; + e->right = idx; + base = e; + continue; + } + if (accept_punct("(")) { + std::vector args; + std::vector> kwargs; + parse_call_args(args, kwargs, depth + 1); + expect_punct(")"); + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Call; + e->operand = base; + e->args = std::move(args); + e->kwargs = std::move(kwargs); + base = e; + continue; + } + if (accept_punct("|")) { + if (peek().kind != ExprToken::Kind::Name) { + throw TemplateError("expected filter name in " + std::string(source_)); + } + std::string fname = peek().text; + ++pos_; + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Filter; + e->name = fname; + e->operand = base; + if (accept_punct("(")) { + e->args = parse_args(depth + 1); + expect_punct(")"); + } + base = e; + continue; + } + return base; + } + } + + ExprPtr parse_primary(std::size_t depth) { + guard_depth(depth); + const ExprToken& t = peek(); + if (t.kind == ExprToken::Kind::Number) { + ++pos_; + if (t.text.find_first_of(".eE") == std::string::npos) { + return lit(validate::Value((long long)std::stoll(t.text))); + } + return lit(validate::Value(std::stod(t.text))); + } + if (t.kind == ExprToken::Kind::String) { + ++pos_; + return lit(validate::Value(t.text)); + } + if (t.kind == ExprToken::Kind::Name) { + ++pos_; + if (t.text == "true" || t.text == "True") return lit(validate::Value(true)); + if (t.text == "false" || t.text == "False") return lit(validate::Value(false)); + if (t.text == "none" || t.text == "None" || t.text == "null") { + return lit(validate::Value::null()); + } + return name_expr(t.text); + } + if (accept_punct("(")) { + ExprPtr e = parse_or(depth + 1); + expect_punct(")"); + return e; + } + if (accept_punct("[")) { + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::List; + if (!accept_punct("]")) { + while (true) { + e->args.push_back(parse_or(depth + 1)); + if (accept_punct("]")) break; + expect_punct(","); + if (peek().kind == ExprToken::Kind::Punct && peek().text == "]") { + ++pos_; + break; + } + } + } + return e; + } + if (accept_punct("{")) { + ExprPtr e = std::make_shared(); + e->kind = Expr::Kind::Dict; + if (!accept_punct("}")) { + while (true) { + ExprPtr k = parse_or(depth + 1); + expect_punct(":"); + ExprPtr v = parse_or(depth + 1); + e->pairs.emplace_back(k, v); + if (accept_punct("}")) break; + expect_punct(","); + if (peek().kind == ExprToken::Kind::Punct && peek().text == "}") { + ++pos_; + break; + } + } + } + return e; + } + throw TemplateError("unexpected token in expression in " + std::string(source_)); + } + + std::vector parse_args(std::size_t depth = 0) { + std::vector args; + if (peek().kind == ExprToken::Kind::Punct && peek().text == ")") return args; + while (true) { + args.push_back(parse_or(depth + 1)); + if (peek().kind == ExprToken::Kind::Punct && peek().text == ")") return args; + expect_punct(","); + } + } + + void parse_call_args(std::vector& args, + std::vector>& kwargs, std::size_t depth) { + if (peek().kind == ExprToken::Kind::Punct && peek().text == ")") return; + while (true) { + if (peek().kind == ExprToken::Kind::Name && peek_at(1).kind == ExprToken::Kind::Op && + peek_at(1).text == "=") { + std::string key = peek().text; + pos_ += 2; + kwargs.emplace_back(key, parse_or(depth + 1)); + } else { + args.push_back(parse_or(depth + 1)); + } + if (peek().kind == ExprToken::Kind::Punct && peek().text == ")") return; + expect_punct(","); + } + } + + const std::vector& toks_; + std::string_view source_; + std::size_t pos_ = 0; +}; + +inline ExprPtr parse_expression(std::string_view text, std::string_view source_name) { + std::vector toks = lex_expr(text, source_name); + ExprParser parser(toks, source_name); + return parser.parse(); +} + +class TemplateParser { + public: + TemplateParser(const std::vector& toks, std::string_view source_name) + : toks_(toks), source_(source_name) {} + + std::vector parse() { + std::vector body = parse_body({}); + if (pos_ != toks_.size()) { + throw TemplateError("unexpected end tag in " + std::string(source_)); + } + return body; + } + + private: + static std::string_view first_word(std::string_view s) { + std::size_t i = 0; + while (i < s.size() && (s[i] == ' ' || s[i] == '\t')) ++i; + std::size_t start = i; + while (i < s.size() && s[i] != ' ' && s[i] != '\t') ++i; + return s.substr(start, i - start); + } + + static std::string_view rest_after_first(std::string_view s) { + std::size_t i = 0; + while (i < s.size() && (s[i] == ' ' || s[i] == '\t')) ++i; + while (i < s.size() && s[i] != ' ' && s[i] != '\t') ++i; + while (i < s.size() && (s[i] == ' ' || s[i] == '\t')) ++i; + return s.substr(i); + } + + std::vector parse_body(const std::set& stop) { + std::vector body; + while (pos_ < toks_.size()) { + const Token& tok = toks_[pos_]; + if (tok.kind == TokenKind::Text) { + body.push_back(text_node(tok.text)); + ++pos_; + continue; + } + if (tok.kind == TokenKind::Var) { + NodePtr n = std::make_shared(); + n->kind = Node::Kind::Output; + n->expr = parse_expression(tok.text, source_); + body.push_back(n); + ++pos_; + continue; + } + std::string kw(first_word(tok.text)); + if (stop.count(kw)) return body; + if (kw == "if") { + body.push_back(parse_if()); + } else if (kw == "for") { + body.push_back(parse_for()); + } else if (kw == "block") { + body.push_back(parse_block()); + } else if (kw == "extends") { + NodePtr n = std::make_shared(); + n->kind = Node::Kind::Extends; + n->expr = parse_expression(rest_after_first(tok.text), source_); + body.push_back(n); + ++pos_; + } else if (kw == "include") { + NodePtr n = std::make_shared(); + n->kind = Node::Kind::Include; + n->expr = parse_expression(rest_after_first(tok.text), source_); + body.push_back(n); + ++pos_; + } else if (kw == "set") { + body.push_back(parse_set()); + } else if (kw == "macro") { + body.push_back(parse_macro()); + } else if (kw == "autoescape") { + body.push_back(parse_autoescape()); + } else { + throw TemplateError("unknown tag '" + kw + "' in " + std::string(source_)); + } + } + if (!stop.empty()) { + throw TemplateError("missing end tag in " + std::string(source_)); + } + return body; + } + + NodePtr parse_if() { + NodePtr n = std::make_shared(); + n->kind = Node::Kind::If; + IfBranch first; + first.cond = parse_expression(rest_after_first(toks_[pos_].text), source_); + ++pos_; + first.body = parse_body({"elif", "else", "endif"}); + n->branches.push_back(std::move(first)); + while (pos_ < toks_.size()) { + std::string kw(first_word(toks_[pos_].text)); + if (kw == "elif") { + IfBranch b; + b.cond = parse_expression(rest_after_first(toks_[pos_].text), source_); + ++pos_; + b.body = parse_body({"elif", "else", "endif"}); + n->branches.push_back(std::move(b)); + } else if (kw == "else") { + ++pos_; + IfBranch b; + b.body = parse_body({"endif"}); + n->branches.push_back(std::move(b)); + break; + } else { + break; + } + } + expect_kw("endif"); + return n; + } + + NodePtr parse_for() { + std::string rest(rest_after_first(toks_[pos_].text)); + std::size_t in_at = rest.find(" in "); + if (in_at == std::string::npos) { + throw TemplateError("for tag needs 'in' in " + std::string(source_)); + } + std::string targets = rest.substr(0, in_at); + std::string seq = rest.substr(in_at + 4); + NodePtr n = std::make_shared(); + n->kind = Node::Kind::For; + std::size_t i = 0; + while (i < targets.size()) { + while (i < targets.size() && (targets[i] == ' ' || targets[i] == '\t' || targets[i] == ',')) { + ++i; + } + std::size_t start = i; + while (i < targets.size() && targets[i] != ' ' && targets[i] != '\t' && + targets[i] != ',') { + ++i; + } + if (i > start) n->targets.push_back(targets.substr(start, i - start)); + } + if (n->targets.empty() || n->targets.size() > 2) { + throw TemplateError("for tag needs one or two targets in " + std::string(source_)); + } + n->seq = parse_expression(seq, source_); + ++pos_; + n->body = parse_body({"else", "endfor"}); + if (pos_ < toks_.size() && first_word(toks_[pos_].text) == "else") { + ++pos_; + n->else_body = parse_body({"endfor"}); + } + expect_kw("endfor"); + return n; + } + + NodePtr parse_block() { + std::string rest(rest_after_first(toks_[pos_].text)); + std::string name(first_word(rest)); + if (name.empty()) throw TemplateError("block tag needs a name in " + std::string(source_)); + NodePtr n = std::make_shared(); + n->kind = Node::Kind::Block; + n->text = name; + ++pos_; + n->body = parse_body({"endblock"}); + expect_kw("endblock"); + return n; + } + + NodePtr parse_set() { + std::string rest(rest_after_first(toks_[pos_].text)); + std::size_t eq = rest.find('='); + if (eq == std::string::npos || eq + 1 >= rest.size() || rest[eq + 1] == '=') { + throw TemplateError("set tag needs name = value in " + std::string(source_)); + } + NodePtr n = std::make_shared(); + n->kind = Node::Kind::Set; + std::string name(rest.substr(0, eq)); + while (!name.empty() && (name.back() == ' ' || name.back() == '\t')) name.pop_back(); + n->text = name; + n->expr = parse_expression(rest.substr(eq + 1), source_); + ++pos_; + return n; + } + + NodePtr parse_macro() { + std::string rest(rest_after_first(toks_[pos_].text)); + std::size_t paren = rest.find('('); + std::size_t close = rest.rfind(')'); + if (paren == std::string::npos || close == std::string::npos || close < paren) { + throw TemplateError("macro tag needs name(args) in " + std::string(source_)); + } + NodePtr n = std::make_shared(); + n->kind = Node::Kind::Macro; + std::string name = rest.substr(0, paren); + while (!name.empty() && (name.back() == ' ' || name.back() == '\t')) name.pop_back(); + n->text = name; + std::string arglist = rest.substr(paren + 1, close - paren - 1); + std::size_t i = 0; + while (i < arglist.size()) { + while (i < arglist.size() && (arglist[i] == ' ' || arglist[i] == '\t' || arglist[i] == ',')) { + ++i; + } + if (i >= arglist.size()) break; + std::size_t start = i; + int depth = 0; + char quote = 0; + while (i < arglist.size()) { + char c = arglist[i]; + if (quote) { + if (c == '\\') { + ++i; + } else if (c == quote) { + quote = 0; + } + ++i; + continue; + } + if (c == '\'' || c == '"') { + quote = c; + ++i; + continue; + } + if (c == '(' || c == '[' || c == '{') ++depth; + if (c == ')' || c == ']' || c == '}') --depth; + if (c == ',' && depth == 0) break; + ++i; + } + std::string piece = arglist.substr(start, i - start); + if (i < arglist.size() && arglist[i] == ',') ++i; + std::size_t eq = piece.find('='); + if (eq == std::string::npos) { + std::string pname = piece; + while (!pname.empty() && (pname.back() == ' ' || pname.back() == '\t')) pname.pop_back(); + std::size_t s = 0; + while (s < pname.size() && (pname[s] == ' ' || pname[s] == '\t')) ++s; + n->params.push_back(pname.substr(s)); + n->defaults.push_back(std::nullopt); + } else { + std::string pname = piece.substr(0, eq); + while (!pname.empty() && (pname.back() == ' ' || pname.back() == '\t')) pname.pop_back(); + std::size_t s = 0; + while (s < pname.size() && (pname[s] == ' ' || pname[s] == '\t')) ++s; + n->params.push_back(pname.substr(s)); + // Defaults are literal-only at definition; full expressions are a + // documented gap (use set plus call with explicit args instead). + std::string def = piece.substr(eq + 1); + n->defaults.push_back(eval_literal_default(def)); + } + if (n->params.size() > caps::kMaxMacroArgs) { + throw TemplateError("too many macro args in " + std::string(source_)); + } + } + ++pos_; + n->body = parse_body({"endmacro"}); + expect_kw("endmacro"); + return n; + } + + validate::Value eval_literal_default(std::string_view def) { + std::string t(def); + std::size_t s = 0; + while (s < t.size() && (t[s] == ' ' || t[s] == '\t')) ++s; + std::size_t e = t.size(); + while (e > s && (t[e - 1] == ' ' || t[e - 1] == '\t')) --e; + std::string v = t.substr(s, e - s); + if (v.size() >= 2 && ((v.front() == '"' && v.back() == '"') || + (v.front() == '\'' && v.back() == '\''))) { + return validate::Value(v.substr(1, v.size() - 2)); + } + if (v == "true" || v == "True") return validate::Value(true); + if (v == "false" || v == "False") return validate::Value(false); + if (v == "none" || v == "None" || v == "null") return validate::Value::null(); + try { + std::size_t used = 0; + long long i = std::stoll(v, &used); + if (used == v.size()) return validate::Value(i); + } catch (...) { + } + try { + std::size_t used = 0; + double d = std::stod(v, &used); + if (used == v.size()) return validate::Value(d); + } catch (...) { + } + throw TemplateError("macro default must be a literal in " + std::string(source_)); + } + + NodePtr parse_autoescape() { + std::string rest(rest_after_first(toks_[pos_].text)); + std::string word(first_word(rest)); + NodePtr n = std::make_shared(); + n->kind = Node::Kind::Autoescape; + if (word == "true" || word == "True") { + n->flag = true; + } else if (word == "false" || word == "False") { + n->flag = false; + } else { + throw TemplateError("autoescape needs true or false in " + std::string(source_)); + } + ++pos_; + n->body = parse_body({"endautoescape"}); + expect_kw("endautoescape"); + return n; + } + + void expect_kw(const std::string& kw) { + if (pos_ >= toks_.size() || toks_[pos_].kind != TokenKind::Block || + first_word(toks_[pos_].text) != kw) { + throw TemplateError("expected '" + kw + "' in " + std::string(source_)); + } + ++pos_; + } + + const std::vector& toks_; + std::string_view source_; + std::size_t pos_ = 0; +}; + +inline std::vector parse_template(const std::vector& toks, + std::string_view source_name) { + TemplateParser parser(toks, source_name); + return parser.parse(); +} + +} // namespace molo::app::templates diff --git a/src/app/templates/predicates.h b/src/app/templates/predicates.h new file mode 100644 index 0000000..f47b561 --- /dev/null +++ b/src/app/templates/predicates.h @@ -0,0 +1,49 @@ +#pragma once +// Spec: molo::app::templates shipped tests (plan 05, section 6 Templates): +// defined, none, string, number, sequence, mapping, even, odd. +// Used as `x is ` with optional `not`. + +#include +#include +#include +#include + +#include "../validate/value.h" + +namespace molo::app::templates { + +using TestFn = + std::function&)>; + +inline std::map builtin_tests() { + std::map t; + t["defined"] = [](const validate::Value& v, const std::vector&) { + return !v.is_undefined(); + }; + t["none"] = [](const validate::Value& v, const std::vector&) { + return v.is_null(); + }; + t["string"] = [](const validate::Value& v, const std::vector&) { + return v.is_string() || v.is_safe(); + }; + t["number"] = [](const validate::Value& v, const std::vector&) { + return v.is_number(); + }; + t["sequence"] = [](const validate::Value& v, const std::vector&) { + return v.is_array() || v.is_string() || v.is_safe(); + }; + t["mapping"] = [](const validate::Value& v, const std::vector&) { + return v.is_object(); + }; + t["even"] = [](const validate::Value& v, const std::vector&) { + if (!v.is_int()) return false; + return (*std::get_if(&v.storage()) % 2) == 0; + }; + t["odd"] = [](const validate::Value& v, const std::vector&) { + if (!v.is_int()) return false; + return (*std::get_if(&v.storage()) % 2) != 0; + }; + return t; +} + +} // namespace molo::app::templates diff --git a/src/app/templates/renderer.h b/src/app/templates/renderer.h new file mode 100644 index 0000000..5c82e0b --- /dev/null +++ b/src/app/templates/renderer.h @@ -0,0 +1,639 @@ +#pragma once +// Spec: molo::app::templates::Renderer. Executes compiled templates: +// extends/block override chains, includes, if/for with loop variable, +// set, macros, filters, tests, autoescape and the undefined policy. +// Caps enforced per render: include depth 16, extends chain 8, loop +// iterations 100000, output 32 MiB. Recursion beyond depth aborts with +// TemplateError (mapped to 500 upstream), never a stack overflow. + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "../http_msg/html.h" +#include "../validate/json.h" +#include "../validate/value.h" +#include "ast.h" +#include "context.h" +#include "environment.h" +#include "filters.h" +#include "lexer.h" + +namespace molo::app::templates { + +class Renderer { + public: + explicit Renderer(Environment* env) : env_(env) {} + + std::string render_template(const std::string& name, validate::Object context = {}) { + ++env_->stats().renders; + include_depth_ = 0; + try { + std::shared_ptr compiled = env_->get_compiled(name); + Context ctx(env_->globals()); + for (const auto& kv : context) ctx.set(kv.first, kv.second); + std::map> no_overrides; + render_with_inheritance(compiled, ctx, no_overrides, 0); + return output_; + } catch (...) { + ++env_->stats().errors; + throw; + } + } + + std::string render_nodes_source(const std::string& source, const std::string& name, + validate::Object context = {}) { + ++env_->stats().renders; + include_depth_ = 0; + try { + std::shared_ptr compiled = env_->compile_source(source, name); + Context ctx(env_->globals()); + for (const auto& kv : context) ctx.set(kv.first, kv.second); + std::map> no_overrides; + render_with_inheritance(compiled, ctx, no_overrides, 0); + return output_; + } catch (...) { + ++env_->stats().errors; + throw; + } + } + + void render_nodes(const std::vector& nodes, Context& ctx, bool autoescape, + const std::map>& overrides) { + for (const NodePtr& node : nodes) { + render_node(node, ctx, autoescape, overrides); + } + } + + validate::Value eval(const ExprPtr& expr, Context& ctx) { + switch (expr->kind) { + case Expr::Kind::Literal: + return expr->literal; + case Expr::Kind::Name: + return ctx.lookup(expr->name); + case Expr::Kind::Attr: + return eval_attr(expr, ctx); + case Expr::Kind::Index: + return eval_index(expr, ctx); + case Expr::Kind::Call: + return eval_call(expr, ctx); + case Expr::Kind::Unary: + return eval_unary(expr, ctx); + case Expr::Kind::Binary: + return eval_binary(expr, ctx); + case Expr::Kind::Compare: + return eval_compare(expr, ctx); + case Expr::Kind::BoolOp: { + validate::Value left = eval(expr->left, ctx); + if (expr->op == "and") { + if (!left.truthy()) return left; + return eval(expr->right, ctx); + } + if (left.truthy()) return left; + return eval(expr->right, ctx); + } + case Expr::Kind::Not: + return validate::Value(!eval(expr->operand, ctx).truthy()); + case Expr::Kind::In: + return eval_in(expr, ctx); + case Expr::Kind::Filter: { + validate::Value base = eval(expr->operand, ctx); + std::vector args; + for (const auto& a : expr->args) args.push_back(eval(a, ctx)); + auto it = env_->filters().find(expr->name); + if (it == env_->filters().end()) { + throw TemplateError("unknown filter '" + expr->name + "'"); + } + return it->second(base, args); + } + case Expr::Kind::Test: { + validate::Value base = eval(expr->operand, ctx); + std::vector args; + for (const auto& a : expr->args) args.push_back(eval(a, ctx)); + auto it = env_->tests().find(expr->name); + if (it == env_->tests().end()) { + throw TemplateError("unknown test '" + expr->name + "'"); + } + bool result = it->second(base, args); + return validate::Value(expr->negated ? !result : result); + } + case Expr::Kind::List: { + validate::Array arr; + for (const auto& a : expr->args) arr.push_back(eval(a, ctx)); + return validate::Value(std::move(arr)); + } + case Expr::Kind::Dict: { + validate::Object obj; + for (const auto& kv : expr->pairs) { + validate::Value k = eval(kv.first, ctx); + obj[value_to_text(k)] = eval(kv.second, ctx); + } + return validate::Value(std::move(obj)); + } + } + return validate::Value::undefined(); + } + + private: + void write(std::string_view s) { + if (output_.size() + s.size() > caps::kMaxRenderOutput) { + throw TemplateError("render output exceeds size cap"); + } + output_.append(s.data(), s.size()); + } + + void write_value(const validate::Value& v, bool autoescape) { + if (v.is_undefined()) { + if (env_->undefined_policy() == UndefinedPolicy::Strict) { + throw UndefinedError("undefined variable rendered"); + } + ++env_->stats().undefined_hits; + return; + } + if (v.is_safe()) { + write(v.as_text()); + return; + } + if (v.is_string()) { + if (autoescape) { + write(http_msg::escape_html(v.as_text())); + } else { + write(v.as_text()); + } + return; + } + if (v.is_array() || v.is_object()) { + std::string text = validate::to_json(v); + if (autoescape) { + write(http_msg::escape_html(text)); + } else { + write(text); + } + return; + } + write(value_to_text(v)); + } + + void collect_blocks(const std::vector& nodes, + std::map>& blocks) { + for (const NodePtr& node : nodes) { + if (node->kind == Node::Kind::Block) { + // First definition wins at each level; nested blocks inside bodies + // are collected too so deep overrides resolve. + if (!blocks.count(node->text)) blocks[node->text] = node->body; + collect_blocks(node->body, blocks); + } else if (node->kind == Node::Kind::If) { + for (const auto& b : node->branches) collect_blocks(b.body, blocks); + } else if (node->kind == Node::Kind::For) { + collect_blocks(node->body, blocks); + collect_blocks(node->else_body, blocks); + } else if (node->kind == Node::Kind::Autoescape) { + collect_blocks(node->body, blocks); + } + } + } + + const Node* find_extends(const std::vector& nodes) { + for (const NodePtr& node : nodes) { + if (node->kind == Node::Kind::Extends) return node.get(); + } + return nullptr; + } + + void render_with_inheritance(std::shared_ptr compiled, Context& ctx, + const std::map>& overrides, + std::size_t chain) { + if (chain > caps::kMaxExtendsChain) { + throw TemplateError("template extends chain exceeds cap"); + } + const Node* ext = find_extends(compiled->nodes); + if (ext == nullptr) { + // Render macro definitions first so they exist before use. + for (const NodePtr& node : compiled->nodes) { + if (node->kind == Node::Kind::Macro) define_macro(node, ctx); + } + render_nodes(compiled->nodes, ctx, compiled->autoescape, overrides); + return; + } + std::map> blocks = overrides; + std::map> child_blocks; + collect_blocks(compiled->nodes, child_blocks); + for (const auto& kv : child_blocks) blocks[kv.first] = kv.second; + for (const NodePtr& node : compiled->nodes) { + if (node->kind == Node::Kind::Macro) define_macro(node, ctx); + } + validate::Value parent_name = eval(ext->expr, ctx); + if (parent_name.is_undefined() || parent_name.is_null()) { + throw TemplateError("extends needs a template name"); + } + std::shared_ptr parent = env_->get_compiled(value_to_text(parent_name)); + render_with_inheritance(parent, ctx, blocks, chain + 1); + } + + void define_macro(const NodePtr& node, Context& ctx) { + auto macro = std::make_shared(); + macro->name = node->text; + macro->params = node->params; + macro->defaults = node->defaults; + macro->body = node->body; + ctx.define_macro(macro); + } + + void render_node(const NodePtr& node, Context& ctx, bool autoescape, + const std::map>& overrides) { + switch (node->kind) { + case Node::Kind::Text: + write(node->text); + return; + case Node::Kind::Output: + write_value(eval(node->expr, ctx), autoescape); + return; + case Node::Kind::Extends: + return; // Handled by render_with_inheritance. + case Node::Kind::Macro: + define_macro(node, ctx); + return; + case Node::Kind::Set: + ctx.set(node->text, eval(node->expr, ctx)); + return; + case Node::Kind::Autoescape: + render_nodes(node->body, ctx, node->flag, overrides); + return; + case Node::Kind::Block: { + auto it = overrides.find(node->text); + if (it != overrides.end()) { + render_nodes(it->second, ctx, autoescape, overrides); + } else { + render_nodes(node->body, ctx, autoescape, overrides); + } + return; + } + case Node::Kind::Include: { + if (include_depth_ >= caps::kMaxIncludeDepth) { + throw TemplateError("template include depth exceeds cap"); + } + validate::Value name = eval(node->expr, ctx); + if (name.is_undefined() || name.is_null()) { + throw TemplateError("include needs a template name"); + } + std::shared_ptr compiled = + env_->get_compiled(value_to_text(name)); + ++include_depth_; + ctx.push(); + render_nodes(compiled->nodes, ctx, compiled->autoescape, overrides); + ctx.pop(); + --include_depth_; + return; + } + case Node::Kind::If: { + for (const IfBranch& branch : node->branches) { + if (!branch.cond || eval(branch.cond, ctx).truthy()) { + render_nodes(branch.body, ctx, autoescape, overrides); + return; + } + } + return; + } + case Node::Kind::For: + render_for(node, ctx, autoescape, overrides); + return; + } + } + + void render_for(const NodePtr& node, Context& ctx, bool autoescape, + const std::map>& overrides) { + validate::Value seq = eval(node->seq, ctx); + if (seq.is_undefined() || seq.is_null()) { + if (seq.is_undefined() && env_->undefined_policy() == UndefinedPolicy::Strict) { + throw UndefinedError("for loop over undefined"); + } + render_nodes(node->else_body, ctx, autoescape, overrides); + return; + } + // Normalize to rows of (key, value); single target binds value (or key + // for mappings, char for strings). + std::vector> rows; + if (const auto* arr = std::get_if(&seq.storage())) { + for (const auto& item : *arr) rows.emplace_back(validate::Value::null(), item); + } else if (const auto* obj = std::get_if(&seq.storage())) { + for (const auto& kv : *obj) { + rows.emplace_back(validate::Value(kv.first), kv.second); + } + } else if (seq.is_string() || seq.is_safe()) { + for (char c : seq.as_text()) { + rows.emplace_back(validate::Value::null(), validate::Value(std::string(1, c))); + } + } else { + throw TemplateError("for loop needs a sequence or mapping"); + } + if (rows.empty()) { + render_nodes(node->else_body, ctx, autoescape, overrides); + return; + } + long long length = (long long)rows.size(); + for (long long i = 0; i < length; ++i) { + ++loop_iterations_; + if (loop_iterations_ > caps::kMaxLoopIterations) { + throw TemplateError("loop iteration cap exceeded"); + } + ctx.push(); + if (node->targets.size() == 1) { + bool is_map = seq.is_object(); + ctx.set(node->targets[0], is_map ? rows[(std::size_t)i].first + : rows[(std::size_t)i].second); + } else { + if (seq.is_object()) { + ctx.set(node->targets[0], rows[(std::size_t)i].first); + ctx.set(node->targets[1], rows[(std::size_t)i].second); + } else { + const validate::Value& item = rows[(std::size_t)i].second; + const auto* pair = std::get_if(&item.storage()); + if (pair == nullptr || pair->size() != 2) { + ctx.pop(); + throw TemplateError("cannot unpack loop item into two targets"); + } + ctx.set(node->targets[0], (*pair)[0]); + ctx.set(node->targets[1], (*pair)[1]); + } + } + validate::Object loop; + loop["index"] = validate::Value(i + 1); + loop["index0"] = validate::Value(i); + loop["first"] = validate::Value(i == 0); + loop["last"] = validate::Value(i == length - 1); + loop["length"] = validate::Value(length); + loop["revindex"] = validate::Value(length - i); + loop["revindex0"] = validate::Value(length - i - 1); + ctx.set("loop", validate::Value(std::move(loop))); + render_nodes(node->body, ctx, autoescape, overrides); + ctx.pop(); + } + } + + validate::Value eval_attr(const ExprPtr& expr, Context& ctx) { + validate::Value base = eval(expr->operand, ctx); + if (base.is_undefined() || base.is_null()) return validate::Value::undefined(); + if (const auto* obj = std::get_if(&base.storage())) { + auto it = obj->find(expr->name); + if (it == obj->end()) return validate::Value::undefined(); + return it->second; + } + return validate::Value::undefined(); + } + + validate::Value eval_index(const ExprPtr& expr, Context& ctx) { + validate::Value base = eval(expr->operand, ctx); + validate::Value idx = eval(expr->right, ctx); + if (base.is_undefined() || base.is_null()) return validate::Value::undefined(); + if (const auto* arr = std::get_if(&base.storage())) { + if (!idx.is_int()) throw TemplateError("sequence index must be int"); + long long i = *std::get_if(&idx.storage()); + long long n = (long long)arr->size(); + if (i < 0) i += n; + if (i < 0 || i >= n) return validate::Value::undefined(); + return (*arr)[(std::size_t)i]; + } + if (const auto* obj = std::get_if(&base.storage())) { + if (!idx.is_string() && !idx.is_safe()) { + throw TemplateError("mapping index must be string"); + } + auto it = obj->find(idx.as_text()); + if (it == obj->end()) return validate::Value::undefined(); + return it->second; + } + if (base.is_string() || base.is_safe()) { + if (!idx.is_int()) throw TemplateError("string index must be int"); + const std::string& s = base.as_text(); + long long i = *std::get_if(&idx.storage()); + long long n = (long long)s.size(); + if (i < 0) i += n; + if (i < 0 || i >= n) return validate::Value::undefined(); + return validate::Value(s.substr((std::size_t)i, 1)); + } + return validate::Value::undefined(); + } + + validate::Value eval_call(const ExprPtr& expr, Context& ctx) { + std::string callee; + if (expr->operand->kind == Expr::Kind::Name) callee = expr->operand->name; + std::shared_ptr macro = + callee.empty() ? nullptr : ctx.lookup_macro(callee); + if (macro) { + if (expr->args.size() > macro->params.size()) { + throw TemplateError("too many args for macro '" + callee + "'"); + } + std::vector bound(macro->params.size(), validate::Value::undefined()); + for (std::size_t i = 0; i < expr->args.size(); ++i) bound[i] = eval(expr->args[i], ctx); + for (const auto& kw : expr->kwargs) { + bool found = false; + for (std::size_t i = 0; i < macro->params.size(); ++i) { + if (macro->params[i] == kw.first) { + bound[i] = eval(kw.second, ctx); + found = true; + break; + } + } + if (!found) throw TemplateError("unknown macro kwarg '" + kw.first + "'"); + } + for (std::size_t i = 0; i < macro->params.size(); ++i) { + if (bound[i].is_undefined()) { + if (i < macro->defaults.size() && macro->defaults[i].has_value()) { + bound[i] = *macro->defaults[i]; + } else { + throw TemplateError("macro '" + callee + "' missing arg '" + macro->params[i] + "'"); + } + } + } + Context inner(env_->globals()); + for (const auto& scope : ctx.scopes()) { + for (const auto& kv : scope.macros) { + if (inner.lookup_macro(kv.first) == nullptr) inner.define_macro(kv.second); + } + } + for (std::size_t i = 0; i < macro->params.size(); ++i) { + inner.set(macro->params[i], bound[i]); + } + std::string saved = std::move(output_); + output_.clear(); + std::map> no_overrides; + render_nodes(macro->body, inner, false, no_overrides); + std::string result = std::move(output_); + output_ = std::move(saved); + return validate::Value::safe(result); + } + validate::Value base = + callee.empty() ? eval(expr->operand, ctx) : ctx.lookup(callee); + if (base.is_undefined()) { + if (env_->undefined_policy() == UndefinedPolicy::Strict) { + throw UndefinedError("call of undefined '" + callee + "'"); + } + ++env_->stats().undefined_hits; + return validate::Value::undefined(); + } + throw TemplateError("value is not callable"); + } + + validate::Value eval_unary(const ExprPtr& expr, Context& ctx) { + validate::Value v = eval(expr->operand, ctx); + if (expr->op == "+") { + if (!v.is_number()) throw TemplateError("unary plus needs a number"); + return v; + } + if (const auto* i = std::get_if(&v.storage())) return validate::Value(-*i); + if (const auto* d = std::get_if(&v.storage())) return validate::Value(-*d); + throw TemplateError("unary minus needs a number"); + } + + validate::Value eval_binary(const ExprPtr& expr, Context& ctx) { + if (expr->op == "~") { + return validate::Value(value_to_text(eval(expr->left, ctx)) + + value_to_text(eval(expr->right, ctx))); + } + validate::Value a = eval(expr->left, ctx); + validate::Value b = eval(expr->right, ctx); + const std::string& op = expr->op; + if (op == "+") { + if (a.is_number() && b.is_number()) { + if (a.is_float() || b.is_float()) return validate::Value(a.as_number() + b.as_number()); + return validate::Value(*std::get_if(&a.storage()) + + *std::get_if(&b.storage())); + } + if ((a.is_string() || a.is_safe()) && (b.is_string() || b.is_safe())) { + return validate::Value(a.as_text() + b.as_text()); + } + if (a.is_array() && b.is_array()) { + validate::Array out = *std::get_if(&a.storage()); + const auto* tail = std::get_if(&b.storage()); + for (const auto& item : *tail) out.push_back(item); + return validate::Value(std::move(out)); + } + throw TemplateError("unsupported operands for +"); + } + if (op == "*") { + if (a.is_number() && b.is_number()) { + if (a.is_float() || b.is_float()) return validate::Value(a.as_number() * b.as_number()); + return validate::Value(*std::get_if(&a.storage()) * + *std::get_if(&b.storage())); + } + if ((a.is_string() || a.is_safe()) && b.is_int()) { + std::string out; + long long n = *std::get_if(&b.storage()); + if (n < 0 || n > 100000) throw TemplateError("string repeat out of range"); + for (long long i = 0; i < n; ++i) out += a.as_text(); + return validate::Value(out); + } + throw TemplateError("unsupported operands for *"); + } + if (!a.is_number() || !b.is_number()) throw TemplateError("arithmetic needs numbers"); + double x = a.as_number(); + double y = b.as_number(); + bool both_int = a.is_int() && b.is_int(); + if (op == "-") { + if (both_int) { + return validate::Value(*std::get_if(&a.storage()) - + *std::get_if(&b.storage())); + } + return validate::Value(x - y); + } + if (op == "/") { + if (y == 0.0) throw TemplateError("division by zero"); + return validate::Value(x / y); + } + if (op == "//") { + if (y == 0.0) throw TemplateError("division by zero"); + return validate::Value((long long)std::floor(x / y)); + } + if (op == "%") { + if (both_int) { + long long d = *std::get_if(&b.storage()); + if (d == 0) throw TemplateError("division by zero"); + return validate::Value(*std::get_if(&a.storage()) % d); + } + if (y == 0.0) throw TemplateError("division by zero"); + return validate::Value(std::fmod(x, y)); + } + throw TemplateError("unknown operator '" + op + "'"); + } + + static bool values_equal(const validate::Value& a, const validate::Value& b) { + if (a.is_number() && b.is_number()) return a.as_number() == b.as_number(); + if ((a.is_string() || a.is_safe()) && (b.is_string() || b.is_safe())) { + return a.as_text() == b.as_text(); + } + return a == b; + } + + validate::Value eval_compare(const ExprPtr& expr, Context& ctx) { + validate::Value a = eval(expr->left, ctx); + validate::Value b = eval(expr->right, ctx); + const std::string& op = expr->op; + if (op == "==") return validate::Value(values_equal(a, b)); + if (op == "!=") return validate::Value(!values_equal(a, b)); + if (a.is_number() && b.is_number()) { + double x = a.as_number(); + double y = b.as_number(); + if (op == "<") return validate::Value(x < y); + if (op == ">") return validate::Value(x > y); + if (op == "<=") return validate::Value(x <= y); + return validate::Value(x >= y); + } + if ((a.is_string() || a.is_safe()) && (b.is_string() || b.is_safe())) { + const std::string& x = a.as_text(); + const std::string& y = b.as_text(); + if (op == "<") return validate::Value(x < y); + if (op == ">") return validate::Value(x > y); + if (op == "<=") return validate::Value(x <= y); + return validate::Value(x >= y); + } + throw TemplateError("cannot compare these values"); + } + + validate::Value eval_in(const ExprPtr& expr, Context& ctx) { + validate::Value left = eval(expr->left, ctx); + validate::Value seq = eval(expr->seq, ctx); + if (seq.is_undefined() || seq.is_null()) { + if (seq.is_undefined() && env_->undefined_policy() == UndefinedPolicy::Strict) { + throw UndefinedError("'in' over undefined"); + } + return validate::Value(expr->negated); + } + bool found = false; + if ((seq.is_string() || seq.is_safe()) && (left.is_string() || left.is_safe())) { + found = seq.as_text().find(left.as_text()) != std::string::npos; + } else if (const auto* arr = std::get_if(&seq.storage())) { + for (const auto& item : *arr) { + if (values_equal(item, left)) { + found = true; + break; + } + } + } else if (const auto* obj = std::get_if(&seq.storage())) { + if (left.is_string() || left.is_safe()) found = obj->count(left.as_text()) > 0; + } else { + throw TemplateError("'in' needs a string, sequence or mapping"); + } + return validate::Value(expr->negated ? !found : found); + } + + Environment* env_; + std::string output_; + std::size_t include_depth_ = 0; + std::size_t loop_iterations_ = 0; +}; + +inline std::string Environment::render(const std::string& name, validate::Object context) { + Renderer renderer(this); + return renderer.render_template(name, std::move(context)); +} + +inline std::string Environment::render_source(const std::string& source, const std::string& name, + validate::Object context) { + Renderer renderer(this); + return renderer.render_nodes_source(source, name, std::move(context)); +} + +} // namespace molo::app::templates diff --git a/src/app/templates/response.h b/src/app/templates/response.h new file mode 100644 index 0000000..5fbfcde --- /dev/null +++ b/src/app/templates/response.h @@ -0,0 +1,23 @@ +#pragma once +// Spec: molo::app::templates::TemplateResponse. Response subclass rendering +// a named template with a context through an Environment (plan 05, E11). +// Renders eagerly; TemplateError propagates and maps to 500 upstream. + +#include +#include + +#include "../http_msg/response.h" +#include "../validate/value.h" +#include "environment.h" + +namespace molo::app::templates { + +class TemplateResponse : public http_msg::Response { + public: + explicit TemplateResponse(Environment& env, std::string name, validate::Object context = {}, + int status = 200, + std::string content_type = "text/html; charset=utf-8") + : Response(status, env.render(name, std::move(context)), std::move(content_type)) {} +}; + +} // namespace molo::app::templates diff --git a/src/app/validate/errors.h b/src/app/validate/errors.h new file mode 100644 index 0000000..7a4281e --- /dev/null +++ b/src/app/validate/errors.h @@ -0,0 +1,51 @@ +#pragma once +// Spec: molo::app::validate::ValidationError. Raised when data cannot be +// parsed into a Model. Lists ALL field failures (plan 05, R3), never +// fail-fast. JSON shape: {"detail":[{"loc":[...],"msg":"...","type":"..."}]} +// with status 422. + +#include +#include +#include + +#include "json.h" +#include "value.h" + +namespace molo::app::validate { + +struct FieldFailure { + std::vector loc; + std::string msg; + std::string type; +}; + +class ValidationError : public std::runtime_error { + public: + explicit ValidationError(std::vector failures) + : std::runtime_error("validation failed"), failures_(std::move(failures)) {} + + const std::vector& failures() const { return failures_; } + + Value to_value() const { + Array detail; + for (const auto& f : failures_) { + Object item; + Array loc; + for (const auto& part : f.loc) loc.emplace_back(part); + item["loc"] = Value(std::move(loc)); + item["msg"] = Value(f.msg); + item["type"] = Value(f.type); + detail.emplace_back(std::move(item)); + } + Object root; + root["detail"] = Value(std::move(detail)); + return Value(std::move(root)); + } + + std::string to_json_string() const { return to_json(to_value()); } + + private: + std::vector failures_; +}; + +} // namespace molo::app::validate diff --git a/src/app/validate/field.h b/src/app/validate/field.h new file mode 100644 index 0000000..5c32254 --- /dev/null +++ b/src/app/validate/field.h @@ -0,0 +1,271 @@ +#pragma once +// Spec: molo::app::validate::Field. One declared field: type, constraints, +// default, required, nullable (plan 05, section 6 Validation). +// Coercion table (Q1 answer, strict booleans, empty string never null): +// - string source: int needs ^[+-]?[0-9]+$ after trim; float needs a full +// number after trim; bool accepts true/false/1/0 case-insensitive after +// trim; string is as-is (no trim). +// - int source: to float ok; to string decimal; to bool rejected. +// - float source: to int only when integral; to string shortest form. +// - bool source: to int/float 1/0; to string "true"/"false". +// - null source: ok only when nullable, else type error. +// - path params use the same strict table; query/form identical (lenient +// part is trim-before-parse only, documented here). + +#include +#include +#include +#include +#include +#include +#include + +#include "errors.h" +#include "value.h" + +namespace molo::app::validate { + +enum class FieldType { Any, Int, Float, Bool, String, List, Dict }; + +inline std::string trim_copy(const std::string& s) { + std::size_t a = 0; + while (a < s.size() && std::isspace((unsigned char)s[a])) ++a; + std::size_t b = s.size(); + while (b > a && std::isspace((unsigned char)s[b - 1])) --b; + return s.substr(a, b - a); +} + +struct Constraints { + std::optional min_value; + std::optional max_value; + std::optional gt; + std::optional ge; + std::optional lt; + std::optional le; + std::optional min_len; + std::optional max_len; + std::optional pattern; + std::vector choices; +}; + +class Field { + public: + Field() = default; + Field(std::string name, FieldType type) : name_(std::move(name)), type_(type) {} + + const std::string& name() const { return name_; } + FieldType type() const { return type_; } + Constraints& constraints() { return constraints_; } + const Constraints& constraints() const { return constraints_; } + + Field& required(bool r = true) { + required_ = r; + return *this; + } + Field& nullable(bool n = true) { + nullable_ = n; + return *this; + } + Field& with_default(Value v) { + default_ = std::move(v); + has_default_ = true; + return *this; + } + bool required() const { return required_; } + bool nullable() const { return nullable_; } + bool has_default() const { return has_default_; } + const Value& default_value() const { return default_; } + + Field& min_value(double v) { constraints_.min_value = v; return *this; } + Field& max_value(double v) { constraints_.max_value = v; return *this; } + Field& gt(double v) { constraints_.gt = v; return *this; } + Field& ge(double v) { constraints_.ge = v; return *this; } + Field& lt(double v) { constraints_.lt = v; return *this; } + Field& le(double v) { constraints_.le = v; return *this; } + Field& min_len(std::size_t v) { constraints_.min_len = v; return *this; } + Field& max_len(std::size_t v) { constraints_.max_len = v; return *this; } + Field& pattern(std::string p) { constraints_.pattern = std::move(p); return *this; } + Field& choices(std::vector c) { constraints_.choices = std::move(c); return *this; } + + // Coerce and check one value. Throws FieldFailure-as-ValidationError with + // a single entry; Model::validate aggregates across fields (R3). + Value parse(const Value& input, const std::vector& loc) const { + if (input.is_undefined() || input.is_null()) { + if (input.is_null() && nullable_) return Value::null(); + if (has_default_) return default_; + if (required_) { + throw ValidationError({{loc, "field required", "missing"}}); + } + return Value::null(); + } + Value coerced = coerce(input, loc); + check_constraints(coerced, loc); + return coerced; + } + + private: + Value coerce(const Value& input, const std::vector& loc) const { + switch (type_) { + case FieldType::Any: + return input; + case FieldType::String: + return coerce_string(input, loc); + case FieldType::Int: + return coerce_int(input, loc); + case FieldType::Float: + return coerce_float(input, loc); + case FieldType::Bool: + return coerce_bool(input, loc); + case FieldType::List: + if (!input.is_array()) fail(loc, "expected array", "type_error"); + return input; + case FieldType::Dict: + if (!input.is_object()) fail(loc, "expected object", "type_error"); + return input; + } + return input; + } + + [[noreturn]] static void fail(const std::vector& loc, const std::string& msg, + const std::string& type) { + throw ValidationError({{loc, msg, type}}); + } + + Value coerce_string(const Value& input, const std::vector& loc) const { + if (input.is_string() || input.is_safe()) return Value(input.as_text()); + if (const auto* i = std::get_if(&input.storage())) { + return Value(std::to_string(*i)); + } + if (const auto* d = std::get_if(&input.storage())) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "%.17g", *d); + return Value(std::string(buf)); + } + if (const auto* b = std::get_if(&input.storage())) { + return Value(*b ? "true" : "false"); + } + fail(loc, "expected string", "string_type"); + } + + static bool full_int(const std::string& t, long long& out) { + if (t.empty()) return false; + std::size_t i = 0; + if (t[0] == '+' || t[0] == '-') i = 1; + if (i == t.size()) return false; + for (std::size_t j = i; j < t.size(); ++j) { + if (!std::isdigit((unsigned char)t[j])) return false; + } + try { + out = std::stoll(t); + } catch (...) { + return false; + } + return true; + } + + static bool full_float(const std::string& t, double& out) { + if (t.empty()) return false; + try { + std::size_t used = 0; + out = std::stod(t, &used); + return used == t.size(); + } catch (...) { + return false; + } + } + + Value coerce_int(const Value& input, const std::vector& loc) const { + if (const auto* i = std::get_if(&input.storage())) return Value(*i); + if (input.is_bool()) { + fail(loc, "bool is not int", "int_type"); + } + if (const auto* d = std::get_if(&input.storage())) { + if (std::trunc(*d) != *d) fail(loc, "float is not integral", "int_from_float"); + return Value((long long)*d); + } + if (input.is_string()) { + long long v = 0; + if (full_int(trim_copy(input.as_text()), v)) return Value(v); + fail(loc, "invalid integer", "int_parsing"); + } + fail(loc, "expected int", "int_type"); + } + + Value coerce_float(const Value& input, const std::vector& loc) const { + if (const auto* d = std::get_if(&input.storage())) return Value(*d); + if (const auto* i = std::get_if(&input.storage())) return Value((double)*i); + if (const auto* b = std::get_if(&input.storage())) return Value(*b ? 1.0 : 0.0); + if (input.is_string()) { + double v = 0; + if (full_float(trim_copy(input.as_text()), v)) return Value(v); + fail(loc, "invalid number", "float_parsing"); + } + fail(loc, "expected number", "float_type"); + } + + Value coerce_bool(const Value& input, const std::vector& loc) const { + if (const auto* b = std::get_if(&input.storage())) return Value(*b); + if (input.is_string()) { + std::string t = trim_copy(input.as_text()); + std::string low; + for (char c : t) low += (char)std::tolower((unsigned char)c); + if (low == "true" || low == "1") return Value(true); + if (low == "false" || low == "0") return Value(false); + fail(loc, "invalid boolean, use true/false/1/0", "bool_parsing"); + } + fail(loc, "expected bool", "bool_type"); + } + + void check_constraints(const Value& v, const std::vector& loc) const { + const Constraints& c = constraints_; + if (v.is_number()) { + double n = v.as_number(); + if (c.min_value && n < *c.min_value) fail(loc, "less than minimum", "greater_than_equal"); + if (c.max_value && n > *c.max_value) fail(loc, "greater than maximum", "less_than_equal"); + if (c.gt && !(n > *c.gt)) fail(loc, "must be greater than bound", "greater_than"); + if (c.ge && !(n >= *c.ge)) fail(loc, "must be at least bound", "greater_than_equal"); + if (c.lt && !(n < *c.lt)) fail(loc, "must be less than bound", "less_than"); + if (c.le && !(n <= *c.le)) fail(loc, "must be at most bound", "less_than_equal"); + } + std::size_t len = 0; + bool has_len = false; + if (v.is_string()) { + len = v.as_text().size(); + has_len = true; + } else if (const auto* a = std::get_if(&v.storage())) { + len = a->size(); + has_len = true; + } else if (const auto* o = std::get_if(&v.storage())) { + len = o->size(); + has_len = true; + } + if (has_len) { + if (c.min_len && len < *c.min_len) fail(loc, "too short", "too_short"); + if (c.max_len && len > *c.max_len) fail(loc, "too long", "too_long"); + } + if (c.pattern && v.is_string()) { + std::regex re(*c.pattern); + if (!std::regex_match(v.as_text(), re)) fail(loc, "pattern mismatch", "string_pattern"); + } + if (!c.choices.empty()) { + bool ok = false; + for (const auto& choice : c.choices) { + if (choice == v) { + ok = true; + break; + } + } + if (!ok) fail(loc, "not one of the allowed choices", "literal_error"); + } + } + + std::string name_; + FieldType type_ = FieldType::Any; + Constraints constraints_; + bool required_ = false; + bool nullable_ = false; + bool has_default_ = false; + Value default_; +}; + +} // namespace molo::app::validate diff --git a/src/app/validate/json.h b/src/app/validate/json.h new file mode 100644 index 0000000..b7ee1e4 --- /dev/null +++ b/src/app/validate/json.h @@ -0,0 +1,298 @@ +#pragma once +// Spec: molo::app::validate JSON codec. Minimal recursive descent parser +// and serializer for form binding, error bodies and template tojson. +// Depth cap kMaxJsonDepth; document size capped by the caller +// (kMaxJsonDocument default, overridable per route down to 0). + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "value.h" + +namespace molo::app::validate { + +class JsonError : public std::runtime_error { + public: + explicit JsonError(const std::string& msg) : std::runtime_error(msg) {} +}; + +class JsonParser { + public: + explicit JsonParser(std::string_view text, std::size_t max_depth = caps::kMaxJsonDepth) + : text_(text), max_depth_(max_depth) {} + + Value parse() { + skip_ws(); + if (pos_ >= text_.size()) throw JsonError("empty JSON document"); + Value v = parse_value(0); + skip_ws(); + if (pos_ != text_.size()) throw JsonError("trailing bytes after JSON document"); + return v; + } + + private: + void skip_ws() { + while (pos_ < text_.size()) { + char c = text_[pos_]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + ++pos_; + } else { + break; + } + } + } + + char peek() const { + if (pos_ >= text_.size()) throw JsonError("unexpected end of JSON"); + return text_[pos_]; + } + + Value parse_value(std::size_t depth) { + if (depth > max_depth_) throw JsonError("JSON nesting exceeds depth cap"); + char c = peek(); + if (c == '{') return parse_object(depth); + if (c == '[') return parse_array(depth); + if (c == '"') return Value(parse_string()); + if (c == 't') return parse_literal("true", Value(true)); + if (c == 'f') return parse_literal("false", Value(false)); + if (c == 'n') return parse_literal("null", Value::null()); + if (c == '-' || (c >= '0' && c <= '9')) return parse_number(); + throw JsonError(std::string("unexpected character in JSON: ") + c); + } + + Value parse_literal(const char* word, Value v) { + for (const char* p = word; *p; ++p) { + if (pos_ >= text_.size() || text_[pos_] != *p) throw JsonError("bad JSON literal"); + ++pos_; + } + return v; + } + + Value parse_object(std::size_t depth) { + ++pos_; // { + Object obj; + skip_ws(); + if (peek() == '}') { + ++pos_; + return Value(std::move(obj)); + } + while (true) { + skip_ws(); + if (peek() != '"') throw JsonError("expected string key in JSON object"); + std::string key = parse_string(); + skip_ws(); + if (peek() != ':') throw JsonError("expected colon in JSON object"); + ++pos_; + skip_ws(); + obj[std::move(key)] = parse_value(depth + 1); + skip_ws(); + char c = peek(); + if (c == ',') { + ++pos_; + continue; + } + if (c == '}') { + ++pos_; + return Value(std::move(obj)); + } + throw JsonError("expected comma or brace in JSON object"); + } + } + + Value parse_array(std::size_t depth) { + ++pos_; // [ + Array arr; + skip_ws(); + if (peek() == ']') { + ++pos_; + return Value(std::move(arr)); + } + while (true) { + skip_ws(); + arr.push_back(parse_value(depth + 1)); + skip_ws(); + char c = peek(); + if (c == ',') { + ++pos_; + continue; + } + if (c == ']') { + ++pos_; + return Value(std::move(arr)); + } + throw JsonError("expected comma or bracket in JSON array"); + } + } + + std::string parse_string() { + ++pos_; // opening quote + std::string out; + while (true) { + if (pos_ >= text_.size()) throw JsonError("unterminated JSON string"); + char c = text_[pos_++]; + if (c == '"') return out; + if (c == '\\') { + if (pos_ >= text_.size()) throw JsonError("bad JSON escape"); + char e = text_[pos_++]; + switch (e) { + case '"': out += '"'; break; + case '\\': out += '\\'; break; + case '/': out += '/'; break; + case 'b': out += '\b'; break; + case 'f': out += '\f'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'u': { + if (pos_ + 4 > text_.size()) throw JsonError("bad unicode escape"); + unsigned code = 0; + for (int i = 0; i < 4; ++i) { + char h = text_[pos_++]; + code <<= 4; + if (h >= '0' && h <= '9') code |= (unsigned)(h - '0'); + else if (h >= 'a' && h <= 'f') code |= (unsigned)(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') code |= (unsigned)(h - 'A' + 10); + else throw JsonError("bad unicode escape"); + } + if (code < 0x80) { + out += (char)code; + } else if (code < 0x800) { + out += (char)(0xC0 | (code >> 6)); + out += (char)(0x80 | (code & 0x3F)); + } else { + out += (char)(0xE0 | (code >> 12)); + out += (char)(0x80 | ((code >> 6) & 0x3F)); + out += (char)(0x80 | (code & 0x3F)); + } + break; + } + default: throw JsonError("bad JSON escape"); + } + } else { + out += c; + } + } + } + + Value parse_number() { + std::size_t start = pos_; + if (peek() == '-') ++pos_; + while (pos_ < text_.size() && std::isdigit((unsigned char)text_[pos_])) ++pos_; + bool is_float = false; + if (pos_ < text_.size() && text_[pos_] == '.') { + is_float = true; + ++pos_; + while (pos_ < text_.size() && std::isdigit((unsigned char)text_[pos_])) ++pos_; + } + if (pos_ < text_.size() && (text_[pos_] == 'e' || text_[pos_] == 'E')) { + is_float = true; + ++pos_; + if (pos_ < text_.size() && (text_[pos_] == '+' || text_[pos_] == '-')) ++pos_; + while (pos_ < text_.size() && std::isdigit((unsigned char)text_[pos_])) ++pos_; + } + std::string tok(text_.substr(start, pos_ - start)); + try { + if (is_float) return Value(std::stod(tok)); + return Value((long long)std::stoll(tok)); + } catch (...) { + throw JsonError("bad JSON number: " + tok); + } + } + + std::string_view text_; + std::size_t pos_ = 0; + std::size_t max_depth_; +}; + +inline Value parse_json(std::string_view text, std::size_t max_depth = caps::kMaxJsonDepth) { + JsonParser parser(text, max_depth); + return parser.parse(); +} + +inline void escape_json_string(const std::string& s, std::string& out) { + out += '"'; + for (unsigned char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += (char)c; + } + } + } + out += '"'; +} + +inline void write_json_value(const Value& v, std::string& out, std::size_t depth) { + if (depth > caps::kMaxJsonDepth) throw JsonError("value exceeds JSON depth cap"); + if (v.is_null() || v.is_undefined()) { + out += "null"; + return; + } + if (const auto* b = std::get_if(&v.storage())) { + out += *b ? "true" : "false"; + return; + } + if (const auto* i = std::get_if(&v.storage())) { + out += std::to_string(*i); + return; + } + if (const auto* d = std::get_if(&v.storage())) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "%.17g", *d); + out += buf; + return; + } + if (v.is_string() || v.is_safe()) { + escape_json_string(v.as_text(), out); + return; + } + if (const auto* a = std::get_if(&v.storage())) { + out += '['; + bool first = true; + for (const auto& item : *a) { + if (!first) out += ','; + first = false; + write_json_value(item, out, depth + 1); + } + out += ']'; + return; + } + if (const auto* o = std::get_if(&v.storage())) { + out += '{'; + bool first = true; + for (const auto& kv : *o) { + if (!first) out += ','; + first = false; + escape_json_string(kv.first, out); + out += ':'; + write_json_value(kv.second, out, depth + 1); + } + out += '}'; + return; + } + out += "null"; +} + +inline std::string to_json(const Value& v) { + std::string out; + write_json_value(v, out, 0); + return out; +} + +} // namespace molo::app::validate diff --git a/src/app/validate/model.h b/src/app/validate/model.h new file mode 100644 index 0000000..58ac27f --- /dev/null +++ b/src/app/validate/model.h @@ -0,0 +1,88 @@ +#pragma once +// Spec: molo::app::validate::Model. Runtime-declared validation model +// (Pydantic-style without reflection): fields are added in code, validate() +// coerces an input object into a clean object or throws one ValidationError +// listing every failure (plan 05, R2, R3, E4). Feeds path, query, header, +// cookie and body binding plus OpenAPI schema generation (08-clients-openapi). + +#include +#include +#include + +#include "errors.h" +#include "field.h" +#include "value.h" + +namespace molo::app::validate { + +class Model { + public: + explicit Model(std::string name = "") : name_(std::move(name)) {} + + Model& field(Field f) { + fields_.push_back(std::move(f)); + return *this; + } + + // Reject unknown keys when true; default keeps and passes them through. + Model& strict(bool s = true) { + strict_ = s; + return *this; + } + + const std::string& name() const { return name_; } + const std::vector& fields() const { return fields_; } + + Object validate(const Object& input, const std::string& source) const { + Object out; + std::vector failures; + for (const Field& f : fields_) { + std::vector loc{source, f.name()}; + auto it = input.find(f.name()); + Value raw = (it == input.end()) ? Value::null() : it->second; + bool missing = (it == input.end()); + try { + if (missing && !f.required() && !f.has_default()) { + continue; + } + out[f.name()] = f.parse(raw, loc); + } catch (const ValidationError& e) { + for (const auto& item : e.failures()) failures.push_back(item); + } + } + if (strict_) { + for (const auto& kv : input) { + bool known = false; + for (const Field& f : fields_) { + if (f.name() == kv.first) { + known = true; + break; + } + } + if (!known) { + failures.push_back({{source, kv.first}, "unknown field", "extra_forbidden"}); + } + } + } else { + for (const auto& kv : input) { + bool known = false; + for (const Field& f : fields_) { + if (f.name() == kv.first) { + known = true; + break; + } + } + if (!known) out[kv.first] = kv.second; + } + } + if (!failures.empty()) throw ValidationError(std::move(failures)); + return out; + } + + private: + std::string name_; + std::vector fields_; + bool strict_ = false; +}; + +} // namespace molo::app::validate diff --git a/src/app/validate/value.h b/src/app/validate/value.h new file mode 100644 index 0000000..6e67ba3 --- /dev/null +++ b/src/app/validate/value.h @@ -0,0 +1,115 @@ +#pragma once +// Spec: molo::app::validate::Value. Single dynamic value type shared by +// validation, forms, negotiation variants and template contexts (DRY: one +// model, plan 05 section 9). JSON-compatible plus SafeString (template +// markup exempt from escaping) and Undefined (missing template variable). + +#include +#include +#include +#include +#include +#include + +namespace molo::app::validate { + +struct Null { + bool operator==(const Null&) const = default; +}; +struct Undefined { + bool operator==(const Undefined&) const = default; +}; +struct SafeString { + std::string text; + bool operator==(const SafeString&) const = default; +}; + +class Value; +using Array = std::vector; +using Object = std::map; + +class Value { + public: + using Storage = std::variant; + + Value() : data_(Null{}) {} + Value(Null v) : data_(v) {} + Value(Undefined v) : data_(v) {} + Value(bool v) : data_(v) {} + Value(int v) : data_((long long)v) {} + Value(long v) : data_((long long)v) {} + Value(long long v) : data_(v) {} + Value(unsigned v) : data_((long long)v) {} + Value(unsigned long v) : data_((long long)v) {} + Value(double v) : data_(v) {} + Value(const char* v) : data_(std::string(v)) {} + Value(std::string v) : data_(std::move(v)) {} + Value(SafeString v) : data_(std::move(v)) {} + Value(Array v) : data_(std::move(v)) {} + Value(Object v) : data_(std::move(v)) {} + + static Value null() { return Value(Null{}); } + static Value undefined() { return Value(Undefined{}); } + static Value safe(std::string s) { return Value(SafeString{std::move(s)}); } + + bool is_null() const { return std::holds_alternative(data_); } + bool is_undefined() const { return std::holds_alternative(data_); } + bool is_bool() const { return std::holds_alternative(data_); } + bool is_int() const { return std::holds_alternative(data_); } + bool is_float() const { return std::holds_alternative(data_); } + bool is_number() const { return is_int() || is_float(); } + bool is_string() const { return std::holds_alternative(data_); } + bool is_safe() const { return std::holds_alternative(data_); } + bool is_array() const { return std::holds_alternative(data_); } + bool is_object() const { return std::holds_alternative(data_); } + + // String content for string or safe values, else "". + std::string as_text() const { + if (const auto* s = std::get_if(&data_)) return *s; + if (const auto* s = std::get_if(&data_)) return s->text; + return ""; + } + + double as_number(double fallback = 0.0) const { + if (const auto* i = std::get_if(&data_)) return (double)*i; + if (const auto* d = std::get_if(&data_)) return *d; + if (const auto* b = std::get_if(&data_)) return *b ? 1.0 : 0.0; + return fallback; + } + + // Jinja truthiness: null/undefined/false/0/""/empty containers are false. + bool truthy() const { + if (is_null() || is_undefined()) return false; + if (const auto* b = std::get_if(&data_)) return *b; + if (const auto* i = std::get_if(&data_)) return *i != 0; + if (const auto* d = std::get_if(&data_)) return *d != 0.0; + if (const auto* s = std::get_if(&data_)) return !s->empty(); + if (const auto* s = std::get_if(&data_)) return !s->text.empty(); + if (const auto* a = std::get_if(&data_)) return !a->empty(); + if (const auto* o = std::get_if(&data_)) return !o->empty(); + return true; + } + + std::string type_name() const { + if (is_null()) return "null"; + if (is_undefined()) return "undefined"; + if (is_bool()) return "bool"; + if (is_int()) return "int"; + if (is_float()) return "float"; + if (is_string() || is_safe()) return "string"; + if (is_array()) return "array"; + if (is_object()) return "object"; + return "unknown"; + } + + const Storage& storage() const { return data_; } + Storage& storage() { return data_; } + + bool operator==(const Value& other) const = default; + + private: + Storage data_; +}; + +} // namespace molo::app::validate diff --git a/src/app/views/adapters.h b/src/app/views/adapters.h new file mode 100644 index 0000000..57c95c1 --- /dev/null +++ b/src/app/views/adapters.h @@ -0,0 +1,34 @@ +#pragma once +// Spec: molo::app::views handler adapters. Function handlers (sync or +// coroutine) and View subclasses adapt to the single routing::Handler type +// (plan 05, section 3). Raw string returns stay rejected: adapters accept +// Response-returning callables only (R4). + +#include +#include + +#include "../async/task.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" +#include "../routing/route.h" +#include "view.h" + +namespace molo::app::views { + +inline routing::Handler adapt(std::function fn) { + return [fn = std::move(fn)](http_msg::Request& req) + -> async::Task { co_return fn(req); }; +} + +inline routing::Handler adapt_task(routing::Handler fn) { return fn; } + +template +routing::Handler view_handler() { + static_assert(std::is_base_of::value, "V must derive from molo::app::views::View"); + return [](http_msg::Request& req) -> async::Task { + V view(req); + co_return co_await view.dispatch(); + }; +} + +} // namespace molo::app::views diff --git a/src/app/views/flash.h b/src/app/views/flash.h new file mode 100644 index 0000000..2ae4ef0 --- /dev/null +++ b/src/app/views/flash.h @@ -0,0 +1,42 @@ +#pragma once +// Spec: molo::app::views flash helpers. flash() stores a one-time message in +// the session store (seam owned by 03-security, default in-memory store held +// by Application); take() returns pending messages and clears them so each +// message renders exactly once (plan 05, section 6 Errors). + +#include +#include +#include +#include +#include + +#include "../errors/http_error.h" +#include "../http_msg/cookies.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" +#include "../seams/session.h" + +namespace molo::app::app { +class Application; +} + +namespace molo::app::views { + +inline std::string generate_session_id() { + static std::atomic counter{1}; + std::uint64_t n = counter.fetch_add(1, std::memory_order_relaxed); + char buf[32]; + std::snprintf(buf, sizeof(buf), "sess-%016llx", (unsigned long long)n); + return std::string(buf); +} + +inline constexpr const char* kSessionCookie = "molo_session"; + +class Flash { + public: + static void flash(http_msg::Request& req, http_msg::Response& res, std::string level, + std::string text); + static std::vector take(http_msg::Request& req); +}; + +} // namespace molo::app::views diff --git a/src/app/views/view.h b/src/app/views/view.h new file mode 100644 index 0000000..8edc79e --- /dev/null +++ b/src/app/views/view.h @@ -0,0 +1,60 @@ +#pragma once +// Spec: molo::app::views::View. aiohttp-style class-based view: constructed +// with the Request, one virtual coroutine per HTTP method, dispatch() +// selects by method (plan 05, E7). Unimplemented methods throw 405 with the +// view's Allow list; override allowed_methods() to list implemented ones. + +#include +#include + +#include "../async/task.h" +#include "../errors/http_error.h" +#include "../http_msg/method.h" +#include "../http_msg/request.h" +#include "../http_msg/response.h" + +namespace molo::app::views { + +class View { + public: + explicit View(http_msg::Request& request) : request_(request) {} + virtual ~View() = default; + + virtual async::Task get() { return method_not_allowed(); } + virtual async::Task post() { return method_not_allowed(); } + virtual async::Task put() { return method_not_allowed(); } + virtual async::Task patch() { return method_not_allowed(); } + virtual async::Task del() { return method_not_allowed(); } + virtual async::Task head() { return method_not_allowed(); } + virtual async::Task options() { return method_not_allowed(); } + + virtual std::vector allowed_methods() const { + return {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}; + } + + async::Task dispatch() { + switch (request_.method()) { + case http_msg::Method::GET: co_return co_await get(); + case http_msg::Method::POST: co_return co_await post(); + case http_msg::Method::PUT: co_return co_await put(); + case http_msg::Method::PATCH: co_return co_await patch(); + case http_msg::Method::DELETE: co_return co_await del(); + case http_msg::Method::HEAD: co_return co_await head(); + case http_msg::Method::OPTIONS: co_return co_await options(); + } + co_return co_await method_not_allowed(); + } + + protected: + http_msg::Request& request() { return request_; } + + private: + async::Task method_not_allowed() { + throw errors::MethodNotAllowed(allowed_methods()); + co_return http_msg::TextResponse("unreachable"); + } + + http_msg::Request& request_; +}; + +} // namespace molo::app::views diff --git a/src/audit.mk b/src/audit.mk new file mode 100644 index 0000000..be10aa3 --- /dev/null +++ b/src/audit.mk @@ -0,0 +1,40 @@ +# MoloApp audit trail and compliance part build fragment. +# Included by the root Makefile. Header-only library; this fragment builds +# the standalone tests. All outputs go under build/. +# +# Expected root variables (defaults work standalone): +# CXX, CXXFLAGS, AUDIT_SRC (default src/audit), AUDIT_TEST_SRC, BUILD_DIR. + +CXX ?= g++ +CXXFLAGS ?= -std=c++23 -O2 -Wall -Wextra -pthread +AUDIT_SRC ?= src/audit +AUDIT_TEST_SRC ?= tests/src/audit +BUILD_DIR ?= build + +AUDIT_TESTS := pii_boot redactor chain_tamper query_guard retention_hold \ + export_bundle erasure_cascade mask_fixture sql_store orm_hook +AUDIT_TEST_BINS := $(addprefix $(BUILD_DIR)/audit-,$(AUDIT_TESTS)) + +# INTEGRATION: with the base tree present the audit headers bind the real +# crypto/auth TU definitions, so test links compile them in. +AUDIT_LINK_SRCS := src/security/crypto/Sha256.cpp src/security/crypto/Hmac.cpp \ + src/security/crypto/SecureRandom.cpp src/security/crypto/Base64.cpp \ + src/security/auth/Guard.cpp + +.PHONY: audit audit-tests audit-clean + +audit: $(BUILD_DIR)/audit-umbrella.o + +# Umbrella compile check: the whole part must compile as one TU. +$(BUILD_DIR)/audit-umbrella.o: $(AUDIT_SRC)/audit.h + @mkdir -p $(BUILD_DIR) + printf '#include "audit/audit.h"\nint main(){return 0;}\n' | $(CXX) $(CXXFLAGS) -I$(AUDIT_SRC)/.. -x c++ -c -o $@ - + +audit-tests: $(AUDIT_TEST_BINS) + +$(BUILD_DIR)/audit-%: $(AUDIT_TEST_SRC)/%.cpp $(AUDIT_SRC)/audit.h + @mkdir -p $(BUILD_DIR) + $(CXX) $(CXXFLAGS) -I$(AUDIT_SRC)/.. -o $@ $< $(AUDIT_LINK_SRCS) + +audit-clean: + rm -f $(BUILD_DIR)/audit-* diff --git a/src/audit/PATCHES.md b/src/audit/PATCHES.md new file mode 100644 index 0000000..0804b2a --- /dev/null +++ b/src/audit/PATCHES.md @@ -0,0 +1,141 @@ +# Audit part: required base-tree edits + +The audit part itself is header-only under `src/audit/` and needs no base +changes to compile. The five edits below wire it into the framework. Apply in +order. Anchors quote the seam API; adjust surrounding context lines to the +real files, keep the added lines verbatim. + +Include style: audit headers live in `src/audit/`, so base files reach them +by relative path (data layer convention). + +## 1. src/data/orm/field.hpp: PII mark on FieldDef + +Gives every field its compliance mark (see `src/audit/pii/mark.h`). The +`Registry::Build` boot check fails the build with the field name when a +schema field has no mark. + +```diff +--- a/src/data/orm/field.hpp ++++ b/src/data/orm/field.hpp +@@ + #pragma once ++ ++#include "../../audit/pii/mark.h" +@@ + struct FieldDef { + std::string name; + // ... existing members (type, length, nullable, default_value, + // has_default, unique, indexed, pk, check) ... ++ // PII classification. Default None keeps unclassified columns compiling ++ // until Registry::Build boot-fails them by name. ++ ::molo::audit::pii::FieldMark pii; + }; +``` + +## 2. src/data/orm/model.hpp: ORM write hook + +Calls the audit hook from `Model::save` and `Model::remove` +after the existing `bus->publish` call. `notify_write` no-ops until +`Appender::Install` wires a store, so ORM boots without audit configured. + +```diff +--- a/src/data/orm/model.hpp ++++ b/src/data/orm/model.hpp +@@ + #pragma once ++ ++#include "../../audit/log/appender.h" +@@ + // Model::save, after the existing bus->publish(inv) line: + if (bus) bus->publish(inv); ++ AUDIT(store, table_name(), pk_string(), "save", before_json(), after_json(), ctx, actor); +@@ + // Model::remove, after the existing bus->publish(inv) line: + if (bus) bus->publish(inv); ++ AUDIT(store, table_name(), pk_string(), "delete", before_json(), "", ctx, actor); +``` + +Notes: + +- `AUDIT` is defined in `src/audit/log/appender.h` and forwards to + `::molo::audit::log::notify_write(table, pk, op, before, after, ctx, actor)`. +- `table_name()`, `pk_string()`, `before_json()`, `after_json()` are the + call-site spellings for the existing `table_name`/`primary_key` statics and + the `get`/`values` accessors; use whatever locals the real `save`/`remove` + already hold. +- `actor` is the ambient subject string when the call site has one, else `""`. + +## 3. src/observability/logging/registry.h + registry.cpp: redact on emit + +Passes every record through the redactor before fan-out to sinks, so PII can +never reach a log file through any logger. + +```diff +--- a/src/observability/logging/registry.h ++++ b/src/observability/logging/registry.h +@@ + #pragma once ++ ++#include "../../audit/redact/redactor.h" +@@ + void emit(Record r) { ++ ::molo::audit::redact::Redactor::RedactRecord(r); + if (hook_) hook_(r); + for (auto& s : sinks_) s->write(r); + } +``` + +If the real `emit` lives in `registry.cpp` instead of the header, put the +include and the added line there. + +## 4. src/app/errors/handler.h: redact error detail + +Runs user-visible detail and internal diagnostics through the redactor before +they reach the response body and the log sink. + +```diff +--- a/src/app/errors/handler.h ++++ b/src/app/errors/handler.h +@@ + #pragma once ++ ++#include "../../audit/redact/redactor.h" +@@ + Response render(const HttpError& e, const Request& req, const std::string& request_id) { ++ const std::string safe_detail = ++ ::molo::audit::redact::Redactor::RedactDetail(e.detail()); + // ... build body from safe_detail instead of e.detail() ... + } +@@ + Response render_unknown(const std::string& internal, const std::string& request_id) { ++ const std::string safe_internal = ++ ::molo::audit::redact::Redactor::RedactDetail(internal); + // ... sink + body use safe_internal ... + } +``` + +## 5. src/security/auth/Permission.h: audit grants + +Declares the three permission names the audit query path enforces via +`Guard::check` (`audit.read` today; `audit.query` and `rights.admin` reserved +for the query service and the rights-request console). + +```diff +--- a/src/security/auth/Permission.h ++++ b/src/security/auth/Permission.h +@@ + namespace molo::security::auth { ++ ++inline const Permission kPermAuditRead{"audit.read"}; ++inline const Permission kPermAuditQuery{"audit.query"}; ++inline const Permission kPermRightsAdmin{"rights.admin"}; ++ + // ... existing permission declarations ... + } +``` + +No `Application` edit is needed: audit middleware and routes register at +deploy composition, and `handle()` already sets the request id pre-chain. +Blocking audit work (`Enforcer::Apply`, `Exporter::Build`, `Eraser::Erase`, +`Fixture::Scrub`) stays synchronous for the supervisor scheduler and is never +called from request handlers. diff --git a/src/audit/adapt.h b/src/audit/adapt.h new file mode 100644 index 0000000..a1cab35 --- /dev/null +++ b/src/audit/adapt.h @@ -0,0 +1,670 @@ +// ADAPTER: minimal local stand-ins for base-tree seam headers that are +// absent in an isolated checkout. Every block below prefers the real header +// via __has_include and only defines a stub when it is missing. Stubs carry +// the exact seam names so the audit code compiles unchanged against the real +// tree. Delete nothing here when integrating: the guards go quiet on their +// own once the base tree is present. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ---------------------------------------------------------------- crypto +#if __has_include("security/crypto/Sha256.h") +#include "security/crypto/Sha256.h" +#elif __has_include("src/security/crypto/Sha256.h") +#include "src/security/crypto/Sha256.h" +#else +// ADAPTER: compact real SHA-256 (FIPS 180-4) so chain/seal code and tests run +// standalone. Interface mirrors the seam: Sha256::hash/hex/update/final. +namespace molo::security::crypto { +class Sha256 { + public: + static std::array hash(std::string_view msg) { + Sha256 h; + h.update(msg); + return h.final(); + } + static std::string hex(const std::array& d) { + static constexpr char kHex[] = "0123456789abcdef"; + std::string out; + out.resize(64); + for (int i = 0; i < 32; ++i) { + out[2 * i] = kHex[(d[i] >> 4) & 15]; + out[2 * i + 1] = kHex[d[i] & 15]; + } + return out; + } + void update(std::string_view msg) { + for (char c : msg) { + block_[used_++] = static_cast(c); + ++total_; + if (used_ == 64) { + compress(); + used_ = 0; + } + } + } + std::array final() { + std::uint64_t bits = total_ * 8; + unsigned char pad = 0x80; + update({reinterpret_cast(&pad), 1}); + unsigned char zero = 0; + while (used_ != 56) update({reinterpret_cast(&zero), 1}); + for (int i = 7; i >= 0; --i) + update({reinterpret_cast(&kZero_) + 0, 0}), feed_byte((bits >> (i * 8)) & 0xff); + std::array d{}; + for (int i = 0; i < 8; ++i) { + d[4 * i] = (s_[i] >> 24) & 0xff; + d[4 * i + 1] = (s_[i] >> 16) & 0xff; + d[4 * i + 2] = (s_[i] >> 8) & 0xff; + d[4 * i + 3] = s_[i] & 0xff; + } + return d; + } + + private: + void feed_byte(unsigned char b) { + block_[used_++] = b; + if (used_ == 64) { + compress(); + used_ = 0; + } + } + static std::uint32_t rotr(std::uint32_t x, int n) { return (x >> n) | (x << (32 - n)); } + void compress() { + static constexpr std::uint32_t k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2}; + std::uint32_t w[64]; + for (int i = 0; i < 16; ++i) + w[i] = (std::uint32_t(block_[4 * i]) << 24) | (std::uint32_t(block_[4 * i + 1]) << 16) | + (std::uint32_t(block_[4 * i + 2]) << 8) | std::uint32_t(block_[4 * i + 3]); + for (int i = 16; i < 64; ++i) { + std::uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + std::uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + std::uint32_t v[8]; + for (int i = 0; i < 8; ++i) v[i] = s_[i]; + for (int i = 0; i < 64; ++i) { + std::uint32_t s1 = rotr(v[4], 6) ^ rotr(v[4], 11) ^ rotr(v[4], 25); + std::uint32_t ch = (v[4] & v[5]) ^ (~v[4] & v[6]); + std::uint32_t t1 = v[7] + s1 + ch + k[i] + w[i]; + std::uint32_t s0 = rotr(v[0], 2) ^ rotr(v[0], 13) ^ rotr(v[0], 22); + std::uint32_t mj = (v[0] & v[1]) ^ (v[0] & v[2]) ^ (v[1] & v[2]); + std::uint32_t t2 = s0 + mj; + v[7] = v[6]; + v[6] = v[5]; + v[5] = v[4]; + v[4] = v[3] + t1; + v[3] = v[2]; + v[2] = v[1]; + v[1] = v[0]; + v[0] = t1 + t2; + } + for (int i = 0; i < 8; ++i) s_[i] += v[i]; + } + std::uint32_t s_[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + unsigned char block_[64] = {}; + std::size_t used_ = 0; + std::uint64_t total_ = 0; + static constexpr char kZero_ = 0; +}; +} // namespace molo::security::crypto +#endif + +#if __has_include("security/crypto/Hmac.h") +#include "security/crypto/Hmac.h" +#elif __has_include("src/security/crypto/Hmac.h") +#include "src/security/crypto/Hmac.h" +#else +// ADAPTER: HMAC-SHA256 over the adapter Sha256 above. +namespace molo::security::crypto { +class Hmac { + public: + static std::array sha256(std::string_view key, std::string_view msg) { + unsigned char kb[64] = {}; + if (key.size() > 64) { + auto h = Sha256::hash(key); + for (int i = 0; i < 32; ++i) kb[i] = h[i]; + } else { + for (std::size_t i = 0; i < key.size(); ++i) kb[i] = static_cast(key[i]); + } + std::string ik, ok; + for (int i = 0; i < 64; ++i) { + ik.push_back(static_cast(kb[i] ^ 0x36)); + ok.push_back(static_cast(kb[i] ^ 0x5c)); + } + Sha256 inner; + inner.update(ik); + inner.update(msg); + auto ih = inner.final(); + Sha256 outer; + outer.update(ok); + outer.update({reinterpret_cast(ih.data()), 32}); + return outer.final(); + } + static std::string hex(const std::array& d) { return Sha256::hex(d); } +}; +} // namespace molo::security::crypto +#endif + +// ---------------------------------------------------------------- data value +#if __has_include("data/value.hpp") +#include "data/value.hpp" +#elif __has_include("src/data/value.hpp") +#include "src/data/value.hpp" +#else +// ADAPTER: seam-compatible Value variant + makers. +namespace molo::data { +using Bytes = std::vector; +using TimePoint = std::chrono::system_clock::time_point; +using Value = std::variant; +inline Value make_null() { return Value{std::monostate{}}; } +inline Value make_int(std::int64_t v) { return Value{v}; } +inline Value make_double(double v) { return Value{v}; } +inline Value make_text(std::string v) { return Value{std::move(v)}; } +inline Value make_bytes(Bytes v) { return Value{std::move(v)}; } +inline Value make_bool(bool v) { return Value{v}; } +inline Value make_time(TimePoint v) { return Value{v}; } +} // namespace molo::data +#endif + +// ---------------------------------------------------------------- data error +#if __has_include("data/error.hpp") +#include "data/error.hpp" +#elif __has_include("src/data/error.hpp") +#include "src/data/error.hpp" +#else +// ADAPTER: seam-compatible DbError. +namespace molo::data { +enum class Backend { kPostgres, kSqlite }; +class DbError { + public: + DbError(Backend b, std::string c, std::string m, std::string n = "") + : backend_(b), code_(std::move(c)), msg_(std::move(m)), native_(std::move(n)) {} + Backend backend() const { return backend_; } + const std::string& code() const { return code_; } + const std::string& detail() const { return msg_; } + bool healable() const { return false; } + bool retryable_txn() const { return false; } + + private: + Backend backend_; + std::string code_, msg_, native_; +}; +} // namespace molo::data +#endif + +// ---------------------------------------------------------------- data context +#if __has_include("data/context.hpp") +#include "data/context.hpp" +#elif __has_include("src/data/context.hpp") +#include "src/data/context.hpp" +#else +// ADAPTER: seam-compatible Context. +namespace molo::data { +class Context { + public: + Context(std::string cid, std::string tenant, int timeout_sec = 30) + : cid_(std::move(cid)), tenant_(std::move(tenant)), timeout_(timeout_sec) {} + static Context background(std::string why) { return Context("bg:" + why, "", 30); } + const std::string& correlation_id() const { return cid_; } + const std::string& tenant() const { return tenant_; } + bool expired() const { return false; } + + private: + std::string cid_, tenant_; + int timeout_; +}; +} // namespace molo::data +#endif + +// ---------------------------------------------------------------- data statement +#if __has_include("data/statement.hpp") +#include "data/statement.hpp" +#elif __has_include("src/data/statement.hpp") +#include "src/data/statement.hpp" +#else +// ADAPTER: seam-compatible BoundStatement + check_identifier. +namespace molo::data { +inline void check_identifier(const std::string& s) { + for (char c : s) + if (!(c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'))) + throw DbError(Backend::kSqlite, "bad_identifier", s); + if (s.empty() || s.size() > 128) throw DbError(Backend::kSqlite, "bad_identifier", s); +} +class BoundStatement { + public: + BoundStatement(std::string sql, std::vector params) + : sql_(std::move(sql)), params_(std::move(params)) {} + static BoundStatement raw(std::string sql, std::vector params = {}) { + return BoundStatement(std::move(sql), std::move(params)); + } + const std::string& sql() const { return sql_; } + const std::vector& params() const { return params_; } + + private: + std::string sql_; + std::vector params_; +}; +} // namespace molo::data +#endif + +// ---------------------------------------------------------------- data connection +#if __has_include("data/connection.hpp") +#include "data/connection.hpp" +#elif __has_include("src/data/connection.hpp") +#include "src/data/connection.hpp" +#else +// ADAPTER: in-memory Connection stub with a tiny SQL surface (CREATE TABLE, +// INSERT, SELECT with WHERE seq range, DELETE with WHERE) sufficient for the +// audit SqlStore standalone. Not a database: unknown statements fail loudly. +namespace molo::data { +class Connection { + public: + using Row = std::vector; + struct Result { + std::vector rows; + std::uint64_t affected = 0; + }; + explicit Connection(Backend b = Backend::kSqlite) : backend_(b) {} + Backend backend() const { return backend_; } + std::string backend_id() const { return backend_ == Backend::kPostgres ? "pg" : "sqlite"; } + bool is_postgres() const { return backend_ == Backend::kPostgres; } + // ADAPTER has no SQL engine: audit SqlStore binds through SqlExec + // instead (see log/sql_store.h). Any direct call fails loudly. + Result exec(const Context&, const BoundStatement&) { + throw DbError(backend_, "adapter_no_sql", "adapter Connection has no SQL engine"); + } + + private: + Backend backend_; + std::map> tables_; +}; +} // namespace molo::data +#endif + +// ---------------------------------------------------------------- kv store +#if __has_include("data/kv/store.hpp") +#include "data/kv/store.hpp" +#elif __has_include("src/data/kv/store.hpp") +#include "src/data/kv/store.hpp" +#else +// ADAPTER: in-memory KvStore with the seam surface. +namespace molo::data::kv { +struct ScanPage { + std::vector> items; + std::string next_cursor; + bool done = true; +}; +class KvStore { + public: + std::optional get(const std::string& k) { + auto it = m_.find(k); + if (it == m_.end()) return std::nullopt; + return it->second; + } + void set(std::string k, Bytes v, int = 0) { m_[std::move(k)] = std::move(v); } + bool del(const std::string& k) { return m_.erase(k) > 0; } + bool exists(const std::string& k) { return m_.count(k) > 0; } + ScanPage scan(const std::string& prefix, const std::string& cursor, std::size_t limit) { + ScanPage p; + bool started = cursor.empty(); + for (auto& [k, v] : m_) { + if (k.rfind(prefix, 0) != 0) continue; + if (!started) { + if (k == cursor) started = true; + continue; + } + p.items.emplace_back(k, v); + if (p.items.size() >= limit) { + p.done = false; + p.next_cursor = k; + break; + } + } + return p; + } + + private: + std::map m_; +}; +} // namespace molo::data::kv +#endif + +// ---------------------------------------------------------------- cache +#if __has_include("data/cache/cache.hpp") +#include "data/cache/cache.hpp" +#elif __has_include("src/data/cache/cache.hpp") +#include "src/data/cache/cache.hpp" +#else +// ADAPTER: in-memory Cache with the seam surface. +namespace molo::data::cache { +inline std::string tag_for_table(const std::string& t) { return "table:" + t; } +class Cache { + public: + std::optional get(const std::string& k) { + auto it = m_.find(k); + if (it == m_.end()) return std::nullopt; + return it->second; + } + void put(std::string k, Bytes v, std::vector tags = {}, int = 0) { + for (auto& t : tags) tags_[t].push_back(k); + m_[std::move(k)] = std::move(v); + } + bool invalidate(const std::string& k) { return m_.erase(k) > 0; } + std::int64_t purge_tag(const std::string& t) { + std::int64_t n = 0; + auto it = tags_.find(t); + if (it == tags_.end()) return 0; + for (auto& k : it->second) n += m_.erase(k); + tags_.erase(it); + return n; + } + + private: + std::map m_; + std::map> tags_; +}; +} // namespace molo::data::cache +#endif + +// ---------------------------------------------------------------- invalidation bus +#if __has_include("data/orm/invalidation.hpp") +#include "data/orm/invalidation.hpp" +#elif __has_include("src/data/orm/invalidation.hpp") +#include "src/data/orm/invalidation.hpp" +#else +// ADAPTER: seam-compatible Invalidation + InvalidationBus. +namespace molo::data::orm { +struct Invalidation { + std::string table, pk, op; + std::int64_t txid = 0; +}; +class InvalidationBus { + public: + using Handler = std::function; + int subscribe(Handler h) { + handlers_[next_] = std::move(h); + return next_++; + } + void unsubscribe(int id) { handlers_.erase(id); } + void publish(const Invalidation& iv) { + for (auto& [_, h] : handlers_) h(iv); + } + + private: + std::map handlers_; + int next_ = 1; +}; +} // namespace molo::data::orm +#endif + +// ---------------------------------------------------------------- metric sink +#if __has_include("data/obs.hpp") +#include "data/obs.hpp" +#elif __has_include("src/data/obs.hpp") +#include "src/data/obs.hpp" +#else +// ADAPTER: seam-compatible MetricSink + null_sink. +namespace molo::data::obs { +class MetricSink { + public: + virtual ~MetricSink() = default; + virtual void counter(const std::string& n, double d, + const std::vector>& l = {}) = 0; + virtual void gauge(const std::string& n, double v, + const std::vector>& l = {}) = 0; + virtual void histogram(const std::string& n, double v, + const std::vector>& l = {}) = 0; +}; +class NullSink : public MetricSink { + public: + void counter(const std::string&, double, + const std::vector>&) override {} + void gauge(const std::string&, double, + const std::vector>&) override {} + void histogram(const std::string&, double, + const std::vector>&) override {} +}; +inline std::shared_ptr null_sink() { return std::make_shared(); } +} // namespace molo::data::obs +#endif + +// ---------------------------------------------------------------- log record/sink +#if __has_include("observability/logging/record.h") +#include "observability/logging/record.h" +#elif __has_include("src/observability/logging/record.h") +#include "src/observability/logging/record.h" +#else +// ADAPTER: seam-compatible log Record/Level/Field. +namespace molo::log { +enum class Level { kTrace, kDebug, kInfo, kWarn, kError, kFatal }; +struct Field { + std::string key, value; +}; +struct Record { + std::int64_t timestamp = 0; + Level level = Level::kInfo; + std::string logger, message, file, func, request_id, trace_id, span_id; + int line = 0; + std::vector fields; + bool truncated = false; +}; +} // namespace molo::log +#endif + +#if __has_include("observability/logging/sink.h") +#include "observability/logging/sink.h" +#elif __has_include("src/observability/logging/sink.h") +#include "src/observability/logging/sink.h" +#else +// ADAPTER: seam-compatible Sink. +namespace molo::log { +class Sink { + public: + virtual ~Sink() = default; + virtual void write(const Record& r) = 0; + virtual void flush() {} +}; +} // namespace molo::log +#endif + +#if __has_include("observability/logging/registry.h") +#include "observability/logging/registry.h" +#elif __has_include("src/observability/logging/registry.h") +#include "src/observability/logging/registry.h" +#else +// ADAPTER: seam-compatible Registry (emit path only). +namespace molo::log { +class Registry { + public: + static Registry& global() { + static Registry r; + return r; + } + void add_sink(std::shared_ptr s) { sinks_.push_back(std::move(s)); } + void emit(Record r) { + if (hook_) hook_(r); + for (auto& s : sinks_) s->write(r); + } + void set_record_hook(std::function h) { hook_ = std::move(h); } + + private: + std::vector> sinks_; + std::function hook_; +}; +} // namespace molo::log +#endif + +#if __has_include("observability/logging/logger.h") +#include "observability/logging/logger.h" +#elif __has_include("src/observability/logging/logger.h") +#include "src/observability/logging/logger.h" +#else +// ADAPTER: seam-compatible Logger surface used by audit. +namespace molo::log { +class Logger { + public: + explicit Logger(std::string n) : name_(std::move(n)) {} + void log(Level lv, std::string_view msg, std::source_location loc = std::source_location::current(), + std::vector f = {}) { + Record r; + r.level = lv; + r.logger = name_; + r.message = std::string(msg); + r.file = loc.file_name(); + r.line = static_cast(loc.line()); + r.func = loc.function_name(); + r.fields = std::move(f); + Registry::global().emit(std::move(r)); + } + + private: + std::string name_; +}; +} // namespace molo::log +#endif + +// ---------------------------------------------------------------- correlation +#if __has_include("observability/correlation/request_context.h") +#include "observability/correlation/request_context.h" +#elif __has_include("src/observability/correlation/request_context.h") +#include "src/observability/correlation/request_context.h" +#else +// ADAPTER: seam-compatible RequestContext. +namespace molo::corr { +class RequestContext { + public: + explicit RequestContext(std::string rid) : rid_(std::move(rid)) {} + const std::string& request_id() const { return rid_; } + + private: + std::string rid_; +}; +} // namespace molo::corr +#endif + +#if __has_include("observability/correlation/redaction.h") +#include "observability/correlation/redaction.h" +#elif __has_include("src/observability/correlation/redaction.h") +#include "src/observability/correlation/redaction.h" +#else +// ADAPTER: seam-compatible sensitivity helpers. +namespace molo::corr { +inline bool is_sensitive_key(std::string_view key) { + static constexpr const char* k[] = {"password", "passwd", "secret", "token", "api_key", + "apikey", "auth", "ssn", "card", "iban", "credential"}; + for (auto* s : k) + if (key.find(s) != std::string_view::npos) return true; + return false; +} +inline std::string redact_value(std::string_view, std::string_view) { return "[redacted]"; } +} // namespace molo::corr +#endif + +// ---------------------------------------------------------------- metrics counter +#if __has_include("observability/metrics/counter.h") +#include "observability/metrics/counter.h" +#elif __has_include("src/observability/metrics/counter.h") +#include "src/observability/metrics/counter.h" +#else +// ADAPTER: seam-compatible Counter. +namespace molo::metrics { +class Counter { + public: + Counter(std::string n, std::string h, std::vector l = {}) + : name_(std::move(n)), help_(std::move(h)), labels_(std::move(l)) {} + void inc(double d = 1.0) { value_ += d; } + double value() const { return value_; } + + private: + std::string name_, help_; + std::vector labels_; + double value_ = 0; +}; +} // namespace molo::metrics +#endif + +// ---------------------------------------------------------------- http request +#if __has_include("app/http_msg/request.h") +#include "app/http_msg/request.h" +#elif __has_include("src/app/http_msg/request.h") +#include "src/app/http_msg/request.h" +#else +// ADAPTER: seam-compatible Request surface used by audit. +namespace molo::app::http_msg { +class Request { + public: + const std::string& request_id() const { return rid_; } + void set_request_id(std::string v) { rid_ = std::move(v); } + std::string route() const { return route_; } + + private: + std::string rid_, route_; +}; +} // namespace molo::app::http_msg +#endif + +// ---------------------------------------------------------------- auth guard +#if __has_include("security/auth/Guard.h") +#include "security/auth/Guard.h" +#elif __has_include("src/security/auth/Guard.h") +#include "src/security/auth/Guard.h" +#else +// ADAPTER: seam-compatible Guard/Permission/Role/Session/Verdict. +namespace molo::security::auth { +struct Permission { + std::string name; +}; +struct Role { + std::string name; + std::vector grants; +}; +struct Session { + std::string subject; + std::string tenant; + std::vector roles; +}; +struct Verdict { + bool allowed = false; + int http_status = 403; + std::string code = "forbidden"; +}; +class Guard { + public: + static Verdict check(const Session& s, const std::vector&, const std::vector& req) { + for (auto& p : req) { + bool ok = false; + for (auto& r : s.roles) + for (auto& g : r.grants) + if (g == p.name) ok = true; + if (!ok) return Verdict{false, 403, "forbidden"}; + } + return Verdict{true, 200, "ok"}; + } +}; +} // namespace molo::security::auth +#endif diff --git a/src/audit/audit.h b/src/audit/audit.h new file mode 100644 index 0000000..c5ea755 --- /dev/null +++ b/src/audit/audit.h @@ -0,0 +1,24 @@ +// MoloApp audit trail and compliance part: single umbrella include. +#pragma once + +#include "adapt.h" +#include "caps.h" +#include "log/appender.h" +#include "log/chain.h" +#include "log/query.h" +#include "log/record.h" +#include "log/sql_store.h" +#include "log/store.h" +#include "mask/fixture.h" +#include "mask/masker.h" +#include "pii/mark.h" +#include "pii/registry.h" +#include "redact/pattern.h" +#include "redact/redactor.h" +#include "retention/enforcer.h" +#include "retention/hold.h" +#include "retention/policy.h" +#include "rights/eraser.h" +#include "rights/exporter.h" +#include "rights/request.h" +#include "rights/subject.h" diff --git a/src/audit/caps.h b/src/audit/caps.h new file mode 100644 index 0000000..c918a9a --- /dev/null +++ b/src/audit/caps.h @@ -0,0 +1,40 @@ +// MoloApp audit trail and compliance part: static capacity ceilings. +// Every knob in this part is a compile-time constant. No autotuning. +#pragma once + +#include +#include + +namespace molo::audit { + +// Single audit record after JSON encoding, including before/after images. +inline constexpr std::size_t kMaxRecordBytes = 64 * 1024; +// Before/after image payloads inside one record. +inline constexpr std::size_t kMaxBeforeBytes = 16 * 1024; +inline constexpr std::size_t kMaxAfterBytes = 16 * 1024; +// Chain seal cadence: every N records or every T seconds, whichever first. +inline constexpr std::uint64_t kSealEveryRecords = 1000; +inline constexpr std::uint64_t kSealEverySeconds = 60; +// Max rows returned by one audit query page. +inline constexpr std::size_t kQueryPageMax = 1000; +// Subject export bundle limits. +inline constexpr std::size_t kExportMaxBytes = 256ULL * 1024 * 1024; +inline constexpr std::size_t kExportMaxRows = 1000000; +// Max subject rows erased by one erasure run. +inline constexpr std::size_t kErasureMaxPerRun = 100000; +// Max rows scanned by one retention enforcement run. +inline constexpr std::size_t kRetentionScanMaxPerRun = 1000000; +// Max input bytes redacted by one Redactor call. +inline constexpr std::size_t kRedactMaxBytesPerCall = 1024 * 1024; +// Max live retention holds process-wide. +inline constexpr std::size_t kMaxHolds = 100000; +// Max open rights requests per tenant. +inline constexpr std::size_t kMaxOpenRequestsPerTenant = 10000; +// Fixture scrub chunk size (1MB streaming, never full buffering). +inline constexpr std::size_t kFixtureChunkBytes = 1024 * 1024; +// Max audit fields per log record redaction pass. +inline constexpr std::size_t kRedactMaxFields = 256; +// Max PII hits processed by one redaction/scrub call. +inline constexpr std::size_t kRedactMaxHitsPerCall = 65536; + +} // namespace molo::audit diff --git a/src/audit/log/appender.h b/src/audit/log/appender.h new file mode 100644 index 0000000..9d9e8c6 --- /dev/null +++ b/src/audit/log/appender.h @@ -0,0 +1,124 @@ +// Appender: the write path for audit records. ORM save()/remove() reach it +// through notify_write (see PATCHES.md entry 2); the AUDIT macro fills +// actor, subject, request and trace ids from the ambient call context. +// Oversize before/after images are truncated to their caps; oversize records +// are rejected loudly, never stored half. +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../adapt.h" +#include "../caps.h" +#include "record.h" +#include "store.h" + +namespace molo::audit::log { + +class Appender { + public: + // INTEGRATION: real null_sink() returns NullSink&, not shared_ptr, and + // real counter() takes (name, int64, labels-string). + explicit Appender(Store& store, ::molo::data::obs::MetricSink& metrics = + ::molo::data::obs::null_sink()) + : store_(store), metrics_(metrics) {} + + struct Write { + std::string actor; + std::string action; + std::string table; + std::string pk; + std::string op; + std::string before_json; + std::string after_json; + std::string tenant; + std::string request_id; + std::string trace_id; + }; + + // Appends one record, returns its seq. Throws on oversize record. + std::uint64_t Append(Write w) { + if (w.before_json.size() > kMaxBeforeBytes) w.before_json.resize(kMaxBeforeBytes); + if (w.after_json.size() > kMaxAfterBytes) w.after_json.resize(kMaxAfterBytes); + AuditRecord rec; + rec.ts = Now(); + rec.tenant = std::move(w.tenant); + rec.actor = std::move(w.actor); + rec.action = std::move(w.action); + rec.table = std::move(w.table); + rec.pk = std::move(w.pk); + rec.op = std::move(w.op); + rec.before_json = std::move(w.before_json); + rec.after_json = std::move(w.after_json); + rec.request_id = std::move(w.request_id); + rec.trace_id = std::move(w.trace_id); + if (rec.Canonical().size() > kMaxRecordBytes) + throw std::runtime_error("audit record exceeds kMaxRecordBytes"); + std::uint64_t seq = store_.Append(rec); + appended_.fetch_add(1); + metrics_.counter("audit_records_appended_total", 1, "table=" + rec.table); + MaybeSeal(seq); + return seq; + } + + // Process-wide writer installed once at deploy composition. Null until set. + static void Install(Appender* a) { Global() = a; } + static Appender* GlobalInstance() { return Global(); } + std::uint64_t appended() const { return appended_.load(); } + + private: + static Appender*& Global() { + static Appender* g = nullptr; + return g; + } + + static std::int64_t Now() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + + void MaybeSeal(std::uint64_t seq) { + if (seq % kSealEveryRecords != 0) return; + Seal s = Chain::MakeSeal(seq / kSealEveryRecords, store_.Tip(), Now(), seal_key_); + store_.SaveSeal(s); + metrics_.counter("audit_seals_total", 1); + } + + Store& store_; + ::molo::data::obs::MetricSink& metrics_; + std::string seal_key_; // empty until deploy composition provisions the key + std::atomic appended_{0}; +}; + +// ORM hook called from Model save()/remove() after bus->publish. No-op until +// Appender::Install wires the store, so ORM boots without audit configured. +inline void notify_write(const std::string& table, const std::string& pk, const std::string& op, + const std::string& before_json, const std::string& after_json, + const ::molo::data::Context& ctx, const std::string& actor = "") { + Appender* a = Appender::GlobalInstance(); + if (!a) return; + Appender::Write w; + w.actor = actor; + w.action = "orm." + op; + w.table = table; + w.pk = pk; + w.op = op; + w.before_json = before_json; + w.after_json = after_json; + w.tenant = ctx.tenant(); + w.request_id = ctx.correlation_id(); + a->Append(std::move(w)); +} + +} // namespace molo::audit::log + +// AUDIT(store, table, pk, op, before, after, ctx, actor): one-line ORM hook +// filling tenant + request id from the data Context. Availability of the +// global Appender decides whether this writes or no-ops. +#define AUDIT(store, table, pk, op, before, after, ctx, actor) \ + ::molo::audit::log::notify_write((table), (pk), (op), (before), (after), (ctx), (actor)) diff --git a/src/audit/log/chain.h b/src/audit/log/chain.h new file mode 100644 index 0000000..e2af20a --- /dev/null +++ b/src/audit/log/chain.h @@ -0,0 +1,112 @@ +// Hash chain over audit records: hash = SHA256hex(prev_hash || canonical). +// Genesis prev_hash is 64 zeros. Seal checkpoints the tip with an HMAC under +// the audit seal key; Verify replays the chain and names the exact sequence +// number of the first gap or tampered record. +#pragma once + +#include +#include +#include +#include + +#include "../adapt.h" +#include "../caps.h" +#include "record.h" + +namespace molo::audit::log { + +inline const std::string& GenesisHash() { + static const std::string k(64, '0'); + return k; +} + +struct Seal { + std::uint64_t seal_seq = 0; + std::string root; // tip hash at seal time + std::int64_t ts = 0; + std::string mac; // HMAC-SHA256 hex over "seal_seq:root:ts" under seal key +}; + +struct VerifyResult { + bool ok = true; + std::uint64_t bad_seq = 0; // first gap or tamper, valid when !ok + std::string reason; // "gap", "prev", "hash", "seal" +}; + +class Chain { + public: + static std::string HashEntry(std::string_view prev_hash, std::string_view canonical) { + std::string in(prev_hash); + in += canonical; + return ::molo::security::crypto::Sha256::hex( + ::molo::security::crypto::Sha256::hash(in)); + } + + // Stamps prev/hash onto rec given the previous tip (genesis for seq 1). + static void Link(AuditRecord& rec, std::string_view prev_tip) { + rec.prev_hash = std::string(prev_tip); + rec.hash = HashEntry(rec.prev_hash, rec.Canonical()); + } + + static Seal MakeSeal(std::uint64_t seal_seq, std::string_view root, std::int64_t ts, + std::string_view seal_key) { + Seal s; + s.seal_seq = seal_seq; + s.root = std::string(root); + s.ts = ts; + std::string body = + std::to_string(seal_seq) + ":" + s.root + ":" + std::to_string(ts); + // INTEGRATION: real Hmac takes (const string&, const string&) and hex + // takes a non-const ref, so materialize both arguments. + auto mac = ::molo::security::crypto::Hmac::sha256(std::string(seal_key), body); + s.mac = ::molo::security::crypto::Hmac::hex(mac); + return s; + } + + static bool CheckSeal(const Seal& s, std::string_view seal_key) { + return MakeSeal(s.seal_seq, s.root, s.ts, seal_key).mac == s.mac; + } + + // Replays ordered records: seq continuity, prev linkage, entry hashes. + // When seals is non-empty the covered tips must match a valid seal root. + static VerifyResult Verify(const std::vector& recs, + const std::vector& seals = {}, + std::string_view seal_key = "") { + VerifyResult r; + std::string tip = GenesisHash(); + std::uint64_t expect = recs.empty() ? 0 : recs.front().seq; + for (const auto& rec : recs) { + if (rec.seq != expect) { + r.ok = false; + r.bad_seq = expect; + r.reason = "gap"; + return r; + } + if (rec.prev_hash != tip) { + r.ok = false; + r.bad_seq = rec.seq; + r.reason = "prev"; + return r; + } + if (rec.hash != HashEntry(rec.prev_hash, rec.Canonical())) { + r.ok = false; + r.bad_seq = rec.seq; + r.reason = "hash"; + return r; + } + tip = rec.hash; + ++expect; + } + for (const auto& s : seals) { + if (!seal_key.empty() && !CheckSeal(s, seal_key)) { + r.ok = false; + r.reason = "seal"; + return r; + } + } + (void)kSealEveryRecords; + return r; + } +}; + +} // namespace molo::audit::log diff --git a/src/audit/log/query.h b/src/audit/log/query.h new file mode 100644 index 0000000..d15bb22 --- /dev/null +++ b/src/audit/log/query.h @@ -0,0 +1,107 @@ +// Query: filtered, paged reads over the audit trail. Every query requires +// the audit.read grant (Guard verdict 403 otherwise) and every query, allowed +// or denied, appends its own self-audit record so trail reads stay visible. +#pragma once + +#include +#include +#include +#include + +#include "../adapt.h" +#include "../caps.h" +#include "appender.h" +#include "record.h" +#include "store.h" + +namespace molo::audit::log { + +struct Filter { + std::string tenant; + std::string table; + std::string actor; + std::string op; + std::uint64_t from_seq = 0; + std::uint64_t to_seq = 0; // 0 means no upper bound + std::size_t limit = 100; + std::string cursor; // seq to resume after, empty starts at from_seq +}; + +struct QueryPage { + int http_status = 200; + std::string code = "ok"; + std::vector rows; + std::string next_cursor; +}; + +class Query { + public: + Query(Store& store, Appender* audit) : store_(store), audit_(audit) {} + + // INTEGRATION: real Guard::check takes (Session, roles, required) with + // Session from security/session; any denial maps to 403 per this API. + QueryPage Run(const Filter& f, const ::molo::security::session::Session& session, + const std::vector<::molo::security::auth::Role>& roles = {}) { + using ::molo::security::auth::Guard; + using ::molo::security::auth::Permission; + VerdictOf(Guard::check(session, roles, {Permission{"audit.read"}})); + if (last_status_ == 403) { + SelfAudit(session, f, "query.denied"); + QueryPage p; + p.http_status = 403; + p.code = "forbidden"; + return p; + } + std::size_t limit = f.limit == 0 ? 100 : f.limit; + if (limit > kQueryPageMax) limit = kQueryPageMax; + std::uint64_t start = f.from_seq == 0 ? 1 : f.from_seq; + if (!f.cursor.empty()) start = std::stoull(f.cursor) + 1; + QueryPage page; + std::uint64_t probe = start; + while (page.rows.size() < limit) { + auto batch = store_.ReadRange(probe, limit - page.rows.size()); + if (batch.empty()) break; + for (auto& r : batch) { + probe = r.seq + 1; + if (f.to_seq != 0 && r.seq > f.to_seq) { + probe = 0; + break; + } + if (!f.tenant.empty() && r.tenant != f.tenant) continue; + if (!f.table.empty() && r.table != f.table) continue; + if (!f.actor.empty() && r.actor != f.actor) continue; + if (!f.op.empty() && r.op != f.op) continue; + page.rows.push_back(r); + if (page.rows.size() >= limit) break; + } + if (probe == 0) break; + } + if (!page.rows.empty()) page.next_cursor = std::to_string(page.rows.back().seq); + SelfAudit(session, f, "query.allowed"); + return page; + } + + private: + void VerdictOf(const ::molo::security::auth::Guard::Verdict& v) { + last_status_ = v.allowed ? 200 : 403; + } + + void SelfAudit(const ::molo::security::session::Session& s, const Filter& f, + const std::string& action) { + if (!audit_) return; + Appender::Write w; + w.actor = s.user_id; + w.action = action; + w.table = "audit_records"; + w.op = "read"; + w.tenant = f.tenant; + w.after_json = "{\"table\":\"" + AuditRecord::Escape(f.table) + "\"}"; + audit_->Append(std::move(w)); + } + + Store& store_; + Appender* audit_; + int last_status_ = 200; +}; + +} // namespace molo::audit::log diff --git a/src/audit/log/record.h b/src/audit/log/record.h new file mode 100644 index 0000000..d558153 --- /dev/null +++ b/src/audit/log/record.h @@ -0,0 +1,81 @@ +// AuditRecord: one immutable audit trail entry with fixed JSON keys. +// Key order is frozen (spec 6.1): v,seq,ts,tenant,actor,action,table,pk,op, +// before,after,req,trace,prev,hash. Canonical() renders exactly those keys +// so the hash chain is stable across backends and releases. +#pragma once + +#include +#include + +namespace molo::audit::log { + +struct AuditRecord { + std::uint64_t seq = 0; + std::int64_t ts = 0; // unix seconds + std::string tenant; + std::string actor; + std::string action; + std::string table; + std::string pk; + std::string op; // insert, update, delete, read, export, erase, seal + std::string before_json; + std::string after_json; + std::string request_id; + std::string trace_id; + std::string prev_hash; // 64 hex chars, genesis is 64 zeros + std::string hash; // 64 hex chars over prev_hash + canonical body + + static std::string Escape(std::string_view s) { + std::string o; + o.reserve(s.size() + 2); + for (char c : s) { + switch (c) { + case '"': + o += "\\\""; + break; + case '\\': + o += "\\\\"; + break; + case '\n': + o += "\\n"; + break; + case '\r': + o += "\\r"; + break; + case '\t': + o += "\\t"; + break; + default: + o += c; + } + } + return o; + } + + // Canonical body: every key except hash, fixed order, no whitespace. + std::string Canonical() const { + std::string o = "{\"v\":1"; + o += ",\"seq\":" + std::to_string(seq); + o += ",\"ts\":" + std::to_string(ts); + o += ",\"tenant\":\"" + Escape(tenant) + "\""; + o += ",\"actor\":\"" + Escape(actor) + "\""; + o += ",\"action\":\"" + Escape(action) + "\""; + o += ",\"table\":\"" + Escape(table) + "\""; + o += ",\"pk\":\"" + Escape(pk) + "\""; + o += ",\"op\":\"" + Escape(op) + "\""; + o += ",\"before\":" + (before_json.empty() ? "null" : before_json); + o += ",\"after\":" + (after_json.empty() ? "null" : after_json); + o += ",\"req\":\"" + Escape(request_id) + "\""; + o += ",\"trace\":\"" + Escape(trace_id) + "\""; + o += ",\"prev\":\"" + Escape(prev_hash) + "\"}"; + return o; + } + + std::string ToJson() const { + std::string c = Canonical(); + c.pop_back(); // strip closing brace + return c + ",\"hash\":\"" + Escape(hash) + "\"}"; + } +}; + +} // namespace molo::audit::log diff --git a/src/audit/log/sql_store.h b/src/audit/log/sql_store.h new file mode 100644 index 0000000..c7d74d3 --- /dev/null +++ b/src/audit/log/sql_store.h @@ -0,0 +1,216 @@ +// SqlStore: production Store over both SQL backends (Postgres, SQLite). +// Tables audit_records and audit_seals use one portable DDL; only the +// parameter placeholder style differs per backend. Binding goes through the +// tiny SqlExec seam so this header compiles standalone; ConnectionExec binds +// it to a real molo::data::Connection, tests bind a FakeExec. +#pragma once + +#include +#include +#include +#include +#include + +#include "../adapt.h" +#include "../caps.h" +#include "chain.h" +#include "record.h" +#include "store.h" + +namespace molo::audit::log { + +// Minimal SQL surface the audit store needs. Params are strings; the +// Connection binding promotes integer-looking params to make_int. +class SqlExec { + public: + virtual ~SqlExec() = default; + virtual void Exec(const std::string& sql, const std::vector& params) = 0; + virtual std::vector> Query(const std::string& sql, + const std::vector& params, + std::size_t max_rows) = 0; + virtual bool IsPostgres() const = 0; +}; + +// Binds SqlExec to a real framework Connection. Both backends supported. +class ConnectionExec : public SqlExec { + public: + ConnectionExec(::molo::data::Connection& c, ::molo::data::Context ctx) + : conn_(c), ctx_(std::move(ctx)) {} + + void Exec(const std::string& sql, const std::vector& params) override { + conn_.exec(ctx_, ::molo::data::BoundStatement(sql, ToValues(params))); + } + + std::vector> Query(const std::string& sql, + const std::vector& params, + std::size_t max_rows) override { + auto res = conn_.exec(ctx_, ::molo::data::BoundStatement(sql, ToValues(params))); + std::vector> out; + // INTEGRATION: real Result exposes size()/raw(i), not a rows member. + for (std::size_t i = 0; i < res.size(); ++i) { + if (out.size() >= max_rows) break; + std::vector r; + for (auto& v : res.raw(i)) r.push_back(FromValue(v)); + out.push_back(std::move(r)); + } + return out; + } + + bool IsPostgres() const override { return conn_.is_postgres(); } + + private: + static std::vector<::molo::data::Value> ToValues(const std::vector& p) { + std::vector<::molo::data::Value> v; + for (auto& s : p) { + char* end = nullptr; + long long n = std::strtoll(s.c_str(), &end, 10); + if (end && *end == '\0' && end != s.c_str()) + v.push_back(::molo::data::make_int(n)); + else + v.push_back(::molo::data::make_text(s)); + } + return v; + } + + static std::string FromValue(const ::molo::data::Value& v) { + if (auto* i = std::get_if(&v)) return std::to_string(*i); + if (auto* s = std::get_if(&v)) return *s; + if (auto* d = std::get_if(&v)) return std::to_string(*d); + if (auto* b = std::get_if(&v)) return *b ? "1" : "0"; + return ""; + } + + ::molo::data::Connection& conn_; + ::molo::data::Context ctx_; +}; + +class SqlStore : public Store { + public: + explicit SqlStore(SqlExec& db) : db_(db) { EnsureSchema(); } + + static std::string DdlRecords() { + return "CREATE TABLE IF NOT EXISTS audit_records (seq INTEGER PRIMARY KEY," + "ts BIGINT NOT NULL,tenant TEXT NOT NULL,actor TEXT NOT NULL," + "action TEXT NOT NULL,tbl TEXT NOT NULL,pk TEXT NOT NULL," + "op TEXT NOT NULL,before_json TEXT NOT NULL,after_json TEXT NOT NULL," + "req TEXT NOT NULL,trace TEXT NOT NULL,prev_hash TEXT NOT NULL," + "hash TEXT NOT NULL)"; + } + + static std::string DdlSeals() { + return "CREATE TABLE IF NOT EXISTS audit_seals (seal_seq INTEGER PRIMARY KEY," + "root TEXT NOT NULL,ts BIGINT NOT NULL,mac TEXT NOT NULL)"; + } + + std::uint64_t Append(AuditRecord rec) override { + std::lock_guard g(mu_); + auto tip_rows = + db_.Query("SELECT seq,hash FROM audit_records ORDER BY seq DESC LIMIT 1", {}, 1); + std::uint64_t seq = 1; + std::string tip = GenesisHash(); + if (!tip_rows.empty() && tip_rows[0].size() == 2) { + seq = std::stoull(tip_rows[0][0]) + 1; + tip = tip_rows[0][1]; + } + rec.seq = seq; + Chain::Link(rec, tip); + const bool pg = db_.IsPostgres(); + std::string sql = + "INSERT INTO audit_records (seq,ts,tenant,actor,action,tbl,pk,op,before_json," + "after_json,req,trace,prev_hash,hash) VALUES (" + + Placeholders(14, pg) + ")"; + db_.Exec(sql, {std::to_string(rec.seq), std::to_string(rec.ts), rec.tenant, rec.actor, + rec.action, rec.table, rec.pk, rec.op, rec.before_json, rec.after_json, + rec.request_id, rec.trace_id, rec.prev_hash, rec.hash}); + return seq; + } + + std::vector ReadRange(std::uint64_t from_seq, std::size_t limit) override { + std::lock_guard g(mu_); + if (limit > kQueryPageMax) limit = kQueryPageMax; + const bool pg = db_.IsPostgres(); + std::string sql = + "SELECT seq,ts,tenant,actor,action,tbl,pk,op,before_json,after_json,req,trace," + "prev_hash,hash FROM audit_records WHERE seq >= " + + Ph(1, pg) + " ORDER BY seq ASC LIMIT " + std::to_string(limit); + std::vector out; + for (auto& r : db_.Query(sql, {std::to_string(from_seq)}, limit)) { + if (r.size() != 14) continue; + AuditRecord rec; + rec.seq = std::stoull(r[0]); + rec.ts = std::stoll(r[1]); + rec.tenant = r[2]; + rec.actor = r[3]; + rec.action = r[4]; + rec.table = r[5]; + rec.pk = r[6]; + rec.op = r[7]; + rec.before_json = r[8]; + rec.after_json = r[9]; + rec.request_id = r[10]; + rec.trace_id = r[11]; + rec.prev_hash = r[12]; + rec.hash = r[13]; + out.push_back(std::move(rec)); + } + return out; + } + + void SaveSeal(const Seal& s) override { + std::lock_guard g(mu_); + const bool pg = db_.IsPostgres(); + db_.Exec("INSERT INTO audit_seals (seal_seq,root,ts,mac) VALUES (" + Placeholders(4, pg) + + ")", + {std::to_string(s.seal_seq), s.root, std::to_string(s.ts), s.mac}); + } + + std::vector Seals() override { + std::lock_guard g(mu_); + std::vector out; + for (auto& r : db_.Query("SELECT seal_seq,root,ts,mac FROM audit_seals ORDER BY seal_seq ASC", + {}, 100000)) { + if (r.size() != 4) continue; + Seal s; + s.seal_seq = std::stoull(r[0]); + s.root = r[1]; + s.ts = std::stoll(r[2]); + s.mac = r[3]; + out.push_back(std::move(s)); + } + return out; + } + + std::uint64_t LatestSeq() override { + std::lock_guard g(mu_); + auto rows = db_.Query("SELECT seq FROM audit_records ORDER BY seq DESC LIMIT 1", {}, 1); + return rows.empty() ? 0 : std::stoull(rows[0][0]); + } + + std::string Tip() override { + std::lock_guard g(mu_); + auto rows = db_.Query("SELECT hash FROM audit_records ORDER BY seq DESC LIMIT 1", {}, 1); + return rows.empty() ? GenesisHash() : rows[0][0]; + } + + private: + void EnsureSchema() { + db_.Exec(DdlRecords(), {}); + db_.Exec(DdlSeals(), {}); + } + + static std::string Ph(int n, bool pg) { return pg ? "$" + std::to_string(n) : "?"; } + + static std::string Placeholders(int n, bool pg) { + std::string o; + for (int i = 1; i <= n; ++i) { + if (i > 1) o += ","; + o += Ph(i, pg); + } + return o; + } + + SqlExec& db_; + std::mutex mu_; +}; + +} // namespace molo::audit::log diff --git a/src/audit/log/store.h b/src/audit/log/store.h new file mode 100644 index 0000000..29bce81 --- /dev/null +++ b/src/audit/log/store.h @@ -0,0 +1,80 @@ +// Store: the only persistence seam for the audit trail. Three operations: +// append, read_range, seal. MemoryStore is the standalone/test backend; +// SqlStore (sql_store.h) is the production backend over both SQL drivers. +#pragma once + +#include +#include +#include +#include + +#include "../caps.h" +#include "chain.h" +#include "record.h" + +namespace molo::audit::log { + +class Store { + public: + virtual ~Store() = default; + // Appends rec, assigns rec.seq, links the chain, returns the seq. + virtual std::uint64_t Append(AuditRecord rec) = 0; + // Reads at most limit records from from_seq (inclusive), ascending. + virtual std::vector ReadRange(std::uint64_t from_seq, std::size_t limit) = 0; + // Persists a chain seal checkpoint. + virtual void SaveSeal(const Seal& s) = 0; + virtual std::vector Seals() = 0; + virtual std::uint64_t LatestSeq() = 0; + // Current chain tip, genesis when empty. + virtual std::string Tip() = 0; +}; + +class MemoryStore : public Store { + public: + std::uint64_t Append(AuditRecord rec) override { + std::lock_guard g(mu_); + rec.seq = recs_.size() + 1; + Chain::Link(rec, tip_); + tip_ = rec.hash; + recs_.push_back(std::move(rec)); + return recs_.size(); + } + + std::vector ReadRange(std::uint64_t from_seq, std::size_t limit) override { + std::lock_guard g(mu_); + if (limit > kQueryPageMax) limit = kQueryPageMax; + std::vector out; + if (from_seq == 0 || from_seq > recs_.size()) return out; + for (std::size_t i = from_seq - 1; i < recs_.size() && out.size() < limit; ++i) + out.push_back(recs_[i]); + return out; + } + + void SaveSeal(const Seal& s) override { + std::lock_guard g(mu_); + seals_.push_back(s); + } + + std::vector Seals() override { + std::lock_guard g(mu_); + return seals_; + } + + std::uint64_t LatestSeq() override { + std::lock_guard g(mu_); + return recs_.size(); + } + + std::string Tip() override { + std::lock_guard g(mu_); + return tip_; + } + + private: + std::mutex mu_; + std::vector recs_; + std::vector seals_; + std::string tip_ = GenesisHash(); +}; + +} // namespace molo::audit::log diff --git a/src/audit/mask/fixture.h b/src/audit/mask/fixture.h new file mode 100644 index 0000000..801b444 --- /dev/null +++ b/src/audit/mask/fixture.h @@ -0,0 +1,84 @@ +// Fixture scrubber: streams fixture blobs through in 1MB chunks, masking +// every PII hit with the Masker, then rescans the output and reports any +// residual raw values. Chunks overlap by 1024 bytes (twice the max match +// length) so matches split across a boundary are still caught whole. +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "../redact/pattern.h" +#include "masker.h" + +namespace molo::audit::mask { + +struct ScrubStats { + std::uint64_t chunks = 0; + std::uint64_t masked = 0; + std::uint64_t residual = 0; // raw hits surviving the rescan, must be 0 +}; + +class Fixture { + public: + static constexpr std::size_t kOverlap = 1024; + + using EmitFn = std::function; + + // Scrubs blob in kFixtureChunkBytes pieces, emitting masked output. + static ScrubStats Scrub(std::string_view blob, const Masker& masker, EmitFn emit) { + ScrubStats st; + std::size_t pos = 0; + std::string carry; + while (pos < blob.size()) { + std::size_t n = kFixtureChunkBytes; + if (n > blob.size() - pos) n = blob.size() - pos; + std::string window = carry + std::string(blob.substr(pos, n)); + pos += n; + ++st.chunks; + std::string masked = MaskWindow(window, masker, st); + if (pos < blob.size() && masked.size() > kOverlap) { + // Hold the overlap back so boundary matches rescan in full context. + std::string hold = masked.substr(masked.size() - kOverlap); + emit(std::string_view(masked.data(), masked.size() - kOverlap)); + carry = hold; + } else { + emit(masked); + carry.clear(); + } + } + return st; + } + + // Counts raw PII pattern hits left in scrubbed output. Zero means clean. + // Masked tokens ("masked::") never match a scanner shape, so + // every hit counted here is a genuine residual. + static std::uint64_t Rescan(std::string_view scrubbed) { + return redact::FindHits(scrubbed).size(); + } + + private: + static std::string MaskWindow(const std::string& window, const Masker& masker, + ScrubStats& st) { + std::vector hits = redact::FindHits(window); + std::sort(hits.begin(), hits.end(), + [](const redact::Hit& a, const redact::Hit& b) { return a.pos < b.pos; }); + std::string out; + out.reserve(window.size() + hits.size() * 8); + std::size_t last = 0; + for (auto& h : hits) { + out.append(window, last, h.pos - last); + out += masker.Mask(window.substr(h.pos, h.len), h.cls); + last = h.pos + h.len; + ++st.masked; + } + out.append(window, last, std::string::npos); + return out; + } +}; + +} // namespace molo::audit::mask diff --git a/src/audit/mask/masker.h b/src/audit/mask/masker.h new file mode 100644 index 0000000..5e0f1f3 --- /dev/null +++ b/src/audit/mask/masker.h @@ -0,0 +1,53 @@ +// Masker: seeded deterministic masking. Same (seed, value, class) always +// yields the same mask, so masked fixtures keep working foreign-key joins +// across tables and runs. Masks carry no recoverable prefix of the input. +#pragma once + +#include +#include +#include + +#include "../adapt.h" +#include "../pii/mark.h" + +namespace molo::audit::mask { + +class Masker { + public: + explicit Masker(std::string seed) : seed_(std::move(seed)) {} + + // Deterministic mask for one value. Format keeps a class tag plus a + // 16-char body derived from seed + class + value, e.g. "masked:email:..". + // Every 4th body char is forced to a letter, so no mask ever contains a + // digit run the phone/card rescanners could flag: masks are rescan-proof + // by construction, for every seed. + std::string Mask(std::string_view value, pii::PiiClass cls) const { + std::string in = seed_ + ":" + pii::ToString(cls) + ":"; + in += value; + std::string hex = ::molo::security::crypto::Sha256::hex( + ::molo::security::crypto::Sha256::hash(in)); + return "masked:" + pii::ToString(cls) + ":" + Body(hex); + } + + // Deterministic join key: stable across tables for the same raw id. + std::string JoinKey(std::string_view raw_id) const { + std::string in = seed_ + ":join:"; + in += raw_id; + std::string hex = ::molo::security::crypto::Sha256::hex( + ::molo::security::crypto::Sha256::hash(in)); + return "join:" + Body(hex); + } + + private: + static std::string Body(const std::string& hex) { + static constexpr char kForce[] = {'g', 'h', 'j', 'k'}; + std::string b = hex.substr(0, 16); + for (std::size_t i = 3; i < b.size(); i += 4) + if (b[i] >= '0' && b[i] <= '9') b[i] = kForce[(b[i] - '0') % 4]; + return b; + } + + std::string seed_; +}; + +} // namespace molo::audit::mask diff --git a/src/audit/pii/mark.h b/src/audit/pii/mark.h new file mode 100644 index 0000000..a87986e --- /dev/null +++ b/src/audit/pii/mark.h @@ -0,0 +1,68 @@ +// Field-level PII classification mark. Attached to ORM FieldDef by the base +// patch (see PATCHES.md entry 1); consumed by Registry, Redactor, Exporter, +// Eraser and Masker so every component agrees on what a field holds. +#pragma once + +#include +#include + +namespace molo::audit::pii { + +// Ten stable classes. Numeric values are persisted in marks and fixtures, +// never renumber, only append. +enum class PiiClass : std::uint8_t { + kNone = 0, + kName = 1, + kEmail = 2, + kPhone = 3, + kAddress = 4, + kNationalId = 5, + kFinancial = 6, + kHealth = 7, + kCredential = 8, + kLocation = 9, +}; + +inline std::string ToString(PiiClass c) { + switch (c) { + case PiiClass::kNone: + return "none"; + case PiiClass::kName: + return "name"; + case PiiClass::kEmail: + return "email"; + case PiiClass::kPhone: + return "phone"; + case PiiClass::kAddress: + return "address"; + case PiiClass::kNationalId: + return "national_id"; + case PiiClass::kFinancial: + return "financial"; + case PiiClass::kHealth: + return "health"; + case PiiClass::kCredential: + return "credential"; + case PiiClass::kLocation: + return "location"; + } + return "none"; +} + +// What erasure must do with a marked field. +enum class ErasureRule : std::uint8_t { + kKeep = 0, // Legal hold or statutory retention: never erase. + kMask = 1, // Replace with deterministic mask, keep row shape. + kDelete = 2, // Null the column or drop the row. +}; + +// One field's compliance mark: its PII class, whether subject export may +// include it, how erasure treats it, and the legal basis string. +struct FieldMark { + PiiClass cls = PiiClass::kNone; + bool exportable = true; + ErasureRule erasure = ErasureRule::kDelete; + std::string basis; +}; + +} // namespace molo::audit::pii diff --git a/src/audit/pii/registry.h b/src/audit/pii/registry.h new file mode 100644 index 0000000..2bf630d --- /dev/null +++ b/src/audit/pii/registry.h @@ -0,0 +1,55 @@ +// Boot-time registry of every ORM field's PII mark. Build fails loudly when +// any schema field lacks a mark, so unclassified columns can never reach +// production silently. +#pragma once + +#include +#include +#include +#include + +#include "mark.h" + +namespace molo::audit::pii { + +class Registry { + public: + // schema maps table -> its column names; marks maps "table.field" -> mark. + // Throws std::runtime_error listing every unmarked field. + static Registry Build(const std::map>& schema, + const std::map& marks) { + Registry r; + std::string missing; + for (const auto& [table, cols] : schema) { + for (const auto& col : cols) { + std::string key = table + "." + col; + auto it = marks.find(key); + if (it == marks.end()) { + if (!missing.empty()) missing += ","; + missing += key; + } else { + r.marks_[key] = it->second; + } + } + } + if (!missing.empty()) throw std::runtime_error("unmarked pii fields: " + missing); + return r; + } + + const FieldMark& Lookup(const std::string& table, const std::string& field) const { + static const FieldMark kDefault; + auto it = marks_.find(table + "." + field); + return it == marks_.end() ? kDefault : it->second; + } + + bool Marked(const std::string& table, const std::string& field) const { + return marks_.count(table + "." + field) > 0; + } + + std::size_t size() const { return marks_.size(); } + + private: + std::map marks_; +}; + +} // namespace molo::audit::pii diff --git a/src/audit/redact/pattern.h b/src/audit/redact/pattern.h new file mode 100644 index 0000000..19bbb3e --- /dev/null +++ b/src/audit/redact/pattern.h @@ -0,0 +1,209 @@ +// Value-shape scanners that spot PII in free text (log messages, error +// detail, fixture blobs). Hand-written matchers, no std::regex: libstdc++ +// regex recurses per input character and overflows the stack on large +// windows. Each pattern names the PiiClass it detects so callers can emit +// [redacted:] or a deterministic mask. Matches are length-capped and +// hits per call are capped, so hostile input cannot blow up time or memory. +#pragma once + +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "../pii/mark.h" + +namespace molo::audit::redact { + +inline constexpr std::size_t kMaxMatchLen = 512; + +struct Hit { + std::size_t pos = 0; + std::size_t len = 0; + pii::PiiClass cls = pii::PiiClass::kNone; +}; + +namespace detail { +inline bool IsEmailLocal(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '.' || c == '_' || c == '%' || c == '+' || c == '-'; +} +inline bool IsEmailDomain(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '.' || c == '-'; +} +inline bool IsAlpha(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} +inline bool IsDigit(char c) { return c >= '0' && c <= '9'; } +inline bool IsPhoneMid(char c) { + return IsDigit(c) || c == ' ' || c == '.' || c == '-' || c == '(' || c == ')'; +} +inline char Lower(char c) { return (c >= 'A' && c <= 'Z') ? c - 'A' + 'a' : c; } + +inline void FindEmail(std::string_view t, std::vector& out) { + for (std::size_t at = t.find('@'); at != std::string_view::npos; at = t.find('@', at + 1)) { + std::size_t l = at; + while (l > 0 && IsEmailLocal(t[l - 1]) && at - l < 64) --l; + if (l == at || (at > 0 && IsEmailLocal(t[l - 1]) && at - l == 64)) continue; + std::size_t r = at + 1; + while (r < t.size() && IsEmailDomain(t[r]) && r - at < 254) ++r; + std::string_view dom = t.substr(at + 1, r - at - 1); + auto dot = dom.rfind('.'); + if (dot == std::string_view::npos || dot == 0 || dot + 1 >= dom.size()) continue; + bool tld_ok = dom.size() - dot - 1 >= 2; + for (std::size_t i = dot + 1; tld_ok && i < dom.size(); ++i) + if (!IsAlpha(dom[i])) tld_ok = false; + if (!tld_ok) continue; + out.push_back({l, r - l, pii::PiiClass::kEmail}); + if (out.size() >= kRedactMaxHitsPerCall) return; + } +} + +inline void FindNationalId(std::string_view t, std::vector& out) { + for (std::size_t i = 0; i + 11 <= t.size(); ++i) { + if (IsDigit(t[i]) && IsDigit(t[i + 1]) && IsDigit(t[i + 2]) && t[i + 3] == '-' && + IsDigit(t[i + 4]) && IsDigit(t[i + 5]) && t[i + 6] == '-' && IsDigit(t[i + 7]) && + IsDigit(t[i + 8]) && IsDigit(t[i + 9]) && IsDigit(t[i + 10]) && + (i == 0 || !IsDigit(t[i - 1])) && (i + 11 >= t.size() || !IsDigit(t[i + 11]))) { + out.push_back({i, 11, pii::PiiClass::kNationalId}); + if (out.size() >= kRedactMaxHitsPerCall) return; + } + } +} + +inline void FindCard(std::string_view t, std::vector& out) { + for (std::size_t i = 0; i < t.size();) { + if (!IsDigit(t[i])) { + ++i; + continue; + } + std::size_t j = i; + int digits = 0; + while (j < t.size() && digits < 19 && + (IsDigit(t[j]) || t[j] == ' ' || t[j] == '-')) { + if (IsDigit(t[j])) ++digits; + ++j; + } + while (j > i && !IsDigit(t[j - 1])) --j; // trailing separator trims + if (digits >= 13 && digits <= 19 && j - i <= kMaxMatchLen && + (i == 0 || !IsDigit(t[i - 1]))) { + out.push_back({i, j - i, pii::PiiClass::kFinancial}); + if (out.size() >= kRedactMaxHitsPerCall) return; + } + i = (j > i) ? j : i + 1; + } +} + +inline void FindIban(std::string_view t, std::vector& out) { + for (std::size_t i = 0; i + 15 <= t.size(); ++i) { + unsigned char a = t[i], b = t[i + 1]; + if (a < 'A' || a > 'Z' || b < 'A' || b > 'Z') continue; + if (!IsDigit(t[i + 2]) || !IsDigit(t[i + 3])) continue; + std::size_t j = i + 4; + while (j < t.size() && j - i < 34 && + ((t[j] >= 'A' && t[j] <= 'Z') || IsDigit(t[j]))) + ++j; + if (j - i >= 15) { + out.push_back({i, j - i, pii::PiiClass::kFinancial}); + if (out.size() >= kRedactMaxHitsPerCall) return; + } + } +} + +inline void FindPhone(std::string_view t, std::vector& out) { + for (std::size_t i = 0; i < t.size();) { + std::size_t s = i; + if (t[s] == '+' && s + 1 < t.size() && IsDigit(t[s + 1])) ++s; + if (!IsDigit(t[s])) { + i = s + 1; + continue; + } + std::size_t j = s; + int digits = 0; + while (j < t.size() && j - i < 40 && IsPhoneMid(t[j])) { + if (IsDigit(t[j])) ++digits; + ++j; + } + while (j > s && !IsDigit(t[j - 1])) --j; + if (digits >= 8 && j - i >= 8) { + out.push_back({i, j - i, pii::PiiClass::kPhone}); + if (out.size() >= kRedactMaxHitsPerCall) return; + } + i = (j > i) ? j : i + 1; + } +} + +inline void FindCredential(std::string_view t, std::vector& out) { + static constexpr const char* kKeys[] = {"password", "passwd", "secret", + "api_key", "api-key", "apikey", "token"}; + for (std::size_t i = 0; i < t.size(); ++i) { + for (const char* k : kKeys) { + std::size_t kl = std::string_view(k).size(); + if (i + kl >= t.size()) continue; + bool eq = true; + for (std::size_t c = 0; eq && c < kl; ++c) eq = Lower(t[i + c]) == k[c]; + if (!eq) continue; + if (i > 0 && (IsAlpha(t[i - 1]) || IsDigit(t[i - 1]) || t[i - 1] == '_')) continue; + std::size_t j = i + kl; + while (j < t.size() && t[j] == ' ') ++j; + if (j >= t.size() || (t[j] != ':' && t[j] != '=')) continue; + ++j; + while (j < t.size() && t[j] == ' ') ++j; + std::size_t v = j; + while (v < t.size() && t[v] != ' ' && t[v] != '\t' && t[v] != '\n' && t[v] != '\r' && + v - i < kMaxMatchLen) + ++v; + if (v == j) continue; + out.push_back({i, v - i, pii::PiiClass::kCredential}); + if (out.size() >= kRedactMaxHitsPerCall) return; + } + } +} +} // namespace detail + +struct Pattern { + pii::PiiClass cls; + std::function&)> find; +}; + +// Priority order: specific shapes first so the phone matcher never eats an +// SSN or card number. +inline const std::vector& Patterns() { + static const std::vector k = { + {pii::PiiClass::kEmail, detail::FindEmail}, + {pii::PiiClass::kNationalId, detail::FindNationalId}, + {pii::PiiClass::kFinancial, detail::FindCard}, + {pii::PiiClass::kFinancial, detail::FindIban}, + {pii::PiiClass::kPhone, detail::FindPhone}, + {pii::PiiClass::kCredential, detail::FindCredential}, + }; + return k; +} + +// Collects hits across all patterns, earlier patterns winning overlaps. +inline std::vector FindHits(std::string_view text) { + std::vector acc; + std::vector covered(text.size(), 0); + for (const auto& p : Patterns()) { + std::vector hs; + p.find(text, hs); + for (auto& h : hs) { + if (acc.size() >= kRedactMaxHitsPerCall) return acc; + bool overlap = false; + for (std::size_t i = h.pos; i < h.pos + h.len && i < covered.size(); ++i) + if (covered[i]) { + overlap = true; + break; + } + if (overlap) continue; + for (std::size_t i = h.pos; i < h.pos + h.len && i < covered.size(); ++i) covered[i] = 1; + acc.push_back(h); + } + } + return acc; +} + +} // namespace molo::audit::redact diff --git a/src/audit/redact/redactor.h b/src/audit/redact/redactor.h new file mode 100644 index 0000000..c3214c8 --- /dev/null +++ b/src/audit/redact/redactor.h @@ -0,0 +1,67 @@ +// Redactor: replaces PII in strings and log records with [redacted:]. +// Wired into the logging Registry emit path and the error renderer by the +// base patches (see PATCHES.md entries 3 and 4). Input is capped at +// kRedactMaxBytesPerCall; oversize input is truncated before scanning. +#pragma once + +#include +#include +#include +#include + +#include "../adapt.h" +#include "../caps.h" +#include "pattern.h" + +namespace molo::audit::redact { + +class Redactor { + public: + // Scans text for every known PII shape and replaces each hit with + // [redacted:]. Pure function, safe to call from any thread. + static std::string Redact(std::string_view text) { + std::string in(text.substr(0, kRedactMaxBytesPerCall)); + std::vector hits = FindHits(in); + std::sort(hits.begin(), hits.end(), + [](const Hit& a, const Hit& b) { return a.pos < b.pos; }); + std::string out; + out.reserve(in.size() + hits.size() * 8); + std::size_t last = 0; + for (auto& h : hits) { + out.append(in, last, h.pos - last); + out += "[redacted:" + pii::ToString(h.cls) + "]"; + last = h.pos + h.len; + } + out.append(in, last, std::string::npos); + return out; + } + + // Redacts one log record in place: message plus structured fields. Fields + // whose key is sensitive are replaced wholesale; other string values are + // scanned. Non-string values pass through (nothing to redact). Already + // redacted values ("***" from the logger, "[redacted:*]" from an earlier + // pass) are left alone so stacked redaction stays idempotent. + // INTEGRATION: real FieldValue is a variant, not a string. + static void RedactRecord(::molo::log::Record& r) { + r.message = Redact(r.message); + std::size_t n = 0; + for (auto& f : r.fields) { + if (++n > kRedactMaxFields) break; + if (::molo::corr::is_sensitive_key(f.key)) { + if (auto* s = std::get_if(&f.value)) { + if (*s != "***" && s->rfind("[redacted:", 0) != 0) + *s = "[redacted:credential]"; + } else { + f.value = std::string("[redacted:credential]"); + } + } else if (auto* s = std::get_if(&f.value)) { + *s = Redact(*s); + } + } + } + + // Redacts an opaque error detail string for HTTP bodies and sinks. + static std::string RedactDetail(std::string_view detail) { return Redact(detail); } +}; + +} // namespace molo::audit::redact diff --git a/src/audit/retention/enforcer.h b/src/audit/retention/enforcer.h new file mode 100644 index 0000000..f421bee --- /dev/null +++ b/src/audit/retention/enforcer.h @@ -0,0 +1,93 @@ +// Enforcer: retention runs in two phases. DryRun scans rows through the +// caller's scan callback and reports what would be deleted, held or kept, +// without touching anything. Apply replays a DryRun report against the pinned +// policy version and deletes through the caller's delete callback. Both are +// synchronous functions for the supervisor scheduler, never request work. +#pragma once + +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "hold.h" +#include "policy.h" + +namespace molo::audit::retention { + +struct ScannedRow { + std::string tenant; + std::string table; + std::string pk; + std::uint32_t age_days = 0; +}; + +struct DryRunReport { + std::uint64_t policy_version = 0; + std::uint64_t scanned = 0; + std::uint64_t would_delete = 0; + std::uint64_t held = 0; + std::uint64_t kept = 0; + bool truncated = false; // scan hit kRetentionScanMaxPerRun + std::vector deletions; // exact rows apply() will delete +}; + +struct ApplyResult { + std::uint64_t policy_version = 0; + std::uint64_t deleted = 0; + std::uint64_t skipped_hold = 0; + std::uint64_t skipped_policy = 0; // policy changed since dry run +}; + +class Enforcer { + public: + using ScanFn = std::function; // false ends the scan + using DeleteFn = std::function; + + static DryRunReport DryRun(const Policy& policy, const Hold& hold, ScanFn scan) { + DryRunReport r; + r.policy_version = policy.version(); + ScannedRow row; + while (scan(row)) { + if (++r.scanned > kRetentionScanMaxPerRun) { + r.truncated = true; + break; + } + TableRule rule = policy.RuleFor(row.table); + if (rule.ttl_days == 0 || row.age_days < rule.ttl_days + rule.grace_days) { + ++r.kept; + continue; + } + if (hold.Has(Hold::RowKey(row.tenant, row.table, row.pk))) { + ++r.held; + continue; + } + ++r.would_delete; + r.deletions.push_back(row); + } + return r; + } + + static ApplyResult Apply(const Policy& policy, const Hold& hold, const DryRunReport& report, + DeleteFn del) { + ApplyResult r; + r.policy_version = policy.version(); + for (const auto& row : report.deletions) { + if (policy.version() != report.policy_version) { + ++r.skipped_policy; + continue; + } + if (hold.Has(Hold::RowKey(row.tenant, row.table, row.pk))) { + ++r.skipped_hold; + continue; + } + del(row); + ++r.deleted; + } + return r; + } +}; + +} // namespace molo::audit::retention diff --git a/src/audit/retention/hold.h b/src/audit/retention/hold.h new file mode 100644 index 0000000..590be6c --- /dev/null +++ b/src/audit/retention/hold.h @@ -0,0 +1,41 @@ +// Legal hold set: subjects or rows the Enforcer must never delete. +// Capped at kMaxHolds; adding beyond the cap fails loudly so holds stay +// explicit instead of silently dropping. +#pragma once + +#include +#include +#include +#include + +#include "../caps.h" + +namespace molo::audit::retention { + +class Hold { + public: + // Adds a hold on one key ("tenant:table:pk" or "tenant:subject:"). + void Add(const std::string& key, const std::string& reason) { + if (holds_.size() >= kMaxHolds && !holds_.count(key)) + throw std::runtime_error("hold set exceeds kMaxHolds"); + holds_[key] = reason; + } + + void Remove(const std::string& key) { holds_.erase(key); } + bool Has(const std::string& key) const { return holds_.count(key) > 0; } + std::size_t size() const { return holds_.size(); } + + static std::string RowKey(const std::string& tenant, const std::string& table, + const std::string& pk) { + return tenant + ":" + table + ":" + pk; + } + + static std::string SubjectKey(const std::string& tenant, const std::string& subject) { + return tenant + ":subject:" + subject; + } + + private: + std::map holds_; +}; + +} // namespace molo::audit::retention diff --git a/src/audit/retention/policy.h b/src/audit/retention/policy.h new file mode 100644 index 0000000..c3a8bb1 --- /dev/null +++ b/src/audit/retention/policy.h @@ -0,0 +1,39 @@ +// Retention policy: per-table TTL in days plus a pinned version. Enforcement +// always runs against one pinned version so a policy edit mid-run can never +// mix two rule sets in a single report. +#pragma once + +#include +#include +#include + +namespace molo::audit::retention { + +struct TableRule { + std::uint32_t ttl_days = 0; // 0 means keep forever + std::uint32_t grace_days = 0; // extra days before apply() deletes +}; + +class Policy { + public: + Policy(std::uint64_t version, std::map rules) + : version_(version), rules_(std::move(rules)) {} + + std::uint64_t version() const { return version_; } + + TableRule RuleFor(const std::string& table) const { + auto it = rules_.find(table); + if (it != rules_.end()) return it->second; + auto d = rules_.find("*"); + if (d != rules_.end()) return d->second; + return TableRule{}; + } + + const std::map& rules() const { return rules_; } + + private: + std::uint64_t version_; + std::map rules_; +}; + +} // namespace molo::audit::retention diff --git a/src/audit/rights/eraser.h b/src/audit/rights/eraser.h new file mode 100644 index 0000000..7686905 --- /dev/null +++ b/src/audit/rights/eraser.h @@ -0,0 +1,82 @@ +// Eraser: erasure-request cascade across every store that can hold subject +// data: sessions, cache tags, KV prefixes, ORM rows, uploads, tasks, and the +// audit index itself. Each store binds through one deleter callback so this +// header stays decoupled from the owning parts. Held subjects report Held and +// delete nothing; statutorily exempt rows report Exempted per store; the audit +// index row is never removed, it is rewritten to "erased:". +#pragma once + +#include +#include +#include +#include + +#include "../caps.h" +#include "../retention/hold.h" +#include "request.h" + +namespace molo::audit::rights { + +enum class EraseOutcome : std::uint8_t { kDone = 0, kHeld = 1, kExempted = 2 }; + +struct CascadeDeleters { + // Each deleter removes subject rows from one store, returns erased count. + std::function sessions; + std::function cache_tags; + std::function kv_prefix; + std::function orm_rows; + std::function uploads; + std::function tasks; + // Rewrites the audit index row for the subject to "erased:". + std::function audit_index; + // True when the subject carries rows the law forbids erasing. + std::function has_exemption; +}; + +struct EraseReport { + EraseOutcome outcome = EraseOutcome::kDone; + std::uint64_t erased_total = 0; + bool truncated = false; // hit kErasureMaxPerRun + std::map per_store; +}; + +class Eraser { + public: + static std::string ErasedMarker(const std::string& request_id) { + return "erased:" + request_id; + } + + static EraseReport Erase(const RightsRequest& req, const retention::Hold& hold, + const CascadeDeleters& d) { + EraseReport r; + const std::string key = + retention::Hold::SubjectKey(req.subject.tenant, req.subject.key); + if (hold.Has(key)) { + r.outcome = EraseOutcome::kHeld; + return r; + } + auto run = [&](const std::string& name, + const std::function& fn) { + if (!fn || r.truncated) return; + std::uint64_t n = fn(req.subject); + if (r.erased_total + n > kErasureMaxPerRun) { + r.truncated = true; + return; + } + r.per_store[name] = n; + r.erased_total += n; + }; + run("sessions", d.sessions); + run("cache_tags", d.cache_tags); + run("kv_prefix", d.kv_prefix); + run("orm_rows", d.orm_rows); + run("uploads", d.uploads); + run("tasks", d.tasks); + if (d.audit_index && !r.truncated) d.audit_index(req.subject, req.id); + if (d.has_exemption && d.has_exemption(req.subject)) + r.outcome = EraseOutcome::kExempted; + return r; + } +}; + +} // namespace molo::audit::rights diff --git a/src/audit/rights/exporter.h b/src/audit/rights/exporter.h new file mode 100644 index 0000000..0e046be --- /dev/null +++ b/src/audit/rights/exporter.h @@ -0,0 +1,96 @@ +// Exporter: builds a deterministic subject bundle for access/export +// requests. Rows arrive through the caller's row callback in any order; the +// bundle sorts them, splits them into capped chunks, and ships a manifest +// with row counts plus a SHA256 checksum per chunk and over the whole. +// Capped at kExportMaxRows and kExportMaxBytes; exceeding either fails the +// export loudly instead of shipping a partial bundle. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "../adapt.h" +#include "../caps.h" +#include "request.h" + +namespace molo::audit::rights { + +struct ExportRow { + std::string table; + std::string pk; + std::string json; // already redacted/filtered to exportable marks +}; + +struct ExportChunk { + std::uint32_t index = 0; + std::string bytes; // newline-delimited row JSON + std::string sha256; +}; + +struct ExportBundle { + std::string request_id; + std::string manifest_json; + std::vector chunks; + std::string bundle_sha256; + std::uint64_t rows = 0; +}; + +class Exporter { + public: + using RowFn = std::function; // false ends input + static constexpr std::size_t kChunkBytes = 1024 * 1024; + + static ExportBundle Build(const RightsRequest& req, RowFn rows) { + std::vector all; + ExportRow r; + while (rows(r)) { + all.push_back(r); + if (all.size() > kExportMaxRows) + throw std::runtime_error("export exceeds kExportMaxRows"); + } + std::sort(all.begin(), all.end(), [](const ExportRow& a, const ExportRow& b) { + if (a.table != b.table) return a.table < b.table; + return a.pk < b.pk; + }); + ExportBundle b; + b.request_id = req.id; + b.rows = all.size(); + std::string cur; + std::uint32_t idx = 0; + std::size_t total = 0; + auto flush = [&] { + if (cur.empty()) return; + ExportChunk c; + c.index = idx++; + c.bytes = cur; + c.sha256 = ::molo::security::crypto::Sha256::hex( + ::molo::security::crypto::Sha256::hash(cur)); + total += cur.size(); + if (total > kExportMaxBytes) throw std::runtime_error("export exceeds kExportMaxBytes"); + b.chunks.push_back(std::move(c)); + cur.clear(); + }; + for (auto& row : all) { + std::string line = "{\"table\":\"" + row.table + "\",\"pk\":\"" + row.pk + + "\",\"row\":" + row.json + "}\n"; + if (cur.size() + line.size() > kChunkBytes) flush(); + cur += line; + } + flush(); + std::string all_hashes; + for (auto& c : b.chunks) all_hashes += c.sha256; + b.bundle_sha256 = ::molo::security::crypto::Sha256::hex( + ::molo::security::crypto::Sha256::hash(all_hashes)); + b.manifest_json = "{\"request\":\"" + req.id + "\",\"rows\":" + std::to_string(b.rows) + + ",\"chunks\":" + std::to_string(b.chunks.size()) + ",\"sha256\":\"" + + b.bundle_sha256 + "\"}"; + return b; + } +}; + +} // namespace molo::audit::rights diff --git a/src/audit/rights/request.h b/src/audit/rights/request.h new file mode 100644 index 0000000..60bae8c --- /dev/null +++ b/src/audit/rights/request.h @@ -0,0 +1,107 @@ +// RightsRequest: one data-subject request lifecycle. States run received, +// verified, approved, in_progress, fulfilled, rejected, expired. Due date is +// created + 30 days; one extension moves it to + 60 days. Open-request counts +// per tenant are capped at kMaxOpenRequestsPerTenant by the caller-owned +// RequestBook below. +#pragma once + +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "subject.h" + +namespace molo::audit::rights { + +enum class RequestKind : std::uint8_t { kAccess = 0, kExport = 1, kErasure = 2 }; + +enum class RequestState : std::uint8_t { + kReceived = 0, + kVerified = 1, + kApproved = 2, + kInProgress = 3, + kFulfilled = 4, + kRejected = 5, + kExpired = 6, +}; + +inline constexpr std::int64_t kDueDays = 30; +inline constexpr std::int64_t kExtendedDueDays = 60; +inline constexpr std::int64_t kDaySeconds = 86400; + +struct RightsRequest { + std::string id; + Subject subject; + RequestKind kind = RequestKind::kAccess; + RequestState state = RequestState::kReceived; + std::int64_t created_ts = 0; + std::int64_t due_ts = 0; // created + 30d, or + 60d once extended + bool extended = false; + + bool Open() const { + return state == RequestState::kReceived || state == RequestState::kVerified || + state == RequestState::kApproved || state == RequestState::kInProgress; + } +}; + +class RequestBook { + public: + RightsRequest& Open(std::string id, Subject subject, RequestKind kind, std::int64_t now) { + if (OpenCount(subject.tenant) >= kMaxOpenRequestsPerTenant) + throw std::runtime_error("tenant exceeds kMaxOpenRequestsPerTenant"); + RightsRequest r; + r.id = id; + r.subject = std::move(subject); + r.kind = kind; + r.created_ts = now; + r.due_ts = now + kDueDays * kDaySeconds; + reqs_[r.id] = r; + return reqs_[r.id]; + } + + void Transition(const std::string& id, RequestState to) { + auto it = reqs_.find(id); + if (it == reqs_.end()) throw std::runtime_error("unknown rights request: " + id); + it->second.state = to; + } + + void ExtendOnce(const std::string& id) { + auto it = reqs_.find(id); + if (it == reqs_.end()) throw std::runtime_error("unknown rights request: " + id); + if (it->second.extended) throw std::runtime_error("request already extended: " + id); + it->second.extended = true; + it->second.due_ts = it->second.created_ts + kExtendedDueDays * kDaySeconds; + } + + // Marks overdue open requests expired. Returns the expired ids. + std::vector ExpireOverdue(std::int64_t now) { + std::vector out; + for (auto& [id, r] : reqs_) + if (r.Open() && now > r.due_ts) { + r.state = RequestState::kExpired; + out.push_back(id); + } + return out; + } + + std::size_t OpenCount(const std::string& tenant) const { + std::size_t n = 0; + for (auto& [_, r] : reqs_) + if (r.subject.tenant == tenant && r.Open()) ++n; + return n; + } + + const RightsRequest& Get(const std::string& id) const { + auto it = reqs_.find(id); + if (it == reqs_.end()) throw std::runtime_error("unknown rights request: " + id); + return it->second; + } + + private: + std::map reqs_; +}; + +} // namespace molo::audit::rights diff --git a/src/audit/rights/subject.h b/src/audit/rights/subject.h new file mode 100644 index 0000000..14fba7e --- /dev/null +++ b/src/audit/rights/subject.h @@ -0,0 +1,20 @@ +// Data subject identity for rights requests: tenant plus one stable subject +// key (user id, account id). Key format is ":", e.g. "user:42". +#pragma once + +#include + +namespace molo::audit::rights { + +struct Subject { + std::string tenant; + std::string key; + + static Subject User(const std::string& tenant, const std::string& id) { + return Subject{tenant, "user:" + id}; + } + + std::string ToString() const { return tenant + ":" + key; } +}; + +} // namespace molo::audit::rights diff --git a/src/cli/ai/ai_client.cpp b/src/cli/ai/ai_client.cpp new file mode 100644 index 0000000..9dfc529 --- /dev/null +++ b/src/cli/ai/ai_client.cpp @@ -0,0 +1,102 @@ +#include "ai/ai_client.hpp" + +#include +#include + +#include "ai/redactor.hpp" +#include "ai/sse_stream_parser.hpp" +#include "cli/json_writer.hpp" + +namespace molo::ai { + +AiClient::AiClient(AiClientOptions opts, IAiTransport& transport) + : opts_(std::move(opts)), transport_(transport) {} + +std::string AiClient::request_json(const std::string& model, + const std::vector& messages, bool stream) { + std::vector items; + for (const auto& msg : messages) { + std::map fields; + fields["role"] = molo::cli::JsonWriter::quote(msg.role); + fields["content"] = molo::cli::JsonWriter::quote(msg.content); + items.push_back(molo::cli::JsonWriter::object(fields)); + } + std::map body; + body["model"] = molo::cli::JsonWriter::quote(model); + body["messages"] = molo::cli::JsonWriter::array(items); + body["stream"] = stream ? "true" : "false"; + return molo::cli::JsonWriter::object(body); +} + +std::string AiClient::extract_delta(const std::string& data_json) { + // Minimal pull of choices[0].delta.content without a JSON parser. + auto pos = data_json.find("\"content\""); + if (pos == std::string::npos) return ""; + pos = data_json.find(':', pos); + if (pos == std::string::npos) return ""; + pos = data_json.find('"', pos); + if (pos == std::string::npos) return ""; + std::string out; + for (std::size_t i = pos + 1; i < data_json.size(); ++i) { + char c = data_json[i]; + if (c == '\\' && i + 1 < data_json.size()) { + char n = data_json[++i]; + if (n == 'n') { + out += '\n'; + } else if (n == 't') { + out += '\t'; + } else { + out += n; + } + continue; + } + if (c == '"') break; + out += c; + } + return out; +} + +std::string AiClient::chat_stream(const std::vector& messages, + const std::function& on_delta, + std::string& error) const { + const char* key_raw = std::getenv("DEVPLACE_API_KEY"); + if (key_raw == nullptr || *key_raw == '\0') { + error = "DEVPLACE_API_KEY is not set"; + return ""; + } + if (opts_.base_url.rfind("https://", 0) == 0 && !TlsStub::https_supported()) { + error = TlsStub::unavailable_reason(); + return ""; + } + std::string body = request_json(opts_.model, messages, true); + Redactor::Result redacted = Redactor::redact(body); + if (!redacted.ok) { + error = "redaction failed, request aborted"; + return ""; + } + AiHttpRequest req; + req.url = opts_.base_url + "/chat/completions"; + req.bearer_key = key_raw; + req.body_json = redacted.text; + req.connect_timeout_sec = opts_.connect_timeout_sec; + AiHttpResponse resp = transport_.post_stream(req); + if (resp.status != 200) { + error = "AI endpoint status " + std::to_string(resp.status) + ": " + resp.error; + return ""; + } + SseStreamParser parser; + parser.feed(resp.sse_body); + std::string full; + for (const auto& event : parser.take_events()) { + std::string delta = extract_delta(event); + full += delta; + if (on_delta) on_delta(delta); + } + if (!parser.done()) { + error = "stream ended without [DONE]"; + return ""; + } + return full; +} + +} // namespace molo::ai diff --git a/src/cli/ai/ai_client.hpp b/src/cli/ai/ai_client.hpp new file mode 100644 index 0000000..9688164 --- /dev/null +++ b/src/cli/ai/ai_client.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +#include "ai/ai_transport.hpp" + +namespace molo::ai { + +struct ChatMessage { + std::string role; + std::string content; +}; + +struct AiClientOptions { + std::string base_url = "https://devplace.net/openai/v1"; + std::string model = "molodetz"; + int connect_timeout_sec = 30; + int session_cap_sec = 300; +}; + +// OpenAI-compatible chat client. Key comes from DEVPLACE_API_KEY env only, +// never a flag, never stored. Streaming responses parse as SSE. +class AiClient { + public: + AiClient(AiClientOptions opts, IAiTransport& transport); + + // Calls on_delta per content delta; returns full assembled text. + // Returns empty and sets error on failure. + std::string chat_stream(const std::vector& messages, + const std::function& on_delta, + std::string& error) const; + + const AiClientOptions& options() const { return opts_; } + static std::string request_json(const std::string& model, + const std::vector& messages, bool stream); + static std::string extract_delta(const std::string& data_json); + + private: + AiClientOptions opts_; + IAiTransport& transport_; +}; + +} // namespace molo::ai diff --git a/src/cli/ai/ai_session.cpp b/src/cli/ai/ai_session.cpp new file mode 100644 index 0000000..5ed8496 --- /dev/null +++ b/src/cli/ai/ai_session.cpp @@ -0,0 +1,58 @@ +#include "ai/ai_session.hpp" + +#include + +#include "caps.hpp" + +namespace molo::ai { + +AiSession::AiSession(std::string name) + : name_(std::move(name)), started_(std::chrono::steady_clock::now()) {} + +bool AiSession::append(const std::string& role, const std::string& content, std::string& error) { + if (expired()) { + error = "session exceeded 300s cap"; + return false; + } + if (transcript_bytes_ + role.size() + content.size() > caps::kAiTranscriptMaxBytes) { + error = "transcript exceeded 8 MB cap"; + return false; + } + messages_.push_back(ChatMessage{role, content}); + transcript_bytes_ += role.size() + content.size(); + return true; +} + +bool AiSession::expired() const { + auto elapsed = std::chrono::steady_clock::now() - started_; + return elapsed > std::chrono::seconds(caps::kAiSessionCapSec); +} + +std::size_t AiSession::request_write(const std::string& path, const std::string& content) { + if (actions_ >= caps::kAiMaxActions) return static_cast(-1); + ++actions_; + pending_.push_back(PendingWrite{path, content}); + return pending_.size() - 1; +} + +bool AiSession::confirm_write(std::size_t index, std::string& error) { + if (index >= pending_.size()) { + error = "no such pending write"; + return false; + } + const PendingWrite& write = pending_[index]; + std::ofstream out(write.path, std::ios::binary | std::ios::trunc); + if (!out.good()) { + error = "cannot write " + write.path; + return false; + } + out << write.content; + pending_.erase(pending_.begin() + static_cast(index)); + return true; +} + +void AiSession::reject_write(std::size_t index) { + if (index < pending_.size()) pending_.erase(pending_.begin() + static_cast(index)); +} + +} // namespace molo::ai diff --git a/src/cli/ai/ai_session.hpp b/src/cli/ai/ai_session.hpp new file mode 100644 index 0000000..709bbb4 --- /dev/null +++ b/src/cli/ai/ai_session.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +#include "ai/ai_client.hpp" + +namespace molo::ai { + +struct PendingWrite { + std::string path; + std::string content; +}; + +// Plan-then-apply session: the model proposes file writes, a human confirms +// each one, and only confirmation touches the filesystem. Caps: 8 MB +// transcript, 50 actions, 300 s session. +class AiSession { + public: + explicit AiSession(std::string name); + + bool append(const std::string& role, const std::string& content, std::string& error); + std::size_t transcript_bytes() const { return transcript_bytes_; } + std::size_t action_count() const { return actions_; } + bool expired() const; + + std::size_t request_write(const std::string& path, const std::string& content); + const std::vector& pending() const { return pending_; } + bool confirm_write(std::size_t index, std::string& error); + void reject_write(std::size_t index); + + const std::string& name() const { return name_; } + const std::vector& messages() const { return messages_; } + + private: + std::string name_; + std::vector messages_; + std::size_t transcript_bytes_ = 0; + std::size_t actions_ = 0; + std::vector pending_; + std::chrono::steady_clock::time_point started_; +}; + +} // namespace molo::ai diff --git a/src/cli/ai/ai_transport.cpp b/src/cli/ai/ai_transport.cpp new file mode 100644 index 0000000..ba2dc8d --- /dev/null +++ b/src/cli/ai/ai_transport.cpp @@ -0,0 +1,23 @@ +#include "ai/ai_transport.hpp" + +namespace molo::ai { + +StubAiTransport::StubAiTransport(std::vector sse_bodies, int status) + : bodies_(std::move(sse_bodies)), status_(status) {} + +AiHttpResponse StubAiTransport::post_stream(const AiHttpRequest& request) { + seen_.push_back(request); + AiHttpResponse out; + out.status = status_; + if (!bodies_.empty()) { + out.sse_body = bodies_.front(); + bodies_.erase(bodies_.begin()); + } + return out; +} + +std::string TlsStub::unavailable_reason() { + return "https transport needs wolfSSL (not installed); plain http only in this build"; +} + +} // namespace molo::ai diff --git a/src/cli/ai/ai_transport.hpp b/src/cli/ai/ai_transport.hpp new file mode 100644 index 0000000..08247c2 --- /dev/null +++ b/src/cli/ai/ai_transport.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include + +namespace molo::ai { + +struct AiHttpRequest { + std::string url; + std::string bearer_key; + std::string body_json; + int connect_timeout_sec = 30; +}; + +struct AiHttpResponse { + int status = 0; + std::string sse_body; + std::string error; +}; + +// Transport interface. In the full tree this is the 08-clients-openapi.md +// HTTP client; here a minimal socket transport plus a scripted stub. +class IAiTransport { + public: + virtual ~IAiTransport() = default; + virtual AiHttpResponse post_stream(const AiHttpRequest& request) = 0; +}; + +// Scripted transport for tests: returns canned SSE bodies. +class StubAiTransport : public IAiTransport { + public: + explicit StubAiTransport(std::vector sse_bodies, int status = 200); + AiHttpResponse post_stream(const AiHttpRequest& request) override; + const std::vector& seen() const { return seen_; } + + private: + std::vector bodies_; + int status_; + std::vector seen_; +}; + +// MARKED STUB: HTTPS needs wolfSSL (sole dependency), which is not installed +// in this environment. Plain http:// works; https:// reports unavailable. +class TlsStub { + public: + static bool https_supported() { return false; } + static std::string unavailable_reason(); +}; + +} // namespace molo::ai diff --git a/src/cli/ai/prompt_pack.cpp b/src/cli/ai/prompt_pack.cpp new file mode 100644 index 0000000..46ea25d --- /dev/null +++ b/src/cli/ai/prompt_pack.cpp @@ -0,0 +1,44 @@ +#include "ai/prompt_pack.hpp" + +namespace molo::ai { + +std::string PromptPack::system_prompt() { + return "You are Molo, a C++23 web framework assistant. " + "Generate complete, compiling MoloApp code: routers, class-based views, " + "validation models, Jinja-like templates (extends, blocks, includes, if, for, " + "filters, auto-escaping), ORM models with Postgres/SQLite backends, auth, " + "sessions, forms, WebSockets, Socket.IO rooms, SSE streams, WebDAV handlers, " + "supervisor tasks, and integration tests mirroring URL paths. " + "Prefer small files, one class per file, namespaces mirroring directories. " + "Every knob gets a static cap. Never emit secrets; placeholders only. " + "Plan file writes first, then apply only after human confirmation."; +} + +std::string PromptPack::feature_prompt(const std::string& topic) { + if (topic == "routing") { + return "Routing: FastAPI-style routers, path/query/body binding through the " + "validation framework, application chaining with automatic path merging."; + } + if (topic == "templates") { + return "Templates: inheritance with extends and blocks, includes, if/for, filters, " + "auto-escaping with explicit safe opt-out, i18n t() hooks, CSP nonces, " + "responsive mobile-first output."; + } + if (topic == "orm") { + return "ORM: dual Postgres/SQLite backends, auto-sync tables in dev and prod, " + "self-healing schema mismatches, parameterized queries only, pooling, " + "transactions, cache invalidation hooks."; + } + if (topic == "realtime") { + return "Realtime: WebSocket class views on routers, SSE streams, Socket.IO " + "namespaces/rooms/events with long-polling fallback, shared auth/session/" + "middleware/logging, client parity for every protocol."; + } + if (topic == "ops") { + return "Ops: /healthz /readyz /startupz probes, Prometheus 0.0.4 metrics with " + "series caps, OTLP bridge, CSP report endpoint, PO-based i18n."; + } + return "General MoloApp feature guidance: " + topic; +} + +} // namespace molo::ai diff --git a/src/cli/ai/prompt_pack.hpp b/src/cli/ai/prompt_pack.hpp new file mode 100644 index 0000000..571b600 --- /dev/null +++ b/src/cli/ai/prompt_pack.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace molo::ai { + +// Framework feature prompts for vibe-coding sessions. +class PromptPack { + public: + static std::string system_prompt(); + static std::string feature_prompt(const std::string& topic); +}; + +} // namespace molo::ai diff --git a/src/cli/ai/redactor.cpp b/src/cli/ai/redactor.cpp new file mode 100644 index 0000000..d6f6bd9 --- /dev/null +++ b/src/cli/ai/redactor.cpp @@ -0,0 +1,79 @@ +#include "ai/redactor.hpp" + +#include + +namespace molo::ai { + +bool Redactor::key_is_secret(const std::string& key) { + std::string upper; + for (char c : key) upper += static_cast(std::toupper(static_cast(c))); + auto ends_with = [&](const std::string& suffix) { + return upper.size() >= suffix.size() && + upper.compare(upper.size() - suffix.size(), suffix.size(), suffix) == 0; + }; + return ends_with("_KEY") || ends_with("_SECRET") || ends_with("_TOKEN") || + ends_with("APIKEY") || ends_with("API_KEY") || upper == "KEY" || upper == "SECRET" || + upper == "TOKEN" || upper == "PASSWORD"; +} + +Redactor::Result Redactor::redact(const std::string& text) { + Result out; + out.text.reserve(text.size()); + std::size_t i = 0; + while (i < text.size()) { + // Match a quoted or bare key followed by ':' or '='. + std::size_t key_start = i; + bool quoted = i < text.size() && text[i] == '"'; + std::string key; + std::size_t j = i; + if (quoted) { + ++j; + while (j < text.size() && text[j] != '"') { + key += text[j]; + ++j; + } + if (j < text.size()) ++j; // closing quote + } else { + while (j < text.size() && + (std::isalnum(static_cast(text[j])) || text[j] == '_')) { + key += text[j]; + ++j; + } + } + std::size_t k = j; + while (k < text.size() && (text[k] == ' ' || text[k] == '\t')) ++k; + if (!key.empty() && k < text.size() && (text[k] == ':' || text[k] == '=')) { + char sep = text[k]; + std::size_t v = k + 1; + while (v < text.size() && (text[v] == ' ' || text[v] == '\t')) ++v; + if (key_is_secret(key)) { + out.text += text.substr(key_start, v - key_start); + if (v < text.size() && text[v] == '"') { + out.text += "\"***\""; + ++v; + while (v < text.size() && text[v] != '"') { + if (text[v] == '\\') ++v; + ++v; + } + if (v < text.size()) ++v; + } else { + out.text += "***"; + while (v < text.size() && text[v] != '\n' && text[v] != ',' && text[v] != '}') ++v; + } + ++out.redacted; + i = v; + if (i < text.size() && (text[i] == ',' || text[i] == '}')) { + out.text += text[i]; + ++i; + } + (void)sep; + continue; + } + } + out.text += text[i]; + ++i; + } + return out; +} + +} // namespace molo::ai diff --git a/src/cli/ai/redactor.hpp b/src/cli/ai/redactor.hpp new file mode 100644 index 0000000..32073e9 --- /dev/null +++ b/src/cli/ai/redactor.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace molo::ai { + +// Redacts secrets before prompt construction. Keys matching *_KEY, +// *_SECRET, *_TOKEN (case-insensitive) and .env.json-style assignments +// are masked. Redaction failures abort the request (caller checks ok()). +class Redactor { + public: + struct Result { + std::string text; + unsigned long redacted = 0; + bool ok = true; + }; + + static Result redact(const std::string& text); + static bool key_is_secret(const std::string& key); +}; + +} // namespace molo::ai diff --git a/src/cli/ai/socket_transport.cpp b/src/cli/ai/socket_transport.cpp new file mode 100644 index 0000000..d182e8b --- /dev/null +++ b/src/cli/ai/socket_transport.cpp @@ -0,0 +1,127 @@ +#include "ai/socket_transport.hpp" + +#if !defined(_WIN32) +#include +#include +#include +#include +#endif +#include + +namespace molo::ai { + +SocketAiTransport::ParsedUrl SocketAiTransport::parse(const std::string& url) { + ParsedUrl out; + std::string rest = url; + if (rest.rfind("http://", 0) == 0) { + rest = rest.substr(7); + } else if (rest.rfind("https://", 0) == 0) { + out.https = true; + rest = rest.substr(8); + } else { + return out; + } + auto slash = rest.find('/'); + std::string hostport = slash == std::string::npos ? rest : rest.substr(0, slash); + out.path = slash == std::string::npos ? "/" : rest.substr(slash); + auto colon = hostport.find(':'); + if (colon != std::string::npos) { + out.host = hostport.substr(0, colon); + try { + out.port = std::stoi(hostport.substr(colon + 1)); + } catch (...) { + return out; + } + } else { + out.host = hostport; + out.port = out.https ? 443 : 80; + } + if (!out.host.empty()) out.ok = true; + return out; +} + +AiHttpResponse SocketAiTransport::post_stream(const AiHttpRequest& request) { + AiHttpResponse out; + ParsedUrl url = parse(request.url); + if (!url.ok) { + out.error = "bad URL: " + request.url; + return out; + } + if (url.https) { + out.error = TlsStub::unavailable_reason(); + return out; + } +#if defined(_WIN32) + out.error = "socket transport is POSIX only"; + return out; +#else + addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + addrinfo* list = nullptr; + if (getaddrinfo(url.host.c_str(), std::to_string(url.port).c_str(), &hints, &list) != 0) { + out.error = "dns failed for " + url.host; + return out; + } + int fd = -1; + for (addrinfo* ai = list; ai != nullptr; ai = ai->ai_next) { + fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) continue; + struct timeval tv; + tv.tv_sec = request.connect_timeout_sec; + tv.tv_usec = 0; + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) break; + close(fd); + fd = -1; + } + freeaddrinfo(list); + if (fd < 0) { + out.error = "connect failed to " + url.host; + return out; + } + std::string head = "POST " + url.path + " HTTP/1.1\r\nHost: " + url.host + + "\r\nContent-Type: application/json\r\nAccept: text/event-stream\r\n" + + "Authorization: Bearer " + request.bearer_key + "\r\nContent-Length: " + + std::to_string(request.body_json.size()) + "\r\nConnection: close\r\n\r\n"; + std::string wire = head + request.body_json; + std::size_t sent = 0; + while (sent < wire.size()) { + ssize_t n = send(fd, wire.data() + sent, wire.size() - sent, 0); + if (n <= 0) { + close(fd); + out.error = "send failed"; + return out; + } + sent += static_cast(n); + } + std::string raw; + char buf[8192]; + while (true) { + ssize_t n = recv(fd, buf, sizeof(buf), 0); + if (n <= 0) break; + raw.append(buf, static_cast(n)); + } + close(fd); + auto eol = raw.find("\r\n"); + if (eol == std::string::npos) { + out.error = "bad response"; + return out; + } + std::string status_line = raw.substr(0, eol); + auto sp = status_line.find(' '); + if (sp != std::string::npos) { + try { + out.status = std::stoi(status_line.substr(sp + 1)); + } catch (...) { + out.status = 0; + } + } + auto hdr_end = raw.find("\r\n\r\n"); + out.sse_body = hdr_end == std::string::npos ? "" : raw.substr(hdr_end + 4); + return out; +#endif +} + +} // namespace molo::ai diff --git a/src/cli/ai/socket_transport.hpp b/src/cli/ai/socket_transport.hpp new file mode 100644 index 0000000..317c143 --- /dev/null +++ b/src/cli/ai/socket_transport.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include "ai/ai_transport.hpp" + +namespace molo::ai { + +// Minimal blocking HTTP transport over POSIX sockets. Plain http:// only; +// https:// URLs fail with the marked TLS stub reason. +class SocketAiTransport : public IAiTransport { + public: + AiHttpResponse post_stream(const AiHttpRequest& request) override; + + private: + struct ParsedUrl { + std::string host; + int port = 80; + std::string path; + bool https = false; + bool ok = false; + }; + static ParsedUrl parse(const std::string& url); +}; + +} // namespace molo::ai diff --git a/src/cli/ai/sse_stream_parser.cpp b/src/cli/ai/sse_stream_parser.cpp new file mode 100644 index 0000000..61f474d --- /dev/null +++ b/src/cli/ai/sse_stream_parser.cpp @@ -0,0 +1,56 @@ +#include "ai/sse_stream_parser.hpp" + +namespace molo::ai { + +void SseStreamParser::feed(const std::string& chunk) { + buffer_ += chunk; + std::size_t pos = 0; + while (true) { + std::size_t end = buffer_.find("\n\n", pos); + std::size_t sep_len = 2; + if (end == std::string::npos) { + end = buffer_.find("\r\n\r\n", pos); + sep_len = 4; + } + if (end == std::string::npos) break; + std::string block = buffer_.substr(pos, end - pos); + pos = end + sep_len; + std::string data; + std::size_t line_pos = 0; + while (line_pos < block.size()) { + std::size_t line_end = block.find('\n', line_pos); + std::string line = block.substr(line_pos, line_end == std::string::npos + ? std::string::npos + : line_end - line_pos); + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.rfind("data:", 0) == 0) { + std::string payload = line.substr(5); + if (!payload.empty() && payload.front() == ' ') payload.erase(0, 1); + if (!data.empty()) data += "\n"; + data += payload; + } + if (line_end == std::string::npos) break; + line_pos = line_end + 1; + } + if (data == "[DONE]") { + done_ = true; + } else if (!data.empty()) { + events_.push_back(data); + } + } + buffer_.erase(0, pos); +} + +std::vector SseStreamParser::take_events() { + std::vector out = std::move(events_); + events_.clear(); + return out; +} + +void SseStreamParser::reset() { + buffer_.clear(); + events_.clear(); + done_ = false; +} + +} // namespace molo::ai diff --git a/src/cli/ai/sse_stream_parser.hpp b/src/cli/ai/sse_stream_parser.hpp new file mode 100644 index 0000000..35eee85 --- /dev/null +++ b/src/cli/ai/sse_stream_parser.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +namespace molo::ai { + +// SSE parser for OpenAI-compatible chat streams. Shared with the 06-realtime +// SSE client contract: text/event-stream, data lines, blank-line framing, +// end on "data: [DONE]". This file owns the CLI copy; the realtime part +// reuses the same framing rules, never a duplicate parser. +class SseStreamParser { + public: + void feed(const std::string& chunk); + std::vector take_events(); + bool done() const { return done_; } + void reset(); + + private: + std::string buffer_; + std::vector events_; + bool done_ = false; +}; + +} // namespace molo::ai diff --git a/src/cli/caps.hpp b/src/cli/caps.hpp new file mode 100644 index 0000000..79b2214 --- /dev/null +++ b/src/cli/caps.hpp @@ -0,0 +1,56 @@ +#pragma once + +// Central static ceilings for the CLI/ops part (09-cli-ops.md). +// Every knob below has a hard cap. No autotuning. + +#include + +namespace molo::caps { + +// CLI +inline constexpr int kMaxVerbose = 3; +inline constexpr std::size_t kMaxCommandDepth = 8; +inline constexpr std::size_t kMaxHelpWidth = 100; + +// Dev server +inline constexpr std::size_t kMaxWatchFiles = 20000; +inline constexpr int kWatchDebounceMs = 150; +inline constexpr int kMakeTimeoutSec = 300; +inline constexpr int kChildKillGraceSec = 5; +inline constexpr int kBackoffBaseSec = 1; +inline constexpr int kBackoffMaxSec = 30; + +// Embedder +inline constexpr std::size_t kEmbedTotalBytes = 512u * 1024u * 1024u; +inline constexpr std::size_t kEmbedMaxFiles = 50000; +inline constexpr std::size_t kEmbedMaxFileBytes = 64u * 1024u * 1024u; +inline constexpr std::size_t kEmbedChunkBytes = 1024u * 1024u; +inline constexpr std::size_t kGzipMinBytes = 1024; + +// AI harness +inline constexpr int kAiConnectTimeoutSec = 30; +inline constexpr int kAiSessionCapSec = 300; +inline constexpr std::size_t kAiTranscriptMaxBytes = 8u * 1024u * 1024u; +inline constexpr std::size_t kAiMaxActions = 50; + +// Health / metrics +inline constexpr int kHealthCheckTimeoutMs = 200; +inline constexpr int kHealthP99Ms = 50; +inline constexpr std::size_t kMetricsMaxSeries = 10000; +inline constexpr std::size_t kMetricsMaxLabels = 64; +inline constexpr int kMetricsExportFloorSec = 5; +inline constexpr int kMetricsRenderP99Ms = 100; + +// CSP reports +inline constexpr std::size_t kCspReportMaxBytes = 64u * 1024u; +inline constexpr std::size_t kCspReportsPerMinPerIp = 100; +inline constexpr std::size_t kCspReportStoreMax = 1000; +inline constexpr std::size_t kNonceBytes = 16; + +// Scaffolding +inline constexpr std::size_t kMaxProjectName = 64; + +// General +inline constexpr std::size_t kMaxPathBytes = 4096; + +} // namespace molo::caps diff --git a/src/cli/cli.mk b/src/cli/cli.mk new file mode 100644 index 0000000..5a64db5 --- /dev/null +++ b/src/cli/cli.mk @@ -0,0 +1,34 @@ +# Per-part build fragment for 09-cli-ops. Included by the root Makefile. +# Usage: include src/cli/cli.mk, then $(MOLO_BIN) and $(CLI_TEST_BINS). +CXX ?= g++ +CXXFLAGS ?= -std=c++23 -Wall -Wextra +CLI_INC := -Isrc/cli +CLI_SRCS := $(shell find src/cli -name '*.cpp' ! -name 'main.cpp') +CLI_MAIN := src/cli/main.cpp +CLI_OBJS := $(patsubst %.cpp,build/obj/%.o,$(CLI_SRCS)) +CLI_MAIN_OBJ := build/obj/src/cli/main.cpp.o +MOLO_BIN := build/molo +CLI_TEST_SRCS := $(wildcard tests/src/cli/test_*.cpp) +CLI_TEST_BINS := $(patsubst tests/src/cli/%.cpp,build/tests/%,$(CLI_TEST_SRCS)) + +$(MOLO_BIN): $(CLI_OBJS) $(CLI_MAIN_OBJ) + @mkdir -p build + $(CXX) $(CXXFLAGS) $(CLI_OBJS) $(CLI_MAIN_OBJ) $(SUPPLY_LIB) $(OPERABILITY_LIB) -o $(MOLO_BIN) + +build/obj/%.o: %.cpp + @mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) $(CLI_INC) -c $< -o $@ + +build/tests/test_%: tests/src/cli/test_%.cpp $(CLI_OBJS) $(SUPPLY_LIB) $(OPERABILITY_LIB) + @mkdir -p build/tests + $(CXX) $(CXXFLAGS) $(CLI_INC) $< $(CLI_OBJS) $(SUPPLY_LIB) $(OPERABILITY_LIB) -o $@ + +cli-test: $(MOLO_BIN) $(CLI_TEST_BINS) + for t in $(CLI_TEST_BINS); do ./$$t || exit 1; done + sh tests/src/cli/test_cli_binary.sh $(MOLO_BIN) + +# Supply-chain wiring (PATCHES 15, section 2, corrected: -Isrc so the +# "supplychain/..." include spelling resolves; MOLO_BIN link carries the lib). +CLI_INC += -Isrc + +$(MOLO_BIN): $(SUPPLY_LIB) $(OPERABILITY_LIB) diff --git a/src/cli/cli/arg_parser.cpp b/src/cli/cli/arg_parser.cpp new file mode 100644 index 0000000..a30c0c9 --- /dev/null +++ b/src/cli/cli/arg_parser.cpp @@ -0,0 +1,102 @@ +#include "cli/arg_parser.hpp" + +#include "caps.hpp" + +namespace molo::cli { + +ParsedArgs ArgParser::parse(int argc, const char* const* argv) const { + std::vector args; + for (int i = 0; i < argc; ++i) args.emplace_back(argv[i]); + return parse(args); +} + +ParsedArgs ArgParser::parse(const std::vector& argv) const { + // Expand combined short flags: -vv becomes -v -v. + std::vector flat; + flat.reserve(argv.size() * 2); + for (const auto& raw : argv) { + bool cluster = raw.size() > 2 && raw[0] == '-' && raw[1] != '-' && + raw.find('=') == std::string::npos; + for (std::size_t k = 1; cluster && k < raw.size(); ++k) { + char c = raw[k]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) cluster = false; + } + if (cluster) { + for (std::size_t k = 1; k < raw.size(); ++k) flat.push_back(std::string("-") + raw[k]); + } else { + flat.push_back(raw); + } + } + ParsedArgs out; + bool seen_command = false; + bool end_of_flags = false; + for (std::size_t i = 1; i < flat.size(); ++i) { + const std::string& tok = flat[i]; + if (!end_of_flags && tok == "--") { + end_of_flags = true; + continue; + } + if (!end_of_flags && tok.size() > 1 && tok[0] == '-') { + std::string name = tok; + std::string value; + auto eq = tok.find('='); + if (eq != std::string::npos) { + name = tok.substr(0, eq); + value = tok.substr(eq + 1); + } + if (name == "-h" || name == "--help") { + out.global.show_help = true; + continue; + } + if (name == "--version") { + out.global.show_version = true; + continue; + } + if (name == "-v" || name == "--verbose") { + if (out.global.verbose < caps::kMaxVerbose) ++out.global.verbose; + continue; + } + if (name == "-q" || name == "--quiet") { + out.global.quiet = true; + continue; + } + if (name == "--format") { + if (value.empty()) { + if (i + 1 >= flat.size()) { + out.error = "flag --format needs a value (json|text)"; + return out; + } + value = flat[++i]; + } + if (value != "json" && value != "text") { + out.error = "flag --format must be json or text"; + return out; + } + out.global.format = value; + continue; + } + // Command-local flag; value may follow. + std::string key = name; + while (key.size() > 1 && key[0] == '-') key.erase(0, 1); + if (value.empty() && i + 1 < flat.size() && !flat[i + 1].empty() && + flat[i + 1][0] != '-') { + value = flat[++i]; + } + if (value.empty()) value = "true"; + out.flags[key] = value; + continue; + } + if (!seen_command && out.command_path.size() < caps::kMaxCommandDepth) { + // Leading non-flag tokens form the command path until a value-like + // token appears after at least one path element; handlers decide. + out.command_path.push_back(tok); + } else { + seen_command = true; + out.positionals.push_back(tok); + } + if (out.command_path.size() >= 2) seen_command = true; + } + return out; +} + +} // namespace molo::cli diff --git a/src/cli/cli/arg_parser.hpp b/src/cli/cli/arg_parser.hpp new file mode 100644 index 0000000..67cdb2c --- /dev/null +++ b/src/cli/cli/arg_parser.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include + +namespace molo::cli { + +struct GlobalOptions { + std::string format = "text"; + int verbose = 0; + bool quiet = false; + bool show_help = false; + bool show_version = false; +}; + +struct ParsedArgs { + GlobalOptions global; + std::vector command_path; + std::map flags; + std::vector positionals; + std::string error; + bool ok() const { return error.empty(); } +}; + +// Splits argv into global options, command path, flags, and positionals. +// Command path is the leading run of non-flag tokens; the tree resolves them. +class ArgParser { + public: + ParsedArgs parse(int argc, const char* const* argv) const; + ParsedArgs parse(const std::vector& argv) const; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/cli_app.cpp b/src/cli/cli/cli_app.cpp new file mode 100644 index 0000000..60e0f23 --- /dev/null +++ b/src/cli/cli/cli_app.cpp @@ -0,0 +1,763 @@ +#include "cli/cli_app.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "supplychain/cli/SupplyCli.hpp" + +#include "operability/chaos/experiment.h" +#include "operability/chaos/hook.h" +#include "operability/chaos/injector.h" +#include "operability/debug/snapshot.h" +#include "operability/deploy/blue_green.h" +#include "operability/deploy/rolling.h" +#include "operability/deploy/strategy.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +#include "ai/ai_client.hpp" +#include "ai/ai_session.hpp" +#include "ai/prompt_pack.hpp" +#include "ai/socket_transport.hpp" +#include "cli/arg_parser.hpp" +#include "cli/command.hpp" +#include "cli/help_renderer.hpp" +#include "cli/json_writer.hpp" +#include "cli/output.hpp" +#include "cli/scaffolder.hpp" +#include "cli/supervisor_cli.hpp" +#include "cli/table_writer.hpp" +#include "dev/dev_server.hpp" +#include "embed/asset_scanner.hpp" +#include "embed/embed_emitter.hpp" +#include "ops/health_registry.hpp" +#include "ops/liveness_check.hpp" +#include "ops/ops_routes.hpp" +#include "ops/prometheus_exporter.hpp" +#include "ops/readiness_check.hpp" +#include "ops/startup_gate.hpp" +#include "version.hpp" + +namespace molo::cli { + +namespace { +std::string flag_value(const ParsedArgs& args, const std::string& key, const std::string& fallback) { + auto it = args.flags.find(key); + if (it == args.flags.end()) return fallback; + return it->second; +} + +bool flag_bool(const ParsedArgs& args, const std::string& key) { + auto it = args.flags.find(key); + if (it == args.flags.end()) return false; + return it->second == "true" || it->second == "1" || it->second == "yes"; +} + +long flag_long(const ParsedArgs& args, const std::string& key, long fallback) { + auto it = args.flags.find(key); + if (it == args.flags.end()) return fallback; + try { + return std::stol(it->second); + } catch (...) { + return fallback; + } +} + +double flag_double(const ParsedArgs& args, const std::string& key, double fallback) { + auto it = args.flags.find(key); + if (it == args.flags.end()) return fallback; + try { + return std::stod(it->second); + } catch (...) { + return fallback; + } +} + +// --- Operability commands (PATCHES 13, section 10) --- + +class OpsClock : public molo::common::Clock { + public: + std::int64_t wall_ms() const override { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + std::int64_t steady_ms() const override { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + } + void sleep_ms(long ms) override { + if (ms > 0) std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } +}; + +class OpsObserver : public molo::common::Observer { + public: + void event(std::string_view, std::string_view) override {} + void counter(std::string_view, long) override {} + void gauge(std::string_view, double) override {} +}; + +molo::common::Clock& ops_clock() { + static OpsClock c; + return c; +} + +molo::common::Observer& ops_observer() { + static OpsObserver o; + return o; +} + +// Local readiness probe for rollout steps: same checks as cmd_health. +bool local_ready() { + auto live = std::make_shared(); + molo::ops::HealthRegistry registry; + registry.add(live); + molo::ops::ReadinessCheck readiness; + molo::ops::StartupGate gate; + gate.mark_boot_complete(); + molo::ops::PrometheusExporter exporter; + molo::ops::OpsRoutes routes(registry, readiness, gate, exporter); + return routes.healthz().status == 200 && routes.readyz().status == 200; +} + +ExitCode cmd_rollout_start(const ParsedArgs& args, Output& out) { + const std::string strategy = flag_value(args, "strategy", "rolling"); + const long instances = std::clamp(flag_long(args, "instances", 3), 1L, 1000L); + const long batch = std::clamp(flag_long(args, "batch", 1), 1L, 1000L); + const long max_un = std::clamp(flag_long(args, "max-unavailable", 0), 0L, 1000L); + if (strategy != "rolling" && strategy != "blue-green") { + out.error("strategy must be rolling or blue-green\n"); + return ExitCode::Usage; + } + auto probe = [](std::size_t) { return local_ready(); }; + molo::deploy::Strategy::State terminal = molo::deploy::Strategy::State::kFailed; + if (strategy == "rolling") { + molo::deploy::Rolling::Config cfg; + cfg.total_instances = static_cast(instances); + cfg.batch_size = static_cast(batch); + cfg.max_unavailable = static_cast(max_un); + cfg.pause_between_batches_ms = 0; + cfg.ready_timeout_ms = 60000; + molo::deploy::Rolling r(cfg, ops_clock(), ops_observer(), probe, [&](std::size_t i) { + out.print("upgraded instance " + std::to_string(i) + "\n"); + }); + if (!r.start()) { + out.error("rollout failed to start\n"); + return ExitCode::Runtime; + } + std::size_t steps = 0; + const std::size_t cap = static_cast(instances) + 10; + while (r.state() == molo::deploy::Strategy::State::kRunning) { + if (++steps > cap) { + r.abort(); + out.error("rollout step cap exceeded\n"); + return ExitCode::Runtime; + } + r.step(); + } + terminal = r.state(); + out.print(std::string("rollout ") + r.state_name() + "\n"); + } else { + molo::deploy::BlueGreen::Config cfg; + cfg.green_instances = static_cast(instances); + cfg.ready_timeout_ms = 60000; + cfg.rollback_window_ms = 0; // One-shot CLI retires blue immediately. + molo::deploy::BlueGreen bg( + cfg, ops_clock(), ops_observer(), probe, + [&] { + out.print("switched traffic to green\n"); + return true; + }, + [&](bool green) { + out.print(std::string("retired ") + (green ? "green" : "blue") + "\n"); + }); + if (!bg.start()) { + out.error("rollout failed to start\n"); + return ExitCode::Runtime; + } + std::size_t steps = 0; + while (bg.state() == molo::deploy::Strategy::State::kRunning) { + if (++steps > 100) { + bg.abort(); + out.error("rollout step cap exceeded\n"); + return ExitCode::Runtime; + } + bg.step(); + } + terminal = bg.state(); + out.print(std::string("rollout ") + bg.state_name() + "\n"); + } + return terminal == molo::deploy::Strategy::State::kSucceeded ? ExitCode::Ok + : ExitCode::Runtime; +} + +ExitCode cmd_rollout_abort(const ParsedArgs&, Output& out) { + out.error("no rollout running in this process (one-shot CLI: start runs to completion)\n"); + return ExitCode::Runtime; +} + +ExitCode cmd_rollback(const ParsedArgs&, Output& out) { + out.error("no blue-green deployment in this process (one-shot CLI has no rollback window)\n"); + return ExitCode::Runtime; +} + +ExitCode cmd_experiment_start(const ParsedArgs& args, Output& out) { + const long blast = std::clamp(flag_long(args, "blast", 1), 0L, 100L); + const long duration = std::clamp(flag_long(args, "duration", 60000), 0L, 3600000L); + double max_burn = flag_double(args, "max-burn", 0.05); + if (max_burn < 0) max_burn = 0; + molo::chaos::Injector injector(ops_clock(), ops_observer()); + injector.add_hook(molo::chaos::Hook("cli-latency", molo::chaos::Hook::Action::kDelay, + static_cast(blast), 50)); + molo::chaos::Experiment exp(ops_clock(), ops_observer(), "cli", static_cast(blast), + duration, max_burn); + if (!exp.start()) { + out.error("experiment failed to start\n"); + return ExitCode::Runtime; + } + out.print("experiment running: blast=" + std::to_string(blast) + + " hooks=" + std::to_string(injector.size()) + "\n"); + for (long waited = 0; waited < duration; waited += 100) { + ops_clock().sleep_ms(100); + // No burn source in a one-shot CLI; the probe reads zero. + if (exp.abort_if_breached([] { return 0.0; })) { + out.error("experiment aborted: burn ceiling breached\n"); + return ExitCode::Runtime; + } + } + exp.finish(); + out.print("experiment finished\n"); + return ExitCode::Ok; +} + +ExitCode cmd_experiment_abort(const ParsedArgs&, Output& out) { + out.error("no experiment running in this process (one-shot CLI: start runs to completion)\n"); + return ExitCode::Runtime; +} + +ExitCode cmd_debug_snapshot(const ParsedArgs&, Output& out) { + molo::debug::Snapshot snap(ops_clock(), molo::kMoloVersion); + snap.add_section("cli", "{\"argv\":\"debug snapshot\"}"); + snap.add_section("health", local_ready() ? "{\"ok\":true}" : "{\"ok\":false}"); + out.print(snap.capture() + "\n"); + return ExitCode::Ok; +} + +ExitCode cmd_version(const ParsedArgs& args, Output& out) { + if (args.global.format == "json") { + std::map fields; + fields["name"] = JsonWriter::quote(molo::kMoloName); + fields["version"] = JsonWriter::quote(molo::kMoloVersion); + out.print(JsonWriter::object(fields) + "\n"); + } else { + out.print(molo::version_string() + "\n"); + } + return ExitCode::Ok; +} + +ExitCode cmd_health(const ParsedArgs& args, Output& out) { + auto live = std::make_shared(); + molo::ops::HealthRegistry registry; + registry.add(live); + molo::ops::ReadinessCheck readiness; + molo::ops::StartupGate gate; + gate.mark_boot_complete(); + molo::ops::PrometheusExporter exporter; + molo::ops::OpsRoutes routes(registry, readiness, gate, exporter); + molo::ops::OpsResponse hz = routes.healthz(); + molo::ops::OpsResponse rz = routes.readyz(); + if (args.global.format == "json") { + std::map fields; + fields["healthz"] = JsonWriter::quote(hz.body); + fields["readyz"] = JsonWriter::quote(rz.body); + fields["ok"] = (hz.status == 200 && rz.status == 200) ? "true" : "false"; + out.print(JsonWriter::object(fields) + "\n"); + } else { + out.print("healthz: " + std::to_string(hz.status) + " " + hz.body + "\n"); + out.print("readyz: " + std::to_string(rz.status) + " " + rz.body + "\n"); + } + return (hz.status == 200 && rz.status == 200) ? ExitCode::Ok : ExitCode::Unavailable; +} + +ExitCode cmd_new(const ParsedArgs& args, Output& out) { + if (args.positionals.empty()) { + out.error("molo new needs a project name\n"); + return ExitCode::Usage; + } + ScaffoldOptions opts; + opts.name = args.positionals[0]; + opts.backend = flag_value(args, "backend", "sqlite"); + opts.force = flag_bool(args, "force"); + Scaffolder scaffolder; + std::string dir = opts.name; + ScaffoldResult result = scaffolder.scaffold(dir, opts); + if (!result.ok) { + out.error(result.error + "\n"); + return ExitCode::Runtime; + } + if (args.global.format == "json") { + std::vector items; + for (const auto& created : result.created) items.push_back(JsonWriter::quote(created)); + std::map fields; + fields["ok"] = "true"; + fields["dir"] = JsonWriter::quote(dir); + fields["created"] = JsonWriter::array(items); + out.print(JsonWriter::object(fields) + "\n"); + } else { + out.print("created " + dir + " (" + std::to_string(result.created.size()) + " entries)\n"); + } + return ExitCode::Ok; +} + +ExitCode cmd_serve(const ParsedArgs& args, Output& out) { + molo::dev::ServeOptions opts; + try { + opts.port = std::stoi(flag_value(args, "port", "8000")); + } catch (...) { + out.error("--port must be a number\n"); + return ExitCode::Usage; + } + opts.reload = flag_bool(args, "reload"); + opts.config_path = flag_value(args, "config", ".env.json"); + opts.allow_prod = flag_bool(args, "allow-prod"); + molo::dev::DevServer server; + return server.run(opts, out); +} + +ExitCode cmd_embed(const ParsedArgs& args, Output& out) { + if (args.positionals.empty()) { + out.error("molo embed needs a directory\n"); + return ExitCode::Usage; + } + std::string out_cpp = flag_value(args, "out", ""); + if (out_cpp.empty()) { + out.error("molo embed needs --out \n"); + return ExitCode::Usage; + } + std::string out_hpp = flag_value(args, "hpp", ""); + if (out_hpp.empty()) { + auto dot = out_cpp.find_last_of('.'); + out_hpp = (dot == std::string::npos ? out_cpp : out_cpp.substr(0, dot)) + ".hpp"; + } + molo::embed::AssetScanner scanner; + molo::embed::ScanResult scanned = scanner.scan(args.positionals[0]); + if (!scanned.ok()) { + out.error(scanned.error + "\n"); + return ExitCode::Runtime; + } + molo::embed::EmitOptions emit_opts; + emit_opts.namespc = flag_value(args, "namespace", "molo::assets"); + emit_opts.gzip = flag_bool(args, "gzip"); + emit_opts.tool_version = molo::kEmbedToolVersion; + molo::embed::EmbedEmitter emitter; + std::string error; + if (!emitter.emit(scanned.files, out_cpp, out_hpp, emit_opts, error)) { + out.error(error + "\n"); + return ExitCode::Runtime; + } + if (args.global.format == "json") { + std::map fields; + fields["ok"] = "true"; + fields["files"] = std::to_string(scanned.files.size()); + fields["bytes"] = std::to_string(scanned.total_bytes); + fields["cpp"] = JsonWriter::quote(out_cpp); + fields["hpp"] = JsonWriter::quote(out_hpp); + out.print(JsonWriter::object(fields) + "\n"); + } else { + out.print("embedded " + std::to_string(scanned.files.size()) + " files (" + + std::to_string(scanned.total_bytes) + " bytes)\n"); + } + return ExitCode::Ok; +} + +ExitCode cmd_build(const ParsedArgs& args, Output& out) { + std::string embed_dir = flag_value(args, "embed", ""); + std::string out_dir = flag_value(args, "out", "build"); + if (embed_dir.empty()) { + out.print("build: nothing to embed (pass --embed )\n"); + return ExitCode::Ok; + } + ParsedArgs sub = args; + sub.positionals = {embed_dir}; + sub.flags["out"] = out_dir + "/assets.cpp"; + return cmd_embed(sub, out); +} + +ExitCode cmd_ai(const std::string& action, const ParsedArgs& args, Output& out) { + std::string prompt = flag_value(args, "prompt", ""); + if (prompt.empty() && !args.positionals.empty()) prompt = args.positionals[0]; + if (prompt.empty()) { + // Read piped stdin when no prompt flag is given. + if (!Output::stdout_is_tty()) { + std::string line; + while (std::getline(std::cin, line)) prompt += line + "\n"; + } + } + if (prompt.empty()) { + out.error("molo ai " + action + " needs --prompt or a prompt argument\n"); + return ExitCode::Usage; + } + molo::ai::AiClientOptions opts; + opts.base_url = flag_value(args, "base-url", opts.base_url); + opts.model = flag_value(args, "model", opts.model); + bool stream = flag_bool(args, "stream"); + if (!stream && args.flags.find("stream") == args.flags.end()) stream = true; + molo::ai::SocketAiTransport transport; + molo::ai::AiClient client(opts, transport); + molo::ai::AiSession session(action == "continue" && !args.positionals.empty() + ? args.positionals[0] + : "molo-ai"); + std::string error; + if (!session.append("system", molo::ai::PromptPack::system_prompt(), error) || + !session.append("user", prompt, error)) { + out.error(error + "\n"); + return ExitCode::Runtime; + } + out.debug(1, "POST " + opts.base_url + "/chat/completions model=" + opts.model + "\n"); + std::string full = client.chat_stream( + session.messages(), + [&](const std::string& delta) { + if (stream && args.global.format != "json") out.print(delta); + }, + error); + if (!error.empty()) { + if (args.global.format == "json") { + out.print(JsonWriter::object({{"ok", "false"}, {"error", JsonWriter::quote(error)}}) + "\n"); + } else { + out.error(error + "\n"); + } + return ExitCode::Unavailable; + } + if (!session.append("assistant", full, error)) { + out.error(error + "\n"); + return ExitCode::Runtime; + } + if (args.global.format == "json") { + out.print(JsonWriter::object({{"ok", "true"}, {"text", JsonWriter::quote(full)}}) + "\n"); + } else if (!stream) { + out.print(full + "\n"); + } else { + out.print("\n"); + } + return ExitCode::Ok; +} + +std::shared_ptr make_leaf(const std::string& name, const std::string& brief) { + return std::make_shared(name, brief); +} + +} // namespace + +CliApp::CliApp() + : supervisor_(&null_supervisor_), tasks_(&null_tasks_), migrate_(&null_migrate_) { + build_tree(); +} + +CliApp::CliApp(ISupervisorControl& supervisor, ITaskControl& tasks, IMigrateControl& migrate) + : supervisor_(&supervisor), tasks_(&tasks), migrate_(&migrate) { + build_tree(); +} + +void CliApp::build_tree() { + auto version = make_leaf("version", "show version"); + version->usage("[--format json|text]").long_help("Prints the molo version."); + version->example("version --format json").handler(cmd_version); + + auto health = make_leaf("health", "local health summary"); + health->usage("[--format json|text]") + .long_help("Runs the local liveness and readiness checks and prints a summary."); + health->example("health").handler(cmd_health); + + auto molo_new = make_leaf("new", "scaffold a new project"); + molo_new->usage(" [--backend postgres|sqlite] [--force]") + .long_help("Scaffolds Makefile, AGENTS.md, layout, .env.json, .gitignore, example " + "files, and runs git init. Refuses non-empty dirs without --force."); + molo_new->flag(Flag("backend", "", "database backend", true, "sqlite")); + molo_new->flag(Flag("force", "", "overwrite non-empty directory", false, "")); + molo_new->example("new shop --backend postgres").handler(cmd_new); + + auto serve = make_leaf("serve", "run the dev server"); + serve->usage("[--port 8000] [--reload] [--config .env.json] [--allow-prod]") + .long_help("Runs the server once, or watches and rebuilds with --reload. Refuses " + "--reload with production config unless --allow-prod is passed."); + serve->flag(Flag("port", "", "listen port", true, "8000")); + serve->flag(Flag("reload", "", "watch and rebuild", false, "")); + serve->flag(Flag("config", "", "config file", true, ".env.json")); + serve->flag(Flag("allow-prod", "", "allow reload against prod", false, "")); + serve->example("serve --reload").handler(cmd_serve); + + auto build = make_leaf("build", "build the project"); + build->usage("[--embed assets/] [--out build/]") + .long_help("Builds the project. With --embed, embeds the directory first."); + build->flag(Flag("embed", "", "asset dir to embed", true, "")); + build->flag(Flag("out", "", "output dir", true, "build")); + build->example("build --embed assets/ --out build/").handler(cmd_build); + + auto embed = make_leaf("embed", "embed assets into C++ source"); + embed->usage(" --out [--namespace molo::assets] [--gzip]") + .long_help("Converts a static files directory into one .cpp plus one .hpp with " + "constexpr byte arrays. Byte-identical reproducible output."); + embed->flag(Flag("out", "", "output .cpp file", true, "")); + embed->flag(Flag("hpp", "", "output .hpp file", true, "")); + embed->flag(Flag("namespace", "", "C++ namespace", true, "molo::assets")); + embed->flag(Flag("gzip", "", "emit gzip variants for text", false, "")); + embed->example("embed assets/ --out build/assets.cpp --gzip").handler(cmd_embed); + + auto migrate = make_leaf("migrate", "run database migrations"); + migrate->usage("[--check | --apply]") + .long_help("Executes 04-data.md migration plans. --check previews, --apply runs."); + migrate->flag(Flag("check", "", "preview only", false, "")); + migrate->flag(Flag("apply", "", "apply migrations", false, "")); + migrate->handler([this](const ParsedArgs& args, Output& out) { + bool apply = flag_bool(args, "apply"); + MigratePlan plan; + std::string error; + if (!migrate_->check(plan, error)) { + if (args.global.format == "json") { + out.print(JsonWriter::object({{"ok", "false"}, {"error", JsonWriter::quote(error)}}) + + "\n"); + } else { + out.error(error + "\n"); + } + return ExitCode::Unavailable; + } + if (apply && !migrate_->apply(plan, error)) { + out.error(error + "\n"); + return ExitCode::Runtime; + } + if (args.global.format == "json") { + std::vector items; + for (const auto& step : plan.steps) items.push_back(JsonWriter::quote(step)); + out.print(JsonWriter::object({{"ok", "true"}, + {"applied", apply ? "true" : "false"}, + {"steps", JsonWriter::array(items)}}) + + "\n"); + } else { + out.print(apply ? "migrations applied\n" : "migrations checked\n"); + } + return ExitCode::Ok; + }); + + auto supervisor = make_leaf("supervisor", "manage the process supervisor"); + supervisor->usage("start|stop|restart|status|logs").long_help("Drives the 07 process supervisor."); + for (const char* action : {"start", "stop", "restart", "status", "logs"}) { + auto sub = make_leaf(action, std::string("supervisor ") + action); + sub->handler([this, name = std::string(action)](const ParsedArgs& args, Output& out) { + SupervisorCli cli(*supervisor_, *tasks_); + return cli.run_supervisor(name, args, out); + }); + supervisor->child(sub); + } + + auto task = make_leaf("task", "manage background tasks"); + task->usage("enqueue|retry|cancel|inspect").long_help("Drives the 07 task queue."); + for (const char* action : {"enqueue", "retry", "cancel", "inspect"}) { + auto sub = make_leaf(action, std::string("task ") + action); + sub->handler([this, name = std::string(action)](const ParsedArgs& args, Output& out) { + SupervisorCli cli(*supervisor_, *tasks_); + return cli.run_task(name, args, out); + }); + task->child(sub); + } + + auto ai = make_leaf("ai", "AI vibe-coding harness"); + ai->usage("new|continue [--stream] [--base-url URL] [--model NAME]") + .long_help("Builds applications with the DevPlace OpenAI-compatible endpoint. Key " + "from DEVPLACE_API_KEY env only. Streams by default."); + ai->flag(Flag("stream", "", "stream deltas", false, "")); + ai->flag(Flag("base-url", "", "completions base URL", true, "https://devplace.net/openai/v1")); + ai->flag(Flag("model", "", "model name", true, "molodetz")); + ai->flag(Flag("prompt", "", "prompt text", true, "")); + for (const char* action : {"new", "continue"}) { + auto sub = make_leaf(action, std::string("ai ") + action); + sub->handler([name = std::string(action)](const ParsedArgs& args, Output& out) { + return cmd_ai(name, args, out); + }); + ai->child(sub); + } + + auto help = make_leaf("help", "show help"); + help->usage("[]").long_help("Prints the full help page for a command."); + help->example("help serve"); + // Handler installed in run_args (needs tree access); placeholder here. + help->handler([](const ParsedArgs&, Output&) { return ExitCode::Ok; }); + + // Supply-chain leaves (PATCHES 15, section 3). + auto verify = make_leaf("verify", "verify release artifacts offline"); + verify->usage("molo verify [dir]") + .long_help( + "Recompute sha256 digests, check the signature bundle chain, and match the " + "wolfssl pin. Fully offline.") + .example("molo verify ./dist") + .flag(Flag("manifest", "", "subjects file", true, "subjects.txt")) + .flag(Flag("bundle", "", "signature bundle", true, "bundle.json")) + .flag(Flag("pin", "", "allow-list with wolfssl pin", true, "")) + .handler(&molo::supply::cli::SupplyCli::run_verify); + + auto supply_check = make_leaf("supply-check", "pin plus advisory gate"); + supply_check->usage("molo supply-check [dir]") + .long_help("Check the wolfssl pin and allow-list, then evaluate advisories. Blocked " + "exits 1.") + .example("molo supply-check ./dist") + .flag(Flag("allow", "", "allow-list file", true, "supply.allow.json")) + .flag(Flag("advisories", "", "advisories file", true, "advisories.json")) + .flag(Flag("exceptions", "", "exceptions file", true, "")) + .handler(&molo::supply::cli::SupplyCli::run_check); + + auto release = make_leaf("release", "ordered release gate"); + release->usage("molo release --commit --suite-stamp ") + .long_help("Run suite, sbom, scan, provenance, sign, tag in order. Any failed step " + "exits 1.") + .example("molo release --commit abc123 --suite-stamp build/suite.stamp") + .flag(Flag("commit", "", "release commit", true, "")) + .flag(Flag("suite-stamp", "", "file containing PASS", true, "")) + .flag(Flag("dir", "", "release directory", true, ".")) + .handler(&molo::supply::cli::SupplyCli::run_release); + + tree_.add(version); + tree_.add(health); + tree_.add(molo_new); + tree_.add(serve); + tree_.add(build); + tree_.add(embed); + tree_.add(migrate); + tree_.add(supervisor); + tree_.add(task); + tree_.add(ai); + tree_.add(help); + tree_.add(verify); + tree_.add(supply_check); + tree_.add(release); + + // Operability commands (PATCHES 13, section 10). + auto rollout = make_leaf("rollout", "run a deployment rollout"); + rollout->usage("start|abort").long_help("Drives Rolling/BlueGreen step loops in-process."); + auto rollout_start = make_leaf("start", "start a rollout"); + rollout_start + ->usage("[--strategy rolling|blue-green] [--batch N] [--max-unavailable M] " + "[--instances K]") + .long_help("Runs the rollout loop to completion with local readiness probes.") + .flag(Flag("strategy", "", "rolling or blue-green", true, "rolling")) + .flag(Flag("batch", "", "rolling batch size", true, "1")) + .flag(Flag("max-unavailable", "", "max unavailable instances", true, "0")) + .flag(Flag("instances", "", "instance count", true, "3")) + .handler(cmd_rollout_start); + auto rollout_abort = make_leaf("abort", "abort the running rollout"); + rollout_abort->handler(cmd_rollout_abort); + rollout->child(rollout_start); + rollout->child(rollout_abort); + + auto rollback = make_leaf("rollback", "roll back to blue"); + rollback->usage("").long_help("Rolls back a blue-green deployment to blue."); + rollback->handler(cmd_rollback); + + auto experiment = make_leaf("experiment", "run a chaos experiment"); + experiment->usage("start|abort").long_help("Drives Experiment plus Injector hooks."); + auto exp_start = make_leaf("start", "start an experiment"); + exp_start + ->usage("[--blast N] [--duration MS] [--max-burn X]") + .long_help("Runs the experiment for the duration, checking the burn ceiling.") + .flag(Flag("blast", "", "blast radius percent", true, "1")) + .flag(Flag("duration", "", "duration in ms", true, "60000")) + .flag(Flag("max-burn", "", "burn ceiling", true, "0.05")) + .handler(cmd_experiment_start); + auto exp_abort = make_leaf("abort", "abort the running experiment"); + exp_abort->handler(cmd_experiment_abort); + experiment->child(exp_start); + experiment->child(exp_abort); + + auto debug = make_leaf("debug", "debug helpers"); + debug->usage("snapshot").long_help("Local debug helpers."); + auto dbg_snapshot = make_leaf("snapshot", "print a debug snapshot"); + dbg_snapshot->handler(cmd_debug_snapshot); + debug->child(dbg_snapshot); + + tree_.add(rollout); + tree_.add(rollback); + tree_.add(experiment); + tree_.add(debug); +} + +ExitCode CliApp::run(int argc, const char* const* argv) { + std::vector args; + for (int i = 0; i < argc; ++i) args.emplace_back(argv[i]); + return run_args(args); +} + +ExitCode CliApp::run_args(const std::vector& argv) { + ArgParser parser; + ParsedArgs parsed = parser.parse(argv); + std::string prog = argv.empty() ? "molo" : argv[0]; + auto slash = prog.find_last_of('/'); + if (slash != std::string::npos) prog = prog.substr(slash + 1); + Output out(parsed.global.format == "json", parsed.global.verbose, parsed.global.quiet, + Output::stdout_is_tty()); + if (!parsed.ok()) { + out.error(parsed.error + "\n"); + return ExitCode::Usage; + } + if (parsed.global.show_version) return cmd_version(parsed, out); + HelpRenderer help_renderer; + if (parsed.command_path.empty()) { + if (parsed.global.show_help) { + out.print(help_renderer.render_root(tree_, prog)); + return ExitCode::Ok; + } + out.error("no command given\n"); + out.print(help_renderer.render_root(tree_, prog)); + return ExitCode::Usage; + } + // `help ` prints the full page. + if (parsed.command_path[0] == "help") { + if (parsed.command_path.size() < 2 && parsed.positionals.empty()) { + out.print(help_renderer.render_root(tree_, prog)); + return ExitCode::Ok; + } + std::vector target(parsed.command_path.begin() + 1, parsed.command_path.end()); + for (const auto& positional : parsed.positionals) target.push_back(positional); + ResolveResult found = tree_.resolve(target); + if (found.command == nullptr) { + out.error(found.error + "\n"); + return ExitCode::Usage; + } + std::string path; + for (std::size_t i = 0; i < found.consumed; ++i) { + if (i > 0) path += " "; + path += target[i]; + } + out.print(help_renderer.render_full(*found.command, prog, path)); + return ExitCode::Ok; + } + ResolveResult found = tree_.resolve(parsed.command_path); + if (found.command == nullptr) { + out.error(found.error + "\n"); + return ExitCode::Usage; + } + if (parsed.global.show_help) { + std::string path; + for (std::size_t i = 0; i < found.consumed; ++i) { + if (i > 0) path += " "; + path += parsed.command_path[i]; + } + out.print(help_renderer.render_full(*found.command, prog, path)); + return ExitCode::Ok; + } + // Unconsumed path tail becomes leading positionals (e.g. `new `). + ParsedArgs adjusted = parsed; + std::vector tail(parsed.command_path.begin() + found.consumed, + parsed.command_path.end()); + tail.insert(tail.end(), adjusted.positionals.begin(), adjusted.positionals.end()); + adjusted.positionals = std::move(tail); + return found.command->invoke(adjusted, out); +} + +} // namespace molo::cli diff --git a/src/cli/cli/cli_app.hpp b/src/cli/cli/cli_app.hpp new file mode 100644 index 0000000..876b337 --- /dev/null +++ b/src/cli/cli/cli_app.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "cli/arg_parser.hpp" +#include "cli/command_tree.hpp" +#include "cli/exit_code.hpp" +#include "cli/migrate_api.hpp" +#include "cli/supervisor_api.hpp" + +namespace molo::cli { + +// The `molo` application: declares the full command tree (data first), +// then dispatches argv to handlers. +class CliApp { + public: + CliApp(); + CliApp(ISupervisorControl& supervisor, ITaskControl& tasks, IMigrateControl& migrate); + + ExitCode run(int argc, const char* const* argv); + ExitCode run_args(const std::vector& argv); + const CommandTree& tree() const { return tree_; } + + private: + void build_tree(); + + CommandTree tree_; + ISupervisorControl* supervisor_; + ITaskControl* tasks_; + IMigrateControl* migrate_; + NullSupervisorControl null_supervisor_; + NullTaskControl null_tasks_; + NullMigrateControl null_migrate_; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/command.cpp b/src/cli/cli/command.cpp new file mode 100644 index 0000000..c0770c7 --- /dev/null +++ b/src/cli/cli/command.cpp @@ -0,0 +1,57 @@ +#include "cli/command.hpp" + +#include + +#include "cli/output.hpp" + +namespace molo::cli { + +Command::Command(std::string name, std::string brief) + : name_(std::move(name)), brief_(std::move(brief)) {} + +Command& Command::usage(std::string text) { + usage_ = std::move(text); + return *this; +} + +Command& Command::long_help(std::string text) { + long_help_ = std::move(text); + return *this; +} + +Command& Command::example(std::string text) { + examples_.push_back(std::move(text)); + return *this; +} + +Command& Command::flag(Flag f) { + flags_.push_back(std::move(f)); + return *this; +} + +Command& Command::child(std::shared_ptr sub) { + children_.push_back(std::move(sub)); + return *this; +} + +Command& Command::handler(Handler fn) { + handler_ = std::move(fn); + return *this; +} + +const Command* Command::find_child(const std::string& name) const { + for (const auto& kid : children_) { + if (kid->name() == name) return kid.get(); + } + return nullptr; +} + +ExitCode Command::invoke(const ParsedArgs& args, Output& out) const { + if (!handler_) { + out.error("command '" + name_ + "' needs a subcommand"); + return ExitCode::Usage; + } + return handler_(args, out); +} + +} // namespace molo::cli diff --git a/src/cli/cli/command.hpp b/src/cli/cli/command.hpp new file mode 100644 index 0000000..8f317bd --- /dev/null +++ b/src/cli/cli/command.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include + +#include "cli/arg_parser.hpp" +#include "cli/exit_code.hpp" +#include "cli/flag.hpp" + +namespace molo::cli { + +class Output; + +using Handler = std::function; + +// One node of the command tree. Data first: help generates from these. +class Command { + public: + explicit Command(std::string name, std::string brief = ""); + + const std::string& name() const { return name_; } + const std::string& brief() const { return brief_; } + const std::string& usage() const { return usage_; } + const std::string& long_help() const { return long_help_; } + const std::vector& examples() const { return examples_; } + const std::vector& flags() const { return flags_; } + const std::vector>& children() const { return children_; } + + Command& usage(std::string text); + Command& long_help(std::string text); + Command& example(std::string text); + Command& flag(Flag flag); + Command& child(std::shared_ptr sub); + Command& handler(Handler fn); + + const Command* find_child(const std::string& name) const; + bool has_handler() const { return static_cast(handler_); } + ExitCode invoke(const ParsedArgs& args, Output& out) const; + + private: + std::string name_; + std::string brief_; + std::string usage_; + std::string long_help_; + std::vector examples_; + std::vector flags_; + std::vector> children_; + Handler handler_; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/command_tree.cpp b/src/cli/cli/command_tree.cpp new file mode 100644 index 0000000..b94f4c3 --- /dev/null +++ b/src/cli/cli/command_tree.cpp @@ -0,0 +1,45 @@ +#include "cli/command_tree.hpp" + +namespace molo::cli { + +void CommandTree::add(std::shared_ptr root) { roots_.push_back(std::move(root)); } + +ResolveResult CommandTree::resolve(const std::vector& path) const { + ResolveResult out; + if (path.empty()) { + out.error = "no command given"; + return out; + } + const Command* current = nullptr; + for (const auto& root : roots_) { + if (root->name() == path[0]) { + current = root.get(); + break; + } + } + if (current == nullptr) { + out.error = "unknown command '" + path[0] + "'"; + return out; + } + out.command = current; + out.consumed = 1; + for (std::size_t i = 1; i < path.size(); ++i) { + const Command* kid = current->find_child(path[i]); + if (kid == nullptr) break; + current = kid; + out.command = current; + out.consumed = i + 1; + } + return out; +} + +std::vector CommandTree::all_paths() const { + std::vector paths; + for (const auto& root : roots_) { + paths.push_back(root->name()); + for (const auto& kid : root->children()) paths.push_back(root->name() + " " + kid->name()); + } + return paths; +} + +} // namespace molo::cli diff --git a/src/cli/cli/command_tree.hpp b/src/cli/cli/command_tree.hpp new file mode 100644 index 0000000..f56a681 --- /dev/null +++ b/src/cli/cli/command_tree.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +#include "cli/command.hpp" + +namespace molo::cli { + +struct ResolveResult { + const Command* command = nullptr; + std::size_t consumed = 0; + std::string error; +}; + +// Owns the root commands and resolves argv command paths. +class CommandTree { + public: + void add(std::shared_ptr root); + ResolveResult resolve(const std::vector& path) const; + const std::vector>& roots() const { return roots_; } + std::vector all_paths() const; + + private: + std::vector> roots_; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/exit_code.hpp b/src/cli/cli/exit_code.hpp new file mode 100644 index 0000000..291cca7 --- /dev/null +++ b/src/cli/cli/exit_code.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace molo::cli { + +// Exit codes per 09-cli-ops.md rule 2. Scripts rely on this mapping. +enum class ExitCode : int { + Ok = 0, + Runtime = 1, + Usage = 2, + Unavailable = 69, +}; + +inline int to_int(ExitCode code) { return static_cast(code); } + +inline std::string exit_code_name(ExitCode code) { + switch (code) { + case ExitCode::Ok: + return "ok"; + case ExitCode::Runtime: + return "runtime"; + case ExitCode::Usage: + return "usage"; + case ExitCode::Unavailable: + return "unavailable"; + } + return "unknown"; +} + +} // namespace molo::cli diff --git a/src/cli/cli/flag.cpp b/src/cli/cli/flag.cpp new file mode 100644 index 0000000..5b61470 --- /dev/null +++ b/src/cli/cli/flag.cpp @@ -0,0 +1,15 @@ +#include "cli/flag.hpp" + +#include + +namespace molo::cli { + +Flag::Flag(std::string name, std::string short_name, std::string help, bool takes_value, + std::string default_value) + : name_(std::move(name)), + short_name_(std::move(short_name)), + help_(std::move(help)), + takes_value_(takes_value), + default_value_(std::move(default_value)) {} + +} // namespace molo::cli diff --git a/src/cli/cli/flag.hpp b/src/cli/cli/flag.hpp new file mode 100644 index 0000000..dcb0ba5 --- /dev/null +++ b/src/cli/cli/flag.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace molo::cli { + +// One declared command-line flag. +class Flag { + public: + Flag() = default; + Flag(std::string name, std::string short_name, std::string help, bool takes_value, + std::string default_value); + + const std::string& name() const { return name_; } + const std::string& short_name() const { return short_name_; } + const std::string& help() const { return help_; } + bool takes_value() const { return takes_value_; } + const std::string& default_value() const { return default_value_; } + + private: + std::string name_; + std::string short_name_; + std::string help_; + bool takes_value_ = false; + std::string default_value_; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/flag_set.cpp b/src/cli/cli/flag_set.cpp new file mode 100644 index 0000000..fdc5c67 --- /dev/null +++ b/src/cli/cli/flag_set.cpp @@ -0,0 +1,44 @@ +#include "cli/flag_set.hpp" + +namespace molo::cli { + +void FlagSet::add(Flag flag) { decls_.push_back(std::move(flag)); } + +bool FlagSet::declares(const std::string& name) const { return find_decl(name) != nullptr; } + +const Flag* FlagSet::find_decl(const std::string& name) const { + for (const auto& decl : decls_) { + if (decl.name() == name || (!decl.short_name().empty() && decl.short_name() == name)) { + return &decl; + } + } + return nullptr; +} + +void FlagSet::set(const std::string& name, const std::string& value) { + for (auto& pair : values_) { + if (pair.first == name) { + pair.second = value; + return; + } + } + values_.emplace_back(name, value); +} + +bool FlagSet::has(const std::string& name) const { + for (const auto& pair : values_) { + if (pair.first == name) return true; + } + return false; +} + +std::string FlagSet::get(const std::string& name, const std::string& fallback) const { + for (const auto& pair : values_) { + if (pair.first == name) return pair.second; + } + const Flag* decl = find_decl(name); + if (decl != nullptr && !decl->default_value().empty()) return decl->default_value(); + return fallback; +} + +} // namespace molo::cli diff --git a/src/cli/cli/flag_set.hpp b/src/cli/cli/flag_set.hpp new file mode 100644 index 0000000..f354c62 --- /dev/null +++ b/src/cli/cli/flag_set.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include "cli/flag.hpp" + +namespace molo::cli { + +// Declared flags plus parsed values for one command invocation. +class FlagSet { + public: + void add(Flag flag); + bool declares(const std::string& name) const; + const Flag* find_decl(const std::string& name) const; + + void set(const std::string& name, const std::string& value); + bool has(const std::string& name) const; + std::string get(const std::string& name, const std::string& fallback = "") const; + + const std::vector& decls() const { return decls_; } + + private: + std::vector decls_; + std::vector> values_; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/help_renderer.cpp b/src/cli/cli/help_renderer.cpp new file mode 100644 index 0000000..9caa291 --- /dev/null +++ b/src/cli/cli/help_renderer.cpp @@ -0,0 +1,59 @@ +#include "cli/help_renderer.hpp" + +#include + +namespace molo::cli { + +std::string HelpRenderer::render_concise(const Command& cmd, const std::string& prog) const { + std::ostringstream out; + out << "usage: " << prog << " " << cmd.name(); + if (!cmd.usage().empty()) out << " " << cmd.usage(); + out << "\n " << cmd.brief() << "\n"; + return out.str(); +} + +std::string HelpRenderer::render_full(const Command& cmd, const std::string& prog, + const std::string& path) const { + std::ostringstream out; + out << cmd.name() << " - " << cmd.brief() << "\n\n"; + out << "usage:\n " << prog << " " << path; + if (!cmd.usage().empty()) out << " " << cmd.usage(); + out << "\n"; + if (!cmd.long_help().empty()) out << "\n" << cmd.long_help() << "\n"; + if (!cmd.flags().empty()) { + out << "\nflags:\n"; + for (const auto& flag : cmd.flags()) { + out << " --" << flag.name(); + if (!flag.short_name().empty()) out << ", " << flag.short_name(); + if (flag.takes_value()) out << " "; + out << "\n " << flag.help() << "\n"; + } + } + if (!cmd.children().empty()) { + out << "\nsubcommands:\n"; + for (const auto& kid : cmd.children()) out << " " << kid->name() << " " << kid->brief() << "\n"; + } + if (!cmd.examples().empty()) { + out << "\nexamples:\n"; + for (const auto& ex : cmd.examples()) out << " " << prog << " " << ex << "\n"; + } + return out.str(); +} + +std::string HelpRenderer::render_root(const CommandTree& tree, const std::string& prog) const { + std::ostringstream out; + out << "usage: " << prog << " [flags]\n\ncommands:\n"; + for (const auto& root : tree.roots()) { + out << " " << root->name() << " " << root->brief() << "\n"; + } + out << "\nglobal flags:\n"; + out << " -h, --help show help\n"; + out << " --version show version\n"; + out << " --format json or text (default text)\n"; + out << " -v, --verbose repeat up to 3 times for more logs\n"; + out << " -q, --quiet only errors and primary output\n"; + out << "\nrun '" << prog << " help ' for full help.\n"; + return out.str(); +} + +} // namespace molo::cli diff --git a/src/cli/cli/help_renderer.hpp b/src/cli/cli/help_renderer.hpp new file mode 100644 index 0000000..41b0d2d --- /dev/null +++ b/src/cli/cli/help_renderer.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include + +#include "cli/command.hpp" +#include "cli/command_tree.hpp" + +namespace molo::cli { + +// Renders concise (-h) and full (help ) pages from the tree. +class HelpRenderer { + public: + std::string render_concise(const Command& cmd, const std::string& prog) const; + std::string render_full(const Command& cmd, const std::string& prog, + const std::string& path) const; + std::string render_root(const CommandTree& tree, const std::string& prog) const; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/json_writer.cpp b/src/cli/cli/json_writer.cpp new file mode 100644 index 0000000..3729663 --- /dev/null +++ b/src/cli/cli/json_writer.cpp @@ -0,0 +1,69 @@ +#include "cli/json_writer.hpp" + +#include + +namespace molo::cli { + +std::string JsonWriter::escape(std::string_view text) { + std::ostringstream out; + for (char c : text) { + switch (c) { + case '"': + out << "\\\""; + break; + case '\\': + out << "\\\\"; + break; + case '\n': + out << "\\n"; + break; + case '\r': + out << "\\r"; + break; + case '\t': + out << "\\t"; + break; + default: + if (static_cast(c) < 0x20) { + char buf[8]; + snprintf(buf, sizeof(buf), "\\u%04x", c); + out << buf; + } else { + out << c; + } + } + } + return out.str(); +} + +std::string JsonWriter::quote(std::string_view text) { + return "\"" + escape(text) + "\""; +} + +std::string JsonWriter::object(const std::map& raw_fields) { + std::string out = "{"; + bool first = true; + for (const auto& [key, raw] : raw_fields) { + if (!first) out += ","; + first = false; + out += quote(key) + ":" + raw; + } + return out + "}"; +} + +std::string JsonWriter::array(const std::vector& raw_items) { + std::string out = "["; + bool first = true; + for (const auto& item : raw_items) { + if (!first) out += ","; + first = false; + out += item; + } + return out + "]"; +} + +std::string JsonWriter::field(std::string_view key, const std::string& raw_value) { + return quote(key) + ":" + raw_value; +} + +} // namespace molo::cli diff --git a/src/cli/cli/json_writer.hpp b/src/cli/cli/json_writer.hpp new file mode 100644 index 0000000..6b27c79 --- /dev/null +++ b/src/cli/cli/json_writer.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include +#include + +namespace molo::cli { + +// Minimal JSON builder for --format json output. Escapes per RFC 8259. +class JsonWriter { + public: + static std::string escape(std::string_view text); + static std::string quote(std::string_view text); + static std::string str(std::string_view text) { return quote(text); } + static std::string boolean(bool value) { return value ? "true" : "false"; } + static std::string number(long long value) { return std::to_string(value); } + static std::string object(const std::map& raw_fields); + static std::string array(const std::vector& raw_items); + static std::string field(std::string_view key, const std::string& raw_value); +}; + +} // namespace molo::cli diff --git a/src/cli/cli/migrate_api.cpp b/src/cli/cli/migrate_api.cpp new file mode 100644 index 0000000..eeade9c --- /dev/null +++ b/src/cli/cli/migrate_api.cpp @@ -0,0 +1,17 @@ +#include "cli/migrate_api.hpp" + +namespace molo::cli { + +bool NullMigrateControl::check(MigratePlan& plan, std::string& error) { + (void)plan; + error = "migration backend unavailable (04-data.md)"; + return false; +} + +bool NullMigrateControl::apply(const MigratePlan& plan, std::string& error) { + (void)plan; + error = "migration backend unavailable (04-data.md)"; + return false; +} + +} // namespace molo::cli diff --git a/src/cli/cli/migrate_api.hpp b/src/cli/cli/migrate_api.hpp new file mode 100644 index 0000000..b4bbb6b --- /dev/null +++ b/src/cli/cli/migrate_api.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +namespace molo::cli { + +struct MigratePlan { + std::vector steps; +}; + +// Migration plans are owned by 04-data.md; the CLI executes them. +class IMigrateControl { + public: + virtual ~IMigrateControl() = default; + virtual bool check(MigratePlan& plan, std::string& error) = 0; + virtual bool apply(const MigratePlan& plan, std::string& error) = 0; +}; + +class NullMigrateControl : public IMigrateControl { + public: + bool check(MigratePlan& plan, std::string& error) override; + bool apply(const MigratePlan& plan, std::string& error) override; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/output.cpp b/src/cli/cli/output.cpp new file mode 100644 index 0000000..285c5f1 --- /dev/null +++ b/src/cli/cli/output.cpp @@ -0,0 +1,40 @@ +#include "cli/output.hpp" + +#if defined(_WIN32) +#include +#else +#include +#endif + +namespace molo::cli { + +Output::Output(bool json_mode, int verbose, bool quiet, bool tty) + : json_mode_(json_mode), verbose_(verbose), quiet_(quiet), tty_(tty) {} + +bool Output::stdout_is_tty() { +#if defined(_WIN32) + return _isatty(_fileno(stdout)) != 0; +#else + return isatty(STDOUT_FILENO) == 1; +#endif +} + +void Output::print(const std::string& text) { std::cout << text; } + +void Output::info(const std::string& text) { + if (!quiet_) std::cerr << text; +} + +void Output::progress(const std::string& text) { + if (!quiet_) std::cerr << text; +} + +void Output::warn(const std::string& text) { std::cerr << "warning: " << text; } + +void Output::error(const std::string& text) { std::cerr << "error: " << text; } + +void Output::debug(int level, const std::string& text) { + if (!quiet_ && verbose_ >= level) std::cerr << "debug: " << text; +} + +} // namespace molo::cli diff --git a/src/cli/cli/output.hpp b/src/cli/cli/output.hpp new file mode 100644 index 0000000..b09248c --- /dev/null +++ b/src/cli/cli/output.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +namespace molo::cli { + +// Output splits primary results (stdout) from diagnostics (stderr) per +// 09-cli-ops.md rule 3. Color only when stdout is a TTY. +class Output { + public: + Output(bool json_mode, int verbose, bool quiet, bool tty); + + static bool stdout_is_tty(); + + bool json_mode() const { return json_mode_; } + int verbose() const { return verbose_; } + bool quiet() const { return quiet_; } + bool tty() const { return tty_; } + + void print(const std::string& text); // primary output to stdout + void info(const std::string& text); // stderr, suppressed when quiet + void progress(const std::string& text); + void warn(const std::string& text); + void error(const std::string& text); + void debug(int level, const std::string& text); + + private: + bool json_mode_ = false; + int verbose_ = 0; + bool quiet_ = false; + bool tty_ = false; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/scaffolder.cpp b/src/cli/cli/scaffolder.cpp new file mode 100644 index 0000000..7528251 --- /dev/null +++ b/src/cli/cli/scaffolder.cpp @@ -0,0 +1,103 @@ +#include "cli/scaffolder.hpp" + +#include +#include +#include + +#include "caps.hpp" + +namespace molo::cli { +namespace fs = std::filesystem; + +namespace { +bool write_file(const std::string& path, const std::string& content) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out.good()) return false; + out << content; + return true; +} +} // namespace + +ScaffoldResult Scaffolder::scaffold(const std::string& dir, const ScaffoldOptions& opts) const { + ScaffoldResult out; + if (opts.name.empty() || opts.name.size() > caps::kMaxProjectName) { + out.error = "project name must be 1-64 chars"; + return out; + } + if (opts.backend != "sqlite" && opts.backend != "postgres") { + out.error = "backend must be sqlite or postgres"; + return out; + } + std::error_code ec; + bool exists = fs::exists(dir, ec); + if (exists && !opts.force) { + bool empty = fs::is_empty(dir, ec); + if (!empty) { + out.error = "directory is not empty, use --force to overwrite"; + return out; + } + } + fs::create_directories(dir, ec); + if (ec) { + out.error = "cannot create directory " + dir; + return out; + } + auto put = [&](const std::string& rel, const std::string& content) { + std::string full = dir + "/" + rel; + fs::create_directories(fs::path(full).parent_path(), ec); + if (!write_file(full, content)) { + out.error = "cannot write " + full; + return false; + } + out.created.push_back(rel); + return true; + }; + if (!put("Makefile", + "CXX=g++\nCXXFLAGS=-std=c++23 -Wall -Wextra\n\nall:\n\t" + "$(CXX) $(CXXFLAGS) -o build/app src/main.cpp\n\ntest:\n\t" + "@echo no tests yet\n\nclean:\n\trm -rf build\n")) { + return out; + } + if (!put("AGENTS.md", + "# " + opts.name + + "\n\nAutonomous workflow: build, test, docs update, version bump, " + "commit, tag, deploy, verify.\nRules: exact minimal diffs, one class per " + "file, static caps on every knob.\n")) { + return out; + } + if (!put(".env.json", + "{\n \"MOLO_ENV\": \"dev\",\n \"PORT\": 8000,\n \"DATABASE_URL\": \"" + + opts.backend + ":///db/app.db\"\n}\n")) { + return out; + } + if (!put(".gitignore", "build/\n*.o\n.env.json.local\n")) return out; + if (!put("src/main.cpp", + "#include \n\nint main() {\n std::cout << \"hello from " + opts.name + + "\\n\";\n return 0;\n}\n")) { + return out; + } + if (!put("templates/base.html", + "\n\n\n\n" + "\n" + "{% block title %}" + + opts.name + + "{% endblock %}\n\n\n{% block body %}{% endblock %}\n" + "\n\n")) { + return out; + } + if (!put("assets/app.css", "body{margin:0;font-family:system-ui}\n")) return out; + if (!put("locales/en/LC_MESSAGES/app.po", + "msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n")) { + return out; + } + std::string cmd = "git -C \"" + dir + "\" init -q 2>/dev/null"; + if (system(cmd.c_str()) != 0) { + out.error = "git init failed in " + dir; + return out; + } + out.created.push_back(".git/"); + out.ok = true; + return out; +} + +} // namespace molo::cli diff --git a/src/cli/cli/scaffolder.hpp b/src/cli/cli/scaffolder.hpp new file mode 100644 index 0000000..ebe16fd --- /dev/null +++ b/src/cli/cli/scaffolder.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +namespace molo::cli { + +struct ScaffoldOptions { + std::string name; + std::string backend = "sqlite"; // postgres|sqlite + bool force = false; +}; + +struct ScaffoldResult { + bool ok = false; + std::string error; + std::vector created; +}; + +// Implements `molo new`: Makefile, AGENTS.md stub, layout, .env.json, +// .gitignore, git init, plug-and-play examples. Refuses non-empty dirs +// without --force and never touches git history. +class Scaffolder { + public: + ScaffoldResult scaffold(const std::string& dir, const ScaffoldOptions& opts) const; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/supervisor_api.cpp b/src/cli/cli/supervisor_api.cpp new file mode 100644 index 0000000..5260de5 --- /dev/null +++ b/src/cli/cli/supervisor_api.cpp @@ -0,0 +1,57 @@ +#include "cli/supervisor_api.hpp" + +namespace molo::cli { + +namespace { +const char* kUnavailable = "supervisor backend unavailable (07-supervisor-replication.md)"; +} + +bool NullSupervisorControl::start(std::string& error) { + error = kUnavailable; + return false; +} +bool NullSupervisorControl::stop(std::string& error) { + error = kUnavailable; + return false; +} +bool NullSupervisorControl::restart(std::string& error) { + error = kUnavailable; + return false; +} +bool NullSupervisorControl::status(SupervisorStatus& out, std::string& error) { + (void)out; + error = kUnavailable; + return false; +} +bool NullSupervisorControl::logs(int lines, std::vector& out, std::string& error) { + (void)lines; + (void)out; + error = kUnavailable; + return false; +} + +bool NullTaskControl::enqueue(const std::string& name, TaskInfo& out, std::string& error) { + (void)name; + (void)out; + error = kUnavailable; + return false; +} +bool NullTaskControl::retry(const std::string& id, TaskInfo& out, std::string& error) { + (void)id; + (void)out; + error = kUnavailable; + return false; +} +bool NullTaskControl::cancel(const std::string& id, std::string& error) { + (void)id; + error = kUnavailable; + return false; +} +bool NullTaskControl::inspect(const std::string& id, TaskInfo& out, std::string& error) { + (void)id; + (void)out; + error = kUnavailable; + return false; +} + +} // namespace molo::cli diff --git a/src/cli/cli/supervisor_api.hpp b/src/cli/cli/supervisor_api.hpp new file mode 100644 index 0000000..279fbf0 --- /dev/null +++ b/src/cli/cli/supervisor_api.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include +#include + +namespace molo::cli { + +struct SupervisorStatus { + bool running = false; + int workers = 0; + std::string state; +}; + +struct TaskInfo { + std::string id; + std::string name; + std::string state; + int retries = 0; +}; + +// Control API owned by 07-supervisor-replication.md. The CLI drives it; +// null implementations report unavailable until wired. +class ISupervisorControl { + public: + virtual ~ISupervisorControl() = default; + virtual bool start(std::string& error) = 0; + virtual bool stop(std::string& error) = 0; + virtual bool restart(std::string& error) = 0; + virtual bool status(SupervisorStatus& out, std::string& error) = 0; + virtual bool logs(int lines, std::vector& out, std::string& error) = 0; +}; + +class ITaskControl { + public: + virtual ~ITaskControl() = default; + virtual bool enqueue(const std::string& name, TaskInfo& out, std::string& error) = 0; + virtual bool retry(const std::string& id, TaskInfo& out, std::string& error) = 0; + virtual bool cancel(const std::string& id, std::string& error) = 0; + virtual bool inspect(const std::string& id, TaskInfo& out, std::string& error) = 0; +}; + +class NullSupervisorControl : public ISupervisorControl { + public: + bool start(std::string& error) override; + bool stop(std::string& error) override; + bool restart(std::string& error) override; + bool status(SupervisorStatus& out, std::string& error) override; + bool logs(int lines, std::vector& out, std::string& error) override; +}; + +class NullTaskControl : public ITaskControl { + public: + bool enqueue(const std::string& name, TaskInfo& out, std::string& error) override; + bool retry(const std::string& id, TaskInfo& out, std::string& error) override; + bool cancel(const std::string& id, std::string& error) override; + bool inspect(const std::string& id, TaskInfo& out, std::string& error) override; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/supervisor_cli.cpp b/src/cli/cli/supervisor_cli.cpp new file mode 100644 index 0000000..cb96d48 --- /dev/null +++ b/src/cli/cli/supervisor_cli.cpp @@ -0,0 +1,180 @@ +#include "cli/supervisor_cli.hpp" + +#include + +#include "cli/json_writer.hpp" +#include "cli/output.hpp" +#include "cli/table_writer.hpp" + +namespace molo::cli { + +SupervisorCli::SupervisorCli(ISupervisorControl& supervisor, ITaskControl& tasks) + : supervisor_(supervisor), tasks_(tasks) {} + +ExitCode SupervisorCli::run_supervisor(const std::string& action, const ParsedArgs& args, + Output& out) { + bool json = args.global.format == "json"; + std::string error; + if (action == "start" || action == "stop" || action == "restart") { + bool ok = action == "start" ? supervisor_.start(error) + : (action == "stop" ? supervisor_.stop(error) + : supervisor_.restart(error)); + if (!ok) { + if (json) { + out.print(JsonWriter::object({{"ok", "false"}, {"error", JsonWriter::quote(error)}}) + + "\n"); + } else { + out.error(error + "\n"); + } + return ExitCode::Unavailable; + } + if (json) { + out.print(JsonWriter::object({{"ok", "true"}, {"action", JsonWriter::quote(action)}}) + "\n"); + } else { + out.print("supervisor " + action + ": ok\n"); + } + return ExitCode::Ok; + } + if (action == "status") { + SupervisorStatus status; + if (!supervisor_.status(status, error)) { + if (json) { + out.print(JsonWriter::object({{"ok", "false"}, {"error", JsonWriter::quote(error)}}) + + "\n"); + } else { + out.error(error + "\n"); + } + return ExitCode::Unavailable; + } + if (json) { + std::map fields; + fields["ok"] = "true"; + fields["running"] = status.running ? "true" : "false"; + fields["workers"] = std::to_string(status.workers); + fields["state"] = JsonWriter::quote(status.state); + out.print(JsonWriter::object(fields) + "\n"); + } else { + TableWriter table; + table.header({"running", "workers", "state"}); + table.row({status.running ? "yes" : "no", std::to_string(status.workers), status.state}); + out.print(table.render()); + } + return ExitCode::Ok; + } + if (action == "logs") { + std::vector lines; + if (!supervisor_.logs(100, lines, error)) { + if (json) { + out.print(JsonWriter::object({{"ok", "false"}, {"error", JsonWriter::quote(error)}}) + + "\n"); + } else { + out.error(error + "\n"); + } + return ExitCode::Unavailable; + } + if (json) { + std::vector items; + for (const auto& line : lines) items.push_back(JsonWriter::quote(line)); + out.print(JsonWriter::object({{"ok", "true"}, {"lines", JsonWriter::array(items)}}) + "\n"); + } else { + for (const auto& line : lines) out.print(line + "\n"); + } + return ExitCode::Ok; + } + out.error("unknown supervisor action '" + action + "'\n"); + return ExitCode::Usage; +} + +ExitCode SupervisorCli::run_task(const std::string& action, const ParsedArgs& args, Output& out) { + bool json = args.global.format == "json"; + auto flag = [&](const std::string& key) { + auto it = args.flags.find(key); + return it == args.flags.end() ? "" : it->second; + }; + std::string id = flag("id"); + if (id.empty() && !args.positionals.empty()) id = args.positionals[0]; + std::string error; + if (action == "enqueue") { + std::string name = flag("name"); + if (name.empty() && !args.positionals.empty()) name = args.positionals[0]; + if (name.empty()) { + out.error("task enqueue needs --name \n"); + return ExitCode::Usage; + } + TaskInfo info; + if (!tasks_.enqueue(name, info, error)) { + if (json) { + out.print(JsonWriter::object({{"ok", "false"}, {"error", JsonWriter::quote(error)}}) + + "\n"); + } else { + out.error(error + "\n"); + } + return ExitCode::Unavailable; + } + if (json) { + out.print(JsonWriter::object({{"ok", "true"}, + {"id", JsonWriter::quote(info.id)}, + {"state", JsonWriter::quote(info.state)}}) + + "\n"); + } else { + out.print("enqueued " + info.id + " (" + info.state + ")\n"); + } + return ExitCode::Ok; + } + if (action == "retry" || action == "inspect") { + if (id.empty()) { + out.error("task " + action + " needs --id \n"); + return ExitCode::Usage; + } + TaskInfo info; + bool ok = action == "retry" ? tasks_.retry(id, info, error) : tasks_.inspect(id, info, error); + if (!ok) { + if (json) { + out.print(JsonWriter::object({{"ok", "false"}, {"error", JsonWriter::quote(error)}}) + + "\n"); + } else { + out.error(error + "\n"); + } + return ExitCode::Unavailable; + } + if (json) { + out.print(JsonWriter::object({{"ok", "true"}, + {"id", JsonWriter::quote(info.id)}, + {"name", JsonWriter::quote(info.name)}, + {"state", JsonWriter::quote(info.state)}, + {"retries", std::to_string(info.retries)}}) + + "\n"); + } else { + TableWriter table; + table.header({"id", "name", "state", "retries"}); + table.row({info.id, info.name, info.state, std::to_string(info.retries)}); + out.print(table.render()); + } + return ExitCode::Ok; + } + if (action == "cancel") { + if (id.empty()) { + out.error("task cancel needs --id \n"); + return ExitCode::Usage; + } + if (!tasks_.cancel(id, error)) { + if (json) { + out.print(JsonWriter::object({{"ok", "false"}, {"error", JsonWriter::quote(error)}}) + + "\n"); + } else { + out.error(error + "\n"); + } + return ExitCode::Unavailable; + } + if (json) { + out.print(JsonWriter::object({{"ok", "true"}, {"id", JsonWriter::quote(id)}}) + "\n"); + } else { + out.print("cancelled " + id + "\n"); + } + return ExitCode::Ok; + } + out.error("unknown task action '" + action + "'\n"); + return ExitCode::Usage; +} + +} // namespace molo::cli diff --git a/src/cli/cli/supervisor_cli.hpp b/src/cli/cli/supervisor_cli.hpp new file mode 100644 index 0000000..e195ee4 --- /dev/null +++ b/src/cli/cli/supervisor_cli.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include "cli/arg_parser.hpp" +#include "cli/exit_code.hpp" +#include "cli/supervisor_api.hpp" + +namespace molo::cli { + +class Output; + +// CLI surface for `molo supervisor ...` and `molo task ...` with text and +// structured JSON output for scripts. +class SupervisorCli { + public: + SupervisorCli(ISupervisorControl& supervisor, ITaskControl& tasks); + + ExitCode run_supervisor(const std::string& action, const ParsedArgs& args, Output& out); + ExitCode run_task(const std::string& action, const ParsedArgs& args, Output& out); + + private: + ISupervisorControl& supervisor_; + ITaskControl& tasks_; +}; + +} // namespace molo::cli diff --git a/src/cli/cli/table_writer.cpp b/src/cli/cli/table_writer.cpp new file mode 100644 index 0000000..66d6564 --- /dev/null +++ b/src/cli/cli/table_writer.cpp @@ -0,0 +1,34 @@ +#include "cli/table_writer.hpp" + +#include + +namespace molo::cli { + +void TableWriter::header(std::vector cols) { header_ = std::move(cols); } + +void TableWriter::row(std::vector cols) { rows_.push_back(std::move(cols)); } + +std::string TableWriter::render() const { + std::size_t cols = header_.size(); + for (const auto& row : rows_) cols = std::max(cols, row.size()); + std::vector width(cols, 0); + auto widen = [&](const std::vector& r) { + for (std::size_t i = 0; i < r.size(); ++i) width[i] = std::max(width[i], r[i].size()); + }; + widen(header_); + for (const auto& row : rows_) widen(row); + std::ostringstream out; + auto emit = [&](const std::vector& r) { + for (std::size_t i = 0; i < cols; ++i) { + std::string cell = i < r.size() ? r[i] : ""; + out << cell; + if (i + 1 < cols) out << std::string(width[i] - cell.size() + 2, ' '); + } + out << "\n"; + }; + if (!header_.empty()) emit(header_); + for (const auto& row : rows_) emit(row); + return out.str(); +} + +} // namespace molo::cli diff --git a/src/cli/cli/table_writer.hpp b/src/cli/cli/table_writer.hpp new file mode 100644 index 0000000..12239e9 --- /dev/null +++ b/src/cli/cli/table_writer.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace molo::cli { + +// Plain ASCII tables for human-readable command output. No color codes. +class TableWriter { + public: + void header(std::vector cols); + void row(std::vector cols); + std::string render() const; + + private: + std::vector header_; + std::vector> rows_; +}; + +} // namespace molo::cli diff --git a/src/cli/csp/csp_builder.cpp b/src/cli/csp/csp_builder.cpp new file mode 100644 index 0000000..e058e1a --- /dev/null +++ b/src/cli/csp/csp_builder.cpp @@ -0,0 +1,61 @@ +#include "csp/csp_builder.hpp" + +namespace molo::csp { + +CspBuilder::CspBuilder() { + directives_["default-src"] = "'none'"; + directives_["script-src"] = "'self'"; + directives_["style-src"] = "'self'"; + directives_["img-src"] = "'self' data:"; + directives_["connect-src"] = "'self'"; + directives_["frame-ancestors"] = "'none'"; +} + +CspBuilder& CspBuilder::default_src(const std::string& value) { + directives_["default-src"] = value; + return *this; +} + +CspBuilder& CspBuilder::script_src(const std::string& value) { + directives_["script-src"] = value; + return *this; +} + +CspBuilder& CspBuilder::style_src(const std::string& value) { + directives_["style-src"] = value; + return *this; +} + +CspBuilder& CspBuilder::img_src(const std::string& value) { + directives_["img-src"] = value; + return *this; +} + +CspBuilder& CspBuilder::connect_src(const std::string& value) { + directives_["connect-src"] = value; + return *this; +} + +CspBuilder& CspBuilder::frame_ancestors(const std::string& value) { + directives_["frame-ancestors"] = value; + return *this; +} + +CspBuilder& CspBuilder::report_to(const std::string& value) { + directives_["report-uri"] = value; + return *this; +} + +CspBuilder& CspBuilder::script_nonce(const std::string& nonce_base64) { + directives_["script-src"] += " 'nonce-" + nonce_base64 + "'"; + return *this; +} + +CspBuilder& CspBuilder::enforce() { + report_only_ = false; + return *this; +} + +CspPolicy CspBuilder::build() const { return CspPolicy(directives_, report_only_); } + +} // namespace molo::csp diff --git a/src/cli/csp/csp_builder.hpp b/src/cli/csp/csp_builder.hpp new file mode 100644 index 0000000..97115dc --- /dev/null +++ b/src/cli/csp/csp_builder.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include "csp/csp_policy.hpp" + +namespace molo::csp { + +// Fluent builder. Default is deny-first per 09-cli-ops.md rule 10. +class CspBuilder { + public: + CspBuilder(); + + CspBuilder& default_src(const std::string& value); + CspBuilder& script_src(const std::string& value); + CspBuilder& style_src(const std::string& value); + CspBuilder& img_src(const std::string& value); + CspBuilder& connect_src(const std::string& value); + CspBuilder& frame_ancestors(const std::string& value); + CspBuilder& report_to(const std::string& value); + CspBuilder& script_nonce(const std::string& nonce_base64); + CspBuilder& enforce(); + + CspPolicy build() const; + + private: + std::map directives_; + bool report_only_ = true; +}; + +} // namespace molo::csp diff --git a/src/cli/csp/csp_policy.cpp b/src/cli/csp/csp_policy.cpp new file mode 100644 index 0000000..ea2e535 --- /dev/null +++ b/src/cli/csp/csp_policy.cpp @@ -0,0 +1,24 @@ +#include "csp/csp_policy.hpp" + +namespace molo::csp { + +CspPolicy::CspPolicy(std::map directives, bool report_only) + : directives_(std::move(directives)), report_only_(report_only) {} + +std::string CspPolicy::header_name() const { + return report_only_ ? "Content-Security-Policy-Report-Only" : "Content-Security-Policy"; +} + +std::string CspPolicy::header_value() const { + std::string out; + bool first = true; + for (const auto& [key, value] : directives_) { + if (!first) out += "; "; + first = false; + out += key; + if (!value.empty()) out += " " + value; + } + return out; +} + +} // namespace molo::csp diff --git a/src/cli/csp/csp_policy.hpp b/src/cli/csp/csp_policy.hpp new file mode 100644 index 0000000..dcc5605 --- /dev/null +++ b/src/cli/csp/csp_policy.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +namespace molo::csp { + +// Immutable policy value. Report-only until the app opts into enforcement. +class CspPolicy { + public: + CspPolicy(std::map directives, bool report_only); + + const std::map& directives() const { return directives_; } + bool report_only() const { return report_only_; } + std::string header_name() const; + std::string header_value() const; + + private: + std::map directives_; + bool report_only_ = true; +}; + +} // namespace molo::csp diff --git a/src/cli/csp/csp_report_handler.cpp b/src/cli/csp/csp_report_handler.cpp new file mode 100644 index 0000000..6bdfd67 --- /dev/null +++ b/src/cli/csp/csp_report_handler.cpp @@ -0,0 +1,30 @@ +#include "csp/csp_report_handler.hpp" + +#include "caps.hpp" + +namespace molo::csp { + +CspReportResult CspReportHandler::handle(const std::string& client_ip, const std::string& body, + std::int64_t now_sec) { + CspReportResult out; + if (body.size() > caps::kCspReportMaxBytes) { + out.status = 413; + out.body = "{\"status\":\"report too large\"}"; + return out; + } + auto& window = hits_[client_ip]; + while (!window.empty() && now_sec - window.front() >= 60) window.pop_front(); + if (window.size() >= caps::kCspReportsPerMinPerIp) { + out.status = 429; + out.body = "{\"status\":\"rate limited\"}"; + return out; + } + window.push_back(now_sec); + if (store_.size() >= caps::kCspReportStoreMax) store_.erase(store_.begin()); + store_.push_back(body); + out.status = 202; + out.body = "{\"status\":\"accepted\"}"; + return out; +} + +} // namespace molo::csp diff --git a/src/cli/csp/csp_report_handler.hpp b/src/cli/csp/csp_report_handler.hpp new file mode 100644 index 0000000..319b786 --- /dev/null +++ b/src/cli/csp/csp_report_handler.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace molo::csp { + +struct CspReportResult { + int status = 202; + std::string body; +}; + +// POST /csp-report handler. Caps: 64 KB per report, 100 reports/min/IP, +// bounded in-memory store (1000 reports ring). +class CspReportHandler { + public: + CspReportResult handle(const std::string& client_ip, const std::string& body, + std::int64_t now_sec); + std::size_t stored() const { return store_.size(); } + + private: + std::map> hits_; + std::vector store_; +}; + +} // namespace molo::csp diff --git a/src/cli/csp/nonce.cpp b/src/cli/csp/nonce.cpp new file mode 100644 index 0000000..b1c5773 --- /dev/null +++ b/src/cli/csp/nonce.cpp @@ -0,0 +1,41 @@ +#include "csp/nonce.hpp" + +#include +#include + +#if defined(__linux__) +#include +#endif + +#include "caps.hpp" + +namespace molo::csp { + +std::string Nonce::base64_of(const unsigned char* data, std::size_t len) { + static const char* alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + for (std::size_t i = 0; i < len; i += 3) { + unsigned int n = static_cast(data[i]) << 16; + if (i + 1 < len) n |= static_cast(data[i + 1]) << 8; + if (i + 2 < len) n |= data[i + 2]; + out += alphabet[(n >> 18) & 63]; + out += alphabet[(n >> 12) & 63]; + out += (i + 1 < len) ? alphabet[(n >> 6) & 63] : '='; + out += (i + 2 < len) ? alphabet[n & 63] : '='; + } + return out; +} + +std::string Nonce::generate() { + std::array buf{}; +#if defined(__linux__) + ssize_t got = getrandom(buf.data(), buf.size(), 0); + if (got == static_cast(buf.size())) return base64_of(buf.data(), buf.size()); +#endif + std::random_device rd; + for (auto& byte : buf) byte = static_cast(rd() & 0xFF); + return base64_of(buf.data(), buf.size()); +} + +} // namespace molo::csp diff --git a/src/cli/csp/nonce.hpp b/src/cli/csp/nonce.hpp new file mode 100644 index 0000000..cc2a2b9 --- /dev/null +++ b/src/cli/csp/nonce.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace molo::csp { + +// Per-request nonce: 16 random bytes, base64. RNG comes from 03-security.md +// in the full tree; here OS entropy (getrandom / random_device fallback). +class Nonce { + public: + static std::string generate(); + static std::string base64_of(const unsigned char* data, std::size_t len); +}; + +} // namespace molo::csp diff --git a/src/cli/dev/dev_config.cpp b/src/cli/dev/dev_config.cpp new file mode 100644 index 0000000..d1a1f94 --- /dev/null +++ b/src/cli/dev/dev_config.cpp @@ -0,0 +1,85 @@ +#include "dev/dev_config.hpp" + +#include +#include +#include + +namespace molo::dev { + +std::string ConfigLoader::read_file(const std::string& path) { + std::ifstream in(path); + if (!in.good()) return ""; + std::ostringstream buf; + buf << in.rdbuf(); + return buf.str(); +} + +std::string ConfigLoader::json_string(const std::string& json, const std::string& key, + const std::string& fallback) { + std::string needle = "\"" + key + "\""; + auto pos = json.find(needle); + if (pos == std::string::npos) return fallback; + pos = json.find(':', pos + needle.size()); + if (pos == std::string::npos) return fallback; + pos = json.find('"', pos + 1); + if (pos == std::string::npos) return fallback; + auto end = json.find('"', pos + 1); + if (end == std::string::npos) return fallback; + return json.substr(pos + 1, end - pos - 1); +} + +int ConfigLoader::json_int(const std::string& json, const std::string& key, int fallback) { + std::string needle = "\"" + key + "\""; + auto pos = json.find(needle); + if (pos == std::string::npos) return fallback; + pos = json.find(':', pos + needle.size()); + if (pos == std::string::npos) return fallback; + auto start = json.find_first_of("-0123456789", pos + 1); + if (start == std::string::npos) return fallback; + auto end = json.find_first_not_of("0123456789", start + 1); + try { + return std::stoi(json.substr(start, end - start)); + } catch (...) { + return fallback; + } +} + +LoadedConfig ConfigLoader::load(const std::string& config_path, bool reload_flag, + bool allow_prod) const { + LoadedConfig out; + out.allow_prod = allow_prod; + const char* env_raw = std::getenv("MOLO_ENV"); + std::string env_text = env_raw != nullptr ? env_raw : ""; + std::string json = read_file(config_path); + if (env_text.empty()) env_text = json_string(json, "MOLO_ENV", "dev"); + out.env = (env_text == "prod" || env_text == "production") ? Env::Prod : Env::Dev; + + int port = json_int(json, "PORT", 8000); + if (const char* port_raw = std::getenv("PORT"); port_raw != nullptr && *port_raw != '\0') { + try { + port = std::stoi(port_raw); + } catch (...) { + } + } + std::string db_url = json_string(json, "DATABASE_URL", ""); + if (const char* db_raw = std::getenv("DATABASE_URL"); + db_raw != nullptr && *db_raw != '\0') { + db_url = db_raw; + } + out.dev.port = port; + out.dev.reload = reload_flag; + out.dev.config_path = config_path; + out.dev.db_url = db_url; + out.prod.port = port; + out.prod.config_path = config_path; + out.prod.db_url = db_url; + out.prod.tls_cert = json_string(json, "TLS_CERT", ""); + out.prod.tls_key = json_string(json, "TLS_KEY", ""); + + if (out.is_prod() && reload_flag && !allow_prod) { + out.error = "refusing --reload with production config without --allow-prod"; + } + return out; +} + +} // namespace molo::dev diff --git a/src/cli/dev/dev_config.hpp b/src/cli/dev/dev_config.hpp new file mode 100644 index 0000000..45f584f --- /dev/null +++ b/src/cli/dev/dev_config.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include + +namespace molo::dev { + +enum class Env { Dev, Prod }; + +inline std::string env_name(Env env) { return env == Env::Prod ? "prod" : "dev"; } + +struct DevConfig { + int port = 8000; + bool reload = false; + std::string config_path = ".env.json"; + std::string db_url; +}; + +struct ProdConfig { + int port = 8000; + std::string config_path = ".env.json"; + std::string db_url; + std::string tls_cert; + std::string tls_key; +}; + +struct LoadedConfig { + Env env = Env::Dev; + DevConfig dev; + ProdConfig prod; + bool allow_prod = false; + std::string error; + bool ok() const { return error.empty(); } + bool is_prod() const { return env == Env::Prod; } +}; + +// Loads .env.json plus MOLO_ENV overrides. Refuses --reload against prod +// unless --allow-prod is passed (09-cli-ops.md rule 5). +class ConfigLoader { + public: + LoadedConfig load(const std::string& config_path, bool reload_flag, bool allow_prod) const; + + private: + static std::string read_file(const std::string& path); + static std::string json_string(const std::string& json, const std::string& key, + const std::string& fallback); + static int json_int(const std::string& json, const std::string& key, int fallback); +}; + +} // namespace molo::dev diff --git a/src/cli/dev/dev_server.cpp b/src/cli/dev/dev_server.cpp new file mode 100644 index 0000000..dbc0e09 --- /dev/null +++ b/src/cli/dev/dev_server.cpp @@ -0,0 +1,131 @@ +#include "dev/dev_server.hpp" + +#include +#include +#include +#include + +#include "caps.hpp" +#include "cli/output.hpp" +#include "dev/dev_config.hpp" +#include "dev/file_watcher.hpp" +#include "dev/reloader.hpp" + +namespace molo::dev { + +namespace { +std::atomic g_stop{false}; +void on_signal(int) { g_stop.store(true); } +} // namespace + +bool DevServer::should_refuse(bool is_prod, bool reload, bool allow_prod) { + return is_prod && reload && !allow_prod; +} + +molo::cli::ExitCode DevServer::run(const ServeOptions& opts, molo::cli::Output& out) { + ConfigLoader loader; + LoadedConfig cfg = loader.load(opts.config_path, opts.reload, opts.allow_prod); + if (!cfg.ok()) { + out.error(cfg.error + "\n"); + return molo::cli::ExitCode::Usage; + } + if (should_refuse(cfg.is_prod(), opts.reload, opts.allow_prod)) { + out.error("refusing --reload with production config without --allow-prod\n"); + return molo::cli::ExitCode::Usage; + } + if (cfg.is_prod() && !opts.allow_prod && opts.reload) { + out.error("refusing --reload with production config without --allow-prod\n"); + return molo::cli::ExitCode::Usage; + } + if (!opts.reload) return run_once(opts, out); + return run_reload(opts, out, cfg); +} + +molo::cli::ExitCode DevServer::run_once(const ServeOptions& opts, molo::cli::Output& out) { + std::signal(SIGINT, on_signal); + std::signal(SIGTERM, on_signal); + g_stop.store(false); + out.info("molo serve: listening on port " + std::to_string(opts.port) + " (single-shot)\n"); + while (!g_stop.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + out.info("molo serve: shutdown clean\n"); + return molo::cli::ExitCode::Ok; +} + +molo::cli::ExitCode DevServer::run_reload(const ServeOptions& opts, molo::cli::Output& out, + const LoadedConfig& cfg) { + if (cfg.is_prod() && !cfg.dev.db_url.empty()) { + // Rule 5: never run the reloader against a production database URL. + // A prod URL contains "prod"; refuse loudly rather than guessing. + if (cfg.dev.db_url.find("prod") != std::string::npos && !opts.allow_prod) { + out.error("refusing reloader against production database without --allow-prod\n"); + return molo::cli::ExitCode::Usage; + } + } + std::signal(SIGINT, on_signal); + std::signal(SIGTERM, on_signal); + g_stop.store(false); + + auto watcher = create_default_watcher(); + std::string werr; + if (!watcher->start({"src", "templates", "assets"}, werr)) { + // Fall back to polling when the default backend cannot start. + watcher.reset(); + out.warn("watch backend unavailable (" + werr + "), continuing without reload\n"); + return run_once(opts, out); + } + out.info("molo serve: watching src/ templates/ assets/ (" + watcher->backend_name() + ")\n"); + + Reloader reloader; + std::vector child = {opts.child_argv0.empty() ? "./build/molo-app" : opts.child_argv0}; + for (const auto& a : opts.child_args) child.push_back(a); + child.push_back("--port"); + child.push_back(std::to_string(opts.port)); + + std::string merr; + if (!reloader.run_make("", caps::kMakeTimeoutSec, merr)) { + out.error(merr + "\n"); + return molo::cli::ExitCode::Runtime; + } + std::string cerr; + if (!reloader.start_child(child, cerr)) { + out.warn("child start deferred (" + cerr + ")\n"); + } + bool hold_announced = false; + while (!g_stop.load()) { + auto events = watcher->poll(500); + if (!events.empty()) { + out.progress("change detected (" + std::to_string(events.size()) + " events), rebuilding\n"); + if (!reloader.run_make("", caps::kMakeTimeoutSec, merr)) { + out.error(merr + "\n"); + reloader.note_exit(false); + } else if (!reloader.start_child(child, cerr)) { + out.error(cerr + "\n"); + reloader.note_exit(false); + } else { + reloader.note_exit(true); + hold_announced = false; + } + } + if (!reloader.child_running() && !child.empty()) { + reloader.note_exit(false); + int delay = Reloader::backoff_for(reloader.consecutive_failures()); + if (delay >= caps::kBackoffMaxSec && !hold_announced) { + out.error("crash loop: holding at 30s backoff\n"); + hold_announced = true; + } + std::this_thread::sleep_for(std::chrono::seconds(delay)); + if (g_stop.load()) break; + if (!reloader.start_child(child, cerr)) { + out.error(cerr + "\n"); + } + } + } + reloader.stop_child(caps::kChildKillGraceSec); + watcher->stop(); + out.info("molo serve: shutdown clean\n"); + return molo::cli::ExitCode::Ok; +} + +} // namespace molo::dev diff --git a/src/cli/dev/dev_server.hpp b/src/cli/dev/dev_server.hpp new file mode 100644 index 0000000..a5c61f6 --- /dev/null +++ b/src/cli/dev/dev_server.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +#include "cli/exit_code.hpp" +#include "dev/dev_config.hpp" + +namespace molo::cli { +class Output; +} + +namespace molo::dev { + +struct ServeOptions { + int port = 8000; + bool reload = false; + std::string config_path = ".env.json"; + bool allow_prod = false; + std::string child_argv0; // binary to supervise; empty means self-serve stub + std::vector child_args; +}; + +// Dev-only server supervisor: watch, rebuild via make, restart child. +class DevServer { + public: + // Pure refusal check for tests: prod + reload without --allow-prod. + static bool should_refuse(bool is_prod, bool reload, bool allow_prod); + + molo::cli::ExitCode run(const ServeOptions& opts, molo::cli::Output& out); + + private: + molo::cli::ExitCode run_once(const ServeOptions& opts, molo::cli::Output& out); + molo::cli::ExitCode run_reload(const ServeOptions& opts, molo::cli::Output& out, + const LoadedConfig& cfg); +}; + +} // namespace molo::dev diff --git a/src/cli/dev/file_watcher.cpp b/src/cli/dev/file_watcher.cpp new file mode 100644 index 0000000..d4cd656 --- /dev/null +++ b/src/cli/dev/file_watcher.cpp @@ -0,0 +1,16 @@ +#include "dev/file_watcher.hpp" + +#include "dev/inotify_watcher.hpp" +#include "dev/polling_watcher.hpp" + +namespace molo::dev { + +std::unique_ptr create_default_watcher() { +#if defined(__linux__) + return std::make_unique(); +#else + return std::make_unique(); +#endif +} + +} // namespace molo::dev diff --git a/src/cli/dev/file_watcher.hpp b/src/cli/dev/file_watcher.hpp new file mode 100644 index 0000000..17b8d2c --- /dev/null +++ b/src/cli/dev/file_watcher.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + +namespace molo::dev { + +enum class EventKind { Created, Modified, Deleted }; + +struct FileEvent { + std::string path; + EventKind kind = EventKind::Modified; +}; + +// One interface for inotify/kqueue/polling backends. +class FileWatcher { + public: + virtual ~FileWatcher() = default; + virtual bool start(const std::vector& roots, std::string& error) = 0; + virtual std::vector poll(int timeout_ms) = 0; + virtual void stop() = 0; + virtual std::string backend_name() const = 0; +}; + +std::unique_ptr create_default_watcher(); + +} // namespace molo::dev diff --git a/src/cli/dev/inotify_watcher.cpp b/src/cli/dev/inotify_watcher.cpp new file mode 100644 index 0000000..2b4cca9 --- /dev/null +++ b/src/cli/dev/inotify_watcher.cpp @@ -0,0 +1,107 @@ +#include "dev/inotify_watcher.hpp" + +#if defined(__linux__) +#include +#include +#include +#endif +#include + +#include "caps.hpp" + +namespace molo::dev { +namespace fs = std::filesystem; + +InotifyFileWatcher::InotifyFileWatcher() = default; +InotifyFileWatcher::~InotifyFileWatcher() { stop(); } + +bool InotifyFileWatcher::start(const std::vector& roots, std::string& error) { +#if !defined(__linux__) + (void)roots; + error = "inotify backend only on Linux"; + return false; +#else + stop(); + fd_ = inotify_init1(IN_NONBLOCK | IN_CLOEXEC); + if (fd_ < 0) { + error = "inotify_init1 failed"; + return false; + } + std::size_t watched = 0; + for (const auto& root : roots) { + std::error_code ec; + if (!fs::exists(root, ec)) continue; + if (fs::is_regular_file(root, ec)) continue; + fs::recursive_directory_iterator it(root, ec), end; + auto add_watch = [&](const std::string& dir) { + if (watched >= caps::kMaxWatchFiles) return; + int wd = inotify_add_watch(fd_, dir.c_str(), IN_CREATE | IN_MODIFY | IN_DELETE | + IN_MOVED_FROM | IN_MOVED_TO); + if (wd >= 0) { + watch_dirs_[wd] = dir; + ++watched; + } + }; + add_watch(root); + while (it != end && !ec && watched < caps::kMaxWatchFiles) { + std::error_code e2; + if (it->is_directory(e2)) add_watch(it->path().string()); + it.increment(e2); + } + } + if (watch_dirs_.empty()) { + error = "no watch roots exist"; + stop(); + return false; + } + started_ = true; + return true; +#endif +} + +std::vector InotifyFileWatcher::poll(int timeout_ms) { + std::vector out; +#if defined(__linux__) + if (!started_ || fd_ < 0) return out; + struct pollfd pfd; + pfd.fd = fd_; + pfd.events = POLLIN; + int ready = ::poll(&pfd, 1, timeout_ms); + if (ready <= 0) return out; + char buf[8192]; + ssize_t len = read(fd_, buf, sizeof(buf)); + if (len <= 0) return out; + for (char* p = buf; p < buf + len;) { + auto* ev = reinterpret_cast(p); + std::string dir = watch_dirs_.count(ev->wd) ? watch_dirs_[ev->wd] : ""; + std::string name = ev->len > 0 ? std::string(ev->name) : ""; + FileEvent fe; + fe.path = dir.empty() ? name : dir + "/" + name; + if ((ev->mask & IN_DELETE) || (ev->mask & IN_MOVED_FROM)) { + fe.kind = EventKind::Deleted; + } else if (ev->mask & IN_CREATE) { + fe.kind = EventKind::Created; + } else { + fe.kind = EventKind::Modified; + } + if (!fe.path.empty() && !name.empty()) out.push_back(fe); + p += sizeof(struct inotify_event) + ev->len; + } +#else + (void)timeout_ms; +#endif + return out; +} + +void InotifyFileWatcher::stop() { +#if defined(__linux__) + if (fd_ >= 0) { + close(fd_); + fd_ = -1; + } +#endif + watch_dirs_.clear(); + started_ = false; +} + +} // namespace molo::dev diff --git a/src/cli/dev/inotify_watcher.hpp b/src/cli/dev/inotify_watcher.hpp new file mode 100644 index 0000000..85dec28 --- /dev/null +++ b/src/cli/dev/inotify_watcher.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + +#include "dev/file_watcher.hpp" + +namespace molo::dev { + +// Linux inotify backend. On non-Linux builds start() reports unavailable +// and the dev server falls back to the polling backend. +class InotifyFileWatcher : public FileWatcher { + public: + InotifyFileWatcher(); + ~InotifyFileWatcher() override; + bool start(const std::vector& roots, std::string& error) override; + std::vector poll(int timeout_ms) override; + void stop() override; + std::string backend_name() const override { return "inotify"; } + + private: + int fd_ = -1; + std::map watch_dirs_; + bool started_ = false; +}; + +} // namespace molo::dev diff --git a/src/cli/dev/polling_watcher.cpp b/src/cli/dev/polling_watcher.cpp new file mode 100644 index 0000000..c6bf88a --- /dev/null +++ b/src/cli/dev/polling_watcher.cpp @@ -0,0 +1,107 @@ +#include "dev/polling_watcher.hpp" + +#include +#include + +#include "caps.hpp" + +namespace molo::dev { +namespace fs = std::filesystem; + +bool PollingFileWatcher::start(const std::vector& roots, std::string& error) { + roots_.clear(); + mtimes_.clear(); + pending_.clear(); + for (const auto& root : roots) { + std::error_code ec; + if (!fs::exists(root, ec)) continue; + roots_.push_back(root); + } + if (roots_.empty()) { + error = "no watch roots exist"; + return false; + } + std::vector ignored; + scan(ignored); + // Baseline only: drop initial creates. + pending_.clear(); + started_ = true; + return true; +} + +long long PollingFileWatcher::mtime_of(const std::string& path) { + std::error_code ec; + auto t = fs::last_write_time(path, ec); + if (ec) return -1; + return static_cast(t.time_since_epoch().count()); +} + +void PollingFileWatcher::scan(std::vector& out) { + std::map now; + std::size_t count = 0; + for (const auto& root : roots_) { + std::error_code ec; + if (fs::is_regular_file(root, ec)) { + now[root] = mtime_of(root); + continue; + } + fs::recursive_directory_iterator it(root, ec), end; + while (it != end && !ec && count < caps::kMaxWatchFiles) { + const auto& entry = *it; + std::error_code e2; + if (entry.is_regular_file(e2)) { + std::string p = entry.path().string(); + now[p] = mtime_of(p); + ++count; + } + it.increment(e2); + } + } + auto now_tp = std::chrono::steady_clock::now(); + for (const auto& [path, mtime] : now) { + auto prev = mtimes_.find(path); + if (prev == mtimes_.end()) { + pending_.emplace(path, now_tp); + } else if (prev->second != mtime) { + pending_[path] = now_tp; + } + } + for (const auto& [path, _] : mtimes_) { + if (now.find(path) == now.end()) { + FileEvent ev; + ev.path = path; + ev.kind = EventKind::Deleted; + out.push_back(ev); + } + } + auto debounce = std::chrono::milliseconds(caps::kWatchDebounceMs); + for (auto it = pending_.begin(); it != pending_.end();) { + if (now_tp - it->second >= debounce) { + FileEvent ev; + ev.path = it->first; + ev.kind = mtimes_.count(it->first) ? EventKind::Modified : EventKind::Created; + out.push_back(ev); + it = pending_.erase(it); + } else { + ++it; + } + } + mtimes_.swap(now); +} + +std::vector PollingFileWatcher::poll(int timeout_ms) { + std::vector out; + if (!started_) return out; + if (timeout_ms > 0) std::this_thread::sleep_for(std::chrono::milliseconds(timeout_ms)); + scan(out); + return out; +} + +void PollingFileWatcher::stop() { + started_ = false; + roots_.clear(); + mtimes_.clear(); + pending_.clear(); +} + +} // namespace molo::dev diff --git a/src/cli/dev/polling_watcher.hpp b/src/cli/dev/polling_watcher.hpp new file mode 100644 index 0000000..c185fd0 --- /dev/null +++ b/src/cli/dev/polling_watcher.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include "dev/file_watcher.hpp" + +namespace molo::dev { + +// Portable mtime-sweep backend. Caps files at kMaxWatchFiles. +class PollingFileWatcher : public FileWatcher { + public: + bool start(const std::vector& roots, std::string& error) override; + std::vector poll(int timeout_ms) override; + void stop() override; + std::string backend_name() const override { return "poll"; } + + private: + std::vector roots_; + std::map mtimes_; + std::map pending_; + bool started_ = false; + + void scan(std::vector& out); + static long long mtime_of(const std::string& path); +}; + +} // namespace molo::dev diff --git a/src/cli/dev/reloader.cpp b/src/cli/dev/reloader.cpp new file mode 100644 index 0000000..dee4d87 --- /dev/null +++ b/src/cli/dev/reloader.cpp @@ -0,0 +1,136 @@ +#include "dev/reloader.hpp" + +#include +#include + +#if !defined(_WIN32) +#include +#include +#include +#include +#include +#endif + +#include "caps.hpp" + +#if !defined(_WIN32) +extern char** environ; +#endif + +namespace molo::dev { + +Reloader::Reloader() = default; +Reloader::~Reloader() { stop_child(0); } + +int Reloader::backoff_for(int consecutive_failures) { + if (consecutive_failures <= 0) return 0; + int delay = caps::kBackoffBaseSec; + for (int i = 1; i < consecutive_failures && delay < caps::kBackoffMaxSec; ++i) { + delay *= 2; + if (delay > caps::kBackoffMaxSec) { + delay = caps::kBackoffMaxSec; + break; + } + } + return delay; +} + +void Reloader::note_exit(bool clean) { + if (clean) { + failures_ = 0; + } else { + ++failures_; + } +} + +bool Reloader::run_make(const std::string& make_args, int timeout_sec, std::string& error) { +#if defined(_WIN32) + (void)make_args; + (void)timeout_sec; + error = "dev server rebuild only on POSIX"; + return false; +#else + std::string jobs = "1"; + long nproc = sysconf(_SC_NPROCESSORS_ONLN); + if (nproc > 1) jobs = std::to_string(nproc); + std::string cmd = "make -j" + jobs; + if (!make_args.empty()) cmd += " " + make_args; + cmd += " > /tmp/molo-make.log 2>&1"; + auto start = std::chrono::steady_clock::now(); + int rc = system(cmd.c_str()); + auto elapsed = + std::chrono::duration_cast(std::chrono::steady_clock::now() - start) + .count(); + if (elapsed > timeout_sec) { + error = "make exceeded timeout"; + return false; + } + if (rc != 0) { + error = "make failed, see /tmp/molo-make.log"; + return false; + } + return true; +#endif +} + +bool Reloader::start_child(const std::vector& argv, std::string& error) { +#if defined(_WIN32) + (void)argv; + error = "dev server child only on POSIX"; + return false; +#else + if (argv.empty()) { + error = "empty child argv"; + return false; + } + stop_child(caps::kChildKillGraceSec); + std::vector args; + for (const auto& a : argv) args.push_back(const_cast(a.c_str())); + args.push_back(nullptr); + pid_t pid = -1; + int rc = posix_spawn(&pid, args[0], nullptr, nullptr, args.data(), ::environ); + if (rc != 0 || pid < 0) { + error = "posix_spawn failed"; + return false; + } + child_pid_ = pid; + return true; +#endif +} + +void Reloader::stop_child(int grace_sec) { +#if !defined(_WIN32) + if (child_pid_ < 0) return; + pid_t pid = static_cast(child_pid_); + kill(pid, SIGTERM); + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(grace_sec); + while (std::chrono::steady_clock::now() < deadline) { + int status = 0; + pid_t r = waitpid(pid, &status, WNOHANG); + if (r == pid || r < 0) { + child_pid_ = -1; + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + kill(pid, SIGKILL); + int status = 0; + waitpid(pid, &status, 0); + child_pid_ = -1; +#else + (void)grace_sec; +#endif +} + +bool Reloader::child_running() const { +#if defined(_WIN32) + return false; +#else + if (child_pid_ < 0) return false; + int status = 0; + pid_t r = waitpid(static_cast(child_pid_), &status, WNOHANG); + return r == 0; +#endif +} + +} // namespace molo::dev diff --git a/src/cli/dev/reloader.hpp b/src/cli/dev/reloader.hpp new file mode 100644 index 0000000..a1f4e87 --- /dev/null +++ b/src/cli/dev/reloader.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +namespace molo::dev { + +// Runs `make`, supervises the child process, and computes crash backoff. +// Dumb by design: no state is preserved across restarts. +class Reloader { + public: + Reloader(); + ~Reloader(); + + bool run_make(const std::string& make_args, int timeout_sec, std::string& error); + bool start_child(const std::vector& argv, std::string& error); + void stop_child(int grace_sec); + bool child_running() const; + + // Backoff: 1s, 2s, 4s ... capped at kBackoffMaxSec. + static int backoff_for(int consecutive_failures); + void note_exit(bool clean); + int consecutive_failures() const { return failures_; } + + private: + long child_pid_ = -1; + int failures_ = 0; +}; + +} // namespace molo::dev diff --git a/src/cli/embed/asset_scanner.cpp b/src/cli/embed/asset_scanner.cpp new file mode 100644 index 0000000..feca40d --- /dev/null +++ b/src/cli/embed/asset_scanner.cpp @@ -0,0 +1,73 @@ +#include "embed/asset_scanner.hpp" + +#include +#include + +#include "caps.hpp" + +namespace molo::embed { +namespace fs = std::filesystem; + +ScanResult AssetScanner::scan(const std::string& root) const { + ScanResult out; + std::error_code ec; + fs::path root_path(root); + if (!fs::exists(root_path, ec) || !fs::is_directory(root_path, ec)) { + out.error = "embed root is not a directory: " + root; + return out; + } + fs::path canon_root = fs::weakly_canonical(root_path, ec); + if (ec) { + out.error = "cannot resolve embed root: " + root; + return out; + } + std::string root_str = canon_root.string(); + fs::recursive_directory_iterator it(root_path, ec), end; + while (it != end && !ec) { + const auto& entry = *it; + std::error_code e2; + std::string name = entry.path().filename().string(); + if (!name.empty() && name[0] == '.') { + if (entry.is_directory(e2)) it.disable_recursion_pending(); + it.increment(e2); + continue; + } + if (entry.is_symlink(e2)) { + fs::path target = fs::weakly_canonical(entry.path(), e2); + if (e2 || target.string().rfind(root_str, 0) != 0) { + it.increment(e2); + continue; // escaping symlink: skip + } + } + if (entry.is_regular_file(e2)) { + std::uint64_t size = entry.file_size(e2); + if (!e2) { + if (size > caps::kEmbedMaxFileBytes) { + out.error = "file exceeds 64 MB cap: " + entry.path().string(); + return out; + } + if (out.total_bytes + size > caps::kEmbedTotalBytes) { + out.error = "input exceeds 512 MB total cap"; + return out; + } + if (out.files.size() >= caps::kEmbedMaxFiles) { + out.error = "input exceeds 50000 file cap"; + return out; + } + AssetFile file; + file.abs_path = entry.path().string(); + file.rel_path = fs::relative(entry.path(), root_path, e2).generic_string(); + if (e2) file.rel_path = name; + file.size = size; + out.files.push_back(std::move(file)); + out.total_bytes += size; + } + } + it.increment(e2); + } + std::sort(out.files.begin(), out.files.end(), + [](const AssetFile& a, const AssetFile& b) { return a.rel_path < b.rel_path; }); + return out; +} + +} // namespace molo::embed diff --git a/src/cli/embed/asset_scanner.hpp b/src/cli/embed/asset_scanner.hpp new file mode 100644 index 0000000..9cc044c --- /dev/null +++ b/src/cli/embed/asset_scanner.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +namespace molo::embed { + +struct AssetFile { + std::string rel_path; // forward slashes, sorted order + std::string abs_path; + std::uint64_t size = 0; +}; + +struct ScanResult { + std::vector files; + std::uint64_t total_bytes = 0; + std::string error; + bool ok() const { return error.empty(); } +}; + +// Recursive scanner with static caps. Skips dotfiles and symlinks that +// escape the scanned root. +class AssetScanner { + public: + ScanResult scan(const std::string& root) const; +}; + +} // namespace molo::embed diff --git a/src/cli/embed/embed_emitter.cpp b/src/cli/embed/embed_emitter.cpp new file mode 100644 index 0000000..245f787 --- /dev/null +++ b/src/cli/embed/embed_emitter.cpp @@ -0,0 +1,179 @@ +#include "embed/embed_emitter.hpp" + +#include +#include +#include + +#include "caps.hpp" +#include "embed/gzip_stored.hpp" +#include "embed/mime.hpp" +#include "embed/sha256.hpp" + +namespace molo::embed { + +std::string EmbedEmitter::ident_for(std::size_t index) { + return "molo_asset_" + std::to_string(index); +} + +std::string EmbedEmitter::escape_cpp_string(const std::string& text) { + std::string out; + for (char c : text) { + if (c == '"' || c == '\\') { + out += '\\'; + out += c; + } else if (c == '\n') { + out += "\\n"; + } else { + out += c; + } + } + return out; +} + +bool EmbedEmitter::emit(const std::vector& files, const std::string& out_cpp, + const std::string& out_hpp, const EmitOptions& opts, + std::string& error) const { + struct Prepared { + AssetFile file; + std::vector data; + std::vector gz; + std::string sha; + std::string mime; + bool has_gz = false; + }; + std::vector items; + for (const auto& file : files) { + std::ifstream in(file.abs_path, std::ios::binary); + if (!in.good()) { + error = "cannot read " + file.abs_path; + return false; + } + Prepared item; + item.file = file; + item.data.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); + item.sha = Sha256::hex_of_bytes(item.data); + item.mime = Mime::of_path(file.rel_path); + if (opts.gzip && Mime::is_textual(item.mime) && item.data.size() > caps::kGzipMinBytes) { + item.gz = GzipStored::compress(item.data); + item.has_gz = true; + } + items.push_back(std::move(item)); + } + + std::vector parts; + { + std::string token; + for (char c : opts.namespc) { + if (c == ':') { + if (!token.empty()) { + parts.push_back(token); + token.clear(); + } + } else if (c != ' ') { + token += c; + } + } + if (!token.empty()) parts.push_back(token); + if (parts.empty()) parts.push_back("assets"); + } + + std::ostringstream hpp; + hpp << "// Generated by " << opts.tool_version << ". Do not edit.\n"; + hpp << "#pragma once\n\n#include \n\n"; + for (const auto& part : parts) hpp << "namespace " << part << " {\n"; + hpp << "\nstruct EmbeddedChunk {\n"; + hpp << " const unsigned char* data;\n unsigned long size;\n};\n\n"; + hpp << "struct EmbeddedFile {\n"; + hpp << " const char* path;\n const EmbeddedChunk* chunks;\n"; + hpp << " unsigned long chunk_count;\n unsigned long size;\n"; + hpp << " const char* sha256;\n const char* mime;\n"; + hpp << " const unsigned char* gzip_data;\n unsigned long gzip_size;\n};\n\n"; + hpp << "const EmbeddedFile* bundle_files();\nunsigned long bundle_count();\n"; + hpp << "const char* bundle_tool_version();\n"; + for (auto it = parts.rbegin(); it != parts.rend(); ++it) hpp << "} // namespace " << *it << "\n"; + + std::ostringstream cpp; + cpp << "// Generated by " << opts.tool_version << ". Do not edit.\n"; + std::string hpp_base = out_hpp.substr(out_hpp.find_last_of("/\\") + 1); + cpp << "#include \"" << hpp_base << "\"\n\n"; + for (const auto& part : parts) cpp << "namespace " << part << " {\n"; + cpp << "\nnamespace {\n"; + auto emit_bytes = [&](const std::string& name, const std::vector& data, + std::size_t begin, std::size_t end) { + cpp << "constexpr unsigned char " << name << "[] = {"; + if (begin == end) cpp << "0"; + for (std::size_t k = begin; k < end; ++k) { + if ((k - begin) % 16 == 0) cpp << "\n "; + cpp << static_cast(data[k]) << ","; + } + cpp << "\n};\n"; + }; + for (std::size_t i = 0; i < items.size(); ++i) { + const auto& item = items[i]; + std::size_t chunks = + (item.data.size() + caps::kEmbedChunkBytes - 1) / caps::kEmbedChunkBytes; + if (chunks == 0) chunks = 1; + for (std::size_t c = 0; c < chunks; ++c) { + std::size_t begin = c * caps::kEmbedChunkBytes; + std::size_t end = std::min(begin + caps::kEmbedChunkBytes, item.data.size()); + emit_bytes(ident_for(i) + "_c" + std::to_string(c), item.data, begin, end); + } + cpp << "constexpr EmbeddedChunk " << ident_for(i) << "_chunks[] = {\n"; + for (std::size_t c = 0; c < chunks; ++c) { + std::size_t begin = c * caps::kEmbedChunkBytes; + std::size_t end = std::min(begin + caps::kEmbedChunkBytes, item.data.size()); + unsigned long size = static_cast(end - begin); + cpp << " {" << ident_for(i) << "_c" << c << ", " << size << "},\n"; + } + cpp << "};\n"; + if (item.has_gz) emit_bytes(ident_for(i) + "_gz", item.gz, 0, item.gz.size()); + cpp << "constexpr char " << ident_for(i) << "_path[] = \"" + << escape_cpp_string(item.file.rel_path) << "\";\n"; + cpp << "constexpr char " << ident_for(i) << "_sha[] = \"" << item.sha << "\";\n"; + cpp << "constexpr char " << ident_for(i) << "_mime[] = \"" << escape_cpp_string(item.mime) + << "\";\n"; + } + cpp << "} // namespace\n\n"; + cpp << "namespace {\nconst EmbeddedFile kFiles[] = {\n"; + for (std::size_t i = 0; i < items.size(); ++i) { + const auto& item = items[i]; + std::size_t chunks = + (item.data.size() + caps::kEmbedChunkBytes - 1) / caps::kEmbedChunkBytes; + if (chunks == 0) chunks = 1; + cpp << " {" << ident_for(i) << "_path, " << ident_for(i) << "_chunks, " + << static_cast(chunks) << ", " + << static_cast(item.data.size()) << ", " << ident_for(i) << "_sha, " + << ident_for(i) << "_mime, "; + if (item.has_gz) { + cpp << ident_for(i) << "_gz, " << static_cast(item.gz.size()); + } else { + cpp << "nullptr, 0"; + } + cpp << "},\n"; + } + cpp << "};\n} // namespace\n\n"; + cpp << "const EmbeddedFile* bundle_files() { return kFiles; }\n"; + cpp << "unsigned long bundle_count() { return " << items.size() << "; }\n"; + cpp << "const char* bundle_tool_version() { return \"" << escape_cpp_string(opts.tool_version) + << "\"; }\n"; + for (auto it = parts.rbegin(); it != parts.rend(); ++it) cpp << "} // namespace " << *it << "\n"; + + std::error_code ec; + if (auto parent = std::filesystem::path(out_cpp).parent_path(); !parent.empty()) { + std::filesystem::create_directories(parent, ec); + } + if (auto parent = std::filesystem::path(out_hpp).parent_path(); !parent.empty()) { + std::filesystem::create_directories(parent, ec); + } + std::ofstream hpp_out(out_hpp, std::ios::binary | std::ios::trunc); + std::ofstream cpp_out(out_cpp, std::ios::binary | std::ios::trunc); + if (!hpp_out.good() || !cpp_out.good()) { + error = "cannot write output files"; + return false; + } + hpp_out << hpp.str(); + cpp_out << cpp.str(); + return true; +} + +} // namespace molo::embed diff --git a/src/cli/embed/embed_emitter.hpp b/src/cli/embed/embed_emitter.hpp new file mode 100644 index 0000000..bc7c192 --- /dev/null +++ b/src/cli/embed/embed_emitter.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include "embed/asset_scanner.hpp" + +namespace molo::embed { + +struct EmitOptions { + std::string namespc = "molo::assets"; + bool gzip = false; + std::string tool_version = "molo-embed-1"; +}; + +// Emits one .hpp plus one .cpp with constexpr byte arrays. Output is +// byte-identical reproducible: sorted order, no timestamps. +class EmbedEmitter { + public: + bool emit(const std::vector& files, const std::string& out_cpp, + const std::string& out_hpp, const EmitOptions& opts, std::string& error) const; + + private: + static std::string ident_for(std::size_t index); + static std::string escape_cpp_string(const std::string& text); +}; + +} // namespace molo::embed diff --git a/src/cli/embed/embedded_bundle.cpp b/src/cli/embed/embedded_bundle.cpp new file mode 100644 index 0000000..24809c8 --- /dev/null +++ b/src/cli/embed/embedded_bundle.cpp @@ -0,0 +1,19 @@ +#include "embed/embedded_bundle.hpp" + +namespace molo::embed { + +void EmbeddedBundle::add(EmbeddedAsset asset) { assets_[asset.path] = std::move(asset); } + +const EmbeddedAsset* EmbeddedBundle::find(const std::string& path) const { + auto it = assets_.find(path); + if (it == assets_.end()) return nullptr; + return &it->second; +} + +std::vector EmbeddedBundle::paths() const { + std::vector out; + for (const auto& [path, _] : assets_) out.push_back(path); + return out; +} + +} // namespace molo::embed diff --git a/src/cli/embed/embedded_bundle.hpp b/src/cli/embed/embedded_bundle.hpp new file mode 100644 index 0000000..ce3954c --- /dev/null +++ b/src/cli/embed/embedded_bundle.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +namespace molo::embed { + +struct EmbeddedAsset { + std::string path; + std::vector data; + std::string sha256; + std::string mime; + std::vector gzip_data; + bool has_gzip = false; +}; + +// Runtime reader for embedded bundles. Static routes (01-runtime.md) serve +// from this instead of disk. Generated-code adapters fill it by joining the +// emitted 1 MB chunks in order. +class EmbeddedBundle { + public: + void add(EmbeddedAsset asset); + const EmbeddedAsset* find(const std::string& path) const; + std::size_t size() const { return assets_.size(); } + std::vector paths() const; + + private: + std::map assets_; +}; + +} // namespace molo::embed diff --git a/src/cli/embed/gzip_stored.cpp b/src/cli/embed/gzip_stored.cpp new file mode 100644 index 0000000..20a0010 --- /dev/null +++ b/src/cli/embed/gzip_stored.cpp @@ -0,0 +1,44 @@ +#include "embed/gzip_stored.hpp" + +namespace molo::embed { + +uint32_t GzipStored::crc32(const std::vector& data) { + uint32_t crc = 0xFFFFFFFFu; + for (unsigned char b : data) { + crc ^= b; + for (int i = 0; i < 8; ++i) crc = (crc & 1) ? (crc >> 1) ^ 0xEDB88320u : (crc >> 1); + } + return crc ^ 0xFFFFFFFFu; +} + +std::vector GzipStored::compress(const std::vector& data) { + std::vector out; + // gzip header: magic, deflate, flags=0, mtime=0 (reproducible), xfl=0, os=3 + out.insert(out.end(), {0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03}); + std::size_t pos = 0; + if (data.empty()) { + out.insert(out.end(), {0x01, 0x00, 0x00, 0xFF, 0xFF}); + } + while (pos < data.size()) { + std::size_t n = std::min(65535, data.size() - pos); + bool last = (pos + n == data.size()); + out.push_back(last ? 0x01 : 0x00); // BFINAL + BTYPE=00 stored + out.push_back(static_cast(n & 0xFF)); + out.push_back(static_cast((n >> 8) & 0xFF)); + out.push_back(static_cast((~n) & 0xFF)); + out.push_back(static_cast((~n >> 8) & 0xFF)); + out.insert(out.end(), data.begin() + pos, data.begin() + pos + n); + pos += n; + } + uint32_t crc = crc32(data); + uint32_t isize = static_cast(data.size()); + for (int i = 0; i < 4; ++i) out.push_back(static_cast((crc >> (i * 8)) & 0xFF)); + for (int i = 0; i < 4; ++i) out.push_back(static_cast((isize >> (i * 8)) & 0xFF)); + return out; +} + +std::vector GzipStored::compress_string(const std::string& text) { + return compress(std::vector(text.begin(), text.end())); +} + +} // namespace molo::embed diff --git a/src/cli/embed/gzip_stored.hpp b/src/cli/embed/gzip_stored.hpp new file mode 100644 index 0000000..e8f757f --- /dev/null +++ b/src/cli/embed/gzip_stored.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include + +namespace molo::embed { + +// Valid gzip writer using stored (uncompressed) deflate blocks. +// Output is a correct gzip member; ratio work is out of scope for v1. +class GzipStored { + public: + static std::vector compress(const std::vector& data); + static std::vector compress_string(const std::string& text); + static uint32_t crc32(const std::vector& data); +}; + +} // namespace molo::embed diff --git a/src/cli/embed/mime.cpp b/src/cli/embed/mime.cpp new file mode 100644 index 0000000..1651800 --- /dev/null +++ b/src/cli/embed/mime.cpp @@ -0,0 +1,34 @@ +#include "embed/mime.hpp" + +#include +#include + +namespace molo::embed { + +std::string Mime::of_path(const std::string& path) { + static const std::map table = { + {".html", "text/html"}, {".htm", "text/html"}, {".css", "text/css"}, + {".js", "text/javascript"}, {".mjs", "text/javascript"}, {".json", "application/json"}, + {".xml", "application/xml"}, {".txt", "text/plain"}, {".md", "text/markdown"}, + {".csv", "text/csv"}, {".svg", "image/svg+xml"}, {".png", "image/png"}, + {".jpg", "image/jpeg"}, {".jpeg", "image/jpeg"}, {".gif", "image/gif"}, + {".webp", "image/webp"}, {".ico", "image/x-icon"}, {".woff", "font/woff"}, + {".woff2", "font/woff2"}, {".ttf", "font/ttf"}, {".pdf", "application/pdf"}, + {".wasm", "application/wasm"}, + }; + auto dot = path.find_last_of('.'); + if (dot == std::string::npos) return "application/octet-stream"; + std::string ext = path.substr(dot); + std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); + auto it = table.find(ext); + if (it == table.end()) return "application/octet-stream"; + return it->second; +} + +bool Mime::is_textual(const std::string& mime) { + return mime.rfind("text/", 0) == 0 || mime == "application/json" || + mime == "application/xml" || mime == "image/svg+xml" || + mime == "application/javascript" || mime == "text/javascript"; +} + +} // namespace molo::embed diff --git a/src/cli/embed/mime.hpp b/src/cli/embed/mime.hpp new file mode 100644 index 0000000..a5f4e25 --- /dev/null +++ b/src/cli/embed/mime.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace molo::embed { + +// Extension based mime table for embedded assets. +class Mime { + public: + static std::string of_path(const std::string& path); + static bool is_textual(const std::string& mime); +}; + +} // namespace molo::embed diff --git a/src/cli/embed/sha256.cpp b/src/cli/embed/sha256.cpp new file mode 100644 index 0000000..f8750d3 --- /dev/null +++ b/src/cli/embed/sha256.cpp @@ -0,0 +1,100 @@ +#include "embed/sha256.hpp" + +#include +#include +#include + +namespace molo::embed { + +namespace { +uint32_t rotr(uint32_t x, int n) { return (x >> n) | (x << (32 - n)); } + +const std::array kK = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2}; + +std::array digest(const unsigned char* data, std::size_t len) { + std::array h = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + std::vector msg(data, data + len); + uint64_t bit_len = static_cast(len) * 8; + msg.push_back(0x80); + while (msg.size() % 64 != 56) msg.push_back(0x00); + for (int i = 7; i >= 0; --i) msg.push_back(static_cast(bit_len >> (i * 8))); + for (std::size_t off = 0; off < msg.size(); off += 64) { + std::array w{}; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(msg[off + i * 4]) << 24) | + (static_cast(msg[off + i * 4 + 1]) << 16) | + (static_cast(msg[off + i * 4 + 2]) << 8) | + static_cast(msg[off + i * 4 + 3]); + } + for (int i = 16; i < 64; ++i) { + uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3]; + uint32_t e = h[4], f = h[5], g = h[6], hh = h[7]; + for (int i = 0; i < 64; ++i) { + uint32_t s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t t1 = hh + s1 + ch + kK[i] + w[i]; + uint32_t s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = s0 + maj; + hh = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + h[5] += f; + h[6] += g; + h[7] += hh; + } + return h; +} + +std::string to_hex(std::array h) { + char buf[65]; + snprintf(buf, sizeof(buf), "%08x%08x%08x%08x%08x%08x%08x%08x", h[0], h[1], h[2], h[3], h[4], h[5], + h[6], h[7]); + return std::string(buf); +} +} // namespace + +std::string Sha256::hex_of_bytes(const std::vector& data) { + return to_hex(digest(data.data(), data.size())); +} + +std::string Sha256::hex_of_string(const std::string& text) { + return to_hex(digest(reinterpret_cast(text.data()), text.size())); +} + +bool Sha256::hex_of_file(const std::string& path, std::string& hex_out) { + std::ifstream in(path, std::ios::binary); + if (!in.good()) return false; + std::vector data((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + hex_out = hex_of_bytes(data); + return true; +} + +} // namespace molo::embed diff --git a/src/cli/embed/sha256.hpp b/src/cli/embed/sha256.hpp new file mode 100644 index 0000000..60f284a --- /dev/null +++ b/src/cli/embed/sha256.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +namespace molo::embed { + +// In-house SHA-256 (stdlib only). Used for embed manifests. +class Sha256 { + public: + static std::string hex_of_bytes(const std::vector& data); + static std::string hex_of_string(const std::string& text); + static bool hex_of_file(const std::string& path, std::string& hex_out); +}; + +} // namespace molo::embed diff --git a/src/cli/i18n/catalog.cpp b/src/cli/i18n/catalog.cpp new file mode 100644 index 0000000..b773e00 --- /dev/null +++ b/src/cli/i18n/catalog.cpp @@ -0,0 +1,18 @@ +#include "i18n/catalog.hpp" + +namespace molo::i18n { + +Catalog::Catalog(Locale locale) : locale_(locale) {} + +void Catalog::add(CatalogEntry entry) { + std::string key = entry.context + "\x04" + entry.msgid; + entries_[key] = std::move(entry); +} + +const CatalogEntry* Catalog::find(const std::string& msgid, const std::string& context) const { + auto it = entries_.find(context + "\x04" + msgid); + if (it == entries_.end()) return nullptr; + return &it->second; +} + +} // namespace molo::i18n diff --git a/src/cli/i18n/catalog.hpp b/src/cli/i18n/catalog.hpp new file mode 100644 index 0000000..df5273a --- /dev/null +++ b/src/cli/i18n/catalog.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +#include "i18n/locale.hpp" + +namespace molo::i18n { + +struct CatalogEntry { + std::string msgid; + std::string context; + std::vector forms; // forms[0] singular, rest plurals +}; + +// One compiled message catalog for one locale. +class Catalog { + public: + explicit Catalog(Locale locale); + + const Locale& locale() const { return locale_; } + void add(CatalogEntry entry); + const CatalogEntry* find(const std::string& msgid, const std::string& context = "") const; + std::size_t size() const { return entries_.size(); } + + private: + Locale locale_; + std::map entries_; +}; + +} // namespace molo::i18n diff --git a/src/cli/i18n/locale.cpp b/src/cli/i18n/locale.cpp new file mode 100644 index 0000000..11d4887 --- /dev/null +++ b/src/cli/i18n/locale.cpp @@ -0,0 +1,51 @@ +#include "i18n/locale.hpp" + +#include + +namespace molo::i18n { + +Locale Locale::parse(const std::string& tag) { + Locale out; + std::string clean; + for (char c : tag) { + if (c == '_') { + clean += '-'; + } else { + clean += c; + } + } + auto dash = clean.find('-'); + std::string lang = dash == std::string::npos ? clean : clean.substr(0, dash); + std::string region = dash == std::string::npos ? "" : clean.substr(dash + 1); + auto at = region.find_first_of("-@."); + if (at != std::string::npos) region = region.substr(0, at); + for (char c : lang) { + if (!std::isalpha(static_cast(c))) return Locale(); + out.language_ += static_cast(std::tolower(static_cast(c))); + } + for (char c : region) { + if (!std::isalnum(static_cast(c))) break; + out.region_ += static_cast(std::toupper(static_cast(c))); + } + if (out.language_.empty()) return Locale(); + return out; +} + +std::string Locale::str() const { + if (region_.empty()) return language_; + return language_ + "-" + region_; +} + +std::vector Locale::fallback_chain() const { + std::vector chain; + if (empty()) { + chain.push_back(Locale::english()); + return chain; + } + chain.push_back(*this); + if (!region_.empty()) chain.push_back(Locale::parse(language_)); + if (language_ != "en") chain.push_back(Locale::english()); + return chain; +} + +} // namespace molo::i18n diff --git a/src/cli/i18n/locale.hpp b/src/cli/i18n/locale.hpp new file mode 100644 index 0000000..2961b33 --- /dev/null +++ b/src/cli/i18n/locale.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +namespace molo::i18n { + +// BCP 47 locale tag (en, nl-NL). Normalizes language lower, region upper. +class Locale { + public: + static Locale parse(const std::string& tag); + static Locale english() { return parse("en"); } + + const std::string& language() const { return language_; } + const std::string& region() const { return region_; } + std::string str() const; + bool empty() const { return language_.empty(); } + + // Fallback chain: full tag, language, en (unless already en). + std::vector fallback_chain() const; + + private: + std::string language_; + std::string region_; +}; + +} // namespace molo::i18n diff --git a/src/cli/i18n/plural_rule.cpp b/src/cli/i18n/plural_rule.cpp new file mode 100644 index 0000000..4b5b4ab --- /dev/null +++ b/src/cli/i18n/plural_rule.cpp @@ -0,0 +1,41 @@ +#include "i18n/plural_rule.hpp" + +namespace molo::i18n { + +int PluralRule::form_count(const std::string& language) { + if (language == "ru" || language == "uk" || language == "pl") return 3; + if (language == "ar") return 6; + if (language == "ja" || language == "zh" || language == "ko") return 1; + return 2; // en, nl, de, fr, es and default +} + +int PluralRule::form_index(long count, const std::string& language) { + long n = count < 0 ? -count : count; + if (language == "ja" || language == "zh" || language == "ko") return 0; + if (language == "fr") return (n == 0 || n == 1) ? 0 : 1; + if (language == "ru" || language == "uk") { + long mod10 = n % 10; + long mod100 = n % 100; + if (mod10 == 1 && mod100 != 11) return 0; + if (mod10 >= 2 && mod10 <= 4 && (mod100 < 12 || mod100 > 14)) return 1; + return 2; + } + if (language == "pl") { + if (n == 1) return 0; + long mod10 = n % 10; + long mod100 = n % 100; + if (mod10 >= 2 && mod10 <= 4 && (mod100 < 12 || mod100 > 14)) return 1; + return 2; + } + if (language == "ar") { + if (n == 0) return 0; + if (n == 1) return 1; + if (n == 2) return 2; + if (n % 100 >= 3 && n % 100 <= 10) return 3; + if (n % 100 >= 11) return 4; + return 5; + } + return n == 1 ? 0 : 1; // en, nl, de, es and default +} + +} // namespace molo::i18n diff --git a/src/cli/i18n/plural_rule.hpp b/src/cli/i18n/plural_rule.hpp new file mode 100644 index 0000000..0c54103 --- /dev/null +++ b/src/cli/i18n/plural_rule.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace molo::i18n { + +// CLDR cardinal plural rules for shipped locales. +class PluralRule { + public: + static int form_count(const std::string& language); + static int form_index(long count, const std::string& language); +}; + +} // namespace molo::i18n diff --git a/src/cli/i18n/po_loader.cpp b/src/cli/i18n/po_loader.cpp new file mode 100644 index 0000000..f8c0d78 --- /dev/null +++ b/src/cli/i18n/po_loader.cpp @@ -0,0 +1,138 @@ +#include "i18n/po_loader.hpp" + +#include +#include +#include + +namespace molo::i18n { + +namespace { +std::string unescape(const std::string& quoted) { + std::string out; + bool esc = false; + for (std::size_t i = 0; i < quoted.size(); ++i) { + char c = quoted[i]; + if (esc) { + switch (c) { + case 'n': + out += '\n'; + break; + case 't': + out += '\t'; + break; + case 'r': + out += '\r'; + break; + case '\\': + out += '\\'; + break; + case '"': + out += '"'; + break; + default: + out += c; + break; + } + esc = false; + } else if (c == '\\') { + esc = true; + } else { + out += c; + } + } + return out; +} + +std::string strip_quotes(const std::string& line) { + auto first = line.find('"'); + auto last = line.rfind('"'); + if (first == std::string::npos || last == first) return ""; + return unescape(line.substr(first + 1, last - first - 1)); +} +} // namespace + +PoLoadResult PoLoader::load_file(const std::string& path, const Locale& locale) { + std::ifstream in(path); + if (!in.good()) { + PoLoadResult out; + out.catalog = Catalog(locale); + out.error = "cannot open PO file: " + path; + return out; + } + std::ostringstream buf; + buf << in.rdbuf(); + return load_text(buf.str(), locale); +} + +PoLoadResult PoLoader::load_text(const std::string& text, const Locale& locale) { + PoLoadResult out; + out.catalog = Catalog(locale); + std::istringstream lines(text); + std::string line; + CatalogEntry current; + std::map plural_forms; + std::string* target = nullptr; + int target_plural = -1; + auto flush = [&]() { + if (!current.msgid.empty()) { + if (!plural_forms.empty()) { + int max_index = 0; + for (const auto& [index, _] : plural_forms) max_index = std::max(max_index, index); + current.forms.resize(static_cast(max_index) + 1); + for (const auto& [index, value] : plural_forms) { + current.forms[static_cast(index)] = value; + } + } + out.catalog.add(current); + } + current = CatalogEntry(); + plural_forms.clear(); + target = nullptr; + target_plural = -1; + }; + while (std::getline(lines, line)) { + if (line.empty()) { + flush(); + continue; + } + if (line[0] == '#') continue; // translator comments and flags + if (line.rfind("msgctxt", 0) == 0) { + current.context = strip_quotes(line); + target = ¤t.context; + target_plural = -1; + } else if (line.rfind("msgid_plural", 0) == 0) { + target = nullptr; // plural msgid kept implicitly by form order + target_plural = -2; + (void)strip_quotes(line); + } else if (line.rfind("msgid", 0) == 0) { + current.msgid = strip_quotes(line); + if (current.forms.empty()) current.forms.push_back(""); + target = ¤t.msgid; + target_plural = -1; + } else if (line.rfind("msgstr[", 0) == 0) { + auto open = line.find('['); + auto close = line.find(']'); + int index = 0; + try { + index = std::stoi(line.substr(open + 1, close - open - 1)); + } catch (...) { + index = 0; + } + plural_forms[index] = strip_quotes(line); + target = &plural_forms[index]; + target_plural = index; + } else if (line.rfind("msgstr", 0) == 0) { + if (current.forms.empty()) current.forms.push_back(""); + current.forms[0] = strip_quotes(line); + target = ¤t.forms[0]; + target_plural = -1; + } else if (line[0] == '"') { + if (target != nullptr) *target += strip_quotes(line); + (void)target_plural; + } + } + flush(); + return out; +} + +} // namespace molo::i18n diff --git a/src/cli/i18n/po_loader.hpp b/src/cli/i18n/po_loader.hpp new file mode 100644 index 0000000..4b88819 --- /dev/null +++ b/src/cli/i18n/po_loader.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include "i18n/catalog.hpp" + +namespace molo::i18n { + +struct PoLoadResult { + Catalog catalog = Catalog(Locale::english()); + std::string error; + bool ok() const { return error.empty(); } +}; + +// GNU gettext PO loader: msgid, msgstr, msgid_plural, msgstr[n], msgctxt, +// translator comments (#), multiline strings, C escapes. +class PoLoader { + public: + static PoLoadResult load_file(const std::string& path, const Locale& locale); + static PoLoadResult load_text(const std::string& text, const Locale& locale); +}; + +} // namespace molo::i18n diff --git a/src/cli/i18n/translator.cpp b/src/cli/i18n/translator.cpp new file mode 100644 index 0000000..3e3aa65 --- /dev/null +++ b/src/cli/i18n/translator.cpp @@ -0,0 +1,91 @@ +#include "i18n/translator.hpp" + +#include +#include + +#include "i18n/plural_rule.hpp" + +namespace molo::i18n { + +void Translator::add_catalog(Catalog catalog) { + catalogs_.insert_or_assign(catalog.locale().str(), std::move(catalog)); +} + +std::string Translator::translate(const std::string& msgid, const Locale& locale, long count, + const std::string& context) { + for (const auto& candidate : locale.fallback_chain()) { + auto it = catalogs_.find(candidate.str()); + if (it == catalogs_.end() && !candidate.region().empty()) { + it = catalogs_.find(candidate.language()); + } + if (it == catalogs_.end()) continue; + const CatalogEntry* entry = it->second.find(msgid, context); + if (entry == nullptr || entry->forms.empty()) continue; + int index = PluralRule::form_index(count, candidate.language()); + if (index < 0 || static_cast(index) >= entry->forms.size()) { + return entry->forms[0]; + } + const std::string& form = entry->forms[static_cast(index)]; + return form.empty() ? msgid : form; + } + if (missing_.insert(context + "\x04" + msgid).second) { + std::string note = "i18n: missing key '" + msgid + "' for locale " + locale.str() + "\n"; + if (missing_sink_) { + missing_sink_(note); + } else { + std::cerr << note; + } + } + return msgid; +} + +Locale Translator::parse_accept_language(const std::string& header) { + // Honor q-values: pick the highest-q parseable tag. + std::istringstream parts(header); + std::string item; + double best_q = -1.0; + Locale best; + while (std::getline(parts, item, ',')) { + auto semi = item.find(';'); + std::string tag = semi == std::string::npos ? item : item.substr(0, semi); + double q = 1.0; + if (semi != std::string::npos) { + auto qpos = item.find("q=", semi); + if (qpos != std::string::npos) { + try { + q = std::stod(item.substr(qpos + 2)); + } catch (...) { + q = 0.0; + } + } + } + while (!tag.empty() && tag.front() == ' ') tag.erase(0, 1); + while (!tag.empty() && tag.back() == ' ') tag.pop_back(); + if (tag.empty() || tag == "*") continue; + Locale loc = Locale::parse(tag); + if (loc.empty()) continue; + if (q > best_q) { + best_q = q; + best = loc; + } + } + return best; +} + +Locale Translator::resolve(const RequestLocale& request) { + if (!request.route_prefix.empty()) { + Locale loc = Locale::parse(request.route_prefix); + if (!loc.empty()) return loc; + } + if (!request.cookie.empty()) { + Locale loc = Locale::parse(request.cookie); + if (!loc.empty()) return loc; + } + Locale header = parse_accept_language(request.accept_language); + if (!header.empty()) return header; + Locale def = Locale::parse(request.config_default); + if (!def.empty()) return def; + return Locale::english(); +} + +} // namespace molo::i18n diff --git a/src/cli/i18n/translator.hpp b/src/cli/i18n/translator.hpp new file mode 100644 index 0000000..c7fc6e7 --- /dev/null +++ b/src/cli/i18n/translator.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include + +#include "i18n/catalog.hpp" +#include "i18n/locale.hpp" + +namespace molo::i18n { + +struct RequestLocale { + std::string route_prefix; // e.g. "nl" from /nl/... + std::string cookie; // molo_locale cookie value + std::string accept_language; // raw header + std::string config_default = "en"; +}; + +// Translator with GNU gettext fallback: full tag, language, en, msgid. +// Missing keys log once per key per process. Template hooks (05-request- +// routing.md) call translate() via t() filters and functions. +class Translator { + public: + using MissingSink = std::function; + + void add_catalog(Catalog catalog); + void set_missing_sink(MissingSink sink) { missing_sink_ = std::move(sink); } + + std::string translate(const std::string& msgid, const Locale& locale, long count = 1, + const std::string& context = ""); + + // Resolution order: route prefix, cookie, Accept-Language, default. + static Locale resolve(const RequestLocale& request); + static Locale parse_accept_language(const std::string& header); + + std::size_t missing_count() const { return missing_.size(); } + + private: + std::map catalogs_; + std::set missing_; + MissingSink missing_sink_; +}; + +} // namespace molo::i18n diff --git a/src/cli/main.cpp b/src/cli/main.cpp new file mode 100644 index 0000000..a221702 --- /dev/null +++ b/src/cli/main.cpp @@ -0,0 +1,7 @@ +#include "cli/cli_app.hpp" +#include "cli/exit_code.hpp" + +int main(int argc, const char* const* argv) { + molo::cli::CliApp app; + return molo::cli::to_int(app.run(argc, argv)); +} diff --git a/src/cli/ops/health_check.cpp b/src/cli/ops/health_check.cpp new file mode 100644 index 0000000..914c670 --- /dev/null +++ b/src/cli/ops/health_check.cpp @@ -0,0 +1,4 @@ +#include "ops/health_check.hpp" + +// HealthCheck is interface plus value types; kept as a pair for structure. +namespace molo::ops {} // namespace molo::ops diff --git a/src/cli/ops/health_check.hpp b/src/cli/ops/health_check.hpp new file mode 100644 index 0000000..4c057f4 --- /dev/null +++ b/src/cli/ops/health_check.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +namespace molo::ops { + +enum class HealthState { Ok, Degraded, Failed }; + +inline std::string health_state_name(HealthState state) { + switch (state) { + case HealthState::Ok: + return "ok"; + case HealthState::Degraded: + return "degraded"; + case HealthState::Failed: + return "failed"; + } + return "unknown"; +} + +struct HealthResult { + HealthState state = HealthState::Ok; + std::string detail; + long elapsed_ms = 0; + bool timed_out = false; +}; + +// Base class for liveness and readiness checks. +class HealthCheck { + public: + virtual ~HealthCheck() = default; + virtual std::string name() const = 0; + virtual HealthResult check() = 0; +}; + +} // namespace molo::ops diff --git a/src/cli/ops/health_registry.cpp b/src/cli/ops/health_registry.cpp new file mode 100644 index 0000000..75ccfe3 --- /dev/null +++ b/src/cli/ops/health_registry.cpp @@ -0,0 +1,35 @@ +#include "ops/health_registry.hpp" + +#include +#include + +namespace molo::ops { + +void HealthRegistry::add(std::shared_ptr check) { + checks_.push_back(std::move(check)); +} + +std::vector HealthRegistry::run_all(int timeout_ms) const { + std::vector out; + for (const auto& check : checks_) { + NamedResult named; + named.name = check->name(); + auto start = std::chrono::steady_clock::now(); + std::future fut = + std::async(std::launch::async, [&check]() { return check->check(); }); + if (fut.wait_for(std::chrono::milliseconds(timeout_ms)) == std::future_status::ready) { + named.result = fut.get(); + } else { + named.result.state = HealthState::Failed; + named.result.detail = "check timed out"; + named.result.timed_out = true; + } + auto end = std::chrono::steady_clock::now(); + named.result.elapsed_ms = + std::chrono::duration_cast(end - start).count(); + out.push_back(std::move(named)); + } + return out; +} + +} // namespace molo::ops diff --git a/src/cli/ops/health_registry.hpp b/src/cli/ops/health_registry.hpp new file mode 100644 index 0000000..1719b1a --- /dev/null +++ b/src/cli/ops/health_registry.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + +#include "ops/health_check.hpp" + +namespace molo::ops { + +struct NamedResult { + std::string name; + HealthResult result; +}; + +// Registry of checks. Each check is capped at kHealthCheckTimeoutMs; +// slow checks are reported as timed out, never awaited unbounded. +class HealthRegistry { + public: + void add(std::shared_ptr check); + std::vector run_all(int timeout_ms = 200) const; + std::size_t size() const { return checks_.size(); } + + private: + std::vector> checks_; +}; + +} // namespace molo::ops diff --git a/src/cli/ops/liveness_check.cpp b/src/cli/ops/liveness_check.cpp new file mode 100644 index 0000000..3a0e07e --- /dev/null +++ b/src/cli/ops/liveness_check.cpp @@ -0,0 +1,17 @@ +#include "ops/liveness_check.hpp" + +namespace molo::ops { + +HealthResult LivenessCheck::check() { + HealthResult out; + if (shutdown_stuck_.load()) { + out.state = HealthState::Failed; + out.detail = "shutdown stuck"; + } else { + out.state = HealthState::Ok; + out.detail = "loop serving"; + } + return out; +} + +} // namespace molo::ops diff --git a/src/cli/ops/liveness_check.hpp b/src/cli/ops/liveness_check.hpp new file mode 100644 index 0000000..5715f7c --- /dev/null +++ b/src/cli/ops/liveness_check.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include "ops/health_check.hpp" + +namespace molo::ops { + +// Liveness: fails only when shutdown is stuck. Never touches the DB. +class LivenessCheck : public HealthCheck { + public: + std::string name() const override { return "liveness"; } + HealthResult check() override; + void set_shutdown_stuck(bool stuck) { shutdown_stuck_.store(stuck); } + + private: + std::atomic shutdown_stuck_{false}; +}; + +} // namespace molo::ops diff --git a/src/cli/ops/ops_routes.cpp b/src/cli/ops/ops_routes.cpp new file mode 100644 index 0000000..1167752 --- /dev/null +++ b/src/cli/ops/ops_routes.cpp @@ -0,0 +1,147 @@ +#include "ops/ops_routes.hpp" + +#include +#include + +#include "cli/json_writer.hpp" +#include "operability/debug/endpoint.h" +#include "operability/debug/snapshot.h" +#include "ops/health_registry.hpp" +#include "ops/prometheus_exporter.hpp" +#include "ops/readiness_check.hpp" +#include "ops/startup_gate.hpp" +#include "supervisor/common/clock.hpp" +#include "version.hpp" + +namespace molo::ops { + +namespace { + +// Fail-closed admin auth until the 03 security gate plugs in here +// (operability PATCHES section 9): debug stays 403 in debug builds. +[[maybe_unused]] bool ops_admin_auth(const std::string& token) { + (void)token; + return false; +} + +// Local wall clock so the ops TU does not drag supervisor TU definitions +// into CLI links. Behaviorally identical to SystemClock. +class WallClock : public molo::common::Clock { + public: + std::int64_t wall_ms() const override { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + std::int64_t steady_ms() const override { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + } + void sleep_ms(long ms) override { + if (ms > 0) std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } +}; + +} // namespace + +OpsRoutes::OpsRoutes(const HealthRegistry& live_registry, const ReadinessCheck& readiness, + const StartupGate& gate, const PrometheusExporter& exporter) + : live_registry_(live_registry), readiness_(readiness), gate_(gate), exporter_(exporter) {} + +OpsResponse OpsRoutes::healthz() const { + OpsResponse out; + bool failed = false; + for (const auto& named : live_registry_.run_all()) { + if (named.result.state != HealthState::Ok) failed = true; + } + if (failed) { + out.status = 503; + out.body = "{\"status\":\"failed\"}"; + } else { + out.status = 200; + out.body = "{\"status\":\"ok\"}"; + } + return out; +} + +OpsResponse OpsRoutes::readyz() const { + OpsResponse out; + // ReadinessCheck::check is logically const; cast keeps the registry const. + HealthResult r = const_cast(readiness_).check(); + if (r.state == HealthState::Ok) { + out.status = 200; + out.body = "{\"status\":\"ok\"}"; + } else { + out.status = 503; + std::vector items; + for (const auto& name : const_cast(readiness_).failed()) { + items.push_back(molo::cli::JsonWriter::quote(name)); + } + out.body = "{\"status\":\"degraded\",\"failed\":" + + molo::cli::JsonWriter::array(items) + "}"; + } + return out; +} + +OpsResponse OpsRoutes::startupz() const { + OpsResponse out; + if (gate_.ready()) { + out.status = 200; + out.body = "{\"status\":\"ok\"}"; + } else { + out.status = 503; + out.body = "{\"status\":\"starting\"}"; + } + return out; +} + +OpsResponse OpsRoutes::metrics(bool accept_gzip) const { + OpsResponse out; + out.content_type = "text/plain; version=0.0.4"; + if (accept_gzip) { + auto bytes = exporter_.render_gzip(); + out.body.assign(bytes.begin(), bytes.end()); + out.gzip = true; + } else { + out.body = exporter_.render(); + } + out.status = 200; + return out; +} + +std::string OpsRoutes::debug_snapshot() const { + WallClock clock; + molo::debug::Snapshot snap(clock, molo::kMoloVersion); + snap.add_section("health", healthz().body); + snap.add_section("ready", readyz().body); + snap.add_section("startup", startupz().body); + snap.add_section("metrics_bytes", + "{\"bytes\":" + std::to_string(exporter_.render().size()) + "}"); + return snap.capture(); +} + +OpsResponse OpsRoutes::debug(const std::string& path, const std::string& token, + bool admin_listener) const { + OpsResponse r; + r.status = 404; + r.content_type = "text/plain"; + r.body = "not found"; + r.gzip = false; +#ifdef MOLO_ENABLE_DEBUG + if (!admin_listener) return r; + molo::debug::Endpoint ep([this] { return this->debug_snapshot(); }, + [](const std::string& t) { return ops_admin_auth(t); }); + const auto out = ep.handle(path, token); + r.status = out.status; + r.content_type = out.content_type; + r.body = out.body; +#else + (void)path; + (void)token; + (void)admin_listener; +#endif + return r; +} + +} // namespace molo::ops diff --git a/src/cli/ops/ops_routes.hpp b/src/cli/ops/ops_routes.hpp new file mode 100644 index 0000000..204c70f --- /dev/null +++ b/src/cli/ops/ops_routes.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include + +namespace molo::ops { + +// Pure probe handlers with no auth and no unbounded allocation. +// The HTTP layer (01-runtime.md) mounts these at fixed paths. +struct OpsResponse { + int status = 200; + std::string content_type = "application/json"; + std::string body; + bool gzip = false; +}; + +class HealthRegistry; +class StartupGate; +class PrometheusExporter; +class ReadinessCheck; + +class OpsRoutes { + public: + OpsRoutes(const HealthRegistry& live_registry, const ReadinessCheck& readiness, + const StartupGate& gate, const PrometheusExporter& exporter); + + OpsResponse healthz() const; + OpsResponse readyz() const; + OpsResponse startupz() const; + OpsResponse metrics(bool accept_gzip) const; + // Admin-only debug surface. Returns 404 unless the process was built with + // MOLO_ENABLE_DEBUG and the caller is the admin listener (see the .cpp). + OpsResponse debug(const std::string& path, const std::string& token, + bool admin_listener) const; + + private: + std::string debug_snapshot() const; + const HealthRegistry& live_registry_; + const ReadinessCheck& readiness_; + const StartupGate& gate_; + const PrometheusExporter& exporter_; +}; + +} // namespace molo::ops diff --git a/src/cli/ops/otlp_bridge.cpp b/src/cli/ops/otlp_bridge.cpp new file mode 100644 index 0000000..06fd268 --- /dev/null +++ b/src/cli/ops/otlp_bridge.cpp @@ -0,0 +1,32 @@ +#include "ops/otlp_bridge.hpp" + +#include "caps.hpp" + +namespace molo::ops { + +bool OtlpBridge::configure(const OtlpOptions& opts, std::string& error) { + if (opts.endpoint.empty()) { + error = "otlp endpoint is empty"; + return false; + } + if (opts.interval_sec < caps::kMetricsExportFloorSec) { + error = "otlp interval below 5s floor"; + return false; + } + opts_ = opts; + configured_ = true; + return true; +} + +bool OtlpBridge::export_spans(unsigned long span_count, std::string& error) const { + (void)span_count; + if (!configured_) { + error = "otlp bridge not configured"; + return false; + } + // STUB: transport owned by 02-observability.md, not yet wired. + error = "otlp transport unavailable (02-observability.md owns it)"; + return false; +} + +} // namespace molo::ops diff --git a/src/cli/ops/otlp_bridge.hpp b/src/cli/ops/otlp_bridge.hpp new file mode 100644 index 0000000..05eeb42 --- /dev/null +++ b/src/cli/ops/otlp_bridge.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace molo::ops { + +struct OtlpOptions { + std::string endpoint; + int interval_sec = 5; +}; + +// Thin bridge only. Transport is owned by 02-observability.md; this stub +// records configuration and reports unavailable until wired. +class OtlpBridge { + public: + bool configure(const OtlpOptions& opts, std::string& error); + bool export_spans(unsigned long span_count, std::string& error) const; + bool configured() const { return configured_; } + const OtlpOptions& options() const { return opts_; } + + private: + OtlpOptions opts_; + bool configured_ = false; +}; + +} // namespace molo::ops diff --git a/src/cli/ops/prometheus_exporter.cpp b/src/cli/ops/prometheus_exporter.cpp new file mode 100644 index 0000000..a300490 --- /dev/null +++ b/src/cli/ops/prometheus_exporter.cpp @@ -0,0 +1,103 @@ +#include "ops/prometheus_exporter.hpp" + +#include +#include + +#include "caps.hpp" +#include "embed/gzip_stored.hpp" + +namespace molo::ops { + +std::string PrometheusExporter::escape_label(const std::string& text) { + std::string out; + for (char c : text) { + if (c == '"') { + out += "\\\""; + } else if (c == '\\') { + out += "\\\\"; + } else if (c == '\n') { + out += "\\n"; + } else { + out += c; + } + } + return out; +} + +bool PrometheusExporter::add(MetricSample sample) { + if (sample.labels.size() > caps::kMetricsMaxLabels) { + ++dropped_; + return false; + } + if (samples_.size() >= caps::kMetricsMaxSeries) { + ++dropped_; + return false; + } + samples_.push_back(std::move(sample)); + return true; +} + +std::string PrometheusExporter::render() const { + std::ostringstream out; + std::string last_name; + for (const auto& sample : samples_) { + if (sample.name != last_name) { + if (!sample.help.empty()) out << "# HELP " << sample.name << " " << sample.help << "\n"; + out << "# TYPE " << sample.name << " " << metric_type_name(sample.type) << "\n"; + last_name = sample.name; + } + out << sample.name; + if (!sample.labels.empty()) { + out << "{"; + bool first = true; + for (const auto& [key, value] : sample.labels) { + if (!first) out << ","; + first = false; + out << key << "=\"" << escape_label(value) << "\""; + } + out << "}"; + } + out << " " << sample.value << "\n"; + } + out << "# HELP molo_metrics_dropped_total samples dropped by series caps\n"; + out << "# TYPE molo_metrics_dropped_total counter\n"; + out << "molo_metrics_dropped_total " << dropped_ << "\n"; + return out.str(); +} + +std::vector PrometheusExporter::render_gzip() const { + std::string text = render(); + return molo::embed::GzipStored::compress_string(text); +} + +std::string PrometheusExporter::normalize_route(const std::string& path) { + std::ostringstream out; + std::string segment; + auto flush = [&]() { + if (segment.empty()) return; + bool numeric = true; + bool uuid_like = segment.size() == 36; + for (char c : segment) { + if (!std::isdigit(static_cast(c))) numeric = false; + if (!std::isxdigit(static_cast(c)) && c != '-') uuid_like = false; + } + if (numeric || uuid_like) { + out << "/{id}"; + } else { + out << "/" << segment; + } + segment.clear(); + }; + for (char c : path) { + if (c == '/') { + flush(); + } else { + segment += c; + } + } + flush(); + std::string result = out.str(); + return result.empty() ? "/" : result; +} + +} // namespace molo::ops diff --git a/src/cli/ops/prometheus_exporter.hpp b/src/cli/ops/prometheus_exporter.hpp new file mode 100644 index 0000000..bf3e332 --- /dev/null +++ b/src/cli/ops/prometheus_exporter.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include + +namespace molo::ops { + +enum class MetricType { Counter, Gauge, Histogram, Summary, Untyped }; + +inline std::string metric_type_name(MetricType type) { + switch (type) { + case MetricType::Counter: + return "counter"; + case MetricType::Gauge: + return "gauge"; + case MetricType::Histogram: + return "histogram"; + case MetricType::Summary: + return "summary"; + case MetricType::Untyped: + return "untyped"; + } + return "untyped"; +} + +struct MetricSample { + std::string name; + std::map labels; + double value = 0.0; + MetricType type = MetricType::Untyped; + std::string help; +}; + +// Prometheus text 0.0.4 exporter with static cardinality caps. +// Series cap 10000; overflow counts molo_metrics_dropped_total. +class PrometheusExporter { + public: + bool add(MetricSample sample); + std::string render() const; + std::vector render_gzip() const; + std::size_t series() const { return samples_.size(); } + unsigned long dropped() const { return dropped_; } + + // Route templates only: numeric/uuid segments become {id}. + static std::string normalize_route(const std::string& path); + + private: + std::vector samples_; + unsigned long dropped_ = 0; + + static std::string escape_label(const std::string& text); +}; + +} // namespace molo::ops diff --git a/src/cli/ops/readiness_check.cpp b/src/cli/ops/readiness_check.cpp new file mode 100644 index 0000000..0d9ba53 --- /dev/null +++ b/src/cli/ops/readiness_check.cpp @@ -0,0 +1,24 @@ +#include "ops/readiness_check.hpp" + +namespace molo::ops { + +void ReadinessCheck::add_probe(DependencyProbe probe) { probes_.push_back(std::move(probe)); } + +HealthResult ReadinessCheck::check() { + HealthResult out; + failed_.clear(); + for (auto& probe : probes_) { + HealthResult r = probe.probe(); + if (r.state != HealthState::Ok) failed_.push_back(probe.name); + } + if (failed_.empty()) { + out.state = HealthState::Ok; + out.detail = "ready"; + } else { + out.state = HealthState::Failed; + out.detail = "degraded"; + } + return out; +} + +} // namespace molo::ops diff --git a/src/cli/ops/readiness_check.hpp b/src/cli/ops/readiness_check.hpp new file mode 100644 index 0000000..efef5a2 --- /dev/null +++ b/src/cli/ops/readiness_check.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include + +#include "ops/health_check.hpp" + +namespace molo::ops { + +struct DependencyProbe { + std::string name; + std::function probe; +}; + +// Readiness: aggregates DB, KV, and supervisor heartbeats. Each probe is +// short-circuited by the registry timeout (200 ms). +class ReadinessCheck : public HealthCheck { + public: + std::string name() const override { return "readiness"; } + HealthResult check() override; + void add_probe(DependencyProbe probe); + std::vector failed() const { return failed_; } + + private: + std::vector probes_; + std::vector failed_; +}; + +} // namespace molo::ops diff --git a/src/cli/ops/startup_gate.cpp b/src/cli/ops/startup_gate.cpp new file mode 100644 index 0000000..2366f37 --- /dev/null +++ b/src/cli/ops/startup_gate.cpp @@ -0,0 +1,4 @@ +#include "ops/startup_gate.hpp" + +// StartupGate is inline atomic logic; kept as a pair for structure. +namespace molo::ops {} // namespace molo::ops diff --git a/src/cli/ops/startup_gate.hpp b/src/cli/ops/startup_gate.hpp new file mode 100644 index 0000000..033a5cf --- /dev/null +++ b/src/cli/ops/startup_gate.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace molo::ops { + +// Startup gate: /startupz returns 503 until boot completes, then 200. +class StartupGate { + public: + void mark_boot_complete() { ready_.store(true); } + bool ready() const { return ready_.load(); } + void reset_for_tests() { ready_.store(false); } + + private: + std::atomic ready_{false}; +}; + +} // namespace molo::ops diff --git a/src/cli/spec.md b/src/cli/spec.md new file mode 100644 index 0000000..9cb4734 --- /dev/null +++ b/src/cli/spec.md @@ -0,0 +1,89 @@ +# 09-cli-ops component specification + +Part: CLI tooling, dev server, operations. Owned directory: `src/cli/`. +Plan: `plans/09-cli-ops.md`. Version: 0.1.0. + +## Namespaces (mirror directories under src/cli/) + +- `molo::cli`: argument parsing, command tree, help, output, scaffolder, + supervisor/task CLI, migrate CLI, app wiring. +- `molo::dev`: dev server, config split, file watchers, reloader. +- `molo::embed`: asset scanner, embed emitter, runtime bundle, sha256, + stored gzip, mime. +- `molo::ai`: SSE parser, prompt pack, transports, client, redactor, session. +- `molo::ops`: health checks, registry, startup gate, ops routes, + Prometheus exporter, OTLP bridge stub. +- `molo::i18n`: locale, plural rules, catalog, PO loader, translator. +- `molo::csp`: policy, builder, nonce, report handler. + +## Static caps (src/cli/caps.hpp, single source) + +CLI: verbose max 3, command depth max 8, help width 100. +Dev: 20000 watched files, 150 ms debounce, make timeout 300 s, +child SIGKILL grace 5 s, backoff 1/2/4 s capped at 30 s hold. +Embed: 512 MB total, 50000 files, 64 MB per file, 1 MB code chunks, +gzip variants for text over 1 KB. +AI: 30 s connect, 300 s session, 8 MB transcript, 50 actions. +Health: 200 ms per check, 50 ms p99 budget, scrape render 100 ms p99. +Metrics: 10000 series, 64 labels per metric, export floor 5 s, +route templates only, overflow counts molo_metrics_dropped_total. +CSP: 64 KB per report, 100 reports/min/IP, 1000 report ring store, +16 byte nonces. +Scaffold: project name max 64 chars. Paths: 4096 bytes max. + +## Contracts + +Exit codes: 0 ok, 1 runtime, 2 usage, 69 unavailable (SYSEXITS). +Stdout carries primary output only; logs, progress, errors go to stderr. +`--format json` on stdout is parseable JSON with no log interleaving. +TTY-aware: no color codes when piped. +Every command has `-h` concise and `help ` full help from one tree. + +Command tree: new, serve, build, embed, migrate, supervisor +(start|stop|restart|status|logs), task (enqueue|retry|cancel|inspect), +ai (new|continue), health, version, help. + +Health paths (fixed, no auth): GET /healthz (200 ok, 503 only when +shutdown stuck), GET /readyz (200 ok, 503 degraded plus failed list), +GET /startupz (503 until boot complete, then 200), GET /metrics +(Prometheus text 0.0.4, `text/plain; version=0.0.4`, optional gzip). + +Embed output: sorted manifest order, no timestamps, tool version stamped, +constexpr 1 MB chunks, per-file sha256 plus mime, optional stored-gzip +variants for text over 1 KB. Byte-identical across runs. + +AI: key from DEVPLACE_API_KEY env only; POST {base}/chat/completions with +stream true; SSE parsed, end on `data: [DONE]`; secrets redacted before +prompt construction (redaction failure aborts); plan-then-apply with +human-confirmed file writes; default base https://devplace.net/openai/v1, +model molodetz. https transport is a marked stub until wolfSSL lands. + +i18n: BCP 47 tags; resolution route prefix, molo_locale cookie, +Accept-Language q-values, config default; PO catalogs; fallback full tag, +language, en, msgid; missing keys log once per process. + +CSP: default `default-src 'none'; script-src 'self'; style-src 'self'; +img-src 'self' data:; connect-src 'self'; frame-ancestors 'none'`; +report-only until enforce(); per-request 16 byte nonces; report endpoint +POST /csp-report returns 202/413/429. + +## Dependency seams (other parts land later) + +- ISupervisorControl / ITaskControl: null impl exits 69 until 07 wires in. +- IMigrateControl: null impl exits 69 until 04 wires in. +- OtlpBridge: configured but export reports unavailable until 02 wires in. +- TlsStub: https unavailable until wolfSSL (03) lands; http works now. +- OpsRoutes: pure handlers; 01-runtime mounts them at fixed paths. +- Translator/CspBuilder: template hooks consumed by 05-request-routing. +- SseStreamParser framing matches the 06-realtime SSE client contract. +- AiClient transport interface matches the 08 HTTP client shape. +- PrometheusExporter accepts samples from the 02 registry adapter. + +## Gaps (current, next to the code that closes them) + +- Generated multi-chunk assets expose chunk tables; 01-runtime static + serving joins chunks in order (see embed_emitter.cpp). +- `molo serve` single-shot mode parks the loop; full HTTP serving arrives + with 01-runtime lifecycle wiring (see dev_server.cpp run_once). +- `molo ai continue` resumes by name within one process; durable session + store is deferred (see cli_app.cpp cmd_ai). diff --git a/src/cli/version.hpp b/src/cli/version.hpp new file mode 100644 index 0000000..b3d32e6 --- /dev/null +++ b/src/cli/version.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace molo { + +inline constexpr const char* kMoloVersion = "0.1.0"; +inline constexpr const char* kEmbedToolVersion = "molo-embed-1"; +inline constexpr const char* kMoloName = "molo"; + +inline std::string version_string() { return std::string(kMoloName) + " " + kMoloVersion; } + +} // namespace molo diff --git a/src/clients/SPEC.md b/src/clients/SPEC.md new file mode 100644 index 0000000..3084137 --- /dev/null +++ b/src/clients/SPEC.md @@ -0,0 +1,86 @@ +# 08 Clients, OpenAPI, Test Client - Specification + +Owned directory: `src/clients/`. Namespaces mirror directories under `molo::clients`. +Plan file `plans/08-clients-openapi.md` names `src/client`, `src/openapi`, +`src/testing`; the parts map assigns all three to this part, so they live here as +`src/clients/client/` is flattened to `src/clients/` core, plus +`src/clients/rest/`, `src/clients/openapi/`, `src/clients/testing/`, +`src/clients/body/`, `src/clients/transports/`. + +## 1. Module map (one class or concern per file) + +Core (`molo::clients`): +`caps.h`, `task.h` (coroutine Task plus blocking bridge), `timeouts.h`, +`url.h/.cpp`, `headers.h/.cpp`, `request.h/.cpp`, `response.h/.cpp`, +`transport.h`, `pool.h/.cpp`, `retry.h/.cpp`, `auth.h/.cpp`, +`cookies.h/.cpp`, `proxy.h/.cpp`, `client_config.h`, `client.h/.cpp`. + +Bodies (`molo::clients::body`): `json.h/.cpp` (Json DOM plus parse plus dump), +`form.h/.cpp` (urlencoded plus multipart), `xml.h/.cpp` (escape plus builder +plus lite parse), `stream.h/.cpp` (StreamReader plus chunk sink plus FileReader). + +Transports (`molo::clients::transports`): `http_transport.h/.cpp` (real +HTTP/1.1 over POSIX sockets, keep-alive pool), `mock_transport.h/.cpp` +(scripted double), `app_transport.h/.cpp` (in-process dispatch for tests), +`tls.h` plus `tls_stub.cpp` (TLS interface with marked wolfSSL stub). + +REST (`molo::clients::rest`): `resource.h/.cpp`, `paginator.h/.cpp`, +`errors.h/.cpp` (RFC 9457 problem details), `idempotency.h/.cpp`. + +OpenAPI (`molo::clients::openapi`): `document.h/.cpp`, `schema.h/.cpp`, +`builder.h/.cpp`, `routes.h/.cpp` (payloads for doc routes), `extensions.h`. + +Testing (`molo::clients::testing`): `test_client.h/.cpp`, `fixtures.h/.cpp`, +`clock.h/.cpp`, `assertions.h/.cpp`. + +## 2. Static caps (see caps.h, never autotuned) + +| Knob | Value | +|---|---| +| Timeout connect / read / write / pool acquire / total | 5s / 30s / 30s / 5s / 60s | +| Pool max per Client / per host / idle per host / keepalive | 100 / 20 / 10 / 60s | +| Retry attempts / base / cap / statuses | 3 / 200ms / 5s / 429,502,503,504 | +| Redirect hops | 5, GET/HEAD auto, strip auth cross-host | +| In-memory body / multipart part / request total | 10 MB / 512 MB / 1 GB | +| Pagination default / max per page / max pages | 20 / 100 / 100 | +| OpenAPI paths / schemas / document bytes | 5000 / 2000 / 10 MB | +| URL length / headers count / header line / header block | 8192 / 200 / 16 KB / 64 KB | +| Cookie jar entries / cookie bytes | 4096 / 8 KB | +| Idempotency TTL / store entries | 24h / 100000 | +| JSON depth | 32 | + +## 3. Contracts + +- Every public client API has a coroutine form (primary, returns `Task`) + and a sync `_blocking` form sharing one implementation. +- `Task` runs eagerly on the calling thread; suspension points are + reserved for runtime loop integration (part 01). No unbounded wait exists. +- Transports: `HttpTransport` (real network, HTTP/1.1), `MockTransport` + (scripted), `AppTransport` (in-process handler callable). Tests swap the + transport, never the calling code. +- TLS: `ITlsSession` interface; only `StubTlsSession` ships here and throws + `TlsStubError` naming wolfSSL as missing. HTTPS URLs fail loud, never + silently downgrade. +- HTTP/2: `ClientConfig::h2` is accepted and negotiated only when a real TLS + session exists; cleartext h2c returns a marked `H2StubError`. HTTP/1.1 is + the working protocol in this part. +- OpenAPI builder input is adapter structs (`RouteDecl`, `ModelDecl`, + `FieldDecl`, `ParamDecl`) mapping 1:1 from `molo::validate::Model` field + descriptors and router `Route` records owned by part 05. The adapter lives + with part 05; this part never includes app headers. +- `AppTransport` dispatches through a handler callable with the signature + `Task(const ClientRequest&)` plus lifespan hooks. Part 05 + binds its `Application` to that signature; this part never includes it. +- Doc routes serve `/openapi.json`, `/openapi.yaml`, `/docs`, `/redoc`. + This part returns the payload bytes; mounting is owned by the app. + Swagger/ReDoc full UI assets are packed by the part 09 embedder; the HTML + shells here are marked placeholders. +- Observability: `Reporter` hook plus atomic `ClientStats`. Default + `NullReporter` is an inline no-op (near-zero overhead path). + +## 4. Out of scope (other parts) + +Realtime clients (part 06), validation framework and routers (part 05), +event loop and TLS sessions (part 01), spans and metrics backends (part 02), +KV backend for idempotency (part 04, `InMemoryIdempotencyStore` here is the +dev default), CLI and embedder (part 09), living example (part 10). diff --git a/src/clients/auth.cpp b/src/clients/auth.cpp new file mode 100644 index 0000000..38f9faf --- /dev/null +++ b/src/clients/auth.cpp @@ -0,0 +1,58 @@ +#include "clients/auth.h" + +#include + +namespace molo::clients { + +std::string base64_encode(const std::string& input) { + static const char* kTable = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + out.reserve(((input.size() + 2) / 3) * 4); + for (std::size_t i = 0; i < input.size(); i += 3) { + unsigned n = (unsigned char)input[i] << 16; + int bytes = 1; + if (i + 1 < input.size()) { + n |= (unsigned char)input[i + 1] << 8; + ++bytes; + } + if (i + 2 < input.size()) { + n |= (unsigned char)input[i + 2]; + ++bytes; + } + out.push_back(kTable[(n >> 18) & 63]); + out.push_back(kTable[(n >> 12) & 63]); + out.push_back(bytes > 1 ? kTable[(n >> 6) & 63] : '='); + out.push_back(bytes > 2 ? kTable[n & 63] : '='); + } + return out; +} + +BasicAuth::BasicAuth(std::string username, std::string password) + : username_(std::move(username)), password_(std::move(password)) {} + +void BasicAuth::apply(ClientRequest& request) const { + request.headers().set("Authorization", + "Basic " + base64_encode(username_ + ":" + password_)); +} + +BearerAuth::BearerAuth(std::string token) : token_(std::move(token)) {} + +void BearerAuth::apply(ClientRequest& request) const { + request.headers().set("Authorization", "Bearer " + token_); +} + +ApiKeyAuth::ApiKeyAuth(std::string key, std::string header, + std::string query_param) + : key_(std::move(key)), + header_(std::move(header)), + query_param_(std::move(query_param)) {} + +void ApiKeyAuth::apply(ClientRequest& request) const { + if (!query_param_.empty()) + request.url().set_query_arg(query_param_, key_); + else + request.headers().set(header_, key_); +} + +} // namespace molo::clients diff --git a/src/clients/auth.h b/src/clients/auth.h new file mode 100644 index 0000000..a4dbb30 --- /dev/null +++ b/src/clients/auth.h @@ -0,0 +1,57 @@ +#pragma once + +// Outbound auth providers: Basic, Bearer, ApiKey, plus a custom base. + +#include +#include + +#include "clients/request.h" + +namespace molo::clients { + +class Auth { + public: + virtual ~Auth() = default; + virtual void apply(ClientRequest& request) const = 0; + virtual const char* scheme() const = 0; +}; + +class BasicAuth : public Auth { + public: + BasicAuth(std::string username, std::string password); + void apply(ClientRequest& request) const override; + const char* scheme() const override { return "basic"; } + + private: + std::string username_; + std::string password_; +}; + +class BearerAuth : public Auth { + public: + explicit BearerAuth(std::string token); + void apply(ClientRequest& request) const override; + const char* scheme() const override { return "bearer"; } + + private: + std::string token_; +}; + +class ApiKeyAuth : public Auth { + public: + // Places the key in a header (default "X-Api-Key") or, when query_param is + // set, in the URL query string instead. + ApiKeyAuth(std::string key, std::string header = "X-Api-Key", + std::string query_param = ""); + void apply(ClientRequest& request) const override; + const char* scheme() const override { return "apikey"; } + + private: + std::string key_; + std::string header_; + std::string query_param_; +}; + +std::string base64_encode(const std::string& input); + +} // namespace molo::clients diff --git a/src/clients/body/form.cpp b/src/clients/body/form.cpp new file mode 100644 index 0000000..44e1b29 --- /dev/null +++ b/src/clients/body/form.cpp @@ -0,0 +1,110 @@ +#include "clients/body/form.h" + +#include +#include + +#include "clients/caps.h" +#include "clients/url.h" + +namespace molo::clients::body { + +void UrlEncodedForm::add(const std::string& name, const std::string& value) { + if (fields_.size() >= 1000) throw FormError("too many fields"); + fields_.emplace_back(name, value); +} + +std::string UrlEncodedForm::encode() const { + std::string out; + for (const auto& f : fields_) { + if (!out.empty()) out.push_back('&'); + out += Url::encode_component(f.first); + out.push_back('='); + out += Url::encode_component(f.second); + } + return out; +} + +UrlEncodedForm UrlEncodedForm::decode(const std::string& text) { + UrlEncodedForm form; + std::string rest = text; + while (!rest.empty()) { + auto amp = rest.find('&'); + std::string pair = + amp == std::string::npos ? rest : rest.substr(0, amp); + rest = amp == std::string::npos ? "" : rest.substr(amp + 1); + if (pair.empty()) continue; + auto eq = pair.find('='); + if (eq == std::string::npos) + form.add(Url::decode_component(pair), ""); + else + form.add(Url::decode_component(pair.substr(0, eq)), + Url::decode_component(pair.substr(eq + 1))); + } + return form; +} + +namespace { +std::string random_boundary() { + static thread_local std::mt19937 rng{std::random_device{}()}; + static const char* kDigits = "0123456789abcdef"; + std::string b = "molo"; + for (int i = 0; i < 24; ++i) b.push_back(kDigits[rng() % 16]); + return b; +} +} // namespace + +MultipartBody::MultipartBody() : boundary_(random_boundary()) {} +MultipartBody::MultipartBody(std::string boundary) + : boundary_(std::move(boundary)) {} + +void MultipartBody::add_text(const std::string& name, const std::string& value) { + MultipartPart part; + part.name = name; + part.content_type = "text/plain"; + part.data = value; + parts_.push_back(std::move(part)); +} + +void MultipartBody::add_file(const std::string& name, + const std::string& filename, + const std::string& content_type, + const std::string& data) { + if (data.size() > Caps::kMaxMultipartPart) + throw FormError("multipart part exceeds cap"); + MultipartPart part; + part.name = name; + part.filename = filename; + part.content_type = content_type.empty() ? "application/octet-stream" + : content_type; + part.data = data; + parts_.push_back(std::move(part)); +} + +std::string MultipartBody::encode() const { + std::ostringstream out; + for (const auto& part : parts_) { + out << "--" << boundary_ << "\r\n"; + out << "Content-Disposition: form-data; name=\"" << part.name << "\""; + if (part.filename) out << "; filename=\"" << *part.filename << "\""; + out << "\r\nContent-Type: " << part.content_type << "\r\n\r\n"; + out << part.data << "\r\n"; + } + out << "--" << boundary_ << "--\r\n"; + std::string body = out.str(); + if (body.size() > Caps::kMaxRequestTotal) + throw FormError("multipart body exceeds request cap"); + return body; +} + +std::size_t MultipartBody::encoded_size() const { + std::size_t n = 0; + for (const auto& part : parts_) { + n += boundary_.size() + 6 + part.name.size() + 34; + if (part.filename) n += part.filename->size() + 13; + n += part.content_type.size() + 18 + part.data.size() + 2; + } + n += boundary_.size() + 8; + return n; +} + +} // namespace molo::clients::body diff --git a/src/clients/body/form.h b/src/clients/body/form.h new file mode 100644 index 0000000..7996eee --- /dev/null +++ b/src/clients/body/form.h @@ -0,0 +1,69 @@ +#pragma once + +// URL-encoded forms and multipart bodies (file upload parts). + +#include +#include +#include +#include +#include +#include + +namespace molo::clients::body { + +class FormError : public std::runtime_error { + public: + explicit FormError(const std::string& msg) + : std::runtime_error("molo::clients::body::Form: " + msg) {} +}; + +class UrlEncodedForm { + public: + UrlEncodedForm() = default; + + void add(const std::string& name, const std::string& value); + std::string encode() const; + static UrlEncodedForm decode(const std::string& text); + + const std::vector>& fields() const { + return fields_; + } + static const char* content_type() { + return "application/x-www-form-urlencoded"; + } + + private: + std::vector> fields_; +}; + +struct MultipartPart { + std::string name; + std::optional filename; + std::string content_type = "text/plain"; + std::string data; +}; + +class MultipartBody { + public: + MultipartBody(); + explicit MultipartBody(std::string boundary); + + void add_text(const std::string& name, const std::string& value); + void add_file(const std::string& name, const std::string& filename, + const std::string& content_type, const std::string& data); + + const std::string& boundary() const { return boundary_; } + std::string content_type() const { + return "multipart/form-data; boundary=" + boundary_; + } + std::string encode() const; + std::size_t encoded_size() const; + + const std::vector& parts() const { return parts_; } + + private: + std::string boundary_; + std::vector parts_; +}; + +} // namespace molo::clients::body diff --git a/src/clients/body/json.cpp b/src/clients/body/json.cpp new file mode 100644 index 0000000..0fcde05 --- /dev/null +++ b/src/clients/body/json.cpp @@ -0,0 +1,390 @@ +#include "clients/body/json.h" + +#include +#include + +#include "clients/caps.h" + +namespace molo::clients::body { +namespace { + +struct Parser { + const char* p; + const char* end; + int depth = 0; + + void skip_ws() { + while (p < end && (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r')) + ++p; + } + bool consume(char c) { + skip_ws(); + if (p < end && *p == c) { + ++p; + return true; + } + return false; + } + [[noreturn]] void fail(const std::string& msg) { + throw JsonError(msg); + } + Json parse_value() { + skip_ws(); + if (p >= end) fail("unexpected end"); + if (*p == '{') return parse_object(); + if (*p == '[') return parse_array(); + if (*p == '"') return Json(parse_string()); + if (*p == 't') { + expect("true"); + return Json(true); + } + if (*p == 'f') { + expect("false"); + return Json(false); + } + if (*p == 'n') { + expect("null"); + return Json(nullptr); + } + return Json(parse_number()); + } + void expect(const char* word) { + for (const char* w = word; *w; ++w, ++p) { + if (p >= end || *p != *w) fail("bad literal"); + } + } + Json parse_object() { + if (++depth > molo::clients::Caps::kJsonMaxDepth) fail("depth exceeded"); + ++p; // { + Json::Object obj; + skip_ws(); + if (consume('}')) { + --depth; + return Json(std::move(obj)); + } + while (true) { + skip_ws(); + if (p >= end || *p != '"') fail("expected string key"); + std::string key = parse_string(); + if (!consume(':')) fail("expected ':'"); + obj.emplace(std::move(key), parse_value()); + skip_ws(); + if (consume('}')) break; + if (!consume(',')) fail("expected ',' or '}'"); + } + --depth; + return Json(std::move(obj)); + } + Json parse_array() { + if (++depth > molo::clients::Caps::kJsonMaxDepth) fail("depth exceeded"); + ++p; // [ + Json::Array arr; + skip_ws(); + if (consume(']')) { + --depth; + return Json(std::move(arr)); + } + while (true) { + arr.push_back(parse_value()); + skip_ws(); + if (consume(']')) break; + if (!consume(',')) fail("expected ',' or ']'"); + } + --depth; + return Json(std::move(arr)); + } + std::string parse_string() { + ++p; // opening quote + std::string out; + while (p < end && *p != '"') { + char c = *p++; + if (c == '\\') { + if (p >= end) fail("bad escape"); + char e = *p++; + switch (e) { + case '"': out.push_back('"'); break; + case '\\': out.push_back('\\'); break; + case '/': out.push_back('/'); break; + case 'b': out.push_back('\b'); break; + case 'f': out.push_back('\f'); break; + case 'n': out.push_back('\n'); break; + case 'r': out.push_back('\r'); break; + case 't': out.push_back('\t'); break; + case 'u': { + if (p + 4 > end) fail("bad unicode escape"); + unsigned code = 0; + for (int i = 0; i < 4; ++i, ++p) { + code <<= 4; + char h = *p; + if (h >= '0' && h <= '9') code |= (unsigned)(h - '0'); + else if (h >= 'a' && h <= 'f') code |= (unsigned)(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') code |= (unsigned)(h - 'A' + 10); + else fail("bad unicode escape"); + } + if (code < 0x80) { + out.push_back((char)code); + } else if (code < 0x800) { + out.push_back((char)(0xC0 | (code >> 6))); + out.push_back((char)(0x80 | (code & 0x3F))); + } else { + out.push_back((char)(0xE0 | (code >> 12))); + out.push_back((char)(0x80 | ((code >> 6) & 0x3F))); + out.push_back((char)(0x80 | (code & 0x3F))); + } + break; + } + default: fail("bad escape"); + } + } else { + out.push_back(c); + } + } + if (p >= end) fail("unterminated string"); + ++p; // closing quote + return out; + } + double parse_number() { + const char* start = p; + if (p < end && (*p == '-')) ++p; + while (p < end && (std::isdigit((unsigned char)*p) || *p == '.' || + *p == 'e' || *p == 'E' || *p == '+' || *p == '-')) + ++p; + if (p == start) fail("bad value"); + try { + return std::stod(std::string(start, p)); + } catch (...) { + fail("bad number"); + } + } +}; + +void dump_into(const Json& v, std::ostringstream& out) { + if (v.is_null()) { + out << "null"; + } else if (v.is_bool()) { + out << (v.as_bool() ? "true" : "false"); + } else if (v.is_number()) { + double d = v.as_number(); + long as_long = static_cast(d); + if (d == (double)as_long) + out << as_long; + else + out << d; + } else if (v.is_string()) { + out << '"' << json_escape(v.as_string()) << '"'; + } else if (v.is_array()) { + out << '['; + bool first = true; + for (const auto& item : v.as_array()) { + if (!first) out << ','; + first = false; + dump_into(item, out); + } + out << ']'; + } else { + out << '{'; + bool first = true; + for (const auto& kv : v.as_object()) { + if (!first) out << ','; + first = false; + out << '"' << json_escape(kv.first) << "\":"; + dump_into(kv.second, out); + } + out << '}'; + } +} + +bool yaml_needs_quote(const std::string& s) { + if (s.empty()) return true; + for (char c : s) { + if (!(std::isalnum((unsigned char)c) || c == '_' || c == '-' || c == '.' || + c == '/' || c == '{' || c == '}' || c == '+' || c == ':')) + return true; + } + if (s == "true" || s == "false" || s == "null" || s == "~") return true; + if (s[0] == ':' || s[0] == '#' || s[0] == '-' || s[0] == '?' || + s[0] == '[' || s[0] == ']' || s[0] == '{' || s[0] == '}' || + s[0] == ',' || s[0] == '*' || s[0] == '&' || s[0] == '!' || + s[0] == '|' || s[0] == '>' || s[0] == '\'' || s[0] == '"' || + s[0] == '%' || s[0] == '@' || s[0] == '`') + return true; + return false; +} + +void yaml_scalar(const Json& v, std::ostringstream& out) { + if (v.is_null()) { + out << "null"; + } else if (v.is_bool()) { + out << (v.as_bool() ? "true" : "false"); + } else if (v.is_number()) { + double d = v.as_number(); + long as_long = static_cast(d); + if (d == (double)as_long) + out << as_long; + else + out << d; + } else if (v.is_string()) { + const std::string& s = v.as_string(); + if (yaml_needs_quote(s)) + out << '"' << json_escape(s) << '"'; + else + out << s; + } +} + +void yaml_into(const Json& v, std::ostringstream& out, int indent); + +void yaml_block(const Json& v, std::ostringstream& out, int indent, + bool dash_prefix) { + std::string pad((std::size_t)indent * 2, ' '); + if (v.is_object()) { + if (v.as_object().empty()) { + out << pad << (dash_prefix ? "- " : "") << "{}\n"; + return; + } + bool first = true; + for (const auto& kv : v.as_object()) { + out << pad; + if (dash_prefix && first) { + out << "- "; + first = false; + } + const std::string& k = kv.first; + if (yaml_needs_quote(k)) + out << '"' << json_escape(k) << '"'; + else + out << k; + const Json& child = kv.second; + if (child.is_object() || child.is_array()) { + if ((child.is_object() && child.as_object().empty()) || + (child.is_array() && child.as_array().empty())) { + out << ": " << (child.is_object() ? "{}" : "[]") << "\n"; + } else { + out << ":\n"; + yaml_into(child, out, indent + 1); + } + } else { + out << ": "; + yaml_scalar(child, out); + out << "\n"; + } + } + } else if (v.is_array()) { + if (v.as_array().empty()) { + out << pad << "[]\n"; + return; + } + for (const auto& item : v.as_array()) { + if (item.is_object() || item.is_array()) { + yaml_block(item, out, indent, true); + } else { + out << pad << "- "; + yaml_scalar(item, out); + out << "\n"; + } + } + } else { + out << pad; + yaml_scalar(v, out); + out << "\n"; + } +} + +void yaml_into(const Json& v, std::ostringstream& out, int indent) { + yaml_block(v, out, indent, false); +} + +} // namespace + +Json Json::array() { return Json(Array{}); } +Json Json::object() { return Json(Object{}); } + +Json Json::parse(const std::string& text) { + Parser parser{text.data(), text.data() + text.size(), 0}; + Json v = parser.parse_value(); + parser.skip_ws(); + if (parser.p != parser.end) throw JsonError("trailing characters"); + return v; +} + +const Json* Json::find(const std::string& key) const { + if (!is_object()) return nullptr; + auto it = as_object().find(key); + return it == as_object().end() ? nullptr : &it->second; +} + +void Json::set(const std::string& key, Json value) { + if (!is_object()) value_ = Object{}; + as_object()[key] = std::move(value); +} + +void Json::push(Json value) { + if (!is_array()) value_ = Array{}; + as_array().push_back(std::move(value)); +} + +std::string Json::dump() const { + std::ostringstream out; + dump_into(*this, out); + return out.str(); +} + +bool Json::equals(const Json& other) const { + if (value_.index() != other.value_.index()) { + // Numbers compare across int/double spellings via as_number. + if (is_number() && other.is_number()) + return as_number() == other.as_number(); + return false; + } + if (is_null()) return true; + if (is_bool()) return as_bool() == other.as_bool(); + if (is_number()) return as_number() == other.as_number(); + if (is_string()) return as_string() == other.as_string(); + if (is_array()) { + if (as_array().size() != other.as_array().size()) return false; + for (std::size_t i = 0; i < as_array().size(); ++i) { + if (!as_array()[i].equals(other.as_array()[i])) return false; + } + return true; + } + if (as_object().size() != other.as_object().size()) return false; + for (const auto& kv : as_object()) { + auto it = other.as_object().find(kv.first); + if (it == other.as_object().end() || !kv.second.equals(it->second)) + return false; + } + return true; +} + +std::string json_escape(const std::string& text) { + std::string out; + out.reserve(text.size() + 8); + for (char c : text) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if ((unsigned char)c < 0x20) { + char buf[8]; + snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out.push_back(c); + } + } + } + return out; +} + +std::string yaml_dump(const Json& value) { + std::ostringstream out; + yaml_into(value, out, 0); + return out.str(); +} + +} // namespace molo::clients::body diff --git a/src/clients/body/json.h b/src/clients/body/json.h new file mode 100644 index 0000000..48d5d1c --- /dev/null +++ b/src/clients/body/json.h @@ -0,0 +1,82 @@ +#pragma once + +// Minimal JSON DOM with parser and deterministic serializer. +// Objects use std::map so output key order is stable (OpenAPI requirement). + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace molo::clients::body { + +class JsonError : public std::runtime_error { + public: + explicit JsonError(const std::string& msg) + : std::runtime_error("molo::clients::body::Json: " + msg) {} +}; + +class Json { + public: + using Array = std::vector; + using Object = std::map; + + Json() : value_(nullptr) {} + Json(std::nullptr_t) : value_(nullptr) {} + Json(bool b) : value_(b) {} + Json(int i) : value_(static_cast(i)) {} + Json(long l) : value_(static_cast(l)) {} + Json(double d) : value_(d) {} + Json(const char* s) : value_(std::string(s)) {} + Json(const std::string& s) : value_(s) {} + Json(std::string&& s) : value_(std::move(s)) {} + Json(const Array& a) : value_(a) {} + Json(Array&& a) : value_(std::move(a)) {} + Json(const Object& o) : value_(o) {} + Json(Object&& o) : value_(std::move(o)) {} + + static Json array(); + static Json object(); + static Json parse(const std::string& text); + + bool is_null() const { return std::holds_alternative(value_); } + bool is_bool() const { return std::holds_alternative(value_); } + bool is_number() const { return std::holds_alternative(value_); } + bool is_string() const { + return std::holds_alternative(value_); + } + bool is_array() const { return std::holds_alternative(value_); } + bool is_object() const { return std::holds_alternative(value_); } + + bool as_bool() const { return std::get(value_); } + double as_number() const { return std::get(value_); } + long as_int() const { return static_cast(std::get(value_)); } + const std::string& as_string() const { + return std::get(value_); + } + const Array& as_array() const { return std::get(value_); } + const Object& as_object() const { return std::get(value_); } + Array& as_array() { return std::get(value_); } + Object& as_object() { return std::get(value_); } + + const Json* find(const std::string& key) const; + bool has(const std::string& key) const { return find(key) != nullptr; } + void set(const std::string& key, Json value); + void push(Json value); + + std::string dump() const; // Compact, deterministic. + bool equals(const Json& other) const; + + private: + std::variant value_; +}; + +std::string json_escape(const std::string& text); +// Minimal YAML emitter for the Json subset used by OpenAPI output. +std::string yaml_dump(const Json& value); + +} // namespace molo::clients::body diff --git a/src/clients/body/stream.cpp b/src/clients/body/stream.cpp new file mode 100644 index 0000000..7d6c902 --- /dev/null +++ b/src/clients/body/stream.cpp @@ -0,0 +1,92 @@ +#include "clients/body/stream.h" + +#include + +#include + +namespace molo::clients::body { + +StringReader::StringReader(std::string data) : data_(std::move(data)) {} + +std::string StringReader::read_some(std::size_t max_bytes) { + if (eof()) return {}; + if (max_bytes == 0) return {}; + std::size_t n = data_.size() - pos_; + if (n > max_bytes) n = max_bytes; + std::string out = data_.substr(pos_, n); + pos_ += n; + return out; +} + +bool StringReader::eof() const { return pos_ >= data_.size(); } + +std::optional StringReader::known_size() const { + return data_.size(); +} + +FileReader::FileReader(const std::string& path) { + file_ = std::fopen(path.c_str(), "rb"); + if (!file_) throw StreamError("cannot open file: " + path); + struct stat st {}; + if (::stat(path.c_str(), &st) == 0 && st.st_size >= 0) + size_ = static_cast(st.st_size); +} + +FileReader::~FileReader() { + if (file_) std::fclose(file_); +} + +std::string FileReader::read_some(std::size_t max_bytes) { + if (!file_ || eof_ || max_bytes == 0) return {}; + std::string out; + out.resize(max_bytes); + std::size_t n = std::fread(out.data(), 1, max_bytes, file_); + if (n < max_bytes) { + if (std::ferror(file_)) throw StreamError("file read failed"); + eof_ = true; + } + out.resize(n); + pos_ += n; + return out; +} + +bool FileReader::eof() const { + if (eof_) return true; + return size_ > 0 && pos_ >= size_; +} + +std::optional FileReader::known_size() const { + if (size_ == 0 && pos_ == 0) return std::nullopt; + return size_; +} + +CallbackReader::CallbackReader(ChunkReader reader, + std::optional size) + : reader_(std::move(reader)), size_(size) {} + +std::string CallbackReader::read_some(std::size_t max_bytes) { + if (eof_) return {}; + bool done = false; + std::string chunk = reader_(max_bytes, done); + if (done) eof_ = true; + return chunk; +} + +bool CallbackReader::eof() const { return eof_; } + +std::optional CallbackReader::known_size() const { return size_; } + +CappedCollector::CappedCollector(std::size_t cap) : cap_(cap) {} + +bool CappedCollector::push(const std::string& chunk) { + if (data_.size() + chunk.size() > cap_) + throw StreamError("download exceeds cap"); + data_ += chunk; + return true; +} + +ChunkSink CappedCollector::sink() { + return [this](const std::string& chunk) { return this->push(chunk); }; +} + +} // namespace molo::clients::body diff --git a/src/clients/body/stream.h b/src/clients/body/stream.h new file mode 100644 index 0000000..9db99f0 --- /dev/null +++ b/src/clients/body/stream.h @@ -0,0 +1,93 @@ +#pragma once + +// Streaming bodies: bounded chunk readers for uploads and chunk sinks for +// downloads. Files over 1 MB must use this API, never full buffering. + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::clients::body { + +class StreamError : public std::runtime_error { + public: + explicit StreamError(const std::string& msg) + : std::runtime_error("molo::clients::body::Stream: " + msg) {} +}; + +// Pull reader for request bodies. Returns up to max_bytes per call. +class StreamReader { + public: + virtual ~StreamReader() = default; + virtual std::string read_some(std::size_t max_bytes) = 0; + virtual bool eof() const = 0; + virtual std::optional known_size() const { return std::nullopt; } +}; + +class StringReader : public StreamReader { + public: + explicit StringReader(std::string data); + std::string read_some(std::size_t max_bytes) override; + bool eof() const override; + std::optional known_size() const override; + + private: + std::string data_; + std::size_t pos_ = 0; +}; + +class FileReader : public StreamReader { + public: + explicit FileReader(const std::string& path); + ~FileReader() override; + FileReader(const FileReader&) = delete; + FileReader& operator=(const FileReader&) = delete; + std::string read_some(std::size_t max_bytes) override; + bool eof() const override; + std::optional known_size() const override; + + private: + std::FILE* file_ = nullptr; + std::size_t size_ = 0; + std::size_t pos_ = 0; + bool eof_ = false; +}; + +using ChunkReader = std::function; + +class CallbackReader : public StreamReader { + public: + explicit CallbackReader(ChunkReader reader, + std::optional size = std::nullopt); + std::string read_some(std::size_t max_bytes) override; + bool eof() const override; + std::optional known_size() const override; + + private: + ChunkReader reader_; + std::optional size_; + bool eof_ = false; +}; + +// Push sink for streamed downloads. Return false to cancel the transfer. +using ChunkSink = std::function; + +// Collects chunks into a string, enforcing a byte cap. +class CappedCollector { + public: + explicit CappedCollector(std::size_t cap); + bool push(const std::string& chunk); + const std::string& data() const { return data_; } + std::size_t bytes() const { return data_.size(); } + ChunkSink sink(); + + private: + std::size_t cap_; + std::string data_; +}; + +} // namespace molo::clients::body diff --git a/src/clients/body/xml.cpp b/src/clients/body/xml.cpp new file mode 100644 index 0000000..09fa397 --- /dev/null +++ b/src/clients/body/xml.cpp @@ -0,0 +1,147 @@ +#include "clients/body/xml.h" + +#include + +#include "clients/caps.h" + +namespace molo::clients::body { + +std::string xml_escape(const std::string& text) { + std::string out; + out.reserve(text.size()); + for (char c : text) { + switch (c) { + case '&': out += "&"; break; + case '<': out += "<"; break; + case '>': out += ">"; break; + case '"': out += """; break; + case '\'': out += "'"; break; + default: out.push_back(c); + } + } + return out; +} + +std::string xml_unescape(const std::string& text) { + std::string out; + out.reserve(text.size()); + for (std::size_t i = 0; i < text.size();) { + if (text.compare(i, 5, "&") == 0) { + out.push_back('&'); + i += 5; + } else if (text.compare(i, 4, "<") == 0) { + out.push_back('<'); + i += 4; + } else if (text.compare(i, 4, ">") == 0) { + out.push_back('>'); + i += 4; + } else if (text.compare(i, 6, """) == 0) { + out.push_back('"'); + i += 6; + } else if (text.compare(i, 6, "'") == 0) { + out.push_back('\''); + i += 6; + } else { + out.push_back(text[i++]); + } + } + return out; +} + +XmlBuilder::XmlBuilder(std::string root) : root_(std::move(root)) {} + +XmlBuilder& XmlBuilder::attr(const std::string& name, + const std::string& value) { + attrs_[name] = value; + return *this; +} + +XmlBuilder& XmlBuilder::text(const std::string& text) { + text_ = text; + return *this; +} + +XmlBuilder& XmlBuilder::child(const std::string& name, const std::string& text) { + children_.emplace_back(name, text); + return *this; +} + +std::string XmlBuilder::str() const { + std::ostringstream out; + out << "<" << root_; + for (const auto& kv : attrs_) + out << ' ' << kv.first << "=\"" << xml_escape(kv.second) << "\""; + out << ">"; + out << xml_escape(text_); + for (const auto& c : children_) + out << "<" << c.first << ">" << xml_escape(c.second) << ""; + out << ""; + return out.str(); +} + +std::map xml_parse_flat(const std::string& text) { + if (text.size() > Caps::kMaxMemoryBody) throw XmlError("document too large"); + std::string body = text; + auto decl = body.find("?>"); + if (body.rfind("'); + if (root_end == std::string::npos) throw XmlError("bad root"); + std::string root_tag = body.substr(1, root_end - 1); + auto space = root_tag.find_first_of(" \t\n\r/"); + std::string root_name = + space == std::string::npos ? root_tag : root_tag.substr(0, space); + std::string close = ""; + if (body.size() < close.size() || + body.compare(body.size() - close.size(), close.size(), close) != 0) + throw XmlError("root not closed"); + std::string inner = body.substr(root_end + 1, body.size() - root_end - 1 - + close.size()); + std::map out; + std::size_t i = 0; + while (i < inner.size()) { + while (i < inner.size() && inner[i] != '<') { + if (inner[i] != ' ' && inner[i] != '\t' && inner[i] != '\n' && + inner[i] != '\r') + throw XmlError("mixed content not supported"); + ++i; + } + if (i >= inner.size()) break; + auto tag_end = inner.find('>', i); + if (tag_end == std::string::npos) throw XmlError("bad child tag"); + std::string tag = inner.substr(i + 1, tag_end - i - 1); + auto sp = tag.find_first_of(" \t\n\r/"); + std::string name = sp == std::string::npos ? tag : tag.substr(0, sp); + if (name.empty()) throw XmlError("empty tag"); + if (!tag.empty() && tag.back() == '/') { + out[name] = ""; + i = tag_end + 1; + continue; + } + std::string child_close = ""; + auto close_at = inner.find(child_close, tag_end + 1); + if (close_at == std::string::npos) throw XmlError("child not closed"); + std::string value = inner.substr(tag_end + 1, close_at - tag_end - 1); + if (value.find('<') != std::string::npos) + throw XmlError("nested markup not supported"); + out[name] = xml_unescape(trim(value)); + i = close_at + child_close.size(); + } + return out; +} + +} // namespace molo::clients::body diff --git a/src/clients/body/xml.h b/src/clients/body/xml.h new file mode 100644 index 0000000..0634a27 --- /dev/null +++ b/src/clients/body/xml.h @@ -0,0 +1,41 @@ +#pragma once + +// XML escape plus a small builder and a lite flat parser for client bodies. + +#include +#include +#include +#include +#include + +namespace molo::clients::body { + +class XmlError : public std::runtime_error { + public: + explicit XmlError(const std::string& msg) + : std::runtime_error("molo::clients::body::Xml: " + msg) {} +}; + +std::string xml_escape(const std::string& text); +std::string xml_unescape(const std::string& text); + +class XmlBuilder { + public: + explicit XmlBuilder(std::string root); + XmlBuilder& attr(const std::string& name, const std::string& value); + XmlBuilder& text(const std::string& text); + XmlBuilder& child(const std::string& name, const std::string& text); + std::string str() const; + + private: + std::string root_; + std::map attrs_; + std::vector> children_; + std::string text_; +}; + +// Parses flat documents: vv. Nested markup inside +// a child value is rejected; attributes are ignored. +std::map xml_parse_flat(const std::string& text); + +} // namespace molo::clients::body diff --git a/src/clients/caps.h b/src/clients/caps.h new file mode 100644 index 0000000..1cedbc9 --- /dev/null +++ b/src/clients/caps.h @@ -0,0 +1,60 @@ +#pragma once + +// Static ceilings for part 08. Every knob has a hard cap; nothing autotunes. + +#include +#include + +namespace molo::clients { + +inline constexpr const char* kFrameworkVersion = "0.1.0"; +inline constexpr const char* kOpenApiVersion = "3.1.0"; +inline constexpr const char* kUserAgent = "MoloApp-Client/0.1.0"; + +struct Caps { + // Timeout defaults (overridable per request, always bounded). + static constexpr std::chrono::milliseconds kConnectTimeout{5000}; + static constexpr std::chrono::milliseconds kReadTimeout{30000}; + static constexpr std::chrono::milliseconds kWriteTimeout{30000}; + static constexpr std::chrono::milliseconds kPoolAcquireTimeout{5000}; + static constexpr std::chrono::milliseconds kTotalTimeout{60000}; + // Pool caps. + static constexpr std::size_t kPoolMaxTotal = 100; + static constexpr std::size_t kPoolMaxPerHost = 20; + static constexpr std::size_t kPoolMaxIdlePerHost = 10; + static constexpr std::chrono::seconds kPoolKeepAlive{60}; + // Retry policy. + static constexpr int kMaxAttempts = 3; + static constexpr std::chrono::milliseconds kRetryBase{200}; + static constexpr std::chrono::milliseconds kRetryCap{5000}; + // Redirects. + static constexpr int kMaxRedirects = 5; + // Body caps. + static constexpr std::size_t kMaxMemoryBody = 10u * 1024u * 1024u; + static constexpr std::size_t kMaxMultipartPart = 512u * 1024u * 1024u; + static constexpr std::size_t kMaxRequestTotal = 1024u * 1024u * 1024u; + // Pagination. + static constexpr int kPageDefault = 20; + static constexpr int kPageMax = 100; + static constexpr int kPaginatorMaxPages = 100; + // OpenAPI generation caps. + static constexpr std::size_t kOpenApiMaxPaths = 5000; + static constexpr std::size_t kOpenApiMaxSchemas = 2000; + static constexpr std::size_t kOpenApiMaxBytes = 10u * 1024u * 1024u; + // Wire caps. + static constexpr std::size_t kMaxUrlLength = 8192; + static constexpr std::size_t kMaxHeaders = 200; + static constexpr std::size_t kMaxHeaderLine = 16u * 1024u; + static constexpr std::size_t kMaxHeaderBlock = 64u * 1024u; + static constexpr std::size_t kStreamChunk = 64u * 1024u; + // Cookie jar caps. + static constexpr std::size_t kCookieJarMax = 4096; + static constexpr std::size_t kCookieMaxBytes = 8u * 1024u; + // Idempotency store caps. + static constexpr std::chrono::hours kIdempotencyTtl{24}; + static constexpr std::size_t kIdempotencyMaxEntries = 100000; + // JSON parser cap (matches part 05 form cap). + static constexpr int kJsonMaxDepth = 32; +}; + +} // namespace molo::clients diff --git a/src/clients/client.cpp b/src/clients/client.cpp new file mode 100644 index 0000000..1d4af36 --- /dev/null +++ b/src/clients/client.cpp @@ -0,0 +1,399 @@ +#include "clients/client.h" + +#include +#include +#include +#include + +#include "clients/transports/http_transport.h" +#include "operability/resilience/deadline.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::clients { + +namespace { + +// Process-wide clock and null observer for the client breaker and deadline +// propagation. File-local so clients links stay free of supervisor TUs. +class ChronoClock : public molo::common::Clock { + public: + std::int64_t wall_ms() const override { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + std::int64_t steady_ms() const override { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + } + void sleep_ms(long ms) override { + if (ms > 0) std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } +}; + +class NullClientObserver : public molo::common::Observer { + public: + void event(std::string_view, std::string_view) override {} + void counter(std::string_view, long) override {} + void gauge(std::string_view, double) override {} +}; + +molo::common::Clock& ClientClock() { + static ChronoClock c; + return c; +} + +molo::common::Observer& ClientObserver() { + static NullClientObserver o; + return o; +} + +// Clamp every hop timeout to the propagated budget. (PATCHES names a +// Deadline::budget_for helper that does not exist on the part; min-clamp +// keeps the same "never exceed budget" contract.) +void ClampTimeouts(ClientRequest& req, std::int64_t budget_ms) { + Timeouts t = req.timeouts().value_or(Timeouts::defaults()); + const auto cap = std::chrono::milliseconds(budget_ms); + t.connect = std::min(t.connect, cap); + t.read = std::min(t.read, cap); + t.write = std::min(t.write, cap); + t.pool_acquire = std::min(t.pool_acquire, cap); + t.total = std::min(t.total, cap); + req.timeouts(t); +} + +} // namespace + +Client::Client(std::unique_ptr transport, ClientConfig config) + : config_(std::move(config)), + transport_(std::move(transport)), + pool_(std::make_shared()), + breaker_(ClientClock(), ClientObserver(), "client") { + if (!transport_) throw TransportError("Client needs a transport"); + if (config_.proxy_from_env && config_.proxy.empty()) + config_.proxy = ProxyConfig::from_env(); + // Hand the shared pool and proxy selection to the real transport. + if (auto* http = dynamic_cast(transport_.get())) { + http->set_pool(pool_); + http->set_proxy(config_.proxy); + } +} + +Task Client::send(ClientRequest request) { + // 1. Breaker gate, innermost stage: never retry into an open breaker. + if (!breaker_.allow()) { + co_return ClientResponse(503, HeaderMap{}, "breaker open"); + } + // 2. Deadline propagation: convert the absolute inbound X-Deadline-Ms to + // remaining budget minus the 50 ms floor margin for the outbound hop. + if (const auto abs = request.deadline_ms()) { + molo::resilience::Deadline dl(ClientClock()); + const std::int64_t budget = dl.remaining_ms(*abs); + if (budget == 0) { + breaker_.on_failure(); + co_return ClientResponse(504, HeaderMap{}, "deadline exceeded"); + } + request.headers().set( + "X-Deadline-Ms", + molo::resilience::Deadline::format_header(ClientClock().wall_ms() + budget)); + ClampTimeouts(request, budget); + } + ClientResponse resp; + try { + resp = co_await send_inner(std::move(request)); + } catch (...) { + breaker_.on_failure(); + throw; + } + // Transport errors (above) and 5xx count as breaker failures; everything + // else counts as success. A shed 503 from downstream (Retry-After set) + // is not a local failure: downstream already counted it. + const bool shed = resp.status() == 503 && resp.headers().has("Retry-After"); + if (shed || resp.status() < 500) { + breaker_.on_success(); + } else { + breaker_.on_failure(); + } + co_return resp; +} + +Task Client::send_stream(ClientRequest request, body::ChunkSink sink) { + ClientRequest effective = std::move(request); + if (!effective.url().has_host() && !config_.base_url.empty()) + effective.url() = Url::parse(config_.base_url).join(effective.url().target()); + if (!effective.timeouts()) effective.timeouts(config_.timeouts); + if (config_.auth) config_.auth->apply(effective); + if (config_.persist_cookies) { + std::string cookies = jar_.header_for(effective.url()); + if (!cookies.empty() && !effective.headers().has("Cookie")) + effective.headers().set("Cookie", cookies); + } + if (!effective.headers().has("User-Agent")) + effective.headers().set("User-Agent", config_.user_agent); + co_await transport_->send_stream(effective, std::move(sink)); + co_return; +} + +Task Client::send_inner(ClientRequest request) { + auto start = std::chrono::steady_clock::now(); + RequestSpan span; + span.method = method_name(request.method()); + span.transport = transport_->name(); + stats_.requests++; + int total_attempts = 0; + int hops = 0; + std::size_t bytes_out = 0; + std::size_t bytes_in = 0; + + ClientRequest current = std::move(request); + if (!current.url().has_host() && !config_.base_url.empty()) + current.url() = Url::parse(config_.base_url).join(current.url().target()); + span.url = current.url().str(); + if (!current.timeouts()) current.timeouts(config_.timeouts); + if (current.url().is_https() && config_.h2_https_default) + current.h2(true); + if (config_.h2) current.h2(true); + if (config_.auth) config_.auth->apply(current); + if (!current.headers().has("User-Agent")) + current.headers().set("User-Agent", config_.user_agent); + + auto note_outcome = [&](const ClientResponse& resp, const std::string& err) { + span.status = resp.status(); + span.attempts = total_attempts; + span.redirect_hops = hops; + span.bytes_out = bytes_out; + span.bytes_in = bytes_in; + span.duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + span.error = err; + if (config_.reporter) config_.reporter->on_request(span); + stats_.bytes_out += bytes_out; + stats_.bytes_in += bytes_in; + stats_.redirect_hops += (std::uint64_t)hops; + if (resp.status() >= 200 && resp.status() < 300) stats_.responses_2xx++; + if (resp.status() >= 400 && resp.status() < 500) stats_.responses_4xx++; + if (resp.status() >= 500 && resp.status() < 600) stats_.responses_5xx++; + }; + + try { + while (true) { + if (config_.persist_cookies) { + std::string cookies = jar_.header_for(current.url()); + if (!cookies.empty() && !current.headers().has("Cookie")) + current.headers().set("Cookie", cookies); + } + if (auto* b = std::get_if(¤t.body())) + bytes_out += b->size(); + + // One attempt loop: transport errors and retryable statuses both + // consume the same attempt budget and sleep before the next send. + ClientResponse response; + int attempt = 0; + while (true) { + ++attempt; + ++total_attempts; + bool sent = false; + try { + response = co_await transport_->send(current); + sent = true; + } catch (const TimeoutError& err) { + bool connect_phase = err.phase() == TimeoutPhase::kConnect; + bool may = connect_phase + ? config_.retry.should_retry_connect_failure(attempt) + : (is_idempotent_method(current.method()) || + current.marked_idempotent()) && + attempt < config_.retry.max_attempts(); + // Never retry into an open breaker (state() is side-effect free; + // allow() would consume half-open probes). + if (breaker_.state() == molo::resilience::Breaker::State::kOpen) may = false; + if (!may) throw; + stats_.retries++; + config_.sleeper(config_.retry.backoff_for(attempt)); + continue; + } catch (const TransportError& err) { + bool may = err.connect_phase() + ? config_.retry.should_retry_connect_failure(attempt) + : (is_idempotent_method(current.method()) || + current.marked_idempotent()) && + attempt < config_.retry.max_attempts(); + if (breaker_.state() == molo::resilience::Breaker::State::kOpen) may = false; + if (!may) throw; + stats_.retries++; + config_.sleeper(config_.retry.backoff_for(attempt)); + continue; + } + if (sent) { + bytes_in += response.body().size(); + if (config_.persist_cookies) + jar_.store_from_response(current.url(), response); + if (config_.retry.should_retry_status(current, response.status(), attempt) && + breaker_.state() != molo::resilience::Breaker::State::kOpen) { + std::optional delay; + if (response.status() == 429 || response.status() == 503) { + auto raw = response.headers().get("Retry-After"); + if (raw) delay = RetryPolicy::parse_retry_after(*raw); + } + stats_.retries++; + config_.sleeper(delay ? *delay + : config_.retry.backoff_for(attempt)); + continue; + } + break; + } + } + response.set_attempts(total_attempts); + + // Redirects: GET/HEAD only, bounded hops, auth stripped cross-host. + if (config_.follow_redirects && response.is_redirect() && + (current.method() == Method::kGet || + current.method() == Method::kHead)) { + auto location = response.headers().get("Location"); + if (!location || location->empty()) { + note_outcome(response, ""); + co_return response; + } + if (hops >= config_.max_redirects) + throw RedirectError("too many redirects"); + ++hops; + Url next = current.url().join(*location); + bool cross_host = next.host() != current.url().host() || + next.port() != current.url().port() || + next.scheme() != current.url().scheme(); + // 303 always becomes GET; 301/302 keep GET/HEAD by construction. + current.url() = next; + if (response.status() == 303) { + // Already GET/HEAD; nothing to rewrite. + } + if (cross_host) current.headers().remove("Authorization"); + if (current.url().is_https() && config_.h2_https_default) + current.h2(true); + continue; + } + note_outcome(response, ""); + co_return response; + } + } catch (const std::exception& err) { + stats_.errors++; + span.attempts = total_attempts; + span.redirect_hops = hops; + span.bytes_out = bytes_out; + span.bytes_in = bytes_in; + span.duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + span.error = err.what(); + if (config_.reporter) config_.reporter->on_request(span); + throw; + } +} + +Task Client::get(const std::string& url) { + co_return co_await send(ClientRequest::get(url)); +} + +Task Client::post(const std::string& url, + const std::string& body, + const std::string& content_type) { + ClientRequest req = ClientRequest::post(url); + if (!body.empty() || !content_type.empty()) req.bytes(body, content_type); + co_return co_await send(std::move(req)); +} + +Task Client::post_json(const std::string& url, + const body::Json& doc) { + ClientRequest req = ClientRequest::post(url); + req.json(doc); + co_return co_await send(std::move(req)); +} + +Task Client::put(const std::string& url, + const std::string& body, + const std::string& content_type) { + ClientRequest req = ClientRequest::put(url); + if (!body.empty() || !content_type.empty()) req.bytes(body, content_type); + co_return co_await send(std::move(req)); +} + +Task Client::patch(const std::string& url, + const std::string& body, + const std::string& content_type) { + ClientRequest req = ClientRequest::patch(url); + if (!body.empty() || !content_type.empty()) req.bytes(body, content_type); + co_return co_await send(std::move(req)); +} + +Task Client::remove(const std::string& url) { + co_return co_await send(ClientRequest::remove(url)); +} + +Task Client::head(const std::string& url) { + co_return co_await send(ClientRequest::head(url)); +} + +Task Client::options(const std::string& url) { + co_return co_await send(ClientRequest::options(url)); +} + +ClientResponse Client::send_blocking(ClientRequest request) { + return block_on(send(std::move(request))); +} + +void Client::send_stream_blocking(ClientRequest request, + body::ChunkSink sink) { + block_on(send_stream(std::move(request), std::move(sink))); +} + +ClientResponse Client::get_blocking(const std::string& url) { + return block_on(get(url)); +} + +ClientResponse Client::post_blocking(const std::string& url, + const std::string& body, + const std::string& content_type) { + return block_on(post(url, body, content_type)); +} + +ClientResponse Client::post_json_blocking(const std::string& url, + const body::Json& doc) { + return block_on(post_json(url, doc)); +} + +ClientResponse Client::put_blocking(const std::string& url, + const std::string& body, + const std::string& content_type) { + return block_on(put(url, body, content_type)); +} + +ClientResponse Client::patch_blocking(const std::string& url, + const std::string& body, + const std::string& content_type) { + return block_on(patch(url, body, content_type)); +} + +ClientResponse Client::remove_blocking(const std::string& url) { + return block_on(remove(url)); +} + +ClientResponse Client::head_blocking(const std::string& url) { + return block_on(head(url)); +} + +ClientResponse Client::options_blocking(const std::string& url) { + return block_on(options(url)); +} + +void Client::close() { transport_->close(); } + +Task fetch(ClientRequest request) { + transports::HttpTransportOptions options; + auto transport = std::make_unique(options); + Client client(std::move(transport)); + co_return co_await client.send(std::move(request)); +} + +ClientResponse fetch_blocking(ClientRequest request) { + return block_on(fetch(std::move(request))); +} + +} // namespace molo::clients diff --git a/src/clients/client.h b/src/clients/client.h new file mode 100644 index 0000000..47a7c72 --- /dev/null +++ b/src/clients/client.h @@ -0,0 +1,104 @@ +#pragma once + +// molo::clients::Client: pooling, timeouts, retries, auth, cookies, proxy, +// redirects, observability. Coroutine verbs are primary; _blocking forms +// share the implementation. One-off scripts use fetch()/fetch_blocking(). + +#include +#include + +#include "clients/body/json.h" +#include "clients/client_config.h" +#include "clients/cookies.h" +#include "clients/pool.h" +#include "clients/request.h" +#include "clients/response.h" +#include "clients/task.h" +#include "clients/transport.h" +#include "operability/resilience/breaker.h" + +namespace molo::clients { + +class RedirectError : public std::runtime_error { + public: + explicit RedirectError(const std::string& msg) + : std::runtime_error("molo::clients::Client redirect: " + msg) {} +}; + +class Client { + public: + explicit Client(std::unique_ptr transport, + ClientConfig config = ClientConfig{}); + virtual ~Client() = default; + + Client(const Client&) = delete; + Client& operator=(const Client&) = delete; + + // Coroutine verbs (primary). + Task send(ClientRequest request); + Task send_stream(ClientRequest request, body::ChunkSink sink); + Task get(const std::string& url); + Task post(const std::string& url, + const std::string& body = "", + const std::string& content_type = ""); + Task post_json(const std::string& url, const body::Json& doc); + Task put(const std::string& url, + const std::string& body = "", + const std::string& content_type = ""); + Task patch(const std::string& url, + const std::string& body = "", + const std::string& content_type = ""); + Task remove(const std::string& url); + Task head(const std::string& url); + Task options(const std::string& url); + + // Sync forms sharing the coroutine implementation. + ClientResponse send_blocking(ClientRequest request); + void send_stream_blocking(ClientRequest request, body::ChunkSink sink); + ClientResponse get_blocking(const std::string& url); + ClientResponse post_blocking(const std::string& url, + const std::string& body = "", + const std::string& content_type = ""); + ClientResponse post_json_blocking(const std::string& url, + const body::Json& doc); + ClientResponse put_blocking(const std::string& url, + const std::string& body = "", + const std::string& content_type = ""); + ClientResponse patch_blocking(const std::string& url, + const std::string& body = "", + const std::string& content_type = ""); + ClientResponse remove_blocking(const std::string& url); + ClientResponse head_blocking(const std::string& url); + ClientResponse options_blocking(const std::string& url); + + const ClientConfig& config() const { return config_; } + ClientConfig& config() { return config_; } + ConnectionPool& pool() { return *pool_; } + CookieJar& jar() { return jar_; } + Transport& transport() { return *transport_; } + ClientStatsSnapshot stats() const { return stats_.snapshot(); } + + void close(); + + protected: + ClientConfig config_; + std::unique_ptr transport_; + std::shared_ptr pool_; + CookieJar jar_; + ClientStats stats_; + + private: + Task send_inner(ClientRequest request); + + // Per-client circuit breaker (operability part): fail fast without + // calling downstream while open. Starts closed, so existing behavior + // is unchanged until failures trip it. + molo::resilience::Breaker breaker_; +}; + +// One-shot helpers for scripts. Each call builds a fresh transport: no +// pooling, no cookie persistence. Long-lived code must own a Client. +Task fetch(ClientRequest request); +ClientResponse fetch_blocking(ClientRequest request); + +} // namespace molo::clients diff --git a/src/clients/client_config.cpp b/src/clients/client_config.cpp new file mode 100644 index 0000000..388e6d0 --- /dev/null +++ b/src/clients/client_config.cpp @@ -0,0 +1,19 @@ +#include "clients/client_config.h" + +namespace molo::clients { + +ClientStatsSnapshot ClientStats::snapshot() const { + ClientStatsSnapshot snap; + snap.requests = requests.load(); + snap.responses_2xx = responses_2xx.load(); + snap.responses_4xx = responses_4xx.load(); + snap.responses_5xx = responses_5xx.load(); + snap.retries = retries.load(); + snap.redirect_hops = redirect_hops.load(); + snap.errors = errors.load(); + snap.bytes_out = bytes_out.load(); + snap.bytes_in = bytes_in.load(); + return snap; +} + +} // namespace molo::clients diff --git a/src/clients/client_config.h b/src/clients/client_config.h new file mode 100644 index 0000000..df8cb10 --- /dev/null +++ b/src/clients/client_config.h @@ -0,0 +1,90 @@ +#pragma once + +// Client configuration plus the observability hook (Reporter) and counters. +// The default NullReporter is an inline no-op: the disabled path stays thin. + +#include +#include +#include +#include +#include + +#include "clients/auth.h" +#include "clients/caps.h" +#include "clients/proxy.h" +#include "clients/retry.h" +#include "clients/timeouts.h" + +namespace molo::clients { + +struct RequestSpan { + std::string method; + std::string url; + std::string transport; + int status = 0; + int attempts = 1; + int redirect_hops = 0; + std::size_t bytes_out = 0; + std::size_t bytes_in = 0; + std::chrono::milliseconds duration{0}; + std::string error; +}; + +class Reporter { + public: + virtual ~Reporter() = default; + virtual void on_request(const RequestSpan& span) = 0; +}; + +class NullReporter : public Reporter { + public: + void on_request(const RequestSpan&) override {} +}; + +struct ClientStatsSnapshot { + std::uint64_t requests = 0; + std::uint64_t responses_2xx = 0; + std::uint64_t responses_4xx = 0; + std::uint64_t responses_5xx = 0; + std::uint64_t retries = 0; + std::uint64_t redirect_hops = 0; + std::uint64_t errors = 0; + std::uint64_t bytes_out = 0; + std::uint64_t bytes_in = 0; +}; + +struct ClientStats { + std::atomic requests{0}; + std::atomic responses_2xx{0}; + std::atomic responses_4xx{0}; + std::atomic responses_5xx{0}; + std::atomic retries{0}; + std::atomic redirect_hops{0}; + std::atomic errors{0}; + std::atomic bytes_out{0}; + std::atomic bytes_in{0}; + + ClientStatsSnapshot snapshot() const; +}; + +struct ClientConfig { + Timeouts timeouts; + RetryPolicy retry; + ProxyConfig proxy; + bool proxy_from_env = true; + std::shared_ptr auth; + bool persist_cookies = true; + // Explicit H2 request flag. HTTPS URLs additionally default to H2-capable + // (h2_https_default); both currently resolve against the marked TLS/H2 + // stubs, so HTTPS and h2c fail loud until parts 01/03 land. + bool h2 = false; + bool h2_https_default = true; + int max_redirects = Caps::kMaxRedirects; + bool follow_redirects = true; // GET/HEAD only, auth stripped cross-host. + std::string base_url; + std::string user_agent = kUserAgent; + RetryPolicy::Sleeper sleeper = RetryPolicy::sleep_default; + std::shared_ptr reporter = std::make_shared(); +}; + +} // namespace molo::clients diff --git a/src/clients/clients.mk b/src/clients/clients.mk new file mode 100644 index 0000000..e86e300 --- /dev/null +++ b/src/clients/clients.mk @@ -0,0 +1,47 @@ +# Per-part fragment for 08-clients-openapi (owned dir: src/clients). +# Included by the root Makefile, or run standalone: +# make -f src/clients/clients.mk [clients|clients-test|clients-clean] +# Root wiring: `all` depends on `clients`, `test` depends on `clients-test`, +# `clean` depends on `clients-clean`. + +ROOT ?= . +CXX ?= g++ +CLIENTS_CXXFLAGS ?= -std=c++23 -Wall -Wextra -Werror +CLIENTS_INCS := -I$(ROOT)/src -I$(ROOT)/tests/src/clients + +CLIENTS_SRCS := $(sort $(wildcard $(ROOT)/src/clients/*.cpp) \ + $(wildcard $(ROOT)/src/clients/body/*.cpp) \ + $(wildcard $(ROOT)/src/clients/rest/*.cpp) \ + $(wildcard $(ROOT)/src/clients/openapi/*.cpp) \ + $(wildcard $(ROOT)/src/clients/testing/*.cpp) \ + $(wildcard $(ROOT)/src/clients/transports/*.cpp)) +CLIENTS_TEST_SRCS := $(sort $(wildcard $(ROOT)/tests/src/clients/*.cpp)) + +CLIENTS_BUILD := $(ROOT)/build/clients +CLIENTS_OBJDIR := $(CLIENTS_BUILD)/objs +CLIENTS_LIB := $(CLIENTS_BUILD)/libclients.a +CLIENTS_TEST_BIN := $(ROOT)/build/clients_test + +CLIENTS_OBJS := $(patsubst $(ROOT)/%.cpp,$(CLIENTS_OBJDIR)/%.o,$(CLIENTS_SRCS)) + +.PHONY: clients clients-test clients-clean + +clients: $(CLIENTS_LIB) + +$(CLIENTS_OBJDIR)/%.o: $(ROOT)/%.cpp + @mkdir -p $(dir $@) + $(CXX) $(CLIENTS_CXXFLAGS) -I$(ROOT)/src -c $< -o $@ + +$(CLIENTS_LIB): $(CLIENTS_OBJS) + @mkdir -p $(dir $@) + $(AR) rcs $@ $^ + +clients-test: $(CLIENTS_TEST_BIN) + $(CLIENTS_TEST_BIN) + +$(CLIENTS_TEST_BIN): $(CLIENTS_OBJS) $(CLIENTS_TEST_SRCS) $(OPERABILITY_LIB) + @mkdir -p $(dir $@) + $(CXX) $(CLIENTS_CXXFLAGS) $(CLIENTS_INCS) $(CLIENTS_OBJS) $(CLIENTS_TEST_SRCS) $(OPERABILITY_LIB) -o $@ + +clients-clean: + rm -rf $(CLIENTS_BUILD) $(CLIENTS_TEST_BIN) diff --git a/src/clients/cookies.cpp b/src/clients/cookies.cpp new file mode 100644 index 0000000..0324260 --- /dev/null +++ b/src/clients/cookies.cpp @@ -0,0 +1,173 @@ +#include "clients/cookies.h" + +#include + +#include "clients/caps.h" + +namespace molo::clients { +namespace { + +std::string trim(std::string s) { + while (!s.empty() && (s.front() == ' ' || s.front() == '\t')) s.erase(s.begin()); + while (!s.empty() && (s.back() == ' ' || s.back() == '\t')) s.pop_back(); + return s; +} + +std::string to_lower(std::string s) { + for (auto& c : s) c = static_cast(std::tolower((unsigned char)c)); + return s; +} + +bool domain_match(const std::string& host, const std::string& cookie_domain) { + if (host == cookie_domain) return true; + if (host.size() > cookie_domain.size() + 1 && + host.compare(host.size() - cookie_domain.size(), cookie_domain.size(), + cookie_domain) == 0 && + host[host.size() - cookie_domain.size() - 1] == '.') + return true; + return false; +} + +bool path_match(const std::string& request_path, const std::string& cookie_path) { + if (cookie_path.empty() || cookie_path == "/") return true; + if (request_path.compare(0, cookie_path.size(), cookie_path) != 0) + return false; + return request_path.size() == cookie_path.size() || + request_path[cookie_path.size()] == '/'; +} + +} // namespace + +void CookieJar::clear() { + std::lock_guard lock(mutex_); + cookies_.clear(); +} + +std::size_t CookieJar::size() const { + std::lock_guard lock(mutex_); + return cookies_.size(); +} + +std::optional CookieJar::parse_set_cookie(const Url& url, + const std::string& text) { + if (text.size() > Caps::kCookieMaxBytes) return std::nullopt; + auto semi = text.find(';'); + std::string pair = trim(semi == std::string::npos ? text : text.substr(0, semi)); + std::string attrs = semi == std::string::npos ? "" : text.substr(semi + 1); + auto eq = pair.find('='); + if (eq == std::string::npos || eq == 0) return std::nullopt; + Cookie cookie; + cookie.name = trim(pair.substr(0, eq)); + cookie.value = trim(pair.substr(eq + 1)); + cookie.domain = to_lower(url.host()); + cookie.path = "/"; + // Default path: request path up to the last slash. + auto slash = url.path().rfind('/'); + if (slash != std::string::npos && slash > 0) + cookie.path = url.path().substr(0, slash); + while (!attrs.empty()) { + auto next = attrs.find(';'); + std::string attr = + trim(next == std::string::npos ? attrs : attrs.substr(0, next)); + attrs = next == std::string::npos ? "" : attrs.substr(next + 1); + auto aeq = attr.find('='); + std::string aname = to_lower(trim(aeq == std::string::npos + ? attr + : attr.substr(0, aeq))); + std::string avalue = + aeq == std::string::npos ? "" : trim(attr.substr(aeq + 1)); + if (aname == "domain") { + std::string d = to_lower(avalue); + if (!d.empty() && d.front() == '.') d.erase(d.begin()); + if (d.empty() || !domain_match(to_lower(url.host()), d)) return std::nullopt; + cookie.domain = d; + } else if (aname == "path") { + if (!avalue.empty() && avalue.front() == '/') cookie.path = avalue; + } else if (aname == "max-age") { + try { + long secs = std::stol(avalue); + if (secs <= 0) + cookie.expires_at = std::chrono::steady_clock::now(); + else + cookie.expires_at = + std::chrono::steady_clock::now() + std::chrono::seconds(secs); + } catch (...) { + return std::nullopt; + } + } else if (aname == "secure") { + cookie.secure = true; + } else if (aname == "httponly") { + cookie.http_only = true; + } else if (aname == "samesite") { + cookie.same_site = avalue.empty() ? "Lax" : avalue; + } + } + return cookie; +} + +void CookieJar::store_from_response(const Url& url, + const ClientResponse& response) { + std::lock_guard lock(mutex_); + auto now = std::chrono::steady_clock::now(); + for (const auto& raw : response.headers().set_cookies()) { + auto parsed = parse_set_cookie(url, raw); + if (!parsed) continue; + if (parsed->expires_at && *parsed->expires_at <= now) { + // Expired: delete matching cookie. + for (auto it = cookies_.begin(); it != cookies_.end();) { + if (it->name == parsed->name && it->domain == parsed->domain && + it->path == parsed->path) + it = cookies_.erase(it); + else + ++it; + } + continue; + } + bool replaced = false; + for (auto& existing : cookies_) { + if (existing.name == parsed->name && existing.domain == parsed->domain && + existing.path == parsed->path) { + existing = *parsed; + replaced = true; + break; + } + } + if (!replaced) { + if (cookies_.size() >= Caps::kCookieJarMax) continue; // Loud skip: full. + cookies_.push_back(*parsed); + } + } +} + +std::string CookieJar::header_for(const Url& url) { + std::lock_guard lock(mutex_); + auto now = std::chrono::steady_clock::now(); + std::string host = to_lower(url.host()); + std::string out; + for (auto it = cookies_.begin(); it != cookies_.end();) { + if (it->expires_at && *it->expires_at <= now) { + it = cookies_.erase(it); + continue; + } + if (it->secure && !url.is_https()) { + ++it; + continue; + } + if (!domain_match(host, it->domain) || !path_match(url.path(), it->path)) { + ++it; + continue; + } + // SameSite=Strict cookies are same-site only: host must equal the cookie + // domain exactly (this client never navigates cross-site top-level). + if (to_lower(it->same_site) == "strict" && host != it->domain) { + ++it; + continue; + } + if (!out.empty()) out += "; "; + out += it->name + "=" + it->value; + ++it; + } + return out; +} + +} // namespace molo::clients diff --git a/src/clients/cookies.h b/src/clients/cookies.h new file mode 100644 index 0000000..a48da28 --- /dev/null +++ b/src/clients/cookies.h @@ -0,0 +1,50 @@ +#pragma once + +// Per-client cookie jar with Secure, HttpOnly, SameSite, Domain, Path and +// expiry semantics on read-back. + +#include +#include +#include +#include +#include + +#include "clients/response.h" +#include "clients/url.h" + +namespace molo::clients { + +struct Cookie { + std::string name; + std::string value; + std::string domain; // Lowercased, without leading dot. + std::string path = "/"; + bool secure = false; + bool http_only = false; + std::string same_site; // "", "Lax", "Strict", "None". + // Expiry as steady-clock deadline; nullopt means session cookie. + std::optional expires_at; +}; + +class CookieJar { + public: + CookieJar() = default; + + void clear(); + std::size_t size() const; + + // Stores cookies from response Set-Cookie headers for the request URL. + void store_from_response(const Url& url, const ClientResponse& response); + // Renders the Cookie header for a request URL. Empty means no header. + std::string header_for(const Url& url); + + // Direct parse helper (also used by tests). + static std::optional parse_set_cookie(const Url& url, + const std::string& text); + + private: + mutable std::mutex mutex_; + std::vector cookies_; +}; + +} // namespace molo::clients diff --git a/src/clients/headers.cpp b/src/clients/headers.cpp new file mode 100644 index 0000000..e676e43 --- /dev/null +++ b/src/clients/headers.cpp @@ -0,0 +1,87 @@ +#include "clients/headers.h" + +#include + +#include "clients/caps.h" + +namespace molo::clients { + +void HeaderMap::set(const std::string& name, const std::string& value) { + if (name.size() > Caps::kMaxHeaderLine || value.size() > Caps::kMaxHeaderLine) + throw std::length_error("molo::clients::HeaderMap: header line too long"); + HeaderLess less; + for (auto it = map_.begin(); it != map_.end();) { + if (!less(it->first, name) && !less(name, it->first)) + it = map_.erase(it); + else + ++it; + } + if (map_.size() >= Caps::kMaxHeaders) + throw std::length_error("molo::clients::HeaderMap: too many headers"); + map_[name] = value; +} + +void HeaderMap::add(const std::string& name, const std::string& value) { + HeaderLess less; + std::string key = name; + for (const auto& kv : map_) { + if (!less(kv.first, name) && !less(name, kv.first)) { + key = kv.first; + break; + } + } + std::string lower = name; + for (auto& c : lower) c = static_cast(std::tolower((unsigned char)c)); + if (lower == "set-cookie") { + set_cookies_.push_back(value); + auto it = map_.find(key); + if (it == map_.end()) { + set(key, value); + } else { + // Keep first entry authoritative; full list lives in set_cookies(). + } + return; + } + auto it = map_.find(key); + if (it == map_.end()) { + set(key, value); + } else { + set(key, it->second + ", " + value); + } +} + +bool HeaderMap::has(const std::string& name) const { + return map_.find(name) != map_.end(); +} + +std::optional HeaderMap::get(const std::string& name) const { + auto it = map_.find(name); + if (it == map_.end()) return std::nullopt; + return it->second; +} + +std::string HeaderMap::get_or(const std::string& name, + const std::string& fallback) const { + auto v = get(name); + return v ? *v : fallback; +} + +bool HeaderMap::remove(const std::string& name) { + return map_.erase(name) > 0; +} + +std::string HeaderMap::serialize() const { + std::ostringstream out; + for (const auto& kv : map_) { + std::string lower = kv.first; + for (auto& c : lower) c = static_cast(std::tolower((unsigned char)c)); + if (lower == "set-cookie") { + for (const auto& v : set_cookies_) out << kv.first << ": " << v << "\r\n"; + } else { + out << kv.first << ": " << kv.second << "\r\n"; + } + } + return out.str(); +} + +} // namespace molo::clients diff --git a/src/clients/headers.h b/src/clients/headers.h new file mode 100644 index 0000000..0c6ce3f --- /dev/null +++ b/src/clients/headers.h @@ -0,0 +1,56 @@ +#pragma once + +// Case-insensitive header map with deterministic (sorted) iteration. + +#include +#include +#include +#include +#include + +namespace molo::clients { + +struct HeaderLess { + bool operator()(const std::string& a, const std::string& b) const { + std::size_t n = a.size() < b.size() ? a.size() : b.size(); + for (std::size_t i = 0; i < n; ++i) { + char ca = static_cast(std::tolower((unsigned char)a[i])); + char cb = static_cast(std::tolower((unsigned char)b[i])); + if (ca != cb) return ca < cb; + } + return a.size() < b.size(); + } +}; + +class HeaderMap { + public: + HeaderMap() = default; + + void set(const std::string& name, const std::string& value); + // Appends a value; Set-Cookie style headers keep separate entries joined + // with a newline-safe ", " only for list headers. Repeated Set-Cookie + // values are stored as one entry per call in insertion order via cookies(). + void add(const std::string& name, const std::string& value); + bool has(const std::string& name) const; + std::optional get(const std::string& name) const; + std::string get_or(const std::string& name, + const std::string& fallback) const; + bool remove(const std::string& name); + std::size_t size() const { return map_.size(); } + bool empty() const { return map_.empty(); } + + const std::map& entries() const { + return map_; + } + + // All Set-Cookie values in insertion order. + const std::vector& set_cookies() const { return set_cookies_; } + + std::string serialize() const; // "Name: value\r\n" per entry, sorted. + + private: + std::map map_; + std::vector set_cookies_; +}; + +} // namespace molo::clients diff --git a/src/clients/openapi/builder.cpp b/src/clients/openapi/builder.cpp new file mode 100644 index 0000000..45e3d2a --- /dev/null +++ b/src/clients/openapi/builder.cpp @@ -0,0 +1,97 @@ +#include "clients/openapi/builder.h" + +#include +#include + +#include "clients/caps.h" + +namespace molo::clients::openapi { +namespace { + +std::string to_lower(std::string s) { + for (auto& c : s) c = static_cast(std::tolower((unsigned char)c)); + return s; +} + +} // namespace + +Builder& Builder::extend_hook(ExtendHook hook) { + hook_ = std::move(hook); + return *this; +} + +void Builder::use_governance_declarations() { govern_decls_ = true; } + +// Governance mapping (PATCHES 14, section 6, adapted: build() takes input +// by const ref, so the mapping applies per route instead of mutating input). +// key/key_scopes declare the apiKey scheme; session auth keeps whatever the +// input declared (its scheme is app-defined). +void ApplyGovernanceDeclarations(const RouteDecl& route, OperationDecl& operation) { + if ((route.gov_auth == "key" || route.gov_auth == "key_scopes") && + operation.security.empty()) { + operation.security = {"apiKey"}; + } + if (route.gov_deprecated) operation.deprecated = true; +} + +Document Builder::from_routes(const std::vector& routes, + const std::vector& models) const { + BuilderInput input; + input.routes = routes; + input.models = models; + return build(input); +} + +Document Builder::build(const BuilderInput& input) const { + if (input.routes.size() > Caps::kOpenApiMaxPaths) + throw OpenApiError("route count exceeds 5000 path cap"); + if (input.models.size() > Caps::kOpenApiMaxSchemas) + throw OpenApiError("model count exceeds 2000 schema cap"); + Document doc; + doc.info() = input.info; + doc.servers() = input.servers; + doc.tags() = input.tags; + for (const auto& model : input.models) + doc.schemas()[model.name] = SchemaConverter::convert(model); + doc.security_schemes() = input.security_schemes; + for (const auto& route : input.routes) { + OperationDecl operation; + operation.operation_id = route.operation_id; + operation.summary = route.summary; + operation.description = + route.description.empty() ? "TBD" : route.description; + operation.tags = route.tags; + operation.parameters = route.parameters; + operation.security = route.security; + operation.deprecated = route.deprecated; + if (govern_decls_) ApplyGovernanceDeclarations(route, operation); + if (!route.request_model.empty()) { + operation.has_request_body = true; + operation.request_body.required = route.request_required; + MediaTypeDecl media; + media.schema = SchemaConverter::ref_for(route.request_model); + operation.request_body.content[route.request_content_type] = + std::move(media); + } + for (const auto& resp : route.responses) { + ResponseDecl response; + response.description = "TBD"; + if (!resp.second.empty()) { + MediaTypeDecl media; + media.schema = SchemaConverter::ref_for(resp.second); + response.content["application/json"] = std::move(media); + } + operation.responses[resp.first] = std::move(response); + } + doc.paths()[route.path].operations[to_lower(route.method)] = + std::move(operation); + } + doc.extensions()["x-molo-framework-version"] = + body::Json(std::string(kFrameworkVersion)); + if (hook_) hook_(doc); + // Enforce the byte cap eagerly so oversize apps fail at build time. + (void)doc.to_json_string(); + return doc; +} + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/builder.h b/src/clients/openapi/builder.h new file mode 100644 index 0000000..a64ab43 --- /dev/null +++ b/src/clients/openapi/builder.h @@ -0,0 +1,73 @@ +#pragma once + +// Builds the OpenAPI document from router plus validation-model adapter +// structs. Output is deterministic: stable key order, stable schema names, +// byte-identical for identical input. + +#include +#include +#include + +#include "clients/openapi/document.h" +#include "clients/openapi/schema.h" + +namespace molo::clients::openapi { + +// Adapter for one router Route record (methods, path template, handler meta, +// validation models). Part 05 fills these from its Route records. +struct RouteDecl { + std::string method = "GET"; // Uppercase HTTP method. + std::string path; // Path template, e.g. /v1/users/{id}. + std::string operation_id; + std::string summary; + std::string description; // Empty renders as "TBD", never invented. + std::vector tags; + // Path/query/header/cookie bindings from the validation layer. + std::vector parameters; + // Request body model name (becomes a $ref), empty means no body. + std::string request_model; + std::string request_content_type = "application/json"; + bool request_required = true; + // Status code to response model name (empty model means no content). + std::vector> responses; + std::vector security; // Auth guard scheme names. + bool deprecated = false; + // Governance declarations (molo::govern). Mapped into security/deprecated + // by build() when use_governance_declarations() is on. Scope enforcement + // stays runtime (KeyAuth); the document carries the scheme. + std::string gov_auth; // "", "key", "key_scopes", "session" + bool gov_deprecated = false; +}; + +struct BuilderInput { + ApiInfo info; + std::vector servers; + std::vector routes; + std::vector models; + std::map security_schemes; + std::vector tags; +}; + +class Builder { + public: + using ExtendHook = std::function; + + Builder& extend_hook(ExtendHook hook); + + Document build(const BuilderInput& input) const; + + // Governance: registers the hook that copies RouteDecl gov_* fields + // into OperationDecl::deprecated and OperationDecl::security so the OpenAPI + // document and the runtime headers share one source. + void use_governance_declarations(); + + // Convenience: build from routes plus models with default info. + Document from_routes(const std::vector& routes, + const std::vector& models) const; + + private: + ExtendHook hook_; + bool govern_decls_ = false; +}; + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/document.cpp b/src/clients/openapi/document.cpp new file mode 100644 index 0000000..9805ff5 --- /dev/null +++ b/src/clients/openapi/document.cpp @@ -0,0 +1,168 @@ +#include "clients/openapi/document.h" + +#include "clients/caps.h" + +namespace molo::clients::openapi { +namespace { + +body::Json media_json(const std::map& content) { + body::Json out = body::Json::object(); + for (const auto& kv : content) { + body::Json media = body::Json::object(); + media.set("schema", kv.second.schema); + if (!kv.second.example.empty()) + media.set("example", body::Json(kv.second.example)); + out.set(kv.first, std::move(media)); + } + return out; +} + +body::Json parameter_json(const ParameterDecl& param) { + body::Json out = body::Json::object(); + out.set("name", body::Json(param.name)); + out.set("in", body::Json(param.in)); + out.set("required", body::Json(param.required)); + if (!param.description.empty()) + out.set("description", body::Json(param.description)); + out.set("schema", param.schema); + return out; +} + +} // namespace + +body::Json Document::to_json() const { + body::Json doc = body::Json::object(); + doc.set("openapi", body::Json(kOpenApiVersion)); + body::Json info = body::Json::object(); + info.set("title", body::Json(info_.title)); + info.set("version", body::Json(info_.version)); + if (!info_.description.empty()) + info.set("description", body::Json(info_.description)); + doc.set("info", std::move(info)); + if (!servers_.empty()) { + body::Json servers = body::Json::array(); + for (const auto& server : servers_) { + body::Json entry = body::Json::object(); + entry.set("url", body::Json(server.url)); + if (!server.description.empty()) + entry.set("description", body::Json(server.description)); + servers.push(std::move(entry)); + } + doc.set("servers", std::move(servers)); + } + if (!tags_.empty()) { + body::Json tags = body::Json::array(); + for (const auto& tag : tags_) { + body::Json entry = body::Json::object(); + entry.set("name", body::Json(tag)); + tags.push(std::move(entry)); + } + doc.set("tags", std::move(tags)); + } + body::Json paths = body::Json::object(); + for (const auto& path : paths_) { + body::Json item = body::Json::object(); + for (const auto& op : path.second.operations) { + body::Json operation = body::Json::object(); + if (!op.second.operation_id.empty()) + operation.set("operationId", body::Json(op.second.operation_id)); + if (!op.second.summary.empty()) + operation.set("summary", body::Json(op.second.summary)); + operation.set("description", body::Json(op.second.description.empty() + ? "TBD" + : op.second.description)); + if (!op.second.tags.empty()) { + body::Json tags = body::Json::array(); + for (const auto& tag : op.second.tags) tags.push(body::Json(tag)); + operation.set("tags", std::move(tags)); + } + if (!op.second.parameters.empty()) { + body::Json params = body::Json::array(); + for (const auto& param : op.second.parameters) + params.push(parameter_json(param)); + operation.set("parameters", std::move(params)); + } + if (op.second.has_request_body) { + body::Json body = body::Json::object(); + if (!op.second.request_body.description.empty()) + body.set("description", + body::Json(op.second.request_body.description)); + body.set("required", body::Json(op.second.request_body.required)); + body.set("content", media_json(op.second.request_body.content)); + operation.set("requestBody", std::move(body)); + } + body::Json responses = body::Json::object(); + for (const auto& resp : op.second.responses) { + body::Json entry = body::Json::object(); + entry.set("description", body::Json(resp.second.description)); + if (!resp.second.content.empty()) + entry.set("content", media_json(resp.second.content)); + responses.set(resp.first, std::move(entry)); + } + if (responses.as_object().empty()) { + body::Json entry = body::Json::object(); + entry.set("description", body::Json("TBD")); + responses.set("default", std::move(entry)); + } + operation.set("responses", std::move(responses)); + if (!op.second.security.empty()) { + body::Json security = body::Json::array(); + for (const auto& scheme : op.second.security) { + body::Json req = body::Json::object(); + req.set(scheme, body::Json::array()); + security.push(std::move(req)); + } + operation.set("security", std::move(security)); + } + if (op.second.deprecated) operation.set("deprecated", body::Json(true)); + item.set(op.first, std::move(operation)); + } + paths.set(path.first, std::move(item)); + } + doc.set("paths", std::move(paths)); + if (!schemas_.empty() || !security_schemes_.empty()) { + body::Json components = body::Json::object(); + if (!schemas_.empty()) { + body::Json schemas = body::Json::object(); + for (const auto& kv : schemas_) schemas.set(kv.first, kv.second); + components.set("schemas", std::move(schemas)); + } + if (!security_schemes_.empty()) { + body::Json schemes = body::Json::object(); + for (const auto& kv : security_schemes_) { + body::Json entry = body::Json::object(); + entry.set("type", body::Json(kv.second.type)); + if (!kv.second.scheme.empty()) + entry.set("scheme", body::Json(kv.second.scheme)); + if (!kv.second.bearer_format.empty()) + entry.set("bearerFormat", body::Json(kv.second.bearer_format)); + if (!kv.second.name.empty()) + entry.set("name", body::Json(kv.second.name)); + if (!kv.second.in.empty()) entry.set("in", body::Json(kv.second.in)); + if (!kv.second.description.empty()) + entry.set("description", body::Json(kv.second.description)); + schemes.set(kv.first, std::move(entry)); + } + components.set("securitySchemes", std::move(schemes)); + } + doc.set("components", std::move(components)); + } + for (const auto& kv : extensions_) doc.set(kv.first, kv.second); + return doc; +} + +std::string Document::to_json_string() const { + std::string out = to_json().dump(); + if (out.size() > Caps::kOpenApiMaxBytes) + throw OpenApiError("generated document exceeds 10 MB cap"); + return out; +} + +std::string Document::to_yaml_string() const { + std::string out = body::yaml_dump(to_json()); + if (out.size() > Caps::kOpenApiMaxBytes) + throw OpenApiError("generated document exceeds 10 MB cap"); + return out; +} + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/document.h b/src/clients/openapi/document.h new file mode 100644 index 0000000..70f02ea --- /dev/null +++ b/src/clients/openapi/document.h @@ -0,0 +1,121 @@ +#pragma once + +// OpenAPI 3.1.x document model with deterministic JSON and YAML output. +// Only 3.1 emitters exist; no 2.0/3.0 support by design. + +#include +#include +#include +#include + +#include "clients/body/json.h" + +namespace molo::clients::openapi { + +class OpenApiError : public std::runtime_error { + public: + explicit OpenApiError(const std::string& msg) + : std::runtime_error("molo::clients::openapi: " + msg) {} +}; + +struct ApiInfo { + std::string title = "MoloApp"; + std::string version = "0.1.0"; + std::string description; +}; + +struct ServerDecl { + std::string url = "/"; + std::string description; +}; + +struct ParameterDecl { + std::string name; + std::string in = "query"; // path, query, header, cookie. + bool required = false; + body::Json schema; // JSON Schema fragment. + std::string description; +}; + +struct MediaTypeDecl { + body::Json schema; + std::string example; +}; + +struct RequestBodyDecl { + std::map content; // media type to schema. + bool required = false; + std::string description; +}; + +struct ResponseDecl { + std::string description = "TBD"; + std::map content; +}; + +struct OperationDecl { + std::string operation_id; + std::string summary; + std::string description; + std::vector tags; + std::vector parameters; + bool has_request_body = false; + RequestBodyDecl request_body; + std::map responses; // status to response. + std::vector security; // scheme names. + bool deprecated = false; +}; + +struct PathItemDecl { + std::map operations; // lowercase method to op. +}; + +struct SecuritySchemeDecl { + std::string type = "http"; // http, apiKey, oauth2, openIdConnect. + std::string scheme; // bearer, basic (for http). + std::string bearer_format; + std::string name; // header/query/cookie name (for apiKey). + std::string in; // header, query, cookie (for apiKey). + std::string description; +}; + +class Document { + public: + Document() = default; + + ApiInfo& info() { return info_; } + const ApiInfo& info() const { return info_; } + std::vector& servers() { return servers_; } + const std::vector& servers() const { return servers_; } + std::map& paths() { return paths_; } + const std::map& paths() const { return paths_; } + std::map& schemas() { return schemas_; } + const std::map& schemas() const { return schemas_; } + std::map& security_schemes() { + return security_schemes_; + } + const std::map& security_schemes() const { + return security_schemes_; + } + std::vector& tags() { return tags_; } + const std::vector& tags() const { return tags_; } + std::map& extensions() { return extensions_; } + const std::map& extensions() const { + return extensions_; + } + + body::Json to_json() const; + std::string to_json_string() const; // Enforces the document byte cap. + std::string to_yaml_string() const; // Enforces the document byte cap. + + private: + ApiInfo info_; + std::vector servers_; + std::map paths_; + std::map schemas_; + std::map security_schemes_; + std::vector tags_; + std::map extensions_; +}; + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/extensions.cpp b/src/clients/openapi/extensions.cpp new file mode 100644 index 0000000..abc8451 --- /dev/null +++ b/src/clients/openapi/extensions.cpp @@ -0,0 +1,13 @@ +#include "clients/openapi/extensions.h" + +namespace molo::clients::openapi { + +void apply_overrides(Document& doc, const AppOverride& hook) { + if (hook) hook(doc); + for (const auto& kv : doc.extensions()) { + if (!is_extension_key(kv.first)) + throw OpenApiError("extension key must start with x-: " + kv.first); + } +} + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/extensions.h b/src/clients/openapi/extensions.h new file mode 100644 index 0000000..bae340d --- /dev/null +++ b/src/clients/openapi/extensions.h @@ -0,0 +1,23 @@ +#pragma once + +// Vendor extensions plus the app override hook contract for documents. + +#include +#include + +#include "clients/body/json.h" +#include "clients/openapi/document.h" + +namespace molo::clients::openapi { + +// Extension keys must start with "x-". +inline bool is_extension_key(const std::string& key) { + return key.rfind("x-", 0) == 0; +} + +// App override hook: mutates the built document before it is served. +using AppOverride = std::function; + +void apply_overrides(Document& doc, const AppOverride& hook); + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/routes.cpp b/src/clients/openapi/routes.cpp new file mode 100644 index 0000000..2a90ae0 --- /dev/null +++ b/src/clients/openapi/routes.cpp @@ -0,0 +1,59 @@ +#include "clients/openapi/routes.h" + +namespace molo::clients::openapi { +namespace { + +const char* kSwaggerShell = R"HTML( + + + + +API docs + + +
    Swagger UI assets are packed by the part 09 embedder.
    + + + +)HTML"; + +const char* kRedocShell = R"HTML( + + + + +API reference + + +
    ReDoc assets are packed by the part 09 embedder.
    + + + +)HTML"; + +} // namespace + +DocPayloads build_doc_payloads(const Document& doc) { + DocPayloads payloads; + payloads.openapi_json.content_type = "application/json"; + payloads.openapi_json.body = doc.to_json_string(); + payloads.openapi_yaml.content_type = "application/yaml"; + payloads.openapi_yaml.body = doc.to_yaml_string(); + payloads.swagger_html.content_type = "text/html; charset=utf-8"; + payloads.swagger_html.body = kSwaggerShell; + payloads.redoc_html.content_type = "text/html; charset=utf-8"; + payloads.redoc_html.body = kRedocShell; + return payloads; +} + +std::map doc_route_table(const Document& doc) { + DocPayloads payloads = build_doc_payloads(doc); + return { + {kOpenApiJsonPath, payloads.openapi_json}, + {kOpenApiYamlPath, payloads.openapi_yaml}, + {kSwaggerPath, payloads.swagger_html}, + {kRedocPath, payloads.redoc_html}, + }; +} + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/routes.h b/src/clients/openapi/routes.h new file mode 100644 index 0000000..f31594d --- /dev/null +++ b/src/clients/openapi/routes.h @@ -0,0 +1,37 @@ +#pragma once + +// Payloads for the doc routes: /openapi.json, /openapi.yaml, /docs, /redoc. +// This part returns the bytes; mounting and the production on/off switch +// belong to the app. The UI shells are placeholders until the part 09 +// embedder packs the real Swagger/ReDoc assets. + +#include +#include + +#include "clients/openapi/document.h" + +namespace molo::clients::openapi { + +inline constexpr const char* kOpenApiJsonPath = "/openapi.json"; +inline constexpr const char* kOpenApiYamlPath = "/openapi.yaml"; +inline constexpr const char* kSwaggerPath = "/docs"; +inline constexpr const char* kRedocPath = "/redoc"; + +struct DocPayload { + std::string content_type; + std::string body; +}; + +struct DocPayloads { + DocPayload openapi_json; + DocPayload openapi_yaml; + DocPayload swagger_html; + DocPayload redoc_html; +}; + +DocPayloads build_doc_payloads(const Document& doc); + +// Route table for apps to mount: path to payload. +std::map doc_route_table(const Document& doc); + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/schema.cpp b/src/clients/openapi/schema.cpp new file mode 100644 index 0000000..da67cc5 --- /dev/null +++ b/src/clients/openapi/schema.cpp @@ -0,0 +1,58 @@ +#include "clients/openapi/schema.h" + +namespace molo::clients::openapi { + +body::Json SchemaConverter::ref_for(const std::string& model_name) { + body::Json out = body::Json::object(); + out.set("$ref", body::Json("#/components/schemas/" + model_name)); + return out; +} + +body::Json SchemaConverter::field_schema(const FieldDecl& field) { + if (!field.ref.empty()) return ref_for(field.ref); + body::Json out = body::Json::object(); + out.set("type", body::Json(field.type)); + if (!field.format.empty()) out.set("format", body::Json(field.format)); + if (!field.description.empty()) + out.set("description", body::Json(field.description)); + if (!field.example.empty()) out.set("example", body::Json(field.example)); + if (field.type == "array" && !field.items_type.empty()) { + if (field.items_type.rfind("$ref:", 0) == 0) { + out.set("items", ref_for(field.items_type.substr(5))); + } else { + body::Json items = body::Json::object(); + items.set("type", body::Json(field.items_type)); + out.set("items", std::move(items)); + } + } + if (field.minimum) out.set("minimum", body::Json(*field.minimum)); + if (field.maximum) out.set("maximum", body::Json(*field.maximum)); + if (field.min_length) out.set("minLength", body::Json(*field.min_length)); + if (field.max_length) out.set("maxLength", body::Json(*field.max_length)); + if (!field.pattern.empty()) out.set("pattern", body::Json(field.pattern)); + if (!field.enum_values.empty()) { + body::Json values = body::Json::array(); + for (const auto& v : field.enum_values) values.push(body::Json(v)); + out.set("enum", std::move(values)); + } + if (field.has_default) out.set("default", field.default_value); + return out; +} + +body::Json SchemaConverter::convert(const ModelDecl& model) { + body::Json out = body::Json::object(); + out.set("type", body::Json("object")); + if (!model.description.empty()) + out.set("description", body::Json(model.description)); + body::Json properties = body::Json::object(); + body::Json required = body::Json::array(); + for (const auto& field : model.fields) { + properties.set(field.name, field_schema(field)); + if (field.required) required.push(body::Json(field.name)); + } + out.set("properties", std::move(properties)); + if (!required.as_array().empty()) out.set("required", std::move(required)); + return out; +} + +} // namespace molo::clients::openapi diff --git a/src/clients/openapi/schema.h b/src/clients/openapi/schema.h new file mode 100644 index 0000000..29360fb --- /dev/null +++ b/src/clients/openapi/schema.h @@ -0,0 +1,50 @@ +#pragma once + +// Validation-model to JSON Schema converter. ModelDecl/FieldDecl mirror the +// part 05 molo::validate::Model field descriptors 1:1; the adapter that fills +// them from real models lives with part 05, never here. + +#include +#include +#include + +#include "clients/body/json.h" + +namespace molo::clients::openapi { + +struct FieldDecl { + std::string name; + std::string type = "string"; // string, integer, number, boolean, array, + // object, or a $ref model name via ref. + std::string format; + std::string ref; // Non-empty means $ref to a named model schema. + std::string items_type; // Element type for arrays ("$ref:Name" for models). + bool required = false; + std::string description; + std::string example; + std::optional minimum; + std::optional maximum; + std::optional min_length; + std::optional max_length; + std::string pattern; + std::vector enum_values; + body::Json default_value; + bool has_default = false; +}; + +struct ModelDecl { + std::string name; + std::string description; + std::vector fields; +}; + +class SchemaConverter { + public: + // Converts one model to a JSON Schema object schema. + static body::Json convert(const ModelDecl& model); + // Converts a single field to its schema fragment. + static body::Json field_schema(const FieldDecl& field); + static body::Json ref_for(const std::string& model_name); +}; + +} // namespace molo::clients::openapi diff --git a/src/clients/pool.cpp b/src/clients/pool.cpp new file mode 100644 index 0000000..ce60a25 --- /dev/null +++ b/src/clients/pool.cpp @@ -0,0 +1,144 @@ +#include "clients/pool.h" + +#include + +#include "clients/caps.h" + +namespace molo::clients { + +ConnectionPool::ConnectionPool() = default; + +ConnectionPool::~ConnectionPool() { close_all(); } + +void ConnectionPool::prune_locked( + const std::string& key, std::chrono::steady_clock::time_point now) { + auto it = idle_.find(key); + if (it == idle_.end()) return; + auto& vec = it->second; + for (auto vit = vec.begin(); vit != vec.end();) { + if (vit->expires_at <= now || vit->fd < 0) { + if (vit->fd >= 0) { + ::close(vit->fd); + stats_.closed++; + if (total_ > 0) total_--; + } + vit = vec.erase(vit); + } else { + ++vit; + } + } + if (vec.empty()) idle_.erase(it); +} + +std::optional ConnectionPool::acquire( + const std::string& key) { + std::lock_guard lock(mutex_); + auto now = std::chrono::steady_clock::now(); + prune_locked(key, now); + auto it = idle_.find(key); + if (it == idle_.end() || it->second.empty()) return std::nullopt; + PooledConnection conn = it->second.back(); + it->second.pop_back(); + if (it->second.empty()) idle_.erase(it); + in_use_[key]++; + stats_.reused++; + return conn; +} + +void ConnectionPool::release(const std::string& key, int fd, bool reusable) { + std::lock_guard lock(mutex_); + auto use_it = in_use_.find(key); + if (use_it != in_use_.end() && use_it->second > 0) { + use_it->second--; + if (use_it->second == 0) in_use_.erase(use_it); + } + if (fd < 0) return; + if (!reusable) { + ::close(fd); + stats_.closed++; + if (total_ > 0) total_--; + return; + } + auto now = std::chrono::steady_clock::now(); + prune_locked(key, now); + auto& vec = idle_[key]; + if (vec.size() >= Caps::kPoolMaxIdlePerHost) { + ::close(fd); + stats_.closed++; + stats_.evicted_idle_overflow++; + if (total_ > 0) total_--; + return; + } + vec.push_back(PooledConnection{fd, now, now + Caps::kPoolKeepAlive}); +} + +bool ConnectionPool::reserve(const std::string& key) { + std::lock_guard lock(mutex_); + auto now = std::chrono::steady_clock::now(); + prune_locked(key, now); + std::size_t per_host = 0; + auto use_it = in_use_.find(key); + if (use_it != in_use_.end()) per_host += use_it->second; + auto idle_it = idle_.find(key); + if (idle_it != idle_.end()) per_host += idle_it->second.size(); + if (per_host >= Caps::kPoolMaxPerHost) return false; + if (total_ >= Caps::kPoolMaxTotal) { + // Try a global prune before refusing. + for (auto it = idle_.begin(); it != idle_.end();) { + prune_locked(it->first, now); + it = idle_.begin(); + if (total_ < Caps::kPoolMaxTotal) break; + } + if (total_ >= Caps::kPoolMaxTotal) return false; + } + in_use_[key]++; + total_++; + stats_.created++; + return true; +} + +void ConnectionPool::drop_reservation(const std::string& key) { + std::lock_guard lock(mutex_); + auto use_it = in_use_.find(key); + if (use_it != in_use_.end() && use_it->second > 0) { + use_it->second--; + if (use_it->second == 0) in_use_.erase(use_it); + } + if (total_ > 0) total_--; +} + +void ConnectionPool::discard(int fd) { + // Caller already removed its in-use accounting via release(fd, false) or + // drop_reservation; this path only closes stray fds. + if (fd >= 0) { + ::close(fd); + std::lock_guard lock(mutex_); + stats_.closed++; + } +} + +void ConnectionPool::close_all() { + std::lock_guard lock(mutex_); + for (auto& kv : idle_) { + for (auto& conn : kv.second) { + if (conn.fd >= 0) { + ::close(conn.fd); + stats_.closed++; + if (total_ > 0) total_--; + } + } + } + idle_.clear(); +} + +PoolStats ConnectionPool::stats() const { + std::lock_guard lock(mutex_); + PoolStats out = stats_; + out.in_use = 0; + for (const auto& kv : in_use_) out.in_use += kv.second; + out.idle = 0; + for (const auto& kv : idle_) out.idle += kv.second.size(); + return out; +} + +} // namespace molo::clients diff --git a/src/clients/pool.h b/src/clients/pool.h new file mode 100644 index 0000000..ccfc2ad --- /dev/null +++ b/src/clients/pool.h @@ -0,0 +1,65 @@ +#pragma once + +// Keep-alive connection pool. Owns idle socket fds keyed by pool key, +// enforces per-client, per-host, and idle caps with bounded acquire waits. + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::clients { + +struct PooledConnection { + int fd = -1; + std::chrono::steady_clock::time_point idle_since; + std::chrono::steady_clock::time_point expires_at; +}; + +struct PoolStats { + std::size_t in_use = 0; + std::size_t idle = 0; + std::size_t created = 0; + std::size_t reused = 0; + std::size_t closed = 0; + std::size_t evicted_idle_overflow = 0; +}; + +class ConnectionPool { + public: + ConnectionPool(); + ~ConnectionPool(); + + ConnectionPool(const ConnectionPool&) = delete; + ConnectionPool& operator=(const ConnectionPool&) = delete; + + // Takes an idle connection for key, or nullopt when none is reusable. + std::optional acquire(const std::string& key); + // Returns a connection to the idle set (reusable=true) or closes it. + void release(const std::string& key, int fd, bool reusable); + // Records that the caller opened a fresh connection outside the pool. + // Returns false when the per-host or total cap forbids a new connection. + bool reserve(const std::string& key); + // Drops a reservation made by reserve() (connect failed). + void drop_reservation(const std::string& key); + // Closes a connection that bypassed release() (I/O error mid-use). + void discard(int fd); + + void close_all(); + PoolStats stats() const; + + private: + void prune_locked(const std::string& key, + std::chrono::steady_clock::time_point now); + + mutable std::mutex mutex_; + std::map> idle_; + std::map in_use_; + std::size_t total_ = 0; + PoolStats stats_; +}; + +} // namespace molo::clients diff --git a/src/clients/proxy.cpp b/src/clients/proxy.cpp new file mode 100644 index 0000000..366b4b2 --- /dev/null +++ b/src/clients/proxy.cpp @@ -0,0 +1,143 @@ +#include "clients/proxy.h" + +#include +#include +#include + +namespace molo::clients { +namespace { + +std::string getenv_str(const char* name) { + const char* v = std::getenv(name); + return v ? std::string(v) : std::string{}; +} + +std::string trim(std::string s) { + while (!s.empty() && (s.front() == ' ' || s.front() == '\t')) s.erase(s.begin()); + while (!s.empty() && (s.back() == ' ' || s.back() == '\t')) s.pop_back(); + return s; +} + +std::string to_lower(std::string s) { + for (auto& c : s) c = static_cast(std::tolower((unsigned char)c)); + return s; +} + +} // namespace + +ProxyConfig ProxyConfig::from_env() { + ProxyConfig config; + config.http_proxy = getenv_str("HTTP_PROXY"); + if (config.http_proxy.empty()) config.http_proxy = getenv_str("http_proxy"); + config.https_proxy = getenv_str("HTTPS_PROXY"); + if (config.https_proxy.empty()) + config.https_proxy = getenv_str("https_proxy"); + std::string no_proxy = getenv_str("NO_PROXY"); + if (no_proxy.empty()) no_proxy = getenv_str("no_proxy"); + std::string rest = no_proxy; + while (!rest.empty()) { + auto comma = rest.find(','); + std::string entry = + trim(comma == std::string::npos ? rest : rest.substr(0, comma)); + rest = comma == std::string::npos ? "" : rest.substr(comma + 1); + if (!entry.empty()) config.no_proxy.push_back(entry); + } + return config; +} + +ProxySelector::ProxySelector(ProxyConfig config) : config_(std::move(config)) {} + +std::optional ProxySelector::select(const Url& url) const { + if (is_bypassed(url.host())) return std::nullopt; + const std::string& candidate = + url.is_https() ? config_.https_proxy : config_.http_proxy; + if (candidate.empty()) { + // Fall back to the other scheme proxy when only one is configured. + const std::string& other = + url.is_https() ? config_.http_proxy : config_.https_proxy; + if (other.empty()) return std::nullopt; + return other; + } + return candidate; +} + +bool ProxySelector::is_bypassed(const std::string& host) const { + std::string h = to_lower(trim(host)); + // Strip an optional :port suffix for matching. + auto colon = h.rfind(':'); + if (colon != std::string::npos && h.find(':') == colon) h = h.substr(0, colon); + for (const auto& pattern : config_.no_proxy) { + if (no_proxy_match(pattern, h)) return true; + } + return false; +} + +bool parse_ipv4(const std::string& text, std::uint32_t& out) { + unsigned parts[4] = {0, 0, 0, 0}; + std::string rest = text; + for (int i = 0; i < 4; ++i) { + auto dot = rest.find('.'); + std::string piece = + (i == 3 || dot == std::string::npos) ? rest : rest.substr(0, dot); + if (i < 3) { + if (dot == std::string::npos) return false; + rest = rest.substr(dot + 1); + } else if (dot != std::string::npos) { + return false; + } + if (piece.empty() || piece.size() > 3) return false; + for (char c : piece) { + if (c < '0' || c > '9') return false; + } + unsigned v = (unsigned)std::stoul(piece); + if (v > 255) return false; + parts[i] = v; + } + out = (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]; + return true; +} + +bool no_proxy_match(const std::string& pattern, const std::string& host) { + std::string p = to_lower(trim(pattern)); + std::string h = to_lower(trim(host)); + if (p.empty() || h.empty()) return false; + if (p == "*") return true; + // CIDR: 10.0.0.0/8 + auto slash = p.find('/'); + if (slash != std::string::npos) { + std::uint32_t net = 0, addr = 0; + int bits = 0; + try { + bits = std::stoi(p.substr(slash + 1)); + } catch (...) { + return false; + } + if (bits < 0 || bits > 32) return false; + if (!parse_ipv4(p.substr(0, slash), net)) return false; + if (!parse_ipv4(h, addr)) return false; + std::uint32_t mask = + bits == 0 ? 0u : (bits == 32 ? 0xFFFFFFFFu : (~0u << (32 - bits))); + return (net & mask) == (addr & mask); + } + // Wildcard: *.example.com + if (p.rfind("*.", 0) == 0) { + std::string suffix = p.substr(1); // .example.com + if (h.size() > suffix.size() && + h.compare(h.size() - suffix.size(), suffix.size(), suffix) == 0) + return true; + return h == suffix.substr(1); + } + // Leading-dot suffix: .example.com + if (p.front() == '.') { + if (h == p.substr(1)) return true; + return h.size() > p.size() && + h.compare(h.size() - p.size(), p.size(), p) == 0; + } + if (h == p) return true; + // Bare suffix match: example.com matches a.example.com. + return h.size() > p.size() + 1 && + h.compare(h.size() - p.size(), p.size(), p) == 0 && + h[h.size() - p.size() - 1] == '.'; +} + +} // namespace molo::clients diff --git a/src/clients/proxy.h b/src/clients/proxy.h new file mode 100644 index 0000000..d5239fd --- /dev/null +++ b/src/clients/proxy.h @@ -0,0 +1,40 @@ +#pragma once + +// Proxy selection with HTTP_PROXY/HTTPS_PROXY/NO_PROXY support and +// wildcard, suffix, and CIDR NO_PROXY matching. + +#include +#include +#include +#include + +#include "clients/url.h" + +namespace molo::clients { + +struct ProxyConfig { + std::string http_proxy; + std::string https_proxy; + std::vector no_proxy; + + static ProxyConfig from_env(); + bool empty() const { return http_proxy.empty() && https_proxy.empty(); } +}; + +class ProxySelector { + public: + explicit ProxySelector(ProxyConfig config); + + // Returns the proxy URL to use for the target, or nullopt for direct. + std::optional select(const Url& url) const; + bool is_bypassed(const std::string& host) const; + + private: + ProxyConfig config_; +}; + +bool no_proxy_match(const std::string& pattern, const std::string& host); +// Parses dotted IPv4 into host-order 32 bits. Returns false when invalid. +bool parse_ipv4(const std::string& text, std::uint32_t& out); + +} // namespace molo::clients diff --git a/src/clients/request.cpp b/src/clients/request.cpp new file mode 100644 index 0000000..a1a76a2 --- /dev/null +++ b/src/clients/request.cpp @@ -0,0 +1,139 @@ +#include "clients/request.h" + +#include + +#include "operability/resilience/deadline.h" + +namespace molo::clients { + +std::optional ClientRequest::deadline_ms() const { + const auto v = headers_.get("X-Deadline-Ms"); + if (!v) return std::nullopt; + return molo::resilience::Deadline::parse_header(*v); +} + +ClientRequest::ClientRequest(Method method, Url url) + : method_(method), url_(std::move(url)) {} + +ClientRequest ClientRequest::get(const std::string& url) { + return ClientRequest(Method::kGet, Url::parse(url)); +} +ClientRequest ClientRequest::post(const std::string& url) { + return ClientRequest(Method::kPost, Url::parse(url)); +} +ClientRequest ClientRequest::put(const std::string& url) { + return ClientRequest(Method::kPut, Url::parse(url)); +} +ClientRequest ClientRequest::patch(const std::string& url) { + return ClientRequest(Method::kPatch, Url::parse(url)); +} +ClientRequest ClientRequest::remove(const std::string& url) { + return ClientRequest(Method::kDelete, Url::parse(url)); +} +ClientRequest ClientRequest::head(const std::string& url) { + return ClientRequest(Method::kHead, Url::parse(url)); +} +ClientRequest ClientRequest::options(const std::string& url) { + return ClientRequest(Method::kOptions, Url::parse(url)); +} + +ClientRequest& ClientRequest::header(const std::string& name, + const std::string& value) { + headers_.set(name, value); + return *this; +} + +ClientRequest& ClientRequest::bearer(const std::string& token) { + headers_.set("Authorization", "Bearer " + token); + return *this; +} + +ClientRequest& ClientRequest::idempotency_key(const std::string& key) { + headers_.set("Idempotency-Key", key); + idempotent_ = true; + return *this; +} + +ClientRequest& ClientRequest::timeouts(Timeouts timeouts) { + timeouts_ = timeouts; + return *this; +} + +ClientRequest& ClientRequest::h2(bool enabled) { + h2_ = enabled; + return *this; +} + +ClientRequest& ClientRequest::bytes(std::string data, + const std::string& content_type) { + body_ = std::move(data); + if (!content_type.empty() && !headers_.has("Content-Type")) + headers_.set("Content-Type", content_type); + return *this; +} + +ClientRequest& ClientRequest::json_text(const std::string& payload) { + body_ = payload; + if (!headers_.has("Content-Type")) + headers_.set("Content-Type", "application/json"); + return *this; +} + +ClientRequest& ClientRequest::json(const body::Json& value) { + return json_text(value.dump()); +} + +ClientRequest& ClientRequest::form(const std::string& payload) { + body_ = payload; + if (!headers_.has("Content-Type")) + headers_.set("Content-Type", body::UrlEncodedForm::content_type()); + return *this; +} + +ClientRequest& ClientRequest::form(const body::UrlEncodedForm& fields) { + return form(fields.encode()); +} + +ClientRequest& ClientRequest::multipart(const body::MultipartBody& body) { + body_ = body.encode(); + if (!headers_.has("Content-Type")) + headers_.set("Content-Type", body.content_type()); + return *this; +} + +ClientRequest& ClientRequest::xml(const std::string& payload) { + body_ = payload; + if (!headers_.has("Content-Type")) + headers_.set("Content-Type", "application/xml"); + return *this; +} + +ClientRequest& ClientRequest::stream( + std::shared_ptr reader, std::optional length, + const std::string& content_type) { + StreamSource source{std::move(reader), + length ? length : std::optional{}}; + if (!source.length && source.reader) + source.length = source.reader->known_size(); + body_ = std::move(source); + if (!content_type.empty() && !headers_.has("Content-Type")) + headers_.set("Content-Type", content_type); + return *this; +} + +const std::string& ClientRequest::body_bytes() const { + static const std::string kEmpty; + if (auto* s = std::get_if(&body_)) return *s; + return kEmpty; +} + +bool ClientRequest::marked_idempotent() const { + return idempotent_ || headers_.has("Idempotency-Key"); +} + +ClientRequest& ClientRequest::mark_idempotent(bool value) { + idempotent_ = value; + return *this; +} + +} // namespace molo::clients diff --git a/src/clients/request.h b/src/clients/request.h new file mode 100644 index 0000000..f8d8f03 --- /dev/null +++ b/src/clients/request.h @@ -0,0 +1,110 @@ +#pragma once + +// Outbound request value type with a fluent builder. + +#include +#include +#include +#include +#include + +#include "clients/body/form.h" +#include "clients/body/json.h" +#include "clients/body/stream.h" +#include "clients/headers.h" +#include "clients/timeouts.h" +#include "clients/url.h" + +namespace molo::clients { + +enum class Method { kGet, kPost, kPut, kPatch, kDelete, kHead, kOptions }; + +inline const char* method_name(Method method) { + switch (method) { + case Method::kGet: return "GET"; + case Method::kPost: return "POST"; + case Method::kPut: return "PUT"; + case Method::kPatch: return "PATCH"; + case Method::kDelete: return "DELETE"; + case Method::kHead: return "HEAD"; + case Method::kOptions: return "OPTIONS"; + } + return "GET"; +} + +inline bool is_idempotent_method(Method method) { + return method == Method::kGet || method == Method::kHead || + method == Method::kOptions || method == Method::kPut || + method == Method::kDelete; +} + +struct StreamSource { + std::shared_ptr reader; + std::optional length; // Unknown length uses chunked encoding. +}; + +using RequestBody = std::variant; + +class ClientRequest { + public: + ClientRequest() = default; + ClientRequest(Method method, Url url); + + static ClientRequest get(const std::string& url); + static ClientRequest post(const std::string& url); + static ClientRequest put(const std::string& url); + static ClientRequest patch(const std::string& url); + static ClientRequest remove(const std::string& url); + static ClientRequest head(const std::string& url); + static ClientRequest options(const std::string& url); + + ClientRequest& header(const std::string& name, const std::string& value); + ClientRequest& bearer(const std::string& token); + ClientRequest& idempotency_key(const std::string& key); + ClientRequest& timeouts(Timeouts timeouts); + ClientRequest& h2(bool enabled); + + ClientRequest& bytes(std::string data, + const std::string& content_type = ""); + ClientRequest& json_text(const std::string& payload); + ClientRequest& json(const body::Json& value); + ClientRequest& form(const std::string& payload); + ClientRequest& form(const body::UrlEncodedForm& fields); + ClientRequest& multipart(const body::MultipartBody& body); + ClientRequest& xml(const std::string& payload); + ClientRequest& stream(std::shared_ptr reader, + std::optional length = std::nullopt, + const std::string& content_type = ""); + + Method method() const { return method_; } + const Url& url() const { return url_; } + Url& url() { return url_; } + const HeaderMap& headers() const { return headers_; } + HeaderMap& headers() { return headers_; } + const RequestBody& body() const { return body_; } + const std::optional& timeouts() const { return timeouts_; } + bool h2() const { return h2_; } + // Absolute X-Deadline-Ms value when present and well formed. + std::optional deadline_ms() const; + + bool has_body() const { + return !std::holds_alternative(body_); + } + bool is_stream_body() const { + return std::holds_alternative(body_); + } + const std::string& body_bytes() const; + bool marked_idempotent() const; // Idempotency-Key present or explicit flag. + ClientRequest& mark_idempotent(bool value = true); + + private: + Method method_ = Method::kGet; + Url url_; + HeaderMap headers_; + RequestBody body_; + std::optional timeouts_; + bool h2_ = false; + bool idempotent_ = false; +}; + +} // namespace molo::clients diff --git a/src/clients/response.cpp b/src/clients/response.cpp new file mode 100644 index 0000000..617a965 --- /dev/null +++ b/src/clients/response.cpp @@ -0,0 +1,27 @@ +#include "clients/response.h" + +#include + +#include "clients/body/xml.h" + +namespace molo::clients { + +ClientResponse::ClientResponse(int status, HeaderMap headers, std::string body) + : status_(status), headers_(std::move(headers)), body_(std::move(body)) {} + +std::string ClientResponse::content_type() const { + std::string full = headers_.get_or("Content-Type", ""); + auto semi = full.find(';'); + if (semi != std::string::npos) full = full.substr(0, semi); + while (!full.empty() && (full.back() == ' ' || full.back() == '\t')) + full.pop_back(); + return full; +} + +body::Json ClientResponse::json() const { return body::Json::parse(body_); } + +std::map ClientResponse::xml_flat() const { + return body::xml_parse_flat(body_); +} + +} // namespace molo::clients diff --git a/src/clients/response.h b/src/clients/response.h new file mode 100644 index 0000000..ef7d35c --- /dev/null +++ b/src/clients/response.h @@ -0,0 +1,53 @@ +#pragma once + +// Inbound response value type with content helpers. + +#include +#include +#include + +#include "clients/body/json.h" +#include "clients/headers.h" + +namespace molo::clients { + +class ClientResponse { + public: + ClientResponse() = default; + ClientResponse(int status, HeaderMap headers, std::string body); + + int status() const { return status_; } + const HeaderMap& headers() const { return headers_; } + HeaderMap& headers() { return headers_; } + const std::string& body() const { return body_; } + const std::string& text() const { return body_; } + + bool is_success() const { return status_ >= 200 && status_ < 300; } + bool is_redirect() const { + return status_ == 301 || status_ == 302 || status_ == 303 || + status_ == 307 || status_ == 308; + } + bool is_client_error() const { return status_ >= 400 && status_ < 500; } + bool is_server_error() const { return status_ >= 500 && status_ < 600; } + + std::string content_type() const; + body::Json json() const; + std::map xml_flat() const; + + int attempts() const { return attempts_; } + void set_attempts(int n) { attempts_ = n; } + std::chrono::milliseconds elapsed() const { return elapsed_; } + void set_elapsed(std::chrono::milliseconds ms) { elapsed_ = ms; } + bool from_pool_reuse() const { return from_pool_reuse_; } + void set_from_pool_reuse(bool v) { from_pool_reuse_ = v; } + + private: + int status_ = 0; + HeaderMap headers_; + std::string body_; + int attempts_ = 1; + std::chrono::milliseconds elapsed_{0}; + bool from_pool_reuse_ = false; +}; + +} // namespace molo::clients diff --git a/src/clients/rest/errors.cpp b/src/clients/rest/errors.cpp new file mode 100644 index 0000000..4b028f0 --- /dev/null +++ b/src/clients/rest/errors.cpp @@ -0,0 +1,124 @@ +#include "clients/rest/errors.h" + +#include "clients/headers.h" + +namespace molo::clients::rest { +namespace { + +std::string str_field(const body::Json& value, const std::string& key) { + const body::Json* found = value.find(key); + if (found && found->is_string()) return found->as_string(); + return ""; +} + +} // namespace + +ProblemDetails ProblemDetails::parse(const body::Json& value) { + ProblemDetails problem; + if (!value.is_object()) return problem; + std::string type = str_field(value, "type"); + if (!type.empty()) problem.type = type; + problem.title = str_field(value, "title"); + const body::Json* status = value.find("status"); + if (status && status->is_number()) + problem.status = (int)status->as_number(); + problem.detail = str_field(value, "detail"); + problem.instance = str_field(value, "instance"); + const body::Json* errors = value.find("errors"); + if (errors && errors->is_array()) { + for (const auto& item : errors->as_array()) { + if (!item.is_object()) continue; + FieldFailure failure; + failure.field = str_field(item, "field"); + if (failure.field.empty()) failure.field = str_field(item, "loc"); + failure.message = str_field(item, "message"); + if (failure.message.empty()) failure.message = str_field(item, "msg"); + failure.type = str_field(item, "type"); + problem.errors.push_back(failure); + } + } + return problem; +} + +ProblemDetails ProblemDetails::from_response(const ClientResponse& response) { + ProblemDetails problem; + problem.status = response.status(); + problem.title = "HTTP " + std::to_string(response.status()); + if (response.content_type() == content_type() || + response.content_type() == "application/json") { + try { + body::Json parsed = response.json(); + if (parsed.is_object() && + (parsed.has("title") || parsed.has("detail"))) { + problem = ProblemDetails::parse(parsed); + if (problem.status == 0) problem.status = response.status(); + return problem; + } + } catch (...) { + // Fall through to the generic shape below. + } + } + if (!response.body().empty() && response.body().size() < 4096) + problem.detail = response.body(); + return problem; +} + +body::Json ProblemDetails::to_json() const { + body::Json value = body::Json::object(); + value.set("type", body::Json(type)); + value.set("title", body::Json(title)); + value.set("status", body::Json(status)); + if (!detail.empty()) value.set("detail", body::Json(detail)); + if (!instance.empty()) value.set("instance", body::Json(instance)); + if (!errors.empty()) { + body::Json list = body::Json::array(); + for (const auto& item : errors) { + body::Json entry = body::Json::object(); + entry.set("field", body::Json(item.field)); + entry.set("message", body::Json(item.message)); + if (!item.type.empty()) entry.set("type", body::Json(item.type)); + list.push(std::move(entry)); + } + value.set("errors", std::move(list)); + } + return value; +} + +std::string ProblemDetails::summary() const { + std::string out = std::to_string(status); + if (!title.empty()) { + out += " "; + out += title; + } + if (!detail.empty()) { + out += ": "; + out += detail; + } + return out; +} + +RestError::RestError(ProblemDetails problem) + : std::runtime_error("molo::clients::rest: " + problem.summary()), + problem_(std::move(problem)) {} + +void raise_for_status(const ClientResponse& response) { + if (response.status() >= 400) throw RestError(ProblemDetails::from_response(response)); +} + +ClientResponse problem_response(int status, const std::string& title, + const std::string& detail, + const std::string& instance) { + ProblemDetails problem; + problem.type = "https://molo.app/problems/" + std::to_string(status); + problem.title = title; + problem.status = status; + problem.detail = detail; + problem.instance = instance; + HeaderMap headers; + headers.set("Content-Type", ProblemDetails::content_type()); + std::string body = problem.to_json().dump(); + headers.set("Content-Length", std::to_string(body.size())); + return ClientResponse(status, headers, body); +} + +} // namespace molo::clients::rest diff --git a/src/clients/rest/errors.h b/src/clients/rest/errors.h new file mode 100644 index 0000000..bd6a00a --- /dev/null +++ b/src/clients/rest/errors.h @@ -0,0 +1,53 @@ +#pragma once + +// RFC 9457 problem details plus raise-for-status. + +#include +#include +#include + +#include "clients/body/json.h" +#include "clients/response.h" + +namespace molo::clients::rest { + +struct FieldFailure { + std::string field; + std::string message; + std::string type; +}; + +struct ProblemDetails { + std::string type = "about:blank"; + std::string title; + int status = 0; + std::string detail; + std::string instance; + std::vector errors; + + static ProblemDetails parse(const body::Json& value); + static ProblemDetails from_response(const ClientResponse& response); + body::Json to_json() const; + std::string summary() const; + + static const char* content_type() { return "application/problem+json"; } +}; + +class RestError : public std::runtime_error { + public: + explicit RestError(ProblemDetails problem); + const ProblemDetails& problem() const { return problem_; } + int status() const { return problem_.status; } + + private: + ProblemDetails problem_; +}; + +// Throws RestError when status >= 400, carrying parsed problem details. +void raise_for_status(const ClientResponse& response); +// Builds a problem-details ClientResponse (server-side helper for tests). +ClientResponse problem_response(int status, const std::string& title, + const std::string& detail, + const std::string& instance = ""); + +} // namespace molo::clients::rest diff --git a/src/clients/rest/idempotency.cpp b/src/clients/rest/idempotency.cpp new file mode 100644 index 0000000..f0f97ab --- /dev/null +++ b/src/clients/rest/idempotency.cpp @@ -0,0 +1,47 @@ +#include "clients/rest/idempotency.h" + +#include + +#include "clients/caps.h" + +namespace molo::clients::rest { + +std::string IdempotencyKey::generate() { + static thread_local std::mt19937_64 rng{std::random_device{}()}; + static const char* kDigits = "0123456789abcdef"; + std::string key; + key.reserve(32); + for (int i = 0; i < 32; ++i) key.push_back(kDigits[rng() % 16]); + return key; +} + +std::optional InMemoryIdempotencyStore::lookup( + const std::string& key) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) return std::nullopt; + if (std::chrono::steady_clock::now() - it->second.stored_at > + Caps::kIdempotencyTtl) { + entries_.erase(it); + return std::nullopt; + } + return it->second; +} + +void InMemoryIdempotencyStore::store(const std::string& key, + IdempotentResult result) { + std::lock_guard lock(mutex_); + if (entries_.size() >= Caps::kIdempotencyMaxEntries && !entries_.count(key)) { + // Loud skip: evict nothing silently is worse, so drop the oldest entry. + entries_.erase(entries_.begin()); + } + result.stored_at = std::chrono::steady_clock::now(); + entries_[key] = std::move(result); +} + +std::size_t InMemoryIdempotencyStore::size() const { + std::lock_guard lock(mutex_); + return entries_.size(); +} + +} // namespace molo::clients::rest diff --git a/src/clients/rest/idempotency.h b/src/clients/rest/idempotency.h new file mode 100644 index 0000000..ba6fb83 --- /dev/null +++ b/src/clients/rest/idempotency.h @@ -0,0 +1,48 @@ +#pragma once + +// Idempotency-Key sender plus the keyed-result store interface. The in-memory +// store is the single-process dev default; part 04 KV backs production. + +#include +#include +#include +#include +#include +#include + +#include "clients/body/json.h" + +namespace molo::clients::rest { + +class IdempotencyKey { + public: + static std::string generate(); + static const char* header() { return "Idempotency-Key"; } +}; + +struct IdempotentResult { + int status = 0; + std::string body; + std::string content_type; + std::chrono::steady_clock::time_point stored_at; +}; + +class IdempotencyStore { + public: + virtual ~IdempotencyStore() = default; + virtual std::optional lookup(const std::string& key) = 0; + virtual void store(const std::string& key, IdempotentResult result) = 0; +}; + +class InMemoryIdempotencyStore : public IdempotencyStore { + public: + std::optional lookup(const std::string& key) override; + void store(const std::string& key, IdempotentResult result) override; + std::size_t size() const; + + private: + mutable std::mutex mutex_; + std::map entries_; +}; + +} // namespace molo::clients::rest diff --git a/src/clients/rest/paginator.cpp b/src/clients/rest/paginator.cpp new file mode 100644 index 0000000..7b123d2 --- /dev/null +++ b/src/clients/rest/paginator.cpp @@ -0,0 +1,112 @@ +#include "clients/rest/paginator.h" + +#include + +#include "clients/caps.h" +#include "clients/rest/errors.h" +#include "clients/url.h" + +namespace molo::clients::rest { + +Paginator::Paginator(Client& client, std::string collection_url) + : client_(client), url_(std::move(collection_url)) {} + +Paginator& Paginator::per_page(int n) { + if (n < 1) n = 1; + if (n > Caps::kPageMax) n = Caps::kPageMax; + per_page_ = n; + return *this; +} + +Paginator& Paginator::query_arg(const std::string& name, + const std::string& value) { + extra_[name] = value; + return *this; +} + +Page Paginator::parse_envelope(const ClientResponse& response) { + body::Json doc = response.json(); + Page page; + if (doc.is_array()) { + // Bare array: single page, no meta. + page.items = doc; + page.total = (long)doc.as_array().size(); + return page; + } + const body::Json* data = doc.find("data"); + if (data && data->is_array()) page.items = *data; + const body::Json* meta = doc.find("meta"); + if (meta && meta->is_object()) { + const body::Json* p = meta->find("page"); + const body::Json* pp = meta->find("per_page"); + const body::Json* t = meta->find("total"); + const body::Json* cursor = meta->find("next_cursor"); + if (p && p->is_number()) page.page = (int)p->as_number(); + if (pp && pp->is_number()) page.per_page = (int)pp->as_number(); + if (t && t->is_number()) page.total = (long)t->as_number(); + if (cursor && cursor->is_string()) page.next_cursor = cursor->as_string(); + } + const body::Json* links = doc.find("links"); + if (links && links->is_object()) { + const body::Json* next = links->find("next"); + if (next && next->is_string()) page.next_link = next->as_string(); + } + return page; +} + +Task Paginator::fetch_page(int page) { + if (page < 1) page = 1; + Url url = Url::parse(url_); + url.set_query_arg("page", std::to_string(page)); + url.set_query_arg("per_page", std::to_string(per_page_)); + for (const auto& kv : extra_) url.set_query_arg(kv.first, kv.second); + ClientResponse response = co_await client_.get(url.str()); + raise_for_status(response); + co_return parse_envelope(response); +} + +Page Paginator::fetch_page_blocking(int page) { + return block_on(fetch_page(page)); +} + +Task> Paginator::fetch_all() { + std::vector all; + // Cursor style when the first page advertises it, else page numbers. + Page first = co_await fetch_page(1); + for (const auto& item : first.items.as_array()) all.push_back(item); + if (!first.next_cursor.empty() || !first.next_link.empty()) { + std::string cursor = first.next_cursor; + std::string link = first.next_link; + for (int hop = 1; hop < Caps::kPaginatorMaxPages; ++hop) { + if (cursor.empty() && link.empty()) break; + Url next_url = Url::parse(url_); + next_url.set_query_arg("cursor", cursor); + ClientResponse response = + co_await client_.get(!link.empty() ? link : next_url.str()); + raise_for_status(response); + Page next = parse_envelope(response); + if (next.items.as_array().empty()) break; + for (const auto& item : next.items.as_array()) all.push_back(item); + cursor = next.next_cursor; + link = next.next_link; + } + co_return all; + } + if (first.items.as_array().empty()) co_return all; + long total = first.total; + if (total >= 0 && (long)all.size() >= total) co_return all; + for (int page = 2; page <= Caps::kPaginatorMaxPages; ++page) { + Page next = co_await fetch_page(page); + if (next.items.as_array().empty()) break; + for (const auto& item : next.items.as_array()) all.push_back(item); + if (total >= 0 && (long)all.size() >= total) break; + if ((int)next.items.as_array().size() < per_page_) break; + } + co_return all; +} + +std::vector Paginator::fetch_all_blocking() { + return block_on(fetch_all()); +} + +} // namespace molo::clients::rest diff --git a/src/clients/rest/paginator.h b/src/clients/rest/paginator.h new file mode 100644 index 0000000..9d551de --- /dev/null +++ b/src/clients/rest/paginator.h @@ -0,0 +1,51 @@ +#pragma once + +// Pagination walker for the standard envelope: +// {data:[...], meta:{page,per_page,total}, links:{next}}. +// Supports page+per_page and cursor styles with static bounds. + +#include +#include +#include + +#include "clients/body/json.h" +#include "clients/client.h" +#include "clients/response.h" +#include "clients/task.h" + +namespace molo::clients::rest { + +struct Page { + body::Json items = body::Json::array(); // The data array. + int page = 1; + int per_page = 0; + long total = -1; + std::string next_cursor; + std::string next_link; +}; + +class Paginator { + public: + Paginator(Client& client, std::string collection_url); + + Paginator& per_page(int n); + Paginator& query_arg(const std::string& name, const std::string& value); + + // Fetches one page (1-based) and parses the envelope. + Task fetch_page(int page); + Page fetch_page_blocking(int page); + + // Walks pages until data is empty or caps hit. Returns all items. + Task> fetch_all(); + std::vector fetch_all_blocking(); + + static Page parse_envelope(const ClientResponse& response); + + private: + Client& client_; + std::string url_; + int per_page_ = 20; + std::map extra_; +}; + +} // namespace molo::clients::rest diff --git a/src/clients/rest/resource.cpp b/src/clients/rest/resource.cpp new file mode 100644 index 0000000..e671adb --- /dev/null +++ b/src/clients/rest/resource.cpp @@ -0,0 +1,111 @@ +#include "clients/rest/resource.h" + +#include + +#include "clients/rest/idempotency.h" +#include "clients/url.h" + +namespace molo::clients::rest { + +Resource::Resource(Client& client, std::string collection) + : client_(client), collection_(std::move(collection)) {} + +Resource::Resource(Client& client, std::string base_url, + std::string collection) + : client_(client), + base_url_(std::move(base_url)), + collection_(std::move(collection)) {} + +Resource& Resource::idempotency(std::shared_ptr store) { + idempotency_ = std::move(store); + return *this; +} + +std::string Resource::member_url(const std::string& id) const { + return base_url_ + collection_ + "/" + Url::encode_component(id); +} + +std::string Resource::collection_url( + const std::map& query) const { + std::string url = base_url_ + collection_; + bool first = true; + for (const auto& kv : query) { + url.push_back(first ? '?' : '&'); + first = false; + url += Url::encode_component(kv.first); + url.push_back('='); + url += Url::encode_component(kv.second); + } + return url; +} + +Task Resource::list( + const std::map& query) { + co_return co_await client_.get(collection_url(query)); +} + +Task Resource::get(const std::string& id) { + co_return co_await client_.get(member_url(id)); +} + +Task Resource::create(const body::Json& doc) { + ClientRequest req = ClientRequest::post(collection_url()); + req.json(doc); + req.idempotency_key(IdempotencyKey::generate()); + co_return co_await client_.send(std::move(req)); +} + +Task Resource::create(const std::string& body, + const std::string& content_type) { + ClientRequest req = ClientRequest::post(collection_url()); + req.bytes(body, content_type); + req.idempotency_key(IdempotencyKey::generate()); + co_return co_await client_.send(std::move(req)); +} + +Task Resource::update(const std::string& id, + const body::Json& doc) { + ClientRequest req = ClientRequest::put(member_url(id)); + req.json(doc); + co_return co_await client_.send(std::move(req)); +} + +Task Resource::patch(const std::string& id, + const body::Json& doc) { + ClientRequest req = ClientRequest::patch(member_url(id)); + req.json(doc); + co_return co_await client_.send(std::move(req)); +} + +Task Resource::remove(const std::string& id) { + co_return co_await client_.remove(member_url(id)); +} + +ClientResponse Resource::list_blocking( + const std::map& query) { + return block_on(list(query)); +} + +ClientResponse Resource::get_blocking(const std::string& id) { + return block_on(get(id)); +} + +ClientResponse Resource::create_blocking(const body::Json& doc) { + return block_on(create(doc)); +} + +ClientResponse Resource::update_blocking(const std::string& id, + const body::Json& doc) { + return block_on(update(id, doc)); +} + +ClientResponse Resource::patch_blocking(const std::string& id, + const body::Json& doc) { + return block_on(patch(id, doc)); +} + +ClientResponse Resource::remove_blocking(const std::string& id) { + return block_on(remove(id)); +} + +} // namespace molo::clients::rest diff --git a/src/clients/rest/resource.h b/src/clients/rest/resource.h new file mode 100644 index 0000000..c7d9026 --- /dev/null +++ b/src/clients/rest/resource.h @@ -0,0 +1,57 @@ +#pragma once + +// Fluent CRUD helper over a plural-noun collection path. + +#include +#include +#include + +#include "clients/body/json.h" +#include "clients/client.h" +#include "clients/request.h" +#include "clients/response.h" +#include "clients/task.h" + +namespace molo::clients::rest { + +class IdempotencyStore; + +class Resource { + public: + Resource(Client& client, std::string collection); + Resource(Client& client, std::string base_url, std::string collection); + + Resource& idempotency(std::shared_ptr store); + + const std::string& collection() const { return collection_; } + + std::string member_url(const std::string& id) const; + std::string collection_url( + const std::map& query = {}) const; + + Task list( + const std::map& query = {}); + Task get(const std::string& id); + Task create(const body::Json& doc); + Task create(const std::string& body, + const std::string& content_type); + Task update(const std::string& id, const body::Json& doc); + Task patch(const std::string& id, const body::Json& doc); + Task remove(const std::string& id); + + ClientResponse list_blocking( + const std::map& query = {}); + ClientResponse get_blocking(const std::string& id); + ClientResponse create_blocking(const body::Json& doc); + ClientResponse update_blocking(const std::string& id, const body::Json& doc); + ClientResponse patch_blocking(const std::string& id, const body::Json& doc); + ClientResponse remove_blocking(const std::string& id); + + private: + Client& client_; + std::string base_url_; + std::string collection_; + std::shared_ptr idempotency_; +}; + +} // namespace molo::clients::rest diff --git a/src/clients/retry.cpp b/src/clients/retry.cpp new file mode 100644 index 0000000..0c593ff --- /dev/null +++ b/src/clients/retry.cpp @@ -0,0 +1,152 @@ +#include "clients/retry.h" + +#include +#include +#include +#include +#include +#include + +namespace molo::clients { +namespace { + +std::mutex& rng_mutex() { + static std::mutex m; + return m; +} +std::mt19937_64& rng() { + static std::mt19937_64 r{std::random_device{}()}; + return r; +} + +// Parses IMF-fixdate, e.g. "Wed, 21 Oct 2015 07:28:00 GMT". +bool parse_http_date(const std::string& text, std::time_t& out) { + if (text.size() < 29) return false; + int day = 0, year = 0, hour = 0, min = 0, sec = 0; + char mon[4] = {0, 0, 0, 0}; + if (sscanf(text.c_str(), "%*3s, %d %3s %d %d:%d:%d", &day, mon, &year, &hour, + &min, &sec) != 6) + return false; + static const char* kMonths[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + int month = -1; + for (int i = 0; i < 12; ++i) { + if (std::string(mon) == kMonths[i]) { + month = i; + break; + } + } + if (month < 0) return false; + std::tm tm{}; + tm.tm_mday = day; + tm.tm_mon = month; + tm.tm_year = year - 1900; + tm.tm_hour = hour; + tm.tm_min = min; + tm.tm_sec = sec; +#if defined(_WIN32) + out = _mkgmtime(&tm); +#else + out = timegm(&tm); +#endif + return out != (std::time_t)-1; +} + +} // namespace + +RetryPolicy::RetryPolicy() = default; + +RetryPolicy& RetryPolicy::max_attempts(int n) { + if (n < 1) n = 1; + if (n > 10) n = 10; + max_attempts_ = n; + return *this; +} + +RetryPolicy& RetryPolicy::base(std::chrono::milliseconds ms) { + if (ms.count() < 0) ms = std::chrono::milliseconds{0}; + base_ = ms; + return *this; +} + +RetryPolicy& RetryPolicy::cap(std::chrono::milliseconds ms) { + if (ms.count() < 0) ms = std::chrono::milliseconds{0}; + if (ms > Caps::kRetryCap) ms = Caps::kRetryCap; + cap_ = ms; + return *this; +} + +RetryPolicy& RetryPolicy::retry_status(int status, bool enabled) { + if (enabled) + retryable_.insert(status); + else + retryable_.erase(status); + return *this; +} + +RetryPolicy& RetryPolicy::allow_non_idempotent_retry(bool allow) { + allow_non_idempotent_ = allow; + return *this; +} + +bool RetryPolicy::should_retry_status(const ClientRequest& request, int status, + int attempt) const { + if (attempt >= max_attempts_) return false; + if (retryable_.count(status) == 0) return false; + if (is_idempotent_method(request.method())) return true; + if (request.marked_idempotent()) return true; + return allow_non_idempotent_; +} + +bool RetryPolicy::should_retry_connect_failure(int attempt) const { + return attempt < max_attempts_; +} + +std::chrono::milliseconds RetryPolicy::backoff_for(int attempt) const { + long long exp = 1LL << (attempt - 1 > 20 ? 20 : attempt - 1); + long long ceiling = base_.count() * exp; + if (ceiling > cap_.count()) ceiling = cap_.count(); + if (ceiling <= 0) return std::chrono::milliseconds{0}; + std::lock_guard lock(rng_mutex()); + std::uniform_int_distribution dist(0, ceiling); + return std::chrono::milliseconds{dist(rng())}; +} + +std::optional RetryPolicy::parse_retry_after( + const std::string& value) { + std::string v = value; + while (!v.empty() && (v.front() == ' ' || v.front() == '\t')) v.erase(v.begin()); + while (!v.empty() && (v.back() == ' ' || v.back() == '\t')) v.pop_back(); + if (v.empty()) return std::nullopt; + char* end = nullptr; + long secs = std::strtol(v.c_str(), &end, 10); + if (end && *end == '\0' && secs >= 0) { + auto ms = std::chrono::milliseconds{secs * 1000LL}; + if (ms > Caps::kRetryCap) ms = Caps::kRetryCap; + return ms; + } + std::time_t when = 0; + if (parse_http_date(v, when)) { + std::time_t now = std::time(nullptr); + long long diff = (long long)(when - now); + if (diff < 0) diff = 0; + auto ms = std::chrono::milliseconds{diff * 1000LL}; + if (ms > Caps::kRetryCap) ms = Caps::kRetryCap; + return ms; + } + return std::nullopt; +} + +void RetryPolicy::set_test_seed(std::int64_t seed) { + std::lock_guard lock(rng_mutex()); + if (seed < 0) + rng().seed(std::random_device{}()); + else + rng().seed(static_cast(seed)); +} + +void RetryPolicy::sleep_default(std::chrono::milliseconds ms) { + std::this_thread::sleep_for(ms); +} + +} // namespace molo::clients diff --git a/src/clients/retry.h b/src/clients/retry.h new file mode 100644 index 0000000..9d97ad8 --- /dev/null +++ b/src/clients/retry.h @@ -0,0 +1,61 @@ +#pragma once + +// Retry policy: idempotency gate, exponential backoff with full jitter, +// Retry-After support. All ceilings are static. + +#include +#include +#include +#include +#include + +#include "clients/caps.h" +#include "clients/request.h" + +namespace molo::clients { + +class RetryPolicy { + public: + RetryPolicy(); + + RetryPolicy& max_attempts(int n); + RetryPolicy& base(std::chrono::milliseconds ms); + RetryPolicy& cap(std::chrono::milliseconds ms); + RetryPolicy& retry_status(int status, bool enabled = true); + // Allows POST/PUT/PATCH retry after the body was sent. Default false. + RetryPolicy& allow_non_idempotent_retry(bool allow); + + int max_attempts() const { return max_attempts_; } + + // True when another attempt is allowed after a failed attempt that already + // sent the body. attempt is 1-based (attempt 1 just finished). + bool should_retry_status(const ClientRequest& request, int status, + int attempt) const; + // True when another attempt is allowed after a connect-phase failure + // (nothing was sent, so any method may retry). + bool should_retry_connect_failure(int attempt) const; + + // Delay before attempt number `attempt + 1`. Full jitter: uniform in + // [0, min(cap, base * 2^(attempt-1))]. attempt is 1-based (just finished). + std::chrono::milliseconds backoff_for(int attempt) const; + + // Parses Retry-After (delay seconds or HTTP-date) into a bounded delay. + // Returns nullopt when absent or unparsable. + static std::optional parse_retry_after( + const std::string& value); + + // Deterministic jitter for tests. Negative restores random seeding. + static void set_test_seed(std::int64_t seed); + + using Sleeper = std::function; + static void sleep_default(std::chrono::milliseconds ms); + + private: + int max_attempts_ = Caps::kMaxAttempts; + std::chrono::milliseconds base_ = Caps::kRetryBase; + std::chrono::milliseconds cap_ = Caps::kRetryCap; + std::set retryable_ = {429, 502, 503, 504}; + bool allow_non_idempotent_ = false; +}; + +} // namespace molo::clients diff --git a/src/clients/task.h b/src/clients/task.h new file mode 100644 index 0000000..1c8fb25 --- /dev/null +++ b/src/clients/task.h @@ -0,0 +1,127 @@ +#pragma once + +// Minimal eager coroutine task. Coroutine forms are the primary client API; +// sync `_blocking` forms share the implementation through block_on(). +// Tasks run eagerly on the calling thread; suspension points are reserved +// for integration with the part 01 runtime loop. + +#include +#include +#include +#include + +namespace molo::clients { + +template +class Task { + public: + struct promise_type { + std::optional value; + std::exception_ptr error; + Task get_return_object() { + return Task{std::coroutine_handle::from_promise(*this)}; + } + std::suspend_never initial_suspend() noexcept { return {}; } + std::suspend_always final_suspend() noexcept { return {}; } + void return_value(T v) { value.emplace(std::move(v)); } + void unhandled_exception() { error = std::current_exception(); } + }; + + using Handle = std::coroutine_handle; + + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + Task(Task&& other) noexcept : handle_(other.handle_) { + other.handle_ = nullptr; + } + Task& operator=(Task&& other) noexcept { + if (this != &other) { + destroy(); + handle_ = other.handle_; + other.handle_ = nullptr; + } + return *this; + } + ~Task() { destroy(); } + + // Awaitable interface (already complete: eager evaluation). + bool await_ready() const noexcept { return true; } + void await_suspend(std::coroutine_handle<>) noexcept {} + T await_resume() { return get(); } + + T get() { + if (!handle_) throw std::logic_error("molo::clients::Task: empty task"); + auto& promise = handle_.promise(); + if (promise.error) std::rethrow_exception(promise.error); + if (!promise.value) throw std::logic_error("molo::clients::Task: no value"); + return std::move(*promise.value); + } + + private: + explicit Task(Handle h) : handle_(h) {} + void destroy() { + if (handle_) handle_.destroy(); + handle_ = nullptr; + } + Handle handle_{nullptr}; +}; + +template <> +class Task { + public: + struct promise_type { + std::exception_ptr error; + Task get_return_object() { + return Task{std::coroutine_handle::from_promise(*this)}; + } + std::suspend_never initial_suspend() noexcept { return {}; } + std::suspend_always final_suspend() noexcept { return {}; } + void return_void() noexcept {} + void unhandled_exception() { error = std::current_exception(); } + }; + + using Handle = std::coroutine_handle; + + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + Task(Task&& other) noexcept : handle_(other.handle_) { + other.handle_ = nullptr; + } + Task& operator=(Task&& other) noexcept { + if (this != &other) { + destroy(); + handle_ = other.handle_; + other.handle_ = nullptr; + } + return *this; + } + ~Task() { destroy(); } + + bool await_ready() const noexcept { return true; } + void await_suspend(std::coroutine_handle<>) noexcept {} + void await_resume() { get(); } + + void get() { + if (!handle_) throw std::logic_error("molo::clients::Task: empty task"); + if (handle_.promise().error) + std::rethrow_exception(handle_.promise().error); + } + + private: + explicit Task(Handle h) : handle_(h) {} + void destroy() { + if (handle_) handle_.destroy(); + handle_ = nullptr; + } + Handle handle_{nullptr}; +}; + +// Blocking bridge for CLI use. Shares the coroutine implementation. +template +T block_on(Task task) { + return task.get(); +} + +inline void block_on(Task task) { task.get(); } + +} // namespace molo::clients diff --git a/src/clients/testing/assertions.cpp b/src/clients/testing/assertions.cpp new file mode 100644 index 0000000..19110dd --- /dev/null +++ b/src/clients/testing/assertions.cpp @@ -0,0 +1,166 @@ +#include "clients/testing/assertions.h" + +namespace molo::clients::testing { +namespace { + +std::string at(const std::string& path, const std::string& msg) { + return path.empty() ? msg : path + ": " + msg; +} + +void validate_against(const body::Json& instance, const body::Json& schema, + const std::string& path) { + if (!schema.is_object()) return; + const body::Json* ref = schema.find("$ref"); + if (ref && ref->is_string()) return; // Resolved by the caller. + const body::Json* type = schema.find("type"); + if (type && type->is_string()) { + const std::string& t = type->as_string(); + bool ok = (t == "string" && instance.is_string()) || + (t == "boolean" && instance.is_bool()) || + ((t == "integer" || t == "number") && instance.is_number()) || + (t == "array" && instance.is_array()) || + (t == "object" && instance.is_object()) || + (t == "null" && instance.is_null()); + if (!ok) + throw ExpectError(at(path, "expected type " + t)); + if (t == "integer" && instance.is_number()) { + double d = instance.as_number(); + if (d != (double)(long)d) + throw ExpectError(at(path, "expected integer value")); + } + } + const body::Json* enum_values = schema.find("enum"); + if (enum_values && enum_values->is_array()) { + bool found = false; + for (const auto& candidate : enum_values->as_array()) { + if (candidate.equals(instance)) { + found = true; + break; + } + } + if (!found) throw ExpectError(at(path, "value not in enum")); + } + const body::Json* required = schema.find("required"); + if (required && required->is_array()) { + if (!instance.is_object()) + throw ExpectError(at(path, "required needs an object")); + for (const auto& key : required->as_array()) { + if (key.is_string() && !instance.has(key.as_string())) + throw ExpectError(at(path, "missing required: " + key.as_string())); + } + } + const body::Json* properties = schema.find("properties"); + if (properties && properties->is_object() && instance.is_object()) { + for (const auto& kv : properties->as_object()) { + const body::Json* child = instance.find(kv.first); + if (child) + validate_against(*child, kv.second, + path.empty() ? kv.first : path + "." + kv.first); + } + } + const body::Json* items = schema.find("items"); + if (items && instance.is_array()) { + for (std::size_t i = 0; i < instance.as_array().size(); ++i) { + validate_against(instance.as_array()[i], *items, + path + "[" + std::to_string(i) + "]"); + } + } +} + +void validate_resolving(const body::Json& instance, const body::Json& schema, + const openapi::Document& doc, const std::string& path, + int depth) { + if (depth > 16) throw ExpectError(at(path, "schema $ref cycle")); + if (schema.is_object()) { + const body::Json* ref = schema.find("$ref"); + if (ref && ref->is_string()) { + const std::string& target = ref->as_string(); + const std::string prefix = "#/components/schemas/"; + if (target.rfind(prefix, 0) != 0) + throw ExpectError(at(path, "unsupported $ref: " + target)); + std::string name = target.substr(prefix.size()); + auto it = doc.schemas().find(name); + if (it == doc.schemas().end()) + throw ExpectError(at(path, "unknown schema: " + name)); + validate_resolving(instance, it->second, doc, path, depth + 1); + return; + } + } + // Validate this level, then resolve nested $refs manually. + validate_against(instance, schema, path); + if (!schema.is_object()) return; + const body::Json* properties = schema.find("properties"); + if (properties && properties->is_object() && instance.is_object()) { + for (const auto& kv : properties->as_object()) { + const body::Json* child = instance.find(kv.first); + if (child && kv.second.is_object() && kv.second.has("$ref")) + validate_resolving(*child, kv.second, doc, + path.empty() ? kv.first : path + "." + kv.first, + depth + 1); + else if (child && kv.second.is_object()) { + const body::Json* items = kv.second.find("items"); + if (items && items->is_object() && items->has("$ref") && + child->is_array()) { + for (std::size_t i = 0; i < child->as_array().size(); ++i) + validate_resolving(child->as_array()[i], *items, doc, + kv.first + "[" + std::to_string(i) + "]", + depth + 1); + } + } + } + } +} + +} // namespace + +void expect_status(const ClientResponse& response, int status) { + if (response.status() != status) + throw ExpectError("expected status " + std::to_string(status) + ", got " + + std::to_string(response.status()) + " body: " + + response.body().substr(0, 300)); +} + +void expect_header(const ClientResponse& response, const std::string& name, + const std::string& value) { + auto got = response.headers().get(name); + if (!got) + throw ExpectError("missing header: " + name); + if (*got != value) + throw ExpectError("header " + name + ": expected '" + value + "', got '" + + *got + "'"); +} + +void expect_json(const ClientResponse& response, const body::Json& expected) { + body::Json actual = response.json(); + if (!actual.equals(expected)) + throw ExpectError("JSON mismatch: got " + actual.dump() + ", want " + + expected.dump()); +} + +void expect_json_subset(const body::Json& actual, const body::Json& expected) { + if (!expected.is_object() || !actual.is_object()) + throw ExpectError("subset check needs objects"); + for (const auto& kv : expected.as_object()) { + const body::Json* got = actual.find(kv.first); + if (!got) + throw ExpectError("subset missing key: " + kv.first); + if (!got->equals(kv.second)) + throw ExpectError("subset mismatch at " + kv.first + ": got " + + got->dump() + ", want " + kv.second.dump()); + } +} + +void expect_json_schema(const body::Json& instance, const body::Json& schema) { + validate_against(instance, schema, ""); +} + +void expect_schema_by_name(const body::Json& instance, + const std::string& schema_name, + const openapi::Document& doc) { + auto it = doc.schemas().find(schema_name); + if (it == doc.schemas().end()) + throw ExpectError("unknown schema: " + schema_name); + validate_resolving(instance, it->second, doc, "", 0); +} + +} // namespace molo::clients::testing diff --git a/src/clients/testing/assertions.h b/src/clients/testing/assertions.h new file mode 100644 index 0000000..9c7a2e0 --- /dev/null +++ b/src/clients/testing/assertions.h @@ -0,0 +1,35 @@ +#pragma once + +// Assertion helpers for integration tests: status, JSON, headers, and +// OpenAPI schema conformance (subset validator over generated schemas). + +#include +#include + +#include "clients/body/json.h" +#include "clients/openapi/document.h" +#include "clients/response.h" + +namespace molo::clients::testing { + +class ExpectError : public std::runtime_error { + public: + explicit ExpectError(const std::string& msg) + : std::runtime_error("molo::clients::testing::Expect: " + msg) {} +}; + +void expect_status(const ClientResponse& response, int status); +void expect_header(const ClientResponse& response, const std::string& name, + const std::string& value); +void expect_json(const ClientResponse& response, const body::Json& expected); +// Subset check: every key in expected must equal the actual value. +void expect_json_subset(const body::Json& actual, const body::Json& expected); + +// Minimal JSON Schema validator: type, required, properties, items, enum. +// sufficient for generated OpenAPI component schemas. +void expect_json_schema(const body::Json& instance, const body::Json& schema); +void expect_schema_by_name(const body::Json& instance, + const std::string& schema_name, + const openapi::Document& doc); + +} // namespace molo::clients::testing diff --git a/src/clients/testing/clock.cpp b/src/clients/testing/clock.cpp new file mode 100644 index 0000000..98bec69 --- /dev/null +++ b/src/clients/testing/clock.cpp @@ -0,0 +1,50 @@ +#include "clients/testing/clock.h" + +namespace molo::clients::testing { +namespace { + +std::mutex& clock_mutex() { + static std::mutex mutex; + return mutex; +} + +struct ClockState { + bool frozen = false; + std::int64_t value = 0; +}; + +ClockState& clock_state() { + static ClockState state; + return state; +} + +} // namespace + +std::int64_t Clock::now() { + std::lock_guard lock(clock_mutex()); + if (clock_state().frozen) return clock_state().value; + return (std::int64_t)std::time(nullptr); +} + +void Clock::freeze(std::int64_t unix_seconds) { + std::lock_guard lock(clock_mutex()); + clock_state().frozen = true; + clock_state().value = unix_seconds; +} + +void Clock::advance(std::chrono::seconds delta) { + std::lock_guard lock(clock_mutex()); + clock_state().value += delta.count(); +} + +void Clock::unfreeze() { + std::lock_guard lock(clock_mutex()); + clock_state().frozen = false; +} + +bool Clock::frozen() { + std::lock_guard lock(clock_mutex()); + return clock_state().frozen; +} + +} // namespace molo::clients::testing diff --git a/src/clients/testing/clock.h b/src/clients/testing/clock.h new file mode 100644 index 0000000..6089e8e --- /dev/null +++ b/src/clients/testing/clock.h @@ -0,0 +1,23 @@ +#pragma once + +// Frozen and manual test clock. Runtime timers and ORM timestamps read +// Clock::now(); tests freeze it for determinism and advance it by hand. + +#include +#include +#include +#include + +namespace molo::clients::testing { + +class Clock { + public: + // Unix seconds. Frozen value when freeze() is active, else real time. + static std::int64_t now(); + static void freeze(std::int64_t unix_seconds); + static void advance(std::chrono::seconds delta); + static void unfreeze(); + static bool frozen(); +}; + +} // namespace molo::clients::testing diff --git a/src/clients/testing/fixtures.cpp b/src/clients/testing/fixtures.cpp new file mode 100644 index 0000000..6868c68 --- /dev/null +++ b/src/clients/testing/fixtures.cpp @@ -0,0 +1,73 @@ +#include "clients/testing/fixtures.h" + +#include + +namespace molo::clients::testing { + +void FixtureRegistry::define(const std::string& name, FixtureHooks hooks) { + if (hooks_.count(name)) throw FixtureError("duplicate fixture: " + name); + hooks_[name] = std::move(hooks); + order_.push_back(name); +} + +void FixtureRegistry::setup_all() { + for (const auto& name : order_) setup_one(name); +} + +void FixtureRegistry::teardown_all() { + for (auto it = order_.rbegin(); it != order_.rend(); ++it) teardown_one(*it); +} + +void FixtureRegistry::setup_one(const std::string& name) { + auto it = hooks_.find(name); + if (it == hooks_.end()) throw FixtureError("unknown fixture: " + name); + if (it->second.setup) it->second.setup(); +} + +void FixtureRegistry::teardown_one(const std::string& name) { + auto it = hooks_.find(name); + if (it == hooks_.end()) throw FixtureError("unknown fixture: " + name); + if (it->second.teardown) it->second.teardown(); +} + +void InMemoryDb::reset() { + std::lock_guard lock(mutex_); + tables_.clear(); +} + +void InMemoryDb::insert(const std::string& table, body::Json row) { + std::lock_guard lock(mutex_); + tables_[table].push_back(std::move(row)); +} + +std::vector InMemoryDb::all(const std::string& table) const { + std::lock_guard lock(mutex_); + auto it = tables_.find(table); + return it == tables_.end() ? std::vector{} : it->second; +} + +std::size_t InMemoryDb::count(const std::string& table) const { + std::lock_guard lock(mutex_); + auto it = tables_.find(table); + return it == tables_.end() ? 0 : it->second.size(); +} + +void SeededFactory::define(const std::string& name, Builder builder) { + builders_[name] = std::move(builder); + sequences_[name] = 0; +} + +body::Json SeededFactory::create(const std::string& name) { + auto it = builders_.find(name); + if (it == builders_.end()) throw FixtureError("unknown factory: " + name); + return it->second(++sequences_[name]); +} + +std::vector SeededFactory::create_many(const std::string& name, + int count) { + std::vector out; + for (int i = 0; i < count; ++i) out.push_back(create(name)); + return out; +} + +} // namespace molo::clients::testing diff --git a/src/clients/testing/fixtures.h b/src/clients/testing/fixtures.h new file mode 100644 index 0000000..827b847 --- /dev/null +++ b/src/clients/testing/fixtures.h @@ -0,0 +1,67 @@ +#pragma once + +// Test fixtures: named setup/teardown registry, in-memory DB double with +// per-test reset, and a seeded object factory. Part 04 binds the real ORM +// fixtures to the same interface. + +#include +#include +#include +#include +#include + +#include "clients/body/json.h" + +namespace molo::clients::testing { + +class FixtureError : public std::runtime_error { + public: + explicit FixtureError(const std::string& msg) + : std::runtime_error("molo::clients::testing::Fixture: " + msg) {} +}; + +struct FixtureHooks { + std::function setup; + std::function teardown; +}; + +class FixtureRegistry { + public: + void define(const std::string& name, FixtureHooks hooks); + void setup_all(); + void teardown_all(); // Reverse order. + void setup_one(const std::string& name); + void teardown_one(const std::string& name); + + private: + std::map hooks_; + std::vector order_; +}; + +// Minimal table store standing in for the ORM in client-part tests. +class InMemoryDb { + public: + void reset(); + void insert(const std::string& table, body::Json row); + std::vector all(const std::string& table) const; + std::size_t count(const std::string& table) const; + + private: + mutable std::mutex mutex_; + std::map> tables_; +}; + +// Builds seeded Json documents from named factories. +class SeededFactory { + public: + using Builder = std::function; + void define(const std::string& name, Builder builder); + body::Json create(const std::string& name); + std::vector create_many(const std::string& name, int count); + + private: + std::map builders_; + std::map sequences_; +}; + +} // namespace molo::clients::testing diff --git a/src/clients/testing/test_client.cpp b/src/clients/testing/test_client.cpp new file mode 100644 index 0000000..558a793 --- /dev/null +++ b/src/clients/testing/test_client.cpp @@ -0,0 +1,33 @@ +#include "clients/testing/test_client.h" + +#include + +namespace molo::clients::testing { + +TestClient::TestClient(transports::AppHandler handler, ClientConfig config) + : Client(std::make_unique(std::move(handler)), + std::move(config)) { + app_ = static_cast(transport_.get()); +} + +TestClient::TestClient(transports::SyncAppHandler handler, ClientConfig config) + : Client(std::make_unique(std::move(handler)), + std::move(config)) { + app_ = static_cast(transport_.get()); +} + +transports::AppTransport& TestClient::app_transport() { return *app_; } + +TestClient::Lifespan::Lifespan(TestClient& client) : client_(client) { + client_.app_transport().startup(); +} + +TestClient::Lifespan::~Lifespan() { client_.app_transport().shutdown(); } + +TestClient::Lifespan TestClient::lifespan() { return Lifespan(*this); } + +std::size_t TestClient::dispatch_count() const { + return app_->dispatch_count(); +} + +} // namespace molo::clients::testing diff --git a/src/clients/testing/test_client.h b/src/clients/testing/test_client.h new file mode 100644 index 0000000..12e3d49 --- /dev/null +++ b/src/clients/testing/test_client.h @@ -0,0 +1,42 @@ +#pragma once + +// TestClient: a Client fixed to AppTransport. Same verbs as production, +// in-process dispatch, lifespan guard, zero real sockets. + +#include +#include + +#include "clients/client.h" +#include "clients/transports/app_transport.h" + +namespace molo::clients::testing { + +class TestClient : public Client { + public: + explicit TestClient(transports::AppHandler handler, + ClientConfig config = ClientConfig{}); + explicit TestClient(transports::SyncAppHandler handler, + ClientConfig config = ClientConfig{}); + + transports::AppTransport& app_transport(); + + // RAII lifespan guard: startup on construct, shutdown on destroy. + class Lifespan { + public: + explicit Lifespan(TestClient& client); + ~Lifespan(); + Lifespan(const Lifespan&) = delete; + Lifespan& operator=(const Lifespan&) = delete; + + private: + TestClient& client_; + }; + Lifespan lifespan(); + + std::size_t dispatch_count() const; + + private: + transports::AppTransport* app_; +}; + +} // namespace molo::clients::testing diff --git a/src/clients/timeouts.h b/src/clients/timeouts.h new file mode 100644 index 0000000..a2bec17 --- /dev/null +++ b/src/clients/timeouts.h @@ -0,0 +1,102 @@ +#pragma once + +// Timeout phases plus the per-request deadline helper. + +#include +#include +#include + +#include "clients/caps.h" + +namespace molo::clients { + +enum class TimeoutPhase { kConnect, kRead, kWrite, kPoolAcquire, kTotal }; + +inline const char* phase_name(TimeoutPhase phase) { + switch (phase) { + case TimeoutPhase::kConnect: + return "connect"; + case TimeoutPhase::kRead: + return "read"; + case TimeoutPhase::kWrite: + return "write"; + case TimeoutPhase::kPoolAcquire: + return "pool_acquire"; + case TimeoutPhase::kTotal: + return "total"; + } + return "unknown"; +} + +class TimeoutError : public std::runtime_error { + public: + explicit TimeoutError(TimeoutPhase phase) + : std::runtime_error(std::string("molo::clients: timeout in phase ") + + phase_name(phase)), + phase_(phase) {} + TimeoutPhase phase() const noexcept { return phase_; } + + private: + TimeoutPhase phase_; +}; + +struct Timeouts { + std::chrono::milliseconds connect = Caps::kConnectTimeout; + std::chrono::milliseconds read = Caps::kReadTimeout; + std::chrono::milliseconds write = Caps::kWriteTimeout; + std::chrono::milliseconds pool_acquire = Caps::kPoolAcquireTimeout; + std::chrono::milliseconds total = Caps::kTotalTimeout; + + static Timeouts defaults() { return Timeouts{}; } +}; + +// Tracks the total deadline and derives per-phase poll budgets from it. +class Deadline { + public: + explicit Deadline(Timeouts timeouts) + : timeouts_(timeouts), + start_(std::chrono::steady_clock::now()), + total_at_(start_ + timeouts.total) {} + + const Timeouts& timeouts() const { return timeouts_; } + + // Milliseconds left for a phase, bounded by the total deadline. + // Throws TimeoutError(kTotal) when the total budget is spent. + std::chrono::milliseconds budget_for(TimeoutPhase phase) const { + auto now = std::chrono::steady_clock::now(); + if (now >= total_at_) throw TimeoutError(TimeoutPhase::kTotal); + std::chrono::milliseconds phase_budget = timeouts_.total; + switch (phase) { + case TimeoutPhase::kConnect: + phase_budget = timeouts_.connect; + break; + case TimeoutPhase::kRead: + phase_budget = timeouts_.read; + break; + case TimeoutPhase::kWrite: + phase_budget = timeouts_.write; + break; + case TimeoutPhase::kPoolAcquire: + phase_budget = timeouts_.pool_acquire; + break; + case TimeoutPhase::kTotal: + phase_budget = timeouts_.total; + break; + } + auto total_left = + std::chrono::duration_cast(total_at_ - now); + return phase_budget < total_left ? phase_budget : total_left; + } + + std::chrono::milliseconds elapsed() const { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_); + } + + private: + Timeouts timeouts_; + std::chrono::steady_clock::time_point start_; + std::chrono::steady_clock::time_point total_at_; +}; + +} // namespace molo::clients diff --git a/src/clients/transport.cpp b/src/clients/transport.cpp new file mode 100644 index 0000000..08fdd6b --- /dev/null +++ b/src/clients/transport.cpp @@ -0,0 +1,12 @@ +#include "clients/transport.h" + +namespace molo::clients { + +Task Transport::send_stream(const ClientRequest& request, + body::ChunkSink sink) { + ClientResponse response = co_await send(request); + if (!response.body().empty()) sink(response.body()); + co_return; +} + +} // namespace molo::clients diff --git a/src/clients/transport.h b/src/clients/transport.h new file mode 100644 index 0000000..a43c30c --- /dev/null +++ b/src/clients/transport.h @@ -0,0 +1,40 @@ +#pragma once + +// Transport interface. Real network, in-process dispatch, and scripted mocks +// all implement this; callers never change when the transport swaps. + +#include +#include + +#include "clients/body/stream.h" +#include "clients/request.h" +#include "clients/response.h" +#include "clients/task.h" + +namespace molo::clients { + +class TransportError : public std::runtime_error { + public: + explicit TransportError(const std::string& msg, bool connect_phase = false) + : std::runtime_error("molo::clients::Transport: " + msg), + connect_phase_(connect_phase) {} + // True when the failure happened before any request byte was sent, so + // even non-idempotent methods may retry. + bool connect_phase() const noexcept { return connect_phase_; } + + private: + bool connect_phase_; +}; + +class Transport { + public: + virtual ~Transport() = default; + virtual const char* name() const = 0; + virtual Task send(const ClientRequest& request) = 0; + // Streamed download. Default implementation buffers through send(). + virtual Task send_stream(const ClientRequest& request, + body::ChunkSink sink); + virtual void close() {} +}; + +} // namespace molo::clients diff --git a/src/clients/transports/app_transport.cpp b/src/clients/transports/app_transport.cpp new file mode 100644 index 0000000..7568356 --- /dev/null +++ b/src/clients/transports/app_transport.cpp @@ -0,0 +1,61 @@ +#include "clients/transports/app_transport.h" + +#include + +namespace molo::clients::transports { + +AppTransport::AppTransport(AppHandler handler) : handler_(std::move(handler)) { + if (!handler_) throw TransportError("AppTransport needs a handler"); +} + +AppTransport::AppTransport(SyncAppHandler handler) { + if (!handler) throw TransportError("AppTransport needs a handler"); + handler_ = [fn = std::move(handler)]( + const ClientRequest& req) -> Task { + co_return fn(req); + }; +} + +void AppTransport::on_startup(LifespanHook hook) { + std::lock_guard lock(mutex_); + startup_hook_ = std::move(hook); +} + +void AppTransport::on_shutdown(LifespanHook hook) { + std::lock_guard lock(mutex_); + shutdown_hook_ = std::move(hook); +} + +void AppTransport::startup() { + LifespanHook hook; + { + std::lock_guard lock(mutex_); + if (started_) return; + started_ = true; + hook = startup_hook_; + } + if (hook) hook(); +} + +void AppTransport::shutdown() { + LifespanHook hook; + { + std::lock_guard lock(mutex_); + if (!started_) return; + started_ = false; + hook = shutdown_hook_; + } + if (hook) hook(); +} + +bool AppTransport::started() const { + std::lock_guard lock(mutex_); + return started_; +} + +Task AppTransport::send(const ClientRequest& request) { + dispatch_count_++; + co_return co_await handler_(request); +} + +} // namespace molo::clients::transports diff --git a/src/clients/transports/app_transport.h b/src/clients/transports/app_transport.h new file mode 100644 index 0000000..a066553 --- /dev/null +++ b/src/clients/transports/app_transport.h @@ -0,0 +1,50 @@ +#pragma once + +// In-process transport: dispatches requests to an application handler +// callable with no TCP involved. Part 05 binds its Application to +// AppHandler; this part only declares the signature plus lifespan hooks. + +#include +#include +#include +#include + +#include "clients/request.h" +#include "clients/response.h" +#include "clients/task.h" +#include "clients/transport.h" + +namespace molo::clients::transports { + +using AppHandler = std::function(const ClientRequest&)>; +using SyncAppHandler = std::function; +using LifespanHook = std::function; + +class AppTransport : public Transport { + public: + explicit AppTransport(AppHandler handler); + // Wraps a plain sync handler. + explicit AppTransport(SyncAppHandler handler); + + const char* name() const override { return "app"; } + + void on_startup(LifespanHook hook); + void on_shutdown(LifespanHook hook); + void startup(); + void shutdown(); + bool started() const; + + Task send(const ClientRequest& request) override; + + std::size_t dispatch_count() const { return dispatch_count_; } + + private: + AppHandler handler_; + LifespanHook startup_hook_; + LifespanHook shutdown_hook_; + mutable std::mutex mutex_; + bool started_ = false; + std::atomic dispatch_count_{0}; +}; + +} // namespace molo::clients::transports diff --git a/src/clients/transports/http_transport.cpp b/src/clients/transports/http_transport.cpp new file mode 100644 index 0000000..fbc02be --- /dev/null +++ b/src/clients/transports/http_transport.cpp @@ -0,0 +1,682 @@ +#include "clients/transports/http_transport.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "clients/caps.h" + +namespace molo::clients::transports { +namespace { + +// --- Socket helpers with per-phase poll timeouts. --- + +void set_nonblocking(int fd) { + int flags = ::fcntl(fd, F_GETFL, 0); + ::fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +void poll_wait(int fd, short events, std::chrono::milliseconds timeout, + TimeoutPhase phase) { + struct pollfd pfd {fd, events, 0}; + long ms = (long)timeout.count(); + int rc = ::poll(&pfd, 1, ms < 0 ? -1 : (int)ms); + if (rc == 0) throw TimeoutError(phase); + if (rc < 0) { + if (errno == EINTR) throw TimeoutError(phase); + throw TransportError("poll failed: " + std::string(strerror(errno))); + } + if (pfd.revents & (POLLERR | POLLNVAL)) + throw TransportError("socket error while waiting"); +} + +int connect_with_timeout(const std::string& host, std::uint16_t port, + const Deadline& deadline) { + struct addrinfo hints {}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo* list = nullptr; + int rc = ::getaddrinfo(host.c_str(), std::to_string(port).c_str(), &hints, + &list); + if (rc != 0) + throw TransportError("DNS resolve failed for " + host + ": " + + gai_strerror(rc), + true); + TransportError last_error("no address worked for " + host, true); + for (struct addrinfo* ai = list; ai; ai = ai->ai_next) { + int fd = ::socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) continue; + set_nonblocking(fd); + int cr = ::connect(fd, ai->ai_addr, ai->ai_addrlen); + if (cr == 0) { + ::freeaddrinfo(list); + return fd; + } + if (errno != EINPROGRESS) { + ::close(fd); + last_error = TransportError( + "connect failed: " + std::string(strerror(errno)), true); + continue; + } + try { + // Connect poll: POLLERR/POLLHUP just mean the SO_ERROR check below + // will report the failure; only timeouts and bad fds throw here. + struct pollfd pfd {fd, POLLOUT, 0}; + long ms = (long)deadline.budget_for(TimeoutPhase::kConnect).count(); + int prc = ::poll(&pfd, 1, (int)ms); + if (prc == 0) throw TimeoutError(TimeoutPhase::kConnect); + if (prc < 0) { + if (errno == EINTR) throw TimeoutError(TimeoutPhase::kConnect); + throw TransportError("poll failed: " + std::string(strerror(errno)), + true); + } + if (pfd.revents & POLLNVAL) + throw TransportError("socket error while connecting", true); + } catch (...) { + ::close(fd); + throw; + } + int err = 0; + socklen_t len = sizeof(err); + ::getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len); + if (err != 0) { + ::close(fd); + last_error = TransportError( + "connect failed: " + std::string(strerror(err)), true); + continue; + } + ::freeaddrinfo(list); + return fd; + } + ::freeaddrinfo(list); + throw last_error; +} + +void send_all(int fd, const char* data, std::size_t size, + const Deadline& deadline) { + std::size_t sent = 0; + while (sent < size) { + poll_wait(fd, POLLOUT, deadline.budget_for(TimeoutPhase::kWrite), + TimeoutPhase::kWrite); + ssize_t n = ::send(fd, data + sent, size - sent, MSG_NOSIGNAL); + if (n < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) continue; + throw TransportError("send failed: " + std::string(strerror(errno))); + } + if (n == 0) throw TransportError("send hit closed socket"); + sent += (std::size_t)n; + } +} + +bool recv_some(int fd, char* out, std::size_t max_size, std::size_t& got, + const Deadline& deadline) { + poll_wait(fd, POLLIN, deadline.budget_for(TimeoutPhase::kRead), + TimeoutPhase::kRead); + ssize_t n = ::recv(fd, out, max_size, 0); + if (n < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + got = 0; + return true; + } + throw TransportError("recv failed: " + std::string(strerror(errno))); + } + got = (std::size_t)n; + return n > 0; +} + +std::string to_lower(std::string s) { + for (auto& c : s) c = static_cast(std::tolower((unsigned char)c)); + return s; +} + +std::string trim(std::string s) { + while (!s.empty() && (s.front() == ' ' || s.front() == '\t')) s.erase(s.begin()); + while (!s.empty() && (s.back() == ' ' || s.back() == '\t' || s.back() == '\r')) + s.pop_back(); + return s; +} + +struct HeadState { + std::string buffer; + HeaderMap headers; + int status = 0; + int minor_version = 1; +}; + +// Reads until the header block terminator, enforcing the header block cap. +void read_head(int fd, HeadState& head, const Deadline& deadline) { + char chunk[8192]; + while (true) { + auto at = head.buffer.find("\r\n\r\n"); + if (at != std::string::npos) { + std::string block = head.buffer.substr(0, at); + head.buffer.erase(0, at + 4); + std::istringstream lines(block); + std::string line; + bool first = true; + while (std::getline(lines, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (first) { + first = false; + // "HTTP/1.1 200 OK" + if (line.rfind("HTTP/1.", 0) != 0) + throw HttpError("bad status line: " + line); + head.minor_version = line[7] - '0'; + // "HTTP/1.x " is 9 chars; the code starts at offset 9. + auto sp = line.find(' ', 9); + std::string code = sp == std::string::npos + ? line.substr(9) + : line.substr(9, sp - 9); + try { + head.status = std::stoi(code); + } catch (...) { + throw HttpError("bad status code: " + line); + } + continue; + } + if (line.empty()) continue; + auto colon = line.find(':'); + if (colon == std::string::npos) throw HttpError("bad header line"); + head.headers.add(trim(line.substr(0, colon)), + trim(line.substr(colon + 1))); + } + return; + } + if (head.buffer.size() > Caps::kMaxHeaderBlock) + throw HttpError("response header block too large"); + std::size_t got = 0; + if (!recv_some(fd, chunk, sizeof(chunk), got, deadline)) + throw HttpError("connection closed mid headers"); + head.buffer.append(chunk, got); + } +} + +// Reads exactly total bytes, first draining the pre-read head buffer. +void read_exact_total(int fd, HeadState& head, std::size_t total, + std::string& out, const Deadline& deadline, + std::size_t cap) { + std::size_t have = 0; + if (!head.buffer.empty()) { + std::size_t take = + head.buffer.size() < total ? head.buffer.size() : total; + out.append(head.buffer.data(), take); + head.buffer.erase(0, take); + have = take; + } + char chunk[8192]; + while (have < total) { + std::size_t want = total - have; + if (want > sizeof(chunk)) want = sizeof(chunk); + std::size_t got = 0; + if (!recv_some(fd, chunk, want, got, deadline)) + throw HttpError("connection closed mid body"); + if (got == 0) continue; + if (out.size() + got > cap) throw BodyTooLargeError("response body cap hit"); + out.append(chunk, got); + have += got; + } +} + +bool read_line(int fd, HeadState& head, std::string& line, + const Deadline& deadline) { + char chunk[1024]; + while (true) { + auto at = head.buffer.find("\r\n"); + if (at != std::string::npos) { + line = head.buffer.substr(0, at); + head.buffer.erase(0, at + 2); + return true; + } + if (head.buffer.size() > Caps::kMaxHeaderLine) + throw HttpError("chunk line too long"); + std::size_t got = 0; + if (!recv_some(fd, chunk, sizeof(chunk), got, deadline)) return false; + head.buffer.append(chunk, got); + } +} + +long parse_chunk_size(const std::string& line) { + std::string num = line; + auto semi = num.find(';'); + if (semi != std::string::npos) num = num.substr(0, semi); + num = trim(num); + if (num.empty()) throw HttpError("bad chunk size"); + char* end = nullptr; + long v = std::strtol(num.c_str(), &end, 16); + if (!end || *end != '\0' || v < 0) throw HttpError("bad chunk size"); + return v; +} + +} // namespace + +HttpTransport::HttpTransport(HttpTransportOptions options) + : options_(std::move(options)), + pool_(std::make_shared()), + tls_(std::make_shared()) { + if (options_.proxy_from_env && options_.proxy.empty()) + options_.proxy = ProxyConfig::from_env(); +} + +HttpTransport::~HttpTransport() = default; + +void HttpTransport::set_pool(std::shared_ptr pool) { + pool_ = std::move(pool); +} + +void HttpTransport::set_proxy(ProxyConfig proxy) { + options_.proxy = std::move(proxy); + options_.proxy_from_env = false; +} + +void HttpTransport::close() { pool_->close_all(); } + +namespace { + +struct ResolvedRequest { + Url url; + std::string connect_host; + std::uint16_t connect_port = 0; + std::string target; + std::string pool_key; + bool via_proxy = false; +}; + +ResolvedRequest resolve_target(const ClientRequest& request, + const ProxyConfig& proxy) { + const Url& url = request.url(); + if (!url.has_host()) throw HttpError("request URL has no host"); + if (url.scheme() != "http" && url.scheme() != "https") + throw HttpError("unsupported scheme: " + url.scheme()); + ResolvedRequest out; + out.url = url; + out.target = url.target(); + out.pool_key = url.pool_key(); + ProxySelector selector(proxy); + auto proxy_url = selector.select(url); + if (proxy_url && !proxy_url->empty()) { + Url proxy_parsed = Url::parse(*proxy_url); + out.connect_host = proxy_parsed.host(); + out.connect_port = proxy_parsed.port(); + out.via_proxy = true; + out.pool_key = "proxy://" + proxy_parsed.host_port() + "|" + url.pool_key(); + if (url.scheme() == "http") out.target = url.str(); // Absolute-form. + } else { + out.connect_host = url.host(); + out.connect_port = url.port(); + } + return out; +} + +} // namespace + +Task HttpTransport::send(const ClientRequest& request) { + Timeouts timeouts = + request.timeouts().value_or(options_.timeouts); + Deadline deadline(timeouts); + ResolvedRequest target = resolve_target(request, options_.proxy); + + if (target.url.is_https()) { + // TLS is required; only the marked stub is linked here. + tls_->handshake(-1, target.url.host(), + request.h2() || options_.h2); + } + if ((request.h2() || options_.h2) && !target.url.is_https()) { + throw H2StubError("h2c cleartext needs the part 01 H2 framer"); + } + + // Buffered in-memory bodies must fit the cap; bigger payloads stream. + if (auto* bytes = std::get_if(&request.body())) { + if (bytes->size() > Caps::kMaxMemoryBody) + throw BodyTooLargeError( + "body exceeds 10 MB in-memory cap, use stream()"); + } + + std::string key = target.pool_key; + std::optional pooled = pool_->acquire(key); + bool reused = pooled.has_value(); + int fd = -1; + bool reserved = false; + if (pooled) { + fd = pooled->fd; + } else { + if (!pool_->reserve(key)) + throw TransportError("pool exhausted for " + key); + reserved = true; + try { + fd = connect_with_timeout(target.connect_host, target.connect_port, + deadline); + } catch (...) { + pool_->drop_reservation(key); + throw; + } + } + + bool release_reusable = false; + bool released = false; + auto release_fd = [&](bool reusable) { + if (!released && fd >= 0) { + released = true; + pool_->release(key, fd, reusable); + fd = -1; + } + }; + + try { + // Serialize head. + std::ostringstream head; + head << method_name(request.method()) << " " << target.target + << " HTTP/1.1\r\n"; + HeaderMap headers = request.headers(); + std::string host = target.url.host(); + if ((target.url.scheme() == "http" && target.url.port() != 80) || + (target.url.scheme() == "https" && target.url.port() != 443)) + host += ":" + std::to_string(target.url.port()); + if (!headers.has("Host")) headers.set("Host", host); + if (!headers.has("User-Agent")) headers.set("User-Agent", kUserAgent); + if (!headers.has("Accept")) headers.set("Accept", "*/*"); + if (!headers.has("Connection")) headers.set("Connection", "keep-alive"); + const std::string* bytes = + std::get_if(&request.body()); + const StreamSource* stream = + std::get_if(&request.body()); + bool has_body = bytes || (stream && stream->reader); + if (bytes) { + headers.set("Content-Length", std::to_string(bytes->size())); + } else if (stream && stream->reader) { + if (stream->length) + headers.set("Content-Length", std::to_string(*stream->length)); + else + headers.set("Transfer-Encoding", "chunked"); + } else if (request.method() == Method::kPost || + request.method() == Method::kPut || + request.method() == Method::kPatch) { + headers.set("Content-Length", "0"); + } + (void)has_body; + head << headers.serialize() << "\r\n"; + std::string head_str = head.str(); + try { + send_all(fd, head_str.data(), head_str.size(), deadline); + } catch (...) { + // A pooled connection may have died idle; retry once on a fresh socket + // before reporting. Only safe when nothing stateful was attempted yet + // on this attempt: the head send is the first byte batch. + if (reused && !reserved) { + release_fd(false); + if (!pool_->reserve(key)) + throw TransportError("pool exhausted for " + key); + reserved = true; + reused = false; + fd = connect_with_timeout(target.connect_host, target.connect_port, + deadline); + send_all(fd, head_str.data(), head_str.size(), deadline); + } else { + throw; + } + } + if (bytes && !bytes->empty()) { + send_all(fd, bytes->data(), bytes->size(), deadline); + } else if (stream && stream->reader) { + bool chunked = !stream->length.has_value(); + while (!stream->reader->eof()) { + std::string piece = + stream->reader->read_some(Caps::kStreamChunk); + if (piece.empty()) { + if (stream->reader->eof()) break; + continue; + } + if (chunked) { + std::ostringstream prefix; + prefix << std::hex << piece.size() << "\r\n"; + std::string pre = prefix.str(); + send_all(fd, pre.data(), pre.size(), deadline); + } + send_all(fd, piece.data(), piece.size(), deadline); + if (chunked) send_all(fd, "\r\n", 2, deadline); + } + if (chunked) send_all(fd, "0\r\n\r\n", 5, deadline); + } + + HeadState response_head; + read_head(fd, response_head, deadline); + bool is_head = request.method() == Method::kHead; + bool no_body_status = response_head.status == 204 || + response_head.status == 304 || + (response_head.status >= 100 && response_head.status < 200); + std::string body; + bool close_delimited = false; + std::string transfer = + to_lower(response_head.headers.get_or("Transfer-Encoding", "")); + std::string length_text = response_head.headers.get_or("Content-Length", ""); + if (!is_head && !no_body_status) { + if (transfer.find("chunked") != std::string::npos) { + while (true) { + std::string line; + if (!read_line(fd, response_head, line, deadline)) + throw HttpError("connection closed mid chunks"); + long size = parse_chunk_size(line); + if (size == 0) { + std::string trailer; + while (read_line(fd, response_head, trailer, deadline) && + !trailer.empty()) { + } + break; + } + if ((std::size_t)size > Caps::kMaxMemoryBody) + throw BodyTooLargeError("chunk exceeds body cap"); + std::string piece; + read_exact_total(fd, response_head, (std::size_t)size, piece, + deadline, Caps::kMaxMemoryBody - body.size()); + body += piece; + std::string crlf; + if (!read_line(fd, response_head, crlf, deadline)) + throw HttpError("connection closed mid chunks"); + } + } else if (!length_text.empty()) { + std::size_t total = 0; + try { + total = (std::size_t)std::stoul(trim(length_text)); + } catch (...) { + throw HttpError("bad Content-Length"); + } + if (total > Caps::kMaxMemoryBody) + throw BodyTooLargeError("response exceeds 10 MB buffered cap"); + read_exact_total(fd, response_head, total, body, deadline, + Caps::kMaxMemoryBody); + } else { + // Close-delimited: read until EOF, not reusable. + close_delimited = true; + char chunk[8192]; + while (true) { + std::size_t got = 0; + if (!recv_some(fd, chunk, sizeof(chunk), got, deadline)) break; + if (got == 0) continue; + if (body.size() + got > Caps::kMaxMemoryBody) + throw BodyTooLargeError("response exceeds 10 MB buffered cap"); + body.append(chunk, got); + } + } + } + std::string connection = + to_lower(response_head.headers.get_or("Connection", "")); + bool server_close = connection.find("close") != std::string::npos; + release_reusable = !close_delimited && !server_close && + response_head.minor_version >= 1; + ClientResponse response(response_head.status, response_head.headers, body); + response.set_elapsed(deadline.elapsed()); + response.set_from_pool_reuse(reused); + release_fd(release_reusable); + co_return response; + } catch (...) { + release_fd(false); + throw; + } +} + +Task HttpTransport::send_stream(const ClientRequest& request, + body::ChunkSink sink) { + // Buffered path would cap at 10 MB; stream chunks to the sink instead. + // Implemented as a second pass over a fresh connection for clarity: the + // buffered send() above already proves framing, this path reuses the same + // head logic but pushes body chunks out as they arrive. + Timeouts timeouts = + request.timeouts().value_or(options_.timeouts); + Deadline deadline(timeouts); + ResolvedRequest target = resolve_target(request, options_.proxy); + if (target.url.is_https()) { + tls_->handshake(-1, target.url.host(), request.h2() || options_.h2); + } + if ((request.h2() || options_.h2) && !target.url.is_https()) { + throw H2StubError("h2c cleartext needs the part 01 H2 framer"); + } + std::string key = target.pool_key; + std::optional pooled = pool_->acquire(key); + int fd = -1; + bool reserved = false; + if (pooled) { + fd = pooled->fd; + } else { + if (!pool_->reserve(key)) + throw TransportError("pool exhausted for " + key); + reserved = true; + try { + fd = connect_with_timeout(target.connect_host, target.connect_port, + deadline); + } catch (...) { + pool_->drop_reservation(key); + throw; + } + } + (void)reserved; + bool released = false; + auto release_fd = [&](bool reusable) { + if (!released && fd >= 0) { + released = true; + pool_->release(key, fd, reusable); + fd = -1; + } + }; + try { + std::ostringstream head; + head << method_name(request.method()) << " " << target.target + << " HTTP/1.1\r\n"; + HeaderMap headers = request.headers(); + std::string host = target.url.host(); + if ((target.url.scheme() == "http" && target.url.port() != 80) || + (target.url.scheme() == "https" && target.url.port() != 443)) + host += ":" + std::to_string(target.url.port()); + if (!headers.has("Host")) headers.set("Host", host); + if (!headers.has("User-Agent")) headers.set("User-Agent", kUserAgent); + if (!headers.has("Accept")) headers.set("Accept", "*/*"); + if (!headers.has("Connection")) headers.set("Connection", "keep-alive"); + const std::string* bytes = std::get_if(&request.body()); + if (bytes) headers.set("Content-Length", std::to_string(bytes->size())); + head << headers.serialize() << "\r\n"; + std::string head_str = head.str(); + send_all(fd, head_str.data(), head_str.size(), deadline); + if (bytes && !bytes->empty()) + send_all(fd, bytes->data(), bytes->size(), deadline); + HeadState response_head; + read_head(fd, response_head, deadline); + if (response_head.status < 200 || response_head.status == 204 || + response_head.status == 304 || + request.method() == Method::kHead) { + release_fd(true); + co_return; + } + const std::size_t kNoCap = Caps::kMaxRequestTotal; + std::string transfer = + to_lower(response_head.headers.get_or("Transfer-Encoding", "")); + std::string length_text = + response_head.headers.get_or("Content-Length", ""); + bool close_delimited = false; + if (transfer.find("chunked") != std::string::npos) { + while (true) { + std::string line; + if (!read_line(fd, response_head, line, deadline)) + throw HttpError("connection closed mid chunks"); + long size = parse_chunk_size(line); + if (size == 0) { + std::string trailer; + while (read_line(fd, response_head, trailer, deadline) && + !trailer.empty()) { + } + break; + } + std::string piece; + read_exact_total(fd, response_head, (std::size_t)size, piece, deadline, + kNoCap); + std::string crlf; + if (!read_line(fd, response_head, crlf, deadline)) + throw HttpError("connection closed mid chunks"); + if (!sink(piece)) { + release_fd(false); + co_return; + } + } + } else if (!length_text.empty()) { + std::size_t total = (std::size_t)std::stoul(trim(length_text)); + std::size_t emitted = 0; + if (!response_head.buffer.empty()) { + std::string first = std::move(response_head.buffer); + response_head.buffer.clear(); + if (first.size() > total) first.resize(total); + emitted = first.size(); + if (!sink(first)) { + release_fd(false); + co_return; + } + } + char chunk[8192]; + while (emitted < total) { + std::size_t want = total - emitted; + if (want > sizeof(chunk)) want = sizeof(chunk); + std::size_t got = 0; + if (!recv_some(fd, chunk, want, got, deadline)) + throw HttpError("connection closed mid body"); + if (got == 0) continue; + emitted += got; + if (!sink(std::string(chunk, got))) { + release_fd(false); + co_return; + } + } + } else { + close_delimited = true; + if (!response_head.buffer.empty()) { + if (!sink(response_head.buffer)) { + release_fd(false); + co_return; + } + response_head.buffer.clear(); + } + char chunk[8192]; + while (true) { + std::size_t got = 0; + if (!recv_some(fd, chunk, sizeof(chunk), got, deadline)) break; + if (got == 0) continue; + if (!sink(std::string(chunk, got))) { + release_fd(false); + co_return; + } + } + } + std::string connection = + to_lower(response_head.headers.get_or("Connection", "")); + release_fd(!close_delimited && connection.find("close") == std::string::npos); + co_return; + } catch (...) { + release_fd(false); + throw; + } +} + +} // namespace molo::clients::transports diff --git a/src/clients/transports/http_transport.h b/src/clients/transports/http_transport.h new file mode 100644 index 0000000..322f26f --- /dev/null +++ b/src/clients/transports/http_transport.h @@ -0,0 +1,64 @@ +#pragma once + +// Real network transport: HTTP/1.1 over POSIX sockets with keep-alive +// pooling, per-phase timeouts, proxy (absolute-form), and streaming. +// HTTPS needs ITlsSession (only the marked stub ships here, so HTTPS URLs +// fail loud with TlsStubError). H2/h2c returns a marked H2StubError. + +#include +#include + +#include "clients/pool.h" +#include "clients/proxy.h" +#include "clients/request.h" +#include "clients/response.h" +#include "clients/task.h" +#include "clients/timeouts.h" +#include "clients/transports/tls.h" +#include "clients/transport.h" + +namespace molo::clients::transports { + +class HttpError : public std::runtime_error { + public: + explicit HttpError(const std::string& msg) + : std::runtime_error("molo::clients::transports::Http: " + msg) {} +}; + +class BodyTooLargeError : public std::runtime_error { + public: + explicit BodyTooLargeError(const std::string& msg) + : std::runtime_error("molo::clients::transports::BodyTooLarge: " + msg) {} +}; + +struct HttpTransportOptions { + Timeouts timeouts; + ProxyConfig proxy; + bool proxy_from_env = true; + bool h2 = false; // Explicit H2 request; https also tries TLS first. +}; + +class HttpTransport : public Transport { + public: + explicit HttpTransport(HttpTransportOptions options = {}); + ~HttpTransport() override; + + const char* name() const override { return "http"; } + + void set_pool(std::shared_ptr pool); + void set_proxy(ProxyConfig proxy); + std::shared_ptr pool() const { return pool_; } + void set_tls(std::shared_ptr tls) { tls_ = std::move(tls); } + + Task send(const ClientRequest& request) override; + Task send_stream(const ClientRequest& request, + body::ChunkSink sink) override; + void close() override; + + private: + HttpTransportOptions options_; + std::shared_ptr pool_; + std::shared_ptr tls_; +}; + +} // namespace molo::clients::transports diff --git a/src/clients/transports/mock_transport.cpp b/src/clients/transports/mock_transport.cpp new file mode 100644 index 0000000..6e02fd9 --- /dev/null +++ b/src/clients/transports/mock_transport.cpp @@ -0,0 +1,132 @@ +#include "clients/transports/mock_transport.h" + +#include +#include + +namespace molo::clients::transports { + +std::size_t MockTransport::expect(MockExpectation expectation) { + std::lock_guard lock(mutex_); + expectations_.push_back(std::move(expectation)); + return expectations_.size() - 1; +} + +std::size_t MockTransport::expect_response(Method method, + const std::string& path, + ClientResponse response) { + MockExpectation exp; + exp.method = method; + exp.path = path; + exp.response = std::move(response); + return expect(std::move(exp)); +} + +std::size_t MockTransport::expect_status(Method method, const std::string& path, + int status, const std::string& body, + const std::string& content_type) { + HeaderMap headers; + headers.set("Content-Type", content_type); + headers.set("Content-Length", std::to_string(body.size())); + return expect_response(method, path, + ClientResponse(status, headers, body)); +} + +std::size_t MockTransport::expect_error(Method method, const std::string& path, + std::exception_ptr error) { + MockExpectation exp; + exp.method = method; + exp.path = path; + exp.error = error; + return expect(std::move(exp)); +} + +bool MockTransport::path_match(const std::string& pattern, + const std::string& path) { + if (pattern.find('{') == std::string::npos) return pattern == path; + // Template: split on '/', '{name}' matches any single non-empty segment. + auto split = [](const std::string& s) { + std::vector parts; + std::string cur; + for (char c : s) { + if (c == '/') { + parts.push_back(cur); + cur.clear(); + } else { + cur.push_back(c); + } + } + parts.push_back(cur); + return parts; + }; + auto want = split(pattern); + auto got = split(path); + if (want.size() != got.size()) return false; + for (std::size_t i = 0; i < want.size(); ++i) { + const std::string& w = want[i]; + if (w.size() >= 2 && w.front() == '{' && w.back() == '}') { + if (got[i].empty()) return false; + } else if (w != got[i]) { + return false; + } + } + return true; +} + +Task MockTransport::send(const ClientRequest& request) { + SeenRequest seen{method_name(request.method()), request.url().target(), + request.headers(), request.body_bytes()}; + std::lock_guard lock(mutex_); + seen_.push_back(seen); + for (auto& exp : expectations_) { + if (exp.used >= exp.times) continue; + if (exp.method != request.method()) continue; + if (!path_match(exp.path, request.url().path())) continue; + bool headers_ok = true; + for (const auto& kv : exp.required_headers) { + auto got = request.headers().get(kv.first); + if (!got || *got != kv.second) { + headers_ok = false; + break; + } + } + if (!headers_ok) continue; + if (!exp.required_body_contains.empty() && + request.body_bytes().find(exp.required_body_contains) == + std::string::npos) + continue; + if (exp.predicate && !exp.predicate(request)) continue; + exp.used++; + if (exp.error) std::rethrow_exception(exp.error); + co_return exp.response; + } + std::ostringstream msg; + msg << "no matching expectation for " << method_name(request.method()) + << " " << request.url().target(); + throw MockError(msg.str()); +} + +void MockTransport::verify() const { + std::lock_guard lock(mutex_); + for (const auto& exp : expectations_) { + if (exp.used < exp.times) { + std::ostringstream msg; + msg << "unconsumed expectation"; + if (!exp.label.empty()) msg << " (" << exp.label << ")"; + msg << ": " << method_name(exp.method) << " " << exp.path << " used " + << exp.used << " of " << exp.times; + throw MockError(msg.str()); + } + } +} + +std::vector MockTransport::seen() const { + std::lock_guard lock(mutex_); + return seen_; +} + +std::size_t MockTransport::send_count() const { + std::lock_guard lock(mutex_); + return seen_.size(); +} + +} // namespace molo::clients::transports diff --git a/src/clients/transports/mock_transport.h b/src/clients/transports/mock_transport.h new file mode 100644 index 0000000..02fb079 --- /dev/null +++ b/src/clients/transports/mock_transport.h @@ -0,0 +1,79 @@ +#pragma once + +// Scripted request to response double. Matches on method plus path template +// plus header and body predicates; verify() fails on unconsumed expectations. + +#include +#include +#include +#include +#include +#include +#include + +#include "clients/request.h" +#include "clients/response.h" +#include "clients/task.h" +#include "clients/transport.h" + +namespace molo::clients::transports { + +class MockError : public std::runtime_error { + public: + explicit MockError(const std::string& msg) + : std::runtime_error("molo::clients::transports::Mock: " + msg) {} +}; + +struct MockExpectation { + Method method = Method::kGet; + // Exact path ("/v1/users/7") or template ("/v1/users/{id}"). + std::string path; + std::map required_headers; + std::string required_body_contains; + std::function predicate; + ClientResponse response; + std::exception_ptr error; + int times = 1; // How often this expectation may fire. + int used = 0; + std::string label; +}; + +class MockTransport : public Transport { + public: + MockTransport() = default; + + const char* name() const override { return "mock"; } + + // Adds an expectation. Returns its index for label-based debugging. + std::size_t expect(MockExpectation expectation); + std::size_t expect_response(Method method, const std::string& path, + ClientResponse response); + std::size_t expect_status(Method method, const std::string& path, int status, + const std::string& body = "", + const std::string& content_type = "text/plain"); + std::size_t expect_error(Method method, const std::string& path, + std::exception_ptr error); + + Task send(const ClientRequest& request) override; + + // Throws MockError when any expectation still has unused firings. + void verify() const; + // Requests seen so far, in order (method + target + body). + struct SeenRequest { + std::string method; + std::string target; + HeaderMap headers; + std::string body; + }; + std::vector seen() const; + std::size_t send_count() const; + + private: + static bool path_match(const std::string& pattern, const std::string& path); + + mutable std::mutex mutex_; + std::vector expectations_; + std::vector seen_; +}; + +} // namespace molo::clients::transports diff --git a/src/clients/transports/tls.h b/src/clients/transports/tls.h new file mode 100644 index 0000000..043dc02 --- /dev/null +++ b/src/clients/transports/tls.h @@ -0,0 +1,58 @@ +#pragma once + +// TLS session interface. The only implementation shipped in this part is the +// marked wolfSSL stub (tls_stub.cpp): wolfSSL is NOT installed in this +// environment, so HTTPS URLs fail loud through TlsStubError and never +// silently downgrade to cleartext. Part 01/03 provides the real session. + +#include +#include +#include + +namespace molo::clients::transports { + +class TlsError : public std::runtime_error { + public: + explicit TlsError(const std::string& msg) + : std::runtime_error("molo::clients::transports::Tls: " + msg) {} +}; + +// MARKED STUB: thrown whenever a real TLS session is required but only the +// stub backend is linked (wolfSSL missing). +class TlsStubError : public TlsError { + public: + explicit TlsStubError(const std::string& what) + : TlsError("STUB backend, wolfSSL not installed: " + what) {} +}; + +class H2StubError : public std::runtime_error { + public: + explicit H2StubError(const std::string& what) + : std::runtime_error("molo::clients::transports::H2 STUB: " + what) {} +}; + +// Minimal handshake-plus-record interface the HTTP transport needs. +class ITlsSession { + public: + virtual ~ITlsSession() = default; + virtual void handshake(int fd, const std::string& hostname, + bool want_h2) = 0; + virtual void send_all(const char* data, std::size_t size) = 0; + virtual std::size_t recv_some(char* out, std::size_t max_size) = 0; + virtual std::string negotiated_protocol() const = 0; // "h2", "http/1.1", "" + virtual void shutdown() = 0; + virtual const char* backend() const = 0; +}; + +// MARKED STUB backend. Every operation throws TlsStubError. +class StubTlsSession : public ITlsSession { + public: + void handshake(int fd, const std::string& hostname, bool want_h2) override; + void send_all(const char* data, std::size_t size) override; + std::size_t recv_some(char* out, std::size_t max_size) override; + std::string negotiated_protocol() const override; + void shutdown() override; + const char* backend() const override { return "stub-no-wolfssl"; } +}; + +} // namespace molo::clients::transports diff --git a/src/clients/transports/tls_stub.cpp b/src/clients/transports/tls_stub.cpp new file mode 100644 index 0000000..2051f66 --- /dev/null +++ b/src/clients/transports/tls_stub.cpp @@ -0,0 +1,25 @@ +#include "clients/transports/tls.h" + +// MARKED STUB: wolfSSL is not installed, so no TLS session can be created. +// Every method throws TlsStubError. Part 01/03 owns the real backend and +// injects it through ITlsSession without touching callers. + +namespace molo::clients::transports { + +void StubTlsSession::handshake(int, const std::string&, bool) { + throw TlsStubError("TLS handshake needs wolfSSL"); +} + +void StubTlsSession::send_all(const char*, std::size_t) { + throw TlsStubError("TLS send needs wolfSSL"); +} + +std::size_t StubTlsSession::recv_some(char*, std::size_t) { + throw TlsStubError("TLS receive needs wolfSSL"); +} + +std::string StubTlsSession::negotiated_protocol() const { return ""; } + +void StubTlsSession::shutdown() {} + +} // namespace molo::clients::transports diff --git a/src/clients/url.cpp b/src/clients/url.cpp new file mode 100644 index 0000000..66f84c6 --- /dev/null +++ b/src/clients/url.cpp @@ -0,0 +1,290 @@ +#include "clients/url.h" + +#include + +#include "clients/caps.h" + +namespace molo::clients { +namespace { + +bool is_unreserved(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~'; +} + +int hex_val(char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +std::string to_lower(std::string s) { + for (auto& c : s) c = static_cast(std::tolower((unsigned char)c)); + return s; +} + +std::uint16_t default_port(const std::string& scheme) { + if (scheme == "http") return 80; + if (scheme == "https") return 443; + if (scheme == "ws") return 80; + if (scheme == "wss") return 443; + return 0; +} + +// Merges base path with a relative-path reference and removes dot segments +// (RFC 3986 section 5.2.3, simplified for non-opaque paths). +std::string merge_paths(const std::string& base, const std::string& rel) { + std::string merged; + if (rel.empty()) { + merged = base.empty() ? "/" : base; + } else if (rel[0] == '/') { + merged = rel; + } else { + auto slash = base.rfind('/'); + merged = (slash == std::string::npos) ? "/" + rel + : base.substr(0, slash + 1) + rel; + } + std::vector out; + std::string cur; + for (std::size_t i = 0; i <= merged.size(); ++i) { + if (i == merged.size() || merged[i] == '/') { + if (cur == "..") { + if (!out.empty()) out.pop_back(); + } else if (!cur.empty() && cur != ".") { + out.push_back(cur); + } + cur.clear(); + } else { + cur.push_back(merged[i]); + } + } + std::string path = "/"; + for (std::size_t i = 0; i < out.size(); ++i) { + if (i) path.push_back('/'); + path += out[i]; + } + if (merged.size() > 1 && merged.back() == '/' && path.back() != '/') + path.push_back('/'); + return path; +} + +} // namespace + +std::string Url::encode_component(const std::string& text) { + static const char* kHex = "0123456789ABCDEF"; + std::string out; + out.reserve(text.size()); + for (char c : text) { + if (is_unreserved(c)) { + out.push_back(c); + } else { + out.push_back('%'); + out.push_back(kHex[(c >> 4) & 0xF]); + out.push_back(kHex[c & 0xF]); + } + } + return out; +} + +std::string Url::decode_component(const std::string& text) { + std::string out; + out.reserve(text.size()); + for (std::size_t i = 0; i < text.size(); ++i) { + char c = text[i]; + if (c == '%' && i + 2 < text.size()) { + int hi = hex_val(text[i + 1]); + int lo = hex_val(text[i + 2]); + if (hi < 0 || lo < 0) throw UrlError("bad percent-encoding"); + out.push_back(static_cast((hi << 4) | lo)); + i += 2; + } else if (c == '+') { + out.push_back(' '); + } else { + out.push_back(c); + } + } + return out; +} + +Url Url::parse(const std::string& text) { + if (text.size() > Caps::kMaxUrlLength) throw UrlError("URL too long"); + if (text.empty()) throw UrlError("empty URL"); + Url url; + std::string rest = text; + auto scheme_end = rest.find("://"); + if (scheme_end != std::string::npos) { + url.scheme_ = to_lower(rest.substr(0, scheme_end)); + if (url.scheme_.empty()) throw UrlError("empty scheme"); + rest = rest.substr(scheme_end + 3); + auto path_at = rest.find_first_of("/?#"); + std::string authority = + path_at == std::string::npos ? rest : rest.substr(0, path_at); + rest = path_at == std::string::npos ? "" : rest.substr(path_at); + auto at = authority.rfind('@'); + if (at != std::string::npos) { + url.userinfo_ = authority.substr(0, at); + authority = authority.substr(at + 1); + } + if (!authority.empty() && authority[0] == '[') { + auto close = authority.find(']'); + if (close == std::string::npos) throw UrlError("bad IPv6 literal"); + url.host_ = to_lower(authority.substr(0, close + 1)); + if (close + 1 < authority.size() && authority[close + 1] == ':') + url.port_ = static_cast( + std::stoul(authority.substr(close + 2))); + } else { + auto colon = authority.rfind(':'); + if (colon != std::string::npos && authority.find(':') == colon) { + url.host_ = to_lower(authority.substr(0, colon)); + url.port_ = static_cast( + std::stoul(authority.substr(colon + 1))); + } else { + url.host_ = to_lower(authority); + } + } + if (url.host_.empty()) throw UrlError("empty host"); + if (url.port_ == 0) url.port_ = default_port(url.scheme_); + } + auto hash = rest.find('#'); + if (hash != std::string::npos) { + url.fragment_ = rest.substr(hash + 1); + rest = rest.substr(0, hash); + } + auto qmark = rest.find('?'); + std::string query_text; + if (qmark != std::string::npos) { + query_text = rest.substr(qmark + 1); + rest = rest.substr(0, qmark); + } + url.path_ = rest.empty() ? (url.has_host() ? "/" : "") : rest; + while (!query_text.empty()) { + auto amp = query_text.find('&'); + std::string pair = + amp == std::string::npos ? query_text : query_text.substr(0, amp); + query_text = amp == std::string::npos ? "" : query_text.substr(amp + 1); + if (pair.empty()) continue; + auto eq = pair.find('='); + if (eq == std::string::npos) { + url.query_.emplace_back(decode_component(pair), ""); + } else { + url.query_.emplace_back(decode_component(pair.substr(0, eq)), + decode_component(pair.substr(eq + 1))); + } + } + return url; +} + +Url Url::join(const std::string& ref) const { + if (ref.empty()) return *this; + if (ref.find("://") != std::string::npos) return Url::parse(ref); + Url out = *this; + std::string work = ref; + auto hash = work.find('#'); + if (hash != std::string::npos) { + out.fragment_ = work.substr(hash + 1); + work = work.substr(0, hash); + } else { + out.fragment_.clear(); + } + if (work.empty()) return out; + if (work[0] == '?') { + out.query_.clear(); + Url tmp = Url::parse("http://x" + work); + out.query_ = tmp.query_; + return out; + } + auto qmark = work.find('?'); + std::string path_part = work; + std::string query_part; + if (qmark != std::string::npos) { + path_part = work.substr(0, qmark); + query_part = work.substr(qmark + 1); + } + out.path_ = merge_paths(path_, path_part); + out.query_.clear(); + if (!query_part.empty()) { + Url tmp = Url::parse("http://x/?" + query_part); + out.query_ = tmp.query_; + } + return out; +} + +std::string Url::str() const { + std::string out; + if (!scheme_.empty()) { + out += scheme_; + out += "://"; + if (!userinfo_.empty()) { + out += userinfo_; + out += '@'; + } + out += host_; + if (port_ != 0 && port_ != default_port(scheme_)) { + out += ':'; + out += std::to_string(port_); + } + } + out += path_.empty() ? "/" : path_; + std::string qs = query_string(); + if (!qs.empty()) { + out += '?'; + out += qs; + } + return out; +} + +std::string Url::origin() const { + std::string out = scheme_ + "://" + host_; + if (port_ != 0 && port_ != default_port(scheme_)) { + out += ':'; + out += std::to_string(port_); + } + return out; +} + +std::string Url::target() const { + std::string out = path_.empty() ? "/" : path_; + std::string qs = query_string(); + if (!qs.empty()) { + out += '?'; + out += qs; + } + return out; +} + +std::string Url::host_port() const { + return host_ + ":" + std::to_string(port_); +} + +std::string Url::pool_key() const { return scheme_ + "://" + host_port(); } + +std::optional Url::query_value(const std::string& name) const { + for (const auto& kv : query_) { + if (kv.first == name) return kv.second; + } + return std::nullopt; +} + +std::string Url::query_string() const { + std::string out; + for (const auto& kv : query_) { + if (!out.empty()) out.push_back('&'); + out += encode_component(kv.first); + out.push_back('='); + out += encode_component(kv.second); + } + return out; +} + +void Url::set_query_arg(const std::string& name, const std::string& value) { + for (auto& kv : query_) { + if (kv.first == name) { + kv.second = value; + return; + } + } + query_.emplace_back(name, value); +} + +} // namespace molo::clients diff --git a/src/clients/url.h b/src/clients/url.h new file mode 100644 index 0000000..a0f2097 --- /dev/null +++ b/src/clients/url.h @@ -0,0 +1,65 @@ +#pragma once + +// URL parse, join, query encode. Deterministic, bounded, dependency-free. + +#include +#include +#include +#include +#include +#include + +namespace molo::clients { + +class UrlError : public std::runtime_error { + public: + explicit UrlError(const std::string& msg) + : std::runtime_error("molo::clients::Url: " + msg) {} +}; + +class Url { + public: + Url() = default; + + static Url parse(const std::string& text); + static std::string encode_component(const std::string& text); + static std::string decode_component(const std::string& text); + + // Resolves a relative or absolute reference against this URL (RFC 3986, + // section 5.2 subset: absolute URLs, absolute paths, relative paths, + // query-only and fragment-only references). + Url join(const std::string& ref) const; + + std::string str() const; // Full URL without fragment. + std::string origin() const; // scheme://host[:port] + std::string target() const; // origin-form request target (path?query). + std::string host_port() const; // Pool key part: host:port (lowercased). + std::string pool_key() const; // scheme://host:port (lowercased). + + bool is_https() const { return scheme_ == "https"; } + bool has_host() const { return !host_.empty(); } + + const std::string& scheme() const { return scheme_; } + const std::string& host() const { return host_; } + std::uint16_t port() const { return port_; } + const std::string& path() const { return path_; } + const std::string& fragment() const { return fragment_; } + const std::vector>& query() const { + return query_; + } + + std::optional query_value(const std::string& name) const; + std::string query_string() const; + void set_query_arg(const std::string& name, const std::string& value); + + private: + std::string scheme_; + std::string userinfo_; + std::string host_; + std::uint16_t port_ = 0; + std::string path_; + std::vector> query_; + std::string fragment_; +}; + +} // namespace molo::clients diff --git a/src/data/Makefile b/src/data/Makefile new file mode 100644 index 0000000..8496615 --- /dev/null +++ b/src/data/Makefile @@ -0,0 +1,27 @@ +# Standalone build for the data part (04-data.md). Header-only library; +# this compiles and runs the integration tests. Outputs go to build/. +CXX ?= g++ +CXXFLAGS ?= -std=c++23 -Wall -Wextra -O1 -g -pthread +ROOT := ../.. +SRC := $(ROOT)/src/data +TESTDIR := $(ROOT)/tests/src/data +BUILD := $(ROOT)/build/data +INCLUDES := -I$(SRC) -I$(TESTDIR) + +TESTS := value_error pg_wire sqlite pool_txn orm sync_heal_migrate kv_cache pubsub +BINS := $(addprefix $(BUILD)/test_,$(TESTS)) + +all: $(BINS) + +$(BUILD)/test_%: $(TESTDIR)/test_%.cpp $(shell find $(SRC) -name '*.hpp') $(TESTDIR)/mini.hpp + @mkdir -p $(BUILD) + $(CXX) $(CXXFLAGS) $(INCLUDES) $< -o $@ + +test: all + @for t in $(BINS); do echo "== $$t"; "$$t" || exit 1; done + @echo "data: all suites green" + +clean: + rm -rf $(BUILD) + +.PHONY: all test clean diff --git a/src/data/cache/cache.hpp b/src/data/cache/cache.hpp new file mode 100644 index 0000000..a5891ae --- /dev/null +++ b/src/data/cache/cache.hpp @@ -0,0 +1,216 @@ +#pragma once + +// Cache: get-or-compute with TTL over any KvStore, explicit invalidation +// by key or tag, LRU eviction at whichever cap (entries or bytes) hits +// first, and singleflight stampede protection. ORM write hooks reach the +// cache through the InvalidationBus: cached reads tag entries with +// tag_for_table(table) and attach_bus() purges those tags on writes. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../context.hpp" +#include "../kv/memory.hpp" +#include "../kv/store.hpp" +#include "../obs.hpp" +#include "../orm/invalidation.hpp" +#include "policy.hpp" +#include "singleflight.hpp" + +namespace molo::data::cache { + +inline std::string tag_for_table(const std::string& table) { + return "table:" + table; +} + +class Cache { + public: + explicit Cache(kv::KvStore& store, CachePolicy policy = CachePolicy{}, + obs::MetricSink* metrics = &obs::null_sink()) + : store_(store), policy_(policy), metrics_(metrics) { + policy_.validate(); + } + + // Tenant scoping (PATCHES 11.10): entries never cross tenants. Keys + // are prefixed per tenant; tags stay global so bus-driven purges keep + // working (over-invalidation, never a stale read). + static std::string scoped_key(const Context& ctx, const std::string& key) { + return ctx.tenant().empty() ? key : "tenant:" + ctx.tenant() + ":" + key; + } + + std::optional get(const Context& ctx, const std::string& key) { + return get(scoped_key(ctx, key)); + } + + std::optional get(const std::string& key) { + kv::check_key(key); + auto found = store_.get(key); + if (!found) { + ++misses_; + metrics_->counter("db.cache.miss", 1, ""); + return std::nullopt; + } + if (is_tombstone(*found)) { + ++misses_; + metrics_->counter("db.cache.miss", 1, ""); + return std::nullopt; + } + std::lock_guard lock(mutex_); + touch_locked(key, found->size()); + ++hits_; + metrics_->counter("db.cache.hit", 1, ""); + return found; + } + + void put(const Context& ctx, const std::string& key, kv::Bytes value, + const std::vector& tags = {}, int ttl_sec = 0) { + put(scoped_key(ctx, key), std::move(value), tags, ttl_sec); + } + + void put(const std::string& key, kv::Bytes value, + const std::vector& tags = {}, int ttl_sec = 0) { + kv::check_key(key); + kv::check_value(value); + if (static_cast(tags.size()) > policy_.max_tags_per_entry) + throw std::invalid_argument("too many cache tags"); + const int ttl = ttl_sec > 0 ? ttl_sec : policy_.default_ttl_sec; + store_.set(key, value, ttl); + std::lock_guard lock(mutex_); + untag_locked(key); + for (const auto& tag : tags) tag_index_[tag].insert(key); + key_tags_[key] = std::set(tags.begin(), tags.end()); + touch_locked(key, value.size()); + evict_locked(); + } + + kv::Bytes get_or_compute(const Context& ctx, const std::string& key, + const std::vector& tags, + std::function fn, int ttl_sec = 0) { + return get_or_compute(scoped_key(ctx, key), tags, std::move(fn), ttl_sec); + } + + kv::Bytes get_or_compute(const std::string& key, + const std::vector& tags, + std::function fn, int ttl_sec = 0) { + if (auto hit = get(key)) return *hit; + return flight_.compute(key, [&] { + // Recheck inside the flight: a previous compute may have filled it. + if (auto hit = get(key)) return *hit; + try { + kv::Bytes value = fn(); + put(key, value, tags, ttl_sec); + return value; + } catch (...) { + if (policy_.negative_cache) { + store_.set(key, tombstone(), policy_.negative_ttl_sec); + std::lock_guard lock(mutex_); + touch_locked(key, 8); + } + throw; + } + }); + } + + void invalidate(const Context& ctx, const std::string& key) { + invalidate(scoped_key(ctx, key)); + } + + void invalidate(const std::string& key) { + store_.del(key); + std::lock_guard lock(mutex_); + untag_locked(key); + lru_.remove(key); + sizes_.erase(key); + metrics_->counter("db.cache.invalidate", 1, ""); + } + + std::int64_t purge_tag(const std::string& tag) { + std::set keys; + { + std::lock_guard lock(mutex_); + auto it = tag_index_.find(tag); + if (it == tag_index_.end()) return 0; + keys = it->second; + } + for (const auto& key : keys) invalidate(key); + metrics_->counter("db.cache.tag_purge", 1, ""); + return static_cast(keys.size()); + } + + // ORM write-through: any invalidation for a table purges its tag. + void attach_bus(orm::InvalidationBus& bus) { + bus.subscribe([this](const orm::Invalidation& inv) { + (void)purge_tag(tag_for_table(inv.table)); + }); + } + + double hit_ratio() const { + const auto total = hits_ + misses_; + return total == 0 ? 0.0 + : static_cast(hits_) / static_cast(total); + } + std::int64_t hits() const { return hits_; } + std::int64_t misses() const { return misses_; } + + private: + static kv::Bytes tombstone() { return kv::Bytes{0, 'n', 'e', 'g'}; } + static bool is_tombstone(const kv::Bytes& v) { + return v.size() == 4 && v[0] == 0 && v[1] == 'n'; + } + + void touch_locked(const std::string& key, std::size_t bytes) { + lru_.remove(key); + lru_.push_back(key); + bytes_used_ -= sizes_[key]; + sizes_[key] = bytes; + bytes_used_ += bytes; + } + + void untag_locked(const std::string& key) { + auto it = key_tags_.find(key); + if (it == key_tags_.end()) return; + for (const auto& tag : it->second) { + auto jt = tag_index_.find(tag); + if (jt != tag_index_.end()) { + jt->second.erase(key); + if (jt->second.empty()) tag_index_.erase(jt); + } + } + key_tags_.erase(it); + } + + void evict_locked() { + while ((static_cast(sizes_.size()) > policy_.max_entries || + bytes_used_ > policy_.max_bytes) && + !lru_.empty()) { + const std::string victim = lru_.front(); + lru_.pop_front(); + bytes_used_ -= sizes_[victim]; + sizes_.erase(victim); + untag_locked(victim); + store_.del(victim); + metrics_->counter("db.cache.evict", 1, ""); + } + } + + kv::KvStore& store_; + CachePolicy policy_; + obs::MetricSink* metrics_; + SingleFlight flight_; + std::mutex mutex_; + std::list lru_; + std::map sizes_; + std::size_t bytes_used_ = 0; + std::map> tag_index_; + std::map> key_tags_; + std::int64_t hits_ = 0; + std::int64_t misses_ = 0; +}; + +} // namespace molo::data::cache diff --git a/src/data/cache/policy.hpp b/src/data/cache/policy.hpp new file mode 100644 index 0000000..29c235c --- /dev/null +++ b/src/data/cache/policy.hpp @@ -0,0 +1,37 @@ +#pragma once + +// Cache policy ceilings: TTL, entry count, total bytes, tags per entry, +// negative caching bounds. + +#include +#include + +#include "../caps.hpp" + +namespace molo::data::cache { + +struct CachePolicy { + int default_ttl_sec = caps::kCacheDefaultTtlSec; + int max_entries = caps::kCacheMaxEntries; + std::size_t max_bytes = caps::kCacheMaxBytes; + int max_tags_per_entry = caps::kCacheMaxTagsPerEntry; + bool negative_cache = false; + int negative_ttl_sec = caps::kCacheNegativeTtlMaxSec; + + void validate() const { + if (default_ttl_sec <= 0 || default_ttl_sec > caps::kKvMaxTtlSec) + throw std::invalid_argument("bad cache TTL"); + if (max_entries < 1 || max_entries > caps::kCacheMaxEntries) + throw std::invalid_argument("bad cache entry cap"); + if (max_bytes < 1024 || max_bytes > caps::kCacheMaxBytes) + throw std::invalid_argument("bad cache byte cap"); + if (max_tags_per_entry < 0 || + max_tags_per_entry > caps::kCacheMaxTagsPerEntry) + throw std::invalid_argument("bad cache tag cap"); + if (negative_ttl_sec <= 0 || + negative_ttl_sec > caps::kCacheNegativeTtlMaxSec) + throw std::invalid_argument("bad negative TTL"); + } +}; + +} // namespace molo::data::cache diff --git a/src/data/cache/singleflight.hpp b/src/data/cache/singleflight.hpp new file mode 100644 index 0000000..e7a9cf7 --- /dev/null +++ b/src/data/cache/singleflight.hpp @@ -0,0 +1,76 @@ +#pragma once + +// SingleFlight: coalesces concurrent computes for the same key so a cache +// stampede collapses to one execution. Followers block on the in-flight +// call and share its result or exception. + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::data::cache { + +template +class SingleFlight { + public: + using Compute = std::function; + + V compute(const std::string& key, Compute fn) { + std::shared_ptr call; + bool leader = false; // local: the shared Call must not carry this + { + std::lock_guard lock(mutex_); + auto it = calls_.find(key); + if (it != calls_.end()) { + call = it->second; + } else { + call = std::make_shared(); + calls_[key] = call; + leader = true; + } + } + if (!leader) { + std::unique_lock lock(call->mutex); + call->done.wait(lock, [&] { return call->finished; }); + if (call->error) std::rethrow_exception(call->error); + return *call->result; + } + try { + V value = fn(); + std::lock_guard lock(call->mutex); + call->result = std::move(value); + } catch (...) { + std::lock_guard lock(call->mutex); + call->error = std::current_exception(); + } + { + std::lock_guard lock(call->mutex); + call->finished = true; + } + call->done.notify_all(); + { + std::lock_guard lock(mutex_); + calls_.erase(key); + } + if (call->error) std::rethrow_exception(call->error); + return *call->result; + } + + private: + struct Call { + std::mutex mutex; + std::condition_variable done; + bool finished = false; + std::optional result; + std::exception_ptr error; + }; + + std::mutex mutex_; + std::map> calls_; +}; + +} // namespace molo::data::cache diff --git a/src/data/caps.hpp b/src/data/caps.hpp new file mode 100644 index 0000000..5c28dbe --- /dev/null +++ b/src/data/caps.hpp @@ -0,0 +1,59 @@ +#pragma once + +// Static ceilings for every data-layer knob. Tunable only through validated +// config objects that clamp to these ceilings. No magic autotuning. + +#include +#include + +namespace molo::data::caps { + +// Pool +inline constexpr int kPgPoolMin = 2; +inline constexpr int kPgPoolMax = 32; +inline constexpr int kSqliteMaxWriters = 1; +inline constexpr int kSqliteMaxReaders = 8; +inline constexpr int kPoolAcquireTimeoutSec = 5; +inline constexpr int kPoolIdleEvictSec = 60; +inline constexpr int kPoolMaxLifetimeSec = 1800; // 30 min +inline constexpr int kPoolMaxWaiters = 256; + +// Queries +inline constexpr int kStatementTimeoutSec = 30; +inline constexpr int kStatementTimeoutHardSec = 300; +inline constexpr int kMaxBoundParams = 1024; +inline constexpr int kMaxIdentifierLen = 63; +inline constexpr std::size_t kMaxRowPayload = 64u * 1024u * 1024u; +inline constexpr int kMaxRowsAll = 10000; + +// SQLite +inline constexpr int kSqliteBusyTimeoutMs = 5000; +inline constexpr int kSqliteBusyMaxRetries = 5; +inline constexpr int kSqliteCheckpointPages = 1000; + +// KV +inline constexpr int kKvMaxKeyBytes = 512; +inline constexpr int kKvMaxKeys = 1048576; +inline constexpr std::size_t kKvMaxValueBytes = 16u * 1024u * 1024u; +inline constexpr int kKvMaxTtlSec = 2592000; // 30 days +inline constexpr int kKvScanPageMax = 1000; + +// Cache +inline constexpr int kCacheDefaultTtlSec = 60; +inline constexpr int kCacheMaxEntries = 131072; +inline constexpr std::size_t kCacheMaxBytes = 512u * 1024u * 1024u; +inline constexpr int kCacheMaxTagsPerEntry = 16; +inline constexpr int kCacheNegativeTtlMaxSec = 10; + +// Pubsub +inline constexpr int kChannelMaxLen = 256; +inline constexpr std::size_t kPayloadMaxBytes = 1024u * 1024u; +inline constexpr int kSubscriberQueueMax = 1024; +inline constexpr int kBrokerPatternMax = 1024; + +// Healer +inline constexpr int kHealMaxRetries = 1; +inline constexpr int kVarcharExtendFactor = 4; +inline constexpr int kVarcharPromoteToTextAt = 16384; + +} // namespace molo::data::caps diff --git a/src/data/connection.hpp b/src/data/connection.hpp new file mode 100644 index 0000000..2fb6a65 --- /dev/null +++ b/src/data/connection.hpp @@ -0,0 +1,58 @@ +#pragma once + +// Connection: the single declared driver interface hiding Postgres and +// SQLite. Application code never branches on the driver except through +// backend(). Escaping helpers are deliberately absent: bind, do not escape. +// +// Async seam: these calls are synchronous today with poll-based deadlines +// from Context. The 01-runtime.md integration wraps each call in an +// awaitable on the reactor (network drivers) or the bounded off-loop worker +// (SQLite file I/O). No call blocks without a deadline. + +#include + +#include "context.hpp" +#include "error.hpp" +#include "result.hpp" +#include "statement.hpp" + +namespace molo::data { + +enum class Isolation { + ReadCommitted, + RepeatableRead, + Serializable, +}; + +inline const char* isolation_name(Isolation iso) { + switch (iso) { + case Isolation::ReadCommitted: return "READ COMMITTED"; + case Isolation::RepeatableRead: return "REPEATABLE READ"; + case Isolation::Serializable: return "SERIALIZABLE"; + default: return "READ COMMITTED"; + } +} + +struct Prepared { + std::string name; + std::string sql; +}; + +class Connection { + public: + virtual ~Connection() = default; + + virtual Backend backend() const = 0; + virtual std::string backend_id() const = 0; + + virtual Result exec(const Context& ctx, const BoundStatement& stmt) = 0; + virtual Prepared prepare(const Context& ctx, const BoundStatement& stmt) = 0; + virtual Result exec_prepared(const Context& ctx, const Prepared& prep, + const std::vector& params) = 0; + virtual void ping(const Context& ctx) = 0; + virtual void close() = 0; + + bool is_postgres() const { return backend() == Backend::Postgres; } +}; + +} // namespace molo::data diff --git a/src/data/context.hpp b/src/data/context.hpp new file mode 100644 index 0000000..fc8a707 --- /dev/null +++ b/src/data/context.hpp @@ -0,0 +1,83 @@ +#pragma once + +// Operation context: deadline, correlation id, tenant selector. +// Every public data API takes one. This is the data-layer shape of the +// RequestContext owned by 02-observability.md; a thin adapter there can +// construct this from a request scope. No hidden globals. + +#include +#include + +#include "caps.hpp" + +namespace molo::data { + +// Tenant identity carried alongside the legacy tenant string (PATCHES 11, +// section 1). uuid is the stable identity; tenant() reports it. +struct TenantContext { + std::string uuid; + std::string slug; + std::string strategy = "shared_schema"; + bool privileged = false; +}; + +class Context { + public: + Context() = default; + + Context(std::string correlation_id, std::string tenant, int timeout_sec) + : correlation_id_(std::move(correlation_id)), + tenant_(std::move(tenant)), + deadline_(std::chrono::steady_clock::now() + + std::chrono::seconds(clamped_timeout(timeout_sec))) {} + + Context(std::string correlation_id, TenantContext tenant_ctx, int timeout_sec) + : correlation_id_(std::move(correlation_id)), + tenant_(tenant_ctx.uuid.empty() ? tenant_ctx.slug : tenant_ctx.uuid), + tenant_ctx_(std::move(tenant_ctx)), + deadline_(std::chrono::steady_clock::now() + + std::chrono::seconds(clamped_timeout(timeout_sec))) {} + + static Context background(std::string why = "background") { + Context c; + c.correlation_id_ = "bg-" + why; + c.deadline_ = std::chrono::steady_clock::now() + + std::chrono::seconds(caps::kStatementTimeoutSec); + return c; + } + + const std::string& correlation_id() const { return correlation_id_; } + const std::string& tenant() const { return tenant_; } + const TenantContext& tenant_ctx() const { return tenant_ctx_; } + bool privileged() const { return tenant_ctx_.privileged; } + + std::chrono::steady_clock::time_point deadline() const { return deadline_; } + + bool expired() const { return std::chrono::steady_clock::now() >= deadline_; } + + // Milliseconds remaining until deadline, floored at zero. + int remaining_ms() const { + auto now = std::chrono::steady_clock::now(); + if (now >= deadline_) return 0; + return static_cast( + std::chrono::duration_cast(deadline_ - now) + .count()); + } + + private: + static int clamped_timeout(int timeout_sec) { + if (timeout_sec <= 0) return caps::kStatementTimeoutSec; + if (timeout_sec > caps::kStatementTimeoutHardSec) + return caps::kStatementTimeoutHardSec; + return timeout_sec; + } + + std::string correlation_id_; + std::string tenant_; + TenantContext tenant_ctx_; + std::chrono::steady_clock::time_point deadline_ = + std::chrono::steady_clock::now() + + std::chrono::seconds(caps::kStatementTimeoutSec); +}; + +} // namespace molo::data diff --git a/src/data/data.mk b/src/data/data.mk new file mode 100644 index 0000000..6fbea29 --- /dev/null +++ b/src/data/data.mk @@ -0,0 +1,6 @@ +# Per-part include for the root Makefile (shared contract). Included by the +# root Makefile when it exists; standalone builds use src/data/Makefile. +DATA_DIR := src/data +DATA_TEST_DIR := tests/src/data +DATA_HEADERS := $(shell find $(DATA_DIR) -name '*.hpp') +DATA_TESTS := value_error pg_wire sqlite pool_txn orm sync_heal_migrate kv_cache pubsub diff --git a/src/data/drivers/pg/auth.hpp b/src/data/drivers/pg/auth.hpp new file mode 100644 index 0000000..8599976 --- /dev/null +++ b/src/data/drivers/pg/auth.hpp @@ -0,0 +1,186 @@ +#pragma once + +// SCRAM-SHA-256 client (RFC 5802 / RFC 7677) with SASLprep-lite handling. +// Cleartext password fallback exists but is refused by default and must be +// explicitly enabled. Channel binding is a documented v1 gap (see plan). + +#include +#include +#include +#include +#include + +#include "sha256.hpp" + +namespace molo::data::pg { + +inline std::string base64_encode(const std::vector& in) { + static const char* kTable = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + for (std::size_t i = 0; i < in.size(); i += 3) { + std::uint32_t n = static_cast(in[i]) << 16; + if (i + 1 < in.size()) n |= static_cast(in[i + 1]) << 8; + if (i + 2 < in.size()) n |= in[i + 2]; + out.push_back(kTable[(n >> 18) & 63]); + out.push_back(kTable[(n >> 12) & 63]); + out.push_back(i + 1 < in.size() ? kTable[(n >> 6) & 63] : '='); + out.push_back(i + 2 < in.size() ? kTable[n & 63] : '='); + } + return out; +} + +inline std::string base64_encode_str(const std::string& in) { + return base64_encode(std::vector(in.begin(), in.end())); +} + +inline std::vector base64_decode(const std::string& in) { + auto val = [](char c) -> int { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+') return 62; + if (c == '/') return 63; + return -1; + }; + std::vector out; + std::uint32_t buf = 0; + int bits = 0; + for (char c : in) { + if (c == '=') break; + const int v = val(c); + if (v < 0) throw std::runtime_error("bad base64"); + buf = (buf << 6) | static_cast(v); + bits += 6; + if (bits >= 8) { + bits -= 8; + out.push_back(static_cast((buf >> bits) & 0xff)); + } + } + return out; +} + +// Minimal SASLprep: reject NUL and bare control characters, map nothing. +// Full stringprep tables arrive with 03-security.md; passwords outside the +// safe ASCII range are passed through unchanged and documented. +inline std::string saslprep_lite(const std::string& password) { + for (unsigned char c : password) { + if (c == 0 || (c < 0x20 && c != '\t' && c != '\n' && c != '\r')) + throw std::runtime_error("password fails SASLprep"); + if (c == 0x7f) throw std::runtime_error("password fails SASLprep"); + } + return password; +} + +inline std::map parse_scram_attrs(const std::string& msg) { + std::map attrs; + std::size_t start = 0; + while (start < msg.size()) { + const std::size_t end = msg.find(',', start); + const std::string part = msg.substr(start, end == std::string::npos + ? end + : end - start); + if (part.size() >= 3 && part[1] == '=') + attrs[part[0]] = part.substr(2); + if (end == std::string::npos) break; + start = end + 1; + } + return attrs; +} + +inline std::string xor_bytes(const std::array& a, + const std::array& b) { + std::string out(32, '\0'); + for (int i = 0; i < 32; ++i) out[i] = static_cast(a[i] ^ b[i]); + return out; +} + +class ScramSha256Client { + public: + ScramSha256Client(std::string username, std::string password, + std::string client_nonce) + : username_(std::move(username)), + password_(saslprep_lite(password)), + client_nonce_(std::move(client_nonce)) {} + + std::string client_first_message() const { + return "n,," + std::string("n=") + escape_username(username_) + ",r=" + + client_nonce_; + } + + std::string client_first_bare() const { + return "n=" + escape_username(username_) + ",r=" + client_nonce_; + } + + // Processes server-first-message, returns client-final-message. + std::string process_server_first(const std::string& server_first) { + const auto attrs = parse_scram_attrs(server_first); + const std::string nonce = attrs.at('r'); + const std::string salt_b64 = attrs.at('s'); + const int iterations = std::stoi(attrs.at('i')); + if (nonce.rfind(client_nonce_, 0) != 0) + throw std::runtime_error("scram nonce mismatch"); + if (iterations < 4096) throw std::runtime_error("scram iterations too low"); + combined_nonce_ = nonce; + const auto salt_raw = base64_decode(salt_b64); + const std::string salt(salt_raw.begin(), salt_raw.end()); + salted_ = pbkdf2_hmac_sha256(password_, salt, iterations); + const std::string client_final_wo_proof = + "c=" + base64_encode_str("n,,") + ",r=" + combined_nonce_; + auth_message_ = + client_first_bare() + "," + server_first + "," + client_final_wo_proof; + const auto client_key_arr = + hmac_sha256(salted_, std::string("Client Key")); + const std::vector client_key(client_key_arr.begin(), + client_key_arr.end()); + const auto stored_key = Sha256::hash(client_key); + const auto signature = + hmac_sha256(std::vector(stored_key.begin(), + stored_key.end()), + auth_message_); + const std::string proof = + xor_bytes(client_key_arr, signature); + server_key_ = hmac_sha256(salted_, std::string("Server Key")); + return client_final_wo_proof + ",p=" + base64_encode_str(proof); + } + + void verify_server_final(const std::string& server_final) { + const auto attrs = parse_scram_attrs(server_final); + if (attrs.count('e')) + throw std::runtime_error("scram server error: " + attrs.at('e')); + const auto raw = base64_decode(attrs.at('v')); + const auto expected = + hmac_sha256(std::vector(server_key_.begin(), + server_key_.end()), + auth_message_); + if (raw.size() != 32) throw std::runtime_error("scram bad signature"); + for (int i = 0; i < 32; ++i) + if (raw[i] != expected[i]) + throw std::runtime_error("scram server signature mismatch"); + done_ = true; + } + + bool done() const { return done_; } + + private: + static std::string escape_username(const std::string& u) { + std::string out; + for (char c : u) { + if (c == ',') out += "=2C"; + else if (c == '=') out += "=3D"; + else out += c; + } + return out; + } + + std::string username_; + std::string password_; + std::string client_nonce_; + std::string combined_nonce_; + std::vector salted_; + std::string auth_message_; + std::array server_key_{}; + bool done_ = false; +}; + +} // namespace molo::data::pg diff --git a/src/data/drivers/pg/codecs.hpp b/src/data/drivers/pg/codecs.hpp new file mode 100644 index 0000000..4ff186d --- /dev/null +++ b/src/data/drivers/pg/codecs.hpp @@ -0,0 +1,209 @@ +#pragma once + +// Text and binary codecs between Value and the Postgres wire format. +// Oids for core types only; unknown oids decode as text. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../value.hpp" + +namespace molo::data::pg { + +namespace oid { +inline constexpr std::int32_t kBool = 16; +inline constexpr std::int32_t kBytea = 17; +inline constexpr std::int32_t kInt8 = 20; +inline constexpr std::int32_t kInt2 = 21; +inline constexpr std::int32_t kInt4 = 23; +inline constexpr std::int32_t kText = 25; +inline constexpr std::int32_t kFloat4 = 700; +inline constexpr std::int32_t kFloat8 = 701; +inline constexpr std::int32_t kVarchar = 1043; +inline constexpr std::int32_t kTimestampTz = 1184; +inline constexpr std::int32_t kTimestamp = 1114; +} // namespace oid + +inline void put_net_i16(std::vector& out, std::int16_t v) { + out.push_back(static_cast((v >> 8) & 0xff)); + out.push_back(static_cast(v & 0xff)); +} + +inline void put_net_i32(std::vector& out, std::int32_t v) { + for (int i = 3; i >= 0; --i) + out.push_back(static_cast((v >> (8 * i)) & 0xff)); +} + +inline void put_net_i64(std::vector& out, std::int64_t v) { + for (int i = 7; i >= 0; --i) + out.push_back(static_cast((v >> (8 * i)) & 0xff)); +} + +inline std::int16_t get_net_i16(const std::uint8_t* p) { + return static_cast((p[0] << 8) | p[1]); +} + +inline std::int32_t get_net_i32(const std::uint8_t* p) { + std::int32_t v = 0; + for (int i = 0; i < 4; ++i) v = (v << 8) | p[i]; + return v; +} + +inline std::int64_t get_net_i64(const std::uint8_t* p) { + std::int64_t v = 0; + for (int i = 0; i < 8; ++i) v = (v << 8) | p[i]; + return v; +} + +// Postgres epoch is 2000-01-01; Unix epoch offset in microseconds. +inline constexpr std::int64_t kPgEpochOffsetMicros = 946684800LL * 1000000LL; + +// Text encoding of a Value for text-format Bind parameters. +inline std::string encode_text(const Value& v) { + if (is_null(v)) return ""; + if (const auto* i = std::get_if(&v)) + return std::to_string(*i); + if (const auto* d = std::get_if(&v)) { + std::ostringstream os; + os << std::setprecision(17) << *d; + return os.str(); + } + if (const auto* s = std::get_if(&v)) return *s; + if (const auto* b = std::get_if(&v)) return *b ? "t" : "f"; + if (const auto* bytes = std::get_if(&v)) { + static const char* hex = "0123456789abcdef"; + std::string out = "\\x"; + for (auto c : *bytes) { + out.push_back(hex[c >> 4]); + out.push_back(hex[c & 15]); + } + return out; + } + if (const auto* t = std::get_if(&v)) { + const auto micros = + std::chrono::duration_cast(t->time_since_epoch()) + .count(); + const std::time_t sec = micros / 1000000; + std::tm tm{}; + gmtime_r(&sec, &tm); + char buf[64]; + std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tm); + std::ostringstream os; + os << buf << "." << std::setw(6) << std::setfill('0') + << (micros % 1000000) << "+00"; + return os.str(); + } + throw std::runtime_error("unencodable value"); +} + +// Binary encoding of a Value for binary-format Bind parameters. +inline std::vector encode_binary(const Value& v, + std::int32_t hint_oid = 0) { + std::vector out; + if (const auto* i = std::get_if(&v)) { + if (hint_oid == oid::kInt2) put_net_i16(out, static_cast(*i)); + else if (hint_oid == oid::kInt4) + put_net_i32(out, static_cast(*i)); + else + put_net_i64(out, *i); + return out; + } + if (const auto* d = std::get_if(&v)) { + if (hint_oid == oid::kFloat4) { + const float f = static_cast(*d); + std::uint32_t bits = 0; + __builtin_memcpy(&bits, &f, 4); + put_net_i32(out, static_cast(bits)); + } else { + std::uint64_t bits = 0; + __builtin_memcpy(&bits, d, 8); + put_net_i64(out, static_cast(bits)); + } + return out; + } + if (const auto* s = std::get_if(&v)) { + out.assign(s->begin(), s->end()); + return out; + } + if (const auto* bytes = std::get_if(&v)) return *bytes; + if (const auto* b = std::get_if(&v)) { + out.push_back(*b ? 1 : 0); + return out; + } + if (const auto* t = std::get_if(&v)) { + const auto micros = + std::chrono::duration_cast(t->time_since_epoch()) + .count(); + put_net_i64(out, micros - kPgEpochOffsetMicros); + return out; + } + throw std::runtime_error("cannot binary-encode null"); +} + +// Decode one DataRow cell. format 0 is text, 1 is binary. +inline Value decode_cell(const std::vector& cell, bool has_value, + std::int32_t type_oid, std::int16_t format) { + if (!has_value) return make_null(); + if (format == 0) { + const std::string s(cell.begin(), cell.end()); + switch (type_oid) { + case oid::kBool: return make_bool(s == "t" || s == "true" || s == "1"); + case oid::kInt2: + case oid::kInt4: + case oid::kInt8: return make_int(std::stoll(s)); + case oid::kFloat4: + case oid::kFloat8: return make_double(std::stod(s)); + case oid::kBytea: { + Bytes b; + if (s.rfind("\\x", 0) == 0) { + auto hex = [](char c) -> std::uint8_t { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + return c - 'A' + 10; + }; + for (std::size_t i = 2; i + 1 < s.size(); i += 2) + b.push_back((hex(s[i]) << 4) | hex(s[i + 1])); + } else { + b.assign(s.begin(), s.end()); + } + return make_bytes(std::move(b)); + } + default: return make_text(s); + } + } + const std::uint8_t* p = cell.data(); + switch (type_oid) { + case oid::kBool: return make_bool(p[0] != 0); + case oid::kInt2: return make_int(get_net_i16(p)); + case oid::kInt4: return make_int(get_net_i32(p)); + case oid::kInt8: return make_int(get_net_i64(p)); + case oid::kFloat4: { + const std::uint32_t bits = static_cast(get_net_i32(p)); + float f = 0; + __builtin_memcpy(&f, &bits, 4); + return make_double(f); + } + case oid::kFloat8: { + const std::uint64_t bits = static_cast(get_net_i64(p)); + double d = 0; + __builtin_memcpy(&d, &bits, 8); + return make_double(d); + } + case oid::kTimestamp: + case oid::kTimestampTz: { + const auto micros = get_net_i64(p) + kPgEpochOffsetMicros; + return make_time(TimePoint(std::chrono::microseconds(micros))); + } + case oid::kBytea: return make_bytes(Bytes(cell.begin(), cell.end())); + default: + return make_text(std::string(cell.begin(), cell.end())); + } +} + +} // namespace molo::data::pg diff --git a/src/data/drivers/pg/conn.hpp b/src/data/drivers/pg/conn.hpp new file mode 100644 index 0000000..cd2d665 --- /dev/null +++ b/src/data/drivers/pg/conn.hpp @@ -0,0 +1,467 @@ +#pragma once + +// PgConnection: Postgres wire-protocol client over a Transport. +// Startup, SCRAM-SHA-256 auth (cleartext refused by default), simple query +// and the extended Parse/Bind/Execute/Sync flow. TcpTransport uses +// non-blocking sockets with poll deadlines from Context, so no call blocks +// without a deadline; the 01-runtime.md integration swaps the transport for +// a reactor awaitable without touching this logic. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../connection.hpp" +#include "../../context.hpp" +#include "../../error.hpp" +#include "../../obs.hpp" +#include "../../result.hpp" +#include "../../row.hpp" +#include "../../statement.hpp" +#include "../../value.hpp" +#include "auth.hpp" +#include "codecs.hpp" +#include "extended.hpp" +#include "message.hpp" +#include "secret.hpp" + +namespace molo::data::pg { + +class Transport { + public: + virtual ~Transport() = default; + virtual void send_all(const Context& ctx, + const std::vector& bytes) = 0; + virtual BackendMessage recv_msg(const Context& ctx) = 0; + virtual void close() = 0; +}; + +class TcpTransport final : public Transport { + public: + TcpTransport() = default; + TcpTransport(const TcpTransport&) = delete; + TcpTransport& operator=(const TcpTransport&) = delete; + ~TcpTransport() override { close(); } + + void connect(const std::string& host, int port, const Context& ctx) { + addrinfo hints{}; + hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; + addrinfo* list = nullptr; + if (getaddrinfo(host.c_str(), std::to_string(port).c_str(), &hints, &list) != + 0) + throw DbError(Backend::Postgres, "MOLO_NET", "DNS resolve failed"); + DbError last_err(Backend::Postgres, "MOLO_NET", "connect failed"); + for (addrinfo* ai = list; ai; ai = ai->ai_next) { + const int fd = ::socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) continue; + const int flags = fcntl(fd, F_GETFL, 0); + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + int rc = ::connect(fd, ai->ai_addr, ai->ai_addrlen); + if (rc != 0 && errno != EINPROGRESS) { + ::close(fd); + continue; + } + pollfd pfd{fd, POLLOUT, 0}; + const int ms = ctx.remaining_ms(); + rc = poll(&pfd, 1, ms); + int err = 0; + socklen_t len = sizeof(err); + getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len); + if (rc <= 0 || err != 0) { + ::close(fd); + last_err = DbError(Backend::Postgres, "MOLO_NET", + rc == 0 ? "connect timed out" : "connect refused"); + continue; + } + fcntl(fd, F_SETFL, flags); + fd_ = fd; + freeaddrinfo(list); + return; + } + freeaddrinfo(list); + throw last_err; + } + + void send_all(const Context& ctx, + const std::vector& bytes) override { + std::size_t sent = 0; + while (sent < bytes.size()) { + poll_write(ctx); + const ssize_t n = + ::send(fd_, bytes.data() + sent, bytes.size() - sent, MSG_NOSIGNAL); + if (n < 0) { + if (errno == EINTR) continue; + throw DbError(Backend::Postgres, "MOLO_NET", "send failed"); + } + if (n == 0) throw DbError(Backend::Postgres, "MOLO_NET", "send eof"); + sent += static_cast(n); + } + } + + BackendMessage recv_msg(const Context& ctx) override { + const auto hdr = recv_exact(ctx, 5); + BackendMessage m; + m.type = static_cast(hdr[0]); + const std::int32_t len = get_net_i32(hdr.data() + 1); + if (len < 4 || len > 256 * 1024 * 1024) + throw DbError(Backend::Postgres, "MOLO_PROTO", "bad message length"); + m.body = recv_exact(ctx, static_cast(len - 4)); + return m; + } + + void close() override { + if (fd_ >= 0) { + ::close(fd_); + fd_ = -1; + } + } + + private: + std::vector recv_exact(const Context& ctx, std::size_t n) { + std::vector out(n); + std::size_t got = 0; + while (got < n) { + poll_read(ctx); + const ssize_t r = ::recv(fd_, out.data() + got, n - got, 0); + if (r < 0) { + if (errno == EINTR) continue; + throw DbError(Backend::Postgres, "MOLO_NET", "recv failed"); + } + if (r == 0) throw DbError(Backend::Postgres, "MOLO_NET", "recv eof"); + got += static_cast(r); + } + return out; + } + + void poll_read(const Context& ctx) { + pollfd pfd{fd_, POLLIN, 0}; + const int rc = poll(&pfd, 1, ctx.remaining_ms()); + if (rc <= 0) + throw DbError(Backend::Postgres, "MOLO_TIMEOUT", + rc == 0 ? "read timed out" : "poll failed"); + } + + void poll_write(const Context& ctx) { + pollfd pfd{fd_, POLLOUT, 0}; + const int rc = poll(&pfd, 1, ctx.remaining_ms()); + if (rc <= 0) + throw DbError(Backend::Postgres, "MOLO_TIMEOUT", + rc == 0 ? "write timed out" : "poll failed"); + } + + int fd_ = -1; +}; + +struct PgConnectParams { + std::string host = "127.0.0.1"; + int port = 5432; + std::string user; + std::string database; + Secret password; + bool allow_cleartext = false; // refused by default + bool binary = false; +}; + +class PgConnection final : public Connection { + public: + explicit PgConnection(std::unique_ptr transport = + std::make_unique(), + obs::MetricSink* metrics = &obs::null_sink(), + obs::EventSink* events = &obs::null_events()) + : transport_(std::move(transport)), + metrics_(metrics), + events_(events) {} + + void connect(const Context& ctx, PgConnectParams params) { + params_ = std::move(params); + if (auto* tcp = dynamic_cast(transport_.get())) + tcp->connect(params_.host, params_.port, ctx); + transport_->send_all(ctx, msg::startup(params_.user, params_.database)); + authenticate(ctx); + // Consume ParameterStatus / BackendKeyData until ReadyForQuery. + while (true) { + BackendMessage m = transport_->recv_msg(ctx); + if (m.type == 'Z') break; + if (m.type == 'E') fail(msg::parse_error(m.body)); + } + connected_ = true; + } + + Backend backend() const override { return Backend::Postgres; } + std::string backend_id() const override { + return "pg:" + params_.host + ":" + std::to_string(params_.port) + "/" + + params_.database; + } + + Result exec(const Context& ctx, const BoundStatement& stmt) override { + ensure_open(); + const auto start = std::chrono::steady_clock::now(); + const ExtendedScript script = + build_extended(stmt, params_.binary, next_seq()); + transport_->send_all(ctx, script.bytes); + Result result = read_extended(ctx, script.binary_results); + const auto ms = + std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count(); + metrics_->histogram("db.query.latency_ms", ms, "backend=postgres"); + return result; + } + + Prepared prepare(const Context& ctx, const BoundStatement& stmt) override { + ensure_open(); + Prepared p{"molo_prep_" + std::to_string(next_seq()), stmt.sql()}; + std::vector wire; + append_all(wire, msg::parse(p.name, p.sql)); + append_all(wire, msg::sync()); + transport_->send_all(ctx, wire); + drain_until_ready(ctx); + return p; + } + + Result exec_prepared(const Context& ctx, const Prepared& prep, + const std::vector& params) override { + BoundStatement stmt(prep.sql, params); + return exec(ctx, stmt); + } + + Result simple(const Context& ctx, const std::string& sql) { + ensure_open(); + transport_->send_all(ctx, msg::simple_query(sql)); + return read_simple(ctx); + } + + void ping(const Context& ctx) override { + ensure_open(); + (void)simple(ctx, "SELECT 1"); + } + + void close() override { + if (connected_) { + try { + Context ctx = Context::background("pg-close"); + transport_->send_all(ctx, msg::terminate()); + } catch (...) { + } + } + transport_->close(); + connected_ = false; + } + + private: + void ensure_open() const { + if (!connected_) + throw DbError(Backend::Postgres, "MOLO_CONN", "not connected"); + } + + int next_seq() { return seq_++; } + + [[noreturn]] void fail(const msg::ErrorFields& f) { + events_->emit("db.auth_failed", "code=" + f.code); + metrics_->counter("db.errors", 1, "code=" + f.code); + throw DbError(Backend::Postgres, + f.code.empty() ? "MOLO_PG" : f.code, f.message); + } + + void authenticate(const Context& ctx) { + std::string mechanisms; + while (true) { + BackendMessage m = transport_->recv_msg(ctx); + if (m.type == 'E') fail(msg::parse_error(m.body)); + if (m.type != 'R') + throw DbError(Backend::Postgres, "MOLO_PROTO", + "expected auth message"); + Reader r(m.body); + const std::int32_t kind = r.get_i32(); + if (kind == 0) return; // AuthenticationOk + if (kind == 3) { // CleartextPassword + if (!params_.allow_cleartext) { + events_->emit("db.auth_failed", "reason=cleartext_refused"); + throw DbError(Backend::Postgres, "28P01", + "cleartext auth refused by default"); + } + transport_->send_all(ctx, msg::password(params_.password.view())); + continue; + } + if (kind == 10) { // SASL + while (r.remaining() > 0) { + const std::string mech = r.get_cstr(); + if (mech.empty()) break; + mechanisms += mech + " "; + } + if (mechanisms.find("SCRAM-SHA-256") == std::string::npos) + throw DbError(Backend::Postgres, "MOLO_AUTH", + "server lacks SCRAM-SHA-256"); + scram_auth(ctx); + return; + } + throw DbError(Backend::Postgres, "MOLO_AUTH", + "unsupported auth kind " + std::to_string(kind)); + } + } + + void scram_auth(const Context& ctx) { + std::random_device rd; + std::string nonce; + static const char* alphabet = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + for (int i = 0; i < 24; ++i) nonce += alphabet[rd() % 62]; + ScramSha256Client scram(params_.user, params_.password.view(), nonce); + transport_->send_all( + ctx, msg::sasl_initial("SCRAM-SHA-256", scram.client_first_message())); + BackendMessage cont = transport_->recv_msg(ctx); + if (cont.type == 'E') fail(msg::parse_error(cont.body)); + Reader r(cont.body); + if (cont.type != 'R' || r.get_i32() != 11) + throw DbError(Backend::Postgres, "MOLO_AUTH", "expected SASLContinue"); + const std::string server_first = + r.get_str(r.remaining()); + const std::string client_final = scram.process_server_first(server_first); + transport_->send_all(ctx, msg::sasl_response(client_final)); + BackendMessage fin = transport_->recv_msg(ctx); + if (fin.type == 'E') fail(msg::parse_error(fin.body)); + Reader rf(fin.body); + if (fin.type != 'R' || rf.get_i32() != 12) + throw DbError(Backend::Postgres, "MOLO_AUTH", "expected SASLFinal"); + scram.verify_server_final(rf.get_str(rf.remaining())); + BackendMessage ok = transport_->recv_msg(ctx); + Reader ro(ok.body); + if (ok.type != 'R' || ro.get_i32() != 0) + throw DbError(Backend::Postgres, "28P01", "scram auth failed"); + } + + void drain_until_ready(const Context& ctx) { + while (true) { + BackendMessage m = transport_->recv_msg(ctx); + if (m.type == 'Z') return; + if (m.type == 'E') fail(msg::parse_error(m.body)); + } + } + + Result read_extended(const Context& ctx, bool binary) { + std::vector columns; + std::vector oids; + std::vector> rows; + std::int64_t affected = 0; + std::size_t payload = 0; + while (true) { + BackendMessage m = transport_->recv_msg(ctx); + switch (m.type) { + case 'Z': { + return Result(std::move(columns), std::move(rows), affected, 0); + } + case 'E': + fail(msg::parse_error(m.body)); + break; + case 'T': { + const auto descs = msg::parse_row_desc(m.body); + columns.clear(); + oids.clear(); + for (const auto& c : descs) { + columns.push_back(c.name); + oids.push_back(c.type_oid); + } + break; + } + case 'D': { + const msg::DataRow dr = msg::parse_data_row(m.body); + std::vector row; + for (std::size_t i = 0; i < dr.cells.size(); ++i) { + payload += dr.cells[i].size(); + if (payload > caps::kMaxRowPayload) + throw DbError(Backend::Postgres, "MOLO_LIMIT", + "row payload over 64 MiB cap"); + const std::int32_t oid = + i < oids.size() ? oids[i] : oid::kText; + row.push_back(decode_cell(dr.cells[i], dr.has_value[i], oid, + binary ? 1 : 0)); + } + rows.push_back(std::move(row)); + break; + } + case 'C': { + Reader r(m.body); + const std::string tag = r.get_cstr(); + const auto sp = tag.rfind(' '); + if (sp != std::string::npos) { + try { + affected = std::stoll(tag.substr(sp + 1)); + } catch (...) { + } + } + break; + } + case '1': + case '2': + case 'n': + case 'I': + break; // ParseComplete, BindComplete, NoData, EmptyQuery + default: + break; // NoticeResponse etc. ignored here + } + } + } + + Result read_simple(const Context& ctx) { + std::vector columns; + std::vector oids; + std::vector> rows; + std::int64_t affected = 0; + while (true) { + BackendMessage m = transport_->recv_msg(ctx); + if (m.type == 'Z') + return Result(std::move(columns), std::move(rows), affected, 0); + if (m.type == 'E') fail(msg::parse_error(m.body)); + if (m.type == 'T') { + const auto descs = msg::parse_row_desc(m.body); + columns.clear(); + oids.clear(); + for (const auto& c : descs) { + columns.push_back(c.name); + oids.push_back(c.type_oid); + } + } else if (m.type == 'D') { + const msg::DataRow dr = msg::parse_data_row(m.body); + std::vector row; + for (std::size_t i = 0; i < dr.cells.size(); ++i) { + const std::int32_t o = + i < oids.size() ? oids[i] : oid::kText; + row.push_back(decode_cell(dr.cells[i], dr.has_value[i], o, 0)); + } + rows.push_back(std::move(row)); + } else if (m.type == 'C') { + Reader r(m.body); + const std::string tag = r.get_cstr(); + const auto sp = tag.rfind(' '); + if (sp != std::string::npos) { + try { + affected = std::stoll(tag.substr(sp + 1)); + } catch (...) { + } + } + } + } + } + + PgConnectParams params_; + std::unique_ptr transport_; + obs::MetricSink* metrics_; + obs::EventSink* events_; + bool connected_ = false; + int seq_ = 1; +}; + +} // namespace molo::data::pg diff --git a/src/data/drivers/pg/extended.hpp b/src/data/drivers/pg/extended.hpp new file mode 100644 index 0000000..351e2c8 --- /dev/null +++ b/src/data/drivers/pg/extended.hpp @@ -0,0 +1,66 @@ +#pragma once + +// Extended protocol script: Parse, Bind, Describe, Execute, Sync as one +// byte stream. Parameters travel out-of-band from SQL text, which is the +// structural injection defense. Text formats by default, binary on request. + +#include +#include +#include + +#include "../../caps.hpp" +#include "../../statement.hpp" +#include "../../value.hpp" +#include "codecs.hpp" +#include "message.hpp" + +namespace molo::data::pg { + +struct ExtendedScript { + std::vector bytes; + std::string statement_name; + std::string portal_name; + bool binary_results = false; +}; + +inline void append_all(std::vector& out, + std::vector part) { + out.insert(out.end(), part.begin(), part.end()); +} + +inline ExtendedScript build_extended(const BoundStatement& stmt, bool binary, + int seq) { + if (stmt.params().size() > caps::kMaxBoundParams) + throw std::invalid_argument("too many bound parameters"); + ExtendedScript script; + script.statement_name = "molo_s" + std::to_string(seq); + script.portal_name = "molo_p" + std::to_string(seq); + script.binary_results = binary; + append_all(script.bytes, msg::parse(script.statement_name, stmt.sql())); + std::vector formats(stmt.params().size(), binary ? 1 : 0); + std::vector> values; + std::vector is_null_param; + for (const auto& v : stmt.params()) { + if (is_null(v)) { + values.emplace_back(); + is_null_param.push_back(true); + } else if (binary) { + values.push_back(encode_binary(v)); + is_null_param.push_back(false); + } else { + const std::string t = encode_text(v); + values.emplace_back(t.begin(), t.end()); + is_null_param.push_back(false); + } + } + std::vector result_formats = {static_cast(binary ? 1 : 0)}; + append_all(script.bytes, msg::bind(script.portal_name, script.statement_name, + formats, values, is_null_param, + result_formats)); + append_all(script.bytes, msg::describe_portal(script.portal_name)); + append_all(script.bytes, msg::execute(script.portal_name)); + append_all(script.bytes, msg::sync()); + return script; +} + +} // namespace molo::data::pg diff --git a/src/data/drivers/pg/message.hpp b/src/data/drivers/pg/message.hpp new file mode 100644 index 0000000..b9be3cd --- /dev/null +++ b/src/data/drivers/pg/message.hpp @@ -0,0 +1,307 @@ +#pragma once + +// Postgres frontend/backend message framing: network-order ints, C strings, +// typed message envelopes. Covers startup, auth, simple query and the full +// extended flow (Parse, Bind, Describe, Execute, Sync). + +#include +#include +#include +#include + +namespace molo::data::pg { + +class Writer { + public: + void put_u8(std::uint8_t v) { buf_.push_back(v); } + void put_i16(std::int16_t v) { + buf_.push_back(static_cast((v >> 8) & 0xff)); + buf_.push_back(static_cast(v & 0xff)); + } + void put_i32(std::int32_t v) { + for (int i = 3; i >= 0; --i) + buf_.push_back(static_cast((v >> (8 * i)) & 0xff)); + } + void put_bytes(const std::uint8_t* data, std::size_t n) { + buf_.insert(buf_.end(), data, data + n); + } + void put_bytes(const std::vector& v) { + buf_.insert(buf_.end(), v.begin(), v.end()); + } + void put_cstr(const std::string& s) { + buf_.insert(buf_.end(), s.begin(), s.end()); + buf_.push_back(0); + } + void put_str(const std::string& s) { + buf_.insert(buf_.end(), s.begin(), s.end()); + } + + const std::vector& bytes() const { return buf_; } + std::vector take() { return std::move(buf_); } + + private: + std::vector buf_; +}; + +class Reader { + public: + explicit Reader(const std::vector& buf) : buf_(buf) {} + Reader(const std::uint8_t* data, std::size_t n) : buf_(data, data + n) {} + + std::uint8_t get_u8() { + need(1); + return buf_[pos_++]; + } + std::int16_t get_i16() { + need(2); + std::int16_t v = static_cast((buf_[pos_] << 8) | buf_[pos_ + 1]); + pos_ += 2; + return v; + } + std::int32_t get_i32() { + need(4); + std::int32_t v = 0; + for (int i = 0; i < 4; ++i) v = (v << 8) | buf_[pos_ + i]; + pos_ += 4; + return v; + } + std::string get_cstr() { + std::size_t end = pos_; + while (end < buf_.size() && buf_[end] != 0) ++end; + if (end >= buf_.size()) throw std::runtime_error("truncated cstring"); + std::string s(buf_.begin() + pos_, buf_.begin() + end); + pos_ = end + 1; + return s; + } + std::vector get_bytes(std::size_t n) { + need(n); + std::vector v(buf_.begin() + pos_, buf_.begin() + pos_ + n); + pos_ += n; + return v; + } + std::string get_str(std::size_t n) { + need(n); + std::string s(buf_.begin() + pos_, buf_.begin() + pos_ + n); + pos_ += n; + return s; + } + + std::size_t remaining() const { return buf_.size() - pos_; } + std::size_t position() const { return pos_; } + + private: + void need(std::size_t n) { + if (remaining() < n) throw std::runtime_error("truncated message"); + } + std::vector buf_; + std::size_t pos_ = 0; +}; + +// Wrap a body writer into a typed envelope: type byte + int32 length + body. +inline std::vector envelope(char type, + const std::vector& body) { + Writer w; + w.put_u8(static_cast(type)); + w.put_i32(static_cast(body.size() + 4)); + w.put_bytes(body); + return w.take(); +} + +struct BackendMessage { + char type = 0; + std::vector body; +}; + +namespace msg { + +// Frontend builders. +inline std::vector startup(const std::string& user, + const std::string& database) { + Writer body; + body.put_i32(196608); // protocol 3.0 + body.put_cstr("user"); + body.put_cstr(user); + body.put_cstr("database"); + body.put_cstr(database); + body.put_cstr("application_name"); + body.put_cstr("moloapp"); + body.put_u8(0); + Writer w; + const auto b = body.take(); + w.put_i32(static_cast(b.size() + 4)); + w.put_bytes(b); + return w.take(); +} + +inline std::vector ssl_request() { + Writer w; + w.put_i32(8); + w.put_i32(80877103); + return w.take(); +} + +inline std::vector password(const std::string& secret) { + Writer b; + b.put_cstr(secret); + return envelope('p', b.take()); +} + +inline std::vector sasl_initial(const std::string& mechanism, + const std::string& initial) { + Writer b; + b.put_cstr(mechanism); + b.put_i32(static_cast(initial.size())); + b.put_str(initial); + return envelope('p', b.take()); +} + +inline std::vector sasl_response(const std::string& data) { + Writer b; + b.put_str(data); + return envelope('p', b.take()); +} + +inline std::vector simple_query(const std::string& sql) { + Writer b; + b.put_cstr(sql); + return envelope('Q', b.take()); +} + +inline std::vector parse(const std::string& name, + const std::string& sql) { + Writer b; + b.put_cstr(name); + b.put_cstr(sql); + b.put_i16(0); // no explicit param type oids + return envelope('P', b.take()); +} + +inline std::vector bind( + const std::string& portal, const std::string& name, + const std::vector& formats, + const std::vector>& values, + const std::vector& is_null, const std::vector& result_formats) { + Writer b; + b.put_cstr(portal); + b.put_cstr(name); + b.put_i16(static_cast(formats.size())); + for (auto f : formats) b.put_i16(f); + b.put_i16(static_cast(values.size())); + for (std::size_t i = 0; i < values.size(); ++i) { + if (is_null[i]) { + b.put_i32(-1); + } else { + b.put_i32(static_cast(values[i].size())); + b.put_bytes(values[i]); + } + } + b.put_i16(static_cast(result_formats.size())); + for (auto f : result_formats) b.put_i16(f); + return envelope('B', b.take()); +} + +inline std::vector describe_portal(const std::string& portal) { + Writer b; + b.put_u8('P'); + b.put_cstr(portal); + return envelope('D', b.take()); +} + +inline std::vector execute(const std::string& portal, + std::int32_t max_rows = 0) { + Writer b; + b.put_cstr(portal); + b.put_i32(max_rows); // 0 means all rows + return envelope('E', b.take()); +} + +inline std::vector sync() { + return envelope('S', {}); +} + +inline std::vector flush() { return envelope('H', {}); } + +inline std::vector terminate() { return envelope('X', {}); } + +// Backend parsers. +struct ErrorFields { + std::string severity; + std::string code; // SQLSTATE + std::string message; + std::string detail; + std::string table; + std::string column; +}; + +inline ErrorFields parse_error(const std::vector& body) { + Reader r(body); + ErrorFields f; + while (r.remaining() > 0) { + const char field = static_cast(r.get_u8()); + if (field == 0) break; + std::string val = r.get_cstr(); + switch (field) { + case 'S': f.severity = val; break; + case 'C': f.code = val; break; + case 'M': f.message = val; break; + case 'D': f.detail = val; break; + case 't': f.table = val; break; + case 'c': f.column = val; break; + default: break; + } + } + return f; +} + +struct ColumnDesc { + std::string name; + std::int32_t table_oid = 0; + std::int32_t type_oid = 0; + std::int16_t type_size = 0; + std::int16_t format = 0; +}; + +inline std::vector parse_row_desc( + const std::vector& body) { + Reader r(body); + const std::int16_t n = r.get_i16(); + std::vector cols; + for (int i = 0; i < n; ++i) { + ColumnDesc c; + c.name = r.get_cstr(); + c.table_oid = r.get_i32(); + r.get_i16(); // column attr number, unused + c.type_oid = r.get_i32(); + c.type_size = r.get_i16(); + r.get_i32(); // type modifier, unused + c.format = r.get_i16(); + cols.push_back(std::move(c)); + } + return cols; +} + +struct DataRow { + // has_value[i] false means SQL NULL. + std::vector> cells; + std::vector has_value; +}; + +inline DataRow parse_data_row(const std::vector& body) { + Reader r(body); + const std::int16_t n = r.get_i16(); + DataRow row; + for (int i = 0; i < n; ++i) { + const std::int32_t len = r.get_i32(); + if (len < 0) { + row.cells.emplace_back(); + row.has_value.push_back(false); + } else { + row.cells.push_back(r.get_bytes(static_cast(len))); + row.has_value.push_back(true); + } + } + return row; +} + +} // namespace msg +} // namespace molo::data::pg diff --git a/src/data/drivers/pg/secret.hpp b/src/data/drivers/pg/secret.hpp new file mode 100644 index 0000000..28cd9e5 --- /dev/null +++ b/src/data/drivers/pg/secret.hpp @@ -0,0 +1,74 @@ +#pragma once + +// Opaque credential handle. Secrets arrive from 03-security.md; until then +// callers build one from validated config. Copying is disabled, memory is +// zeroed on destruction, and the value never appears in logs or errors. + +#include +#include +#include + +namespace molo::data::pg { + +class Secret { + public: + Secret() = default; + Secret(const Secret&) = delete; + Secret& operator=(const Secret&) = delete; + Secret(Secret&& other) noexcept : bytes_(std::move(other.bytes_)) { + other.bytes_.clear(); + } + Secret& operator=(Secret&& other) noexcept { + if (this != &other) { + clear(); + bytes_ = std::move(other.bytes_); + other.bytes_.clear(); + } + return *this; + } + ~Secret() { clear(); } + + static Secret from_string(const std::string& s) { + Secret secret; + secret.bytes_.assign(s.begin(), s.end()); + return secret; + } + + std::string view() const { + return std::string(bytes_.begin(), bytes_.end()); + } + + bool empty() const { return bytes_.empty(); } + + void clear() { + if (!bytes_.empty()) { + std::memset(bytes_.data(), 0, bytes_.size()); + bytes_.clear(); + } + } + + private: + std::vector bytes_; +}; + +// Redact password values from conninfo-style strings for logs. +inline std::string redact_conninfo(const std::string& conninfo) { + std::string out = conninfo; + const std::string key = "password="; + std::size_t pos = 0; + while ((pos = out.find(key, pos)) != std::string::npos) { + const std::size_t start = pos + key.size(); + std::size_t end = start; + if (end < out.size() && out[end] == '\'') { + end = out.find('\'', end + 1); + end = (end == std::string::npos) ? out.size() : end + 1; + } else { + while (end < out.size() && out[end] != ' ') ++end; + } + out.replace(start, end - start, "***"); + pos = start + 3; + } + return out; +} + +} // namespace molo::data::pg diff --git a/src/data/drivers/pg/sha256.hpp b/src/data/drivers/pg/sha256.hpp new file mode 100644 index 0000000..ad2b1ae --- /dev/null +++ b/src/data/drivers/pg/sha256.hpp @@ -0,0 +1,188 @@ +#pragma once + +// In-house SHA-256, HMAC-SHA-256 and PBKDF2-HMAC-SHA-256 for SCRAM auth. +// Seam note: when 03-security.md crypto lands, this delegates to +// molo::security::crypto and stays only as a fallback. No third-party code. + +#include +#include +#include +#include + +namespace molo::data::pg { + +class Sha256 { + public: + Sha256() { reset(); } + + void reset() { + state_ = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + total_ = 0; + buffered_ = 0; + } + + void update(const std::uint8_t* data, std::size_t len) { + total_ += len; + while (len > 0) { + const std::size_t take = + (64 - buffered_) < len ? (64 - buffered_) : len; + for (std::size_t i = 0; i < take; ++i) block_[buffered_ + i] = *data++; + buffered_ += take; + len -= take; + if (buffered_ == 64) { + compress(block_.data()); + buffered_ = 0; + } + } + } + + void update(const std::string& s) { + update(reinterpret_cast(s.data()), s.size()); + } + + void update(const std::vector& v) { + if (!v.empty()) update(v.data(), v.size()); + } + + std::array final() { + const std::uint64_t bit_len = total_ * 8; + std::uint8_t pad = 0x80; + update(&pad, 1); + std::uint8_t zero = 0; + while (buffered_ != 56) update(&zero, 1); + std::uint8_t len_bytes[8]; + for (int i = 0; i < 8; ++i) + len_bytes[i] = static_cast(bit_len >> (56 - 8 * i)); + // Feed length directly without touching total_. + for (int i = 0; i < 8; ++i) block_[buffered_ + i] = len_bytes[i]; + compress(block_.data()); + buffered_ = 0; + std::array out{}; + for (int i = 0; i < 8; ++i) { + out[4 * i] = static_cast(state_[i] >> 24); + out[4 * i + 1] = static_cast(state_[i] >> 16); + out[4 * i + 2] = static_cast(state_[i] >> 8); + out[4 * i + 3] = static_cast(state_[i]); + } + return out; + } + + static std::array hash(const std::string& s) { + Sha256 h; + h.update(s); + return h.final(); + } + + static std::array hash(const std::vector& v) { + Sha256 h; + h.update(v); + return h.final(); + } + + private: + static std::uint32_t rotr(std::uint32_t x, int n) { + return (x >> n) | (x << (32 - n)); + } + + void compress(const std::uint8_t* block) { + static const std::uint32_t k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, + 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, + 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, + 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, + 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, + 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, + 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, + 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + std::uint32_t w[64]; + for (int i = 0; i < 16; ++i) + w[i] = (static_cast(block[4 * i]) << 24) | + (static_cast(block[4 * i + 1]) << 16) | + (static_cast(block[4 * i + 2]) << 8) | + static_cast(block[4 * i + 3]); + for (int i = 16; i < 64; ++i) { + const std::uint32_t s0 = + rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + const std::uint32_t s1 = + rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + std::uint32_t a = state_[0], b = state_[1], c = state_[2], d = state_[3]; + std::uint32_t e = state_[4], f = state_[5], g = state_[6], h = state_[7]; + for (int i = 0; i < 64; ++i) { + const std::uint32_t s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + const std::uint32_t ch = (e & f) ^ (~e & g); + const std::uint32_t t1 = h + s1 + ch + k[i] + w[i]; + const std::uint32_t s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + const std::uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + const std::uint32_t t2 = s0 + maj; + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + state_[0] += a; + state_[1] += b; + state_[2] += c; + state_[3] += d; + state_[4] += e; + state_[5] += f; + state_[6] += g; + state_[7] += h; + } + + std::array state_{}; + std::array block_{}; + std::size_t buffered_ = 0; + std::uint64_t total_ = 0; +}; + +inline std::array hmac_sha256( + const std::vector& key, const std::string& msg) { + std::vector k = key; + if (k.size() > 64) { + const auto h = Sha256::hash(k); + k.assign(h.begin(), h.end()); + } + k.resize(64, 0); + std::string ipad(64, 0x36), opad(64, 0x5c); + for (int i = 0; i < 64; ++i) { + ipad[i] = static_cast(k[i] ^ 0x36); + opad[i] = static_cast(k[i] ^ 0x5c); + } + Sha256 inner; + inner.update(reinterpret_cast(ipad.data()), 64); + inner.update(msg); + const auto ih = inner.final(); + Sha256 outer; + outer.update(reinterpret_cast(opad.data()), 64); + outer.update(ih.data(), 32); + return outer.final(); +} + +inline std::vector pbkdf2_hmac_sha256(const std::string& password, + const std::string& salt, + int iterations) { + const std::vector key(password.begin(), password.end()); + const std::string first = salt + std::string("\x00\x00\x00\x01", 4); + auto u = hmac_sha256(key, first); + std::vector out(u.begin(), u.end()); + for (int i = 1; i < iterations; ++i) { + const std::string prev(reinterpret_cast(u.data()), 32); + u = hmac_sha256(key, prev); + for (int j = 0; j < 32; ++j) out[j] ^= u[j]; + } + return out; +} + +} // namespace molo::data::pg diff --git a/src/data/drivers/sqlite/busy.hpp b/src/data/drivers/sqlite/busy.hpp new file mode 100644 index 0000000..f9a2a76 --- /dev/null +++ b/src/data/drivers/sqlite/busy.hpp @@ -0,0 +1,47 @@ +#pragma once + +// Busy handling: second line of defense behind the single-writer gate. +// Retries SQLITE_BUSY with bounded backoff; still-busy after retries is a +// loud error with a counter, never silent. + +#include +#include +#include +#include + +#include "../../caps.hpp" +#include "../../error.hpp" +#include "../../obs.hpp" + +namespace molo::data::sqlite { + +struct BusyPolicy { + int timeout_ms = caps::kSqliteBusyTimeoutMs; + int max_retries = caps::kSqliteBusyMaxRetries; +}; + +template +auto with_busy_retry(BusyPolicy policy, Fn&& fn, + obs::MetricSink* metrics = &obs::null_sink()) + -> decltype(fn()) { + int attempt = 0; + while (true) { + try { + return fn(); + } catch (const DbError& e) { + if (e.backend() != Backend::Sqlite || e.code() != "SQLITE_BUSY") + throw; + if (attempt >= policy.max_retries) { + metrics->counter("db.sqlite.busy_failed", 1, ""); + throw DbError(Backend::Sqlite, "SQLITE_BUSY", + "database is locked after retries"); + } + metrics->counter("db.sqlite.busy_retry", 1, ""); + const int slice = policy.timeout_ms / (policy.max_retries + 1); + std::this_thread::sleep_for(std::chrono::milliseconds(slice)); + ++attempt; + } + } +} + +} // namespace molo::data::sqlite diff --git a/src/data/drivers/sqlite/db.hpp b/src/data/drivers/sqlite/db.hpp new file mode 100644 index 0000000..aface4e --- /dev/null +++ b/src/data/drivers/sqlite/db.hpp @@ -0,0 +1,153 @@ +#pragma once + +// SqliteDb: SQLite handling over a MemoryStore. Applies the mandated pragma +// sequence (WAL, NORMAL sync, foreign keys, busy timeout), refuses network +// paths at startup (WAL requires one host), serializes writers through the +// single-writer gate, retries busy through the busy policy, and schedules +// checkpoints through the WAL scheduler. +// +// Known gap: persistence uses our snapshot image, not the native SQLite +// page format, so files are not readable by stock sqlite3. The SQL layer, +// pragma sequence, locking behavior and checkpoint accounting are real and +// covered by integration tests; a page-codec backend can replace +// MemoryStore behind the same Connection interface. + +#include +#include +#include + +#include "../../connection.hpp" +#include "../../context.hpp" +#include "../../error.hpp" +#include "../../obs.hpp" +#include "../../result.hpp" +#include "../../statement.hpp" +#include "busy.hpp" +#include "gate.hpp" +#include "store.hpp" +#include "wal.hpp" + +namespace molo::data::sqlite { + +struct SqliteConfig { + std::string journal_mode = "WAL"; + std::string synchronous = "NORMAL"; + bool foreign_keys = true; + int busy_timeout_ms = caps::kSqliteBusyTimeoutMs; + std::string path; // empty means pure in-memory +}; + +class SqliteDb final : public Connection { + public: + explicit SqliteDb( + SqliteConfig config = SqliteConfig{}, + std::shared_ptr store = std::make_shared(), + obs::MetricSink* metrics = &obs::null_sink()) + : config_(std::move(config)), + store_(std::move(store)), + metrics_(metrics) {} + + void open(const Context& ctx) { + if (!config_.path.empty()) { + if (config_.path.find("://") != std::string::npos || + config_.path.rfind("//", 0) == 0) + throw DbError(Backend::Sqlite, "MOLO_PATH", + "refusing network path for WAL database"); + store_->load(config_.path); + } + exec_pragma(ctx, "PRAGMA journal_mode=" + config_.journal_mode); + exec_pragma(ctx, "PRAGMA synchronous=" + config_.synchronous); + exec_pragma(ctx, std::string("PRAGMA foreign_keys=") + + (config_.foreign_keys ? "ON" : "OFF")); + exec_pragma(ctx, + "PRAGMA busy_timeout=" + std::to_string(config_.busy_timeout_ms)); + open_ = true; + } + + Backend backend() const override { return Backend::Sqlite; } + std::string backend_id() const override { + return "sqlite:" + (config_.path.empty() ? ":memory:" : config_.path); + } + + Result exec(const Context& ctx, const BoundStatement& stmt) override { + ensure_open(); + // Tenant guard (PATCHES 11.9): statements under a tenant context must + // carry tenant scope unless privileged; the tenants catalog is exempt. + // Migrations run privileged or with an empty tenant. + if (!ctx.privileged() && !ctx.tenant().empty() && + stmt.sql().find("tenant_id") == std::string::npos && + stmt.sql().find("tenants") == std::string::npos) + throw DbError(Backend::Sqlite, "tenant_scope_missing", + "pool-mode statement lacks tenant scope"); + const bool write = is_write(stmt.sql()); + BusyPolicy policy; + policy.timeout_ms = config_.busy_timeout_ms; + if (!write) { + return with_busy_retry( + policy, [&] { return store_->execute(ctx, stmt); }, metrics_); + } + auto guard = gate_.write(); + Result r = with_busy_retry( + policy, [&] { return store_->execute(ctx, stmt); }, metrics_); + wal_.note_write(1 + r.affected()); + wal_.maybe_checkpoint([&] { checkpoint(); }); + return r; + } + + Prepared prepare(const Context&, const BoundStatement& stmt) override { + ensure_open(); + return Prepared{"sqlite_inline", stmt.sql()}; + } + + Result exec_prepared(const Context& ctx, const Prepared& prep, + const std::vector& params) override { + return exec(ctx, BoundStatement(prep.sql, params)); + } + + void ping(const Context& ctx) override { + ensure_open(); + (void)store_->execute(ctx, BoundStatement("SELECT 1")); + } + + void close() override { + if (open_) checkpoint(); + open_ = false; + } + + void checkpoint() { + if (!config_.path.empty()) store_->save(config_.path); + checkpoints_run_++; + } + + MemoryStore& store() { return *store_; } + const CheckpointScheduler& wal() const { return wal_; } + std::int64_t checkpoints_run() const { return checkpoints_run_; } + + private: + void ensure_open() const { + if (!open_) throw DbError(Backend::Sqlite, "MOLO_CONN", "not open"); + } + + void exec_pragma(const Context& ctx, const std::string& sql) { + (void)store_->execute(ctx, BoundStatement::raw(sql)); + } + + static bool is_write(const std::string& sql) { + std::string kw; + for (char c : sql) { + if (c == ' ' || c == '\t' || c == '\n' || c == '(') break; + kw += static_cast(std::toupper(c)); + } + return kw != "SELECT" && kw != "PRAGMA" && kw != "EXPLAIN"; + } + + SqliteConfig config_; + std::shared_ptr store_; + obs::MetricSink* metrics_; + WriterGate gate_; + CheckpointScheduler wal_; + bool open_ = false; + std::int64_t checkpoints_run_ = 0; +}; + +} // namespace molo::data::sqlite diff --git a/src/data/drivers/sqlite/gate.hpp b/src/data/drivers/sqlite/gate.hpp new file mode 100644 index 0000000..9d9b9af --- /dev/null +++ b/src/data/drivers/sqlite/gate.hpp @@ -0,0 +1,30 @@ +#pragma once + +// Single-writer gate: one writer at a time per database file. Readers +// proceed concurrently (WAL semantics); writers serialize here. First line +// of defense, busy_timeout is the second. + +#include +#include + +namespace molo::data::sqlite { + +class WriterGate { + public: + WriterGate() : mutex_(std::make_shared()) {} + + class Guard { + public: + explicit Guard(WriterGate& gate) : lock_(*gate.mutex_) {} + private: + std::lock_guard lock_; + }; + + Guard write() { return Guard(*this); } + + private: + // Shared so the gate (and owning database handle) stays movable. + std::shared_ptr mutex_; +}; + +} // namespace molo::data::sqlite diff --git a/src/data/drivers/sqlite/store.hpp b/src/data/drivers/sqlite/store.hpp new file mode 100644 index 0000000..a795108 --- /dev/null +++ b/src/data/drivers/sqlite/store.hpp @@ -0,0 +1,1463 @@ +#pragma once + +// MemoryStore: dependency-free SQL execution for the SQLite path. It speaks +// exactly the constrained dialect the ORM, syncer and healer generate +// (CREATE/ALTER/DROP TABLE, indexes, INSERT/SELECT/UPDATE/DELETE with +// equality filters, one INNER JOIN, ORDER/LIMIT/OFFSET, transactions and +// savepoints, PRAGMAs) with bound parameters only. Anything outside the +// dialect is a loud syntax error. +// +// Error shapes mirror real SQLite messages ("no such table: X") with +// SQLITE_* codes so the healer exercises the true matrix. Snapshots give +// BEGIN/COMMIT/ROLLBACK plus savepoints. Optional file persistence stores +// a snapshot image; the native SQLite page format is the documented gap +// (see db.hpp), the SQL layer here is complete and real. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../caps.hpp" +#include "../../context.hpp" +#include "../../error.hpp" +#include "../../result.hpp" +#include "../../statement.hpp" +#include "../../value.hpp" + +namespace molo::data::sqlite { + +namespace store_detail { + +inline std::string upper(std::string s) { + for (char& c : s) c = static_cast(std::toupper(c)); + return s; +} + +inline std::vector tokenize(const std::string& sql) { + std::vector toks; + std::string cur; + auto flush = [&] { + if (!cur.empty()) { + toks.push_back(cur); + cur.clear(); + } + }; + for (std::size_t i = 0; i < sql.size(); ++i) { + const char c = sql[i]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ';') { + flush(); + } else if (c == '(' || c == ')' || c == ',' || c == '=' || c == '*') { + flush(); + toks.emplace_back(1, c); + } else if (c == '.') { + flush(); + toks.emplace_back(1, c); + } else if (c == '\'') { + flush(); + std::string lit = "'"; + ++i; + while (i < sql.size()) { + if (sql[i] == '\'' && i + 1 < sql.size() && sql[i + 1] == '\'') { + lit += "''"; + i += 2; + } else if (sql[i] == '\'') { + lit += "'"; + break; + } else { + lit += sql[i++]; + } + } + toks.push_back(lit); + } else { + cur += c; + } + } + flush(); + return toks; +} + +inline bool is_quoted(const std::string& t) { + return t.size() >= 2 && t.front() == '\'' && t.back() == '\''; +} + +inline std::string unquote(const std::string& t) { + std::string s = t.substr(1, t.size() - 2); + std::string out; + for (std::size_t i = 0; i < s.size(); ++i) { + if (s[i] == '\'' && i + 1 < s.size() && s[i + 1] == '\'') { + out += '\''; + ++i; + } else { + out += s[i]; + } + } + return out; +} + +} // namespace store_detail + +class MemoryStore { + public: + struct Column { + std::string name; + std::string type = "TEXT"; // BIGINT SMALLINT INTEGER DOUBLE REAL TEXT + // VARCHAR CHAR BLOB BOOLEAN TIME + int length = 0; // for VARCHAR/CHAR + bool nullable = true; + bool pk = false; + bool unique = false; + std::string def; + bool has_def = false; + }; + + struct Table { + std::string name; + std::vector cols; + std::vector> rows; + std::int64_t next_id = 1; + std::vector indexes; + }; + + Result execute(const Context& ctx, const BoundStatement& stmt) { + std::lock_guard lock(mutex_); + if (ctx.expired()) + throw DbError(Backend::Sqlite, "MOLO_TIMEOUT", "statement expired"); + if (busy_left_ > 0) { + --busy_left_; + throw DbError(Backend::Sqlite, "SQLITE_BUSY", "database is locked", 5); + } + auto toks = store_detail::tokenize(stmt.sql()); + if (toks.empty()) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", "empty statement"); + Cursor c{toks, 0, stmt.params(), 0}; + const std::string kw = store_detail::upper(toks[0]); + c.next(); // consume the leading keyword; handlers parse the rest + if (kw == "BEGIN") return do_txn_begin(c); + if (kw == "COMMIT") return do_txn_end(true); + if (kw == "ROLLBACK") return do_rollback(c); + if (kw == "SAVEPOINT") return do_savepoint(c); + if (kw == "RELEASE") return do_release(c); + if (kw == "PRAGMA") return do_pragma(c); + if (kw == "CREATE") return do_create(c); + if (kw == "ALTER") return do_alter(c); + if (kw == "DROP") return do_drop(c); + if (kw == "INSERT") return do_insert(c); + if (kw == "SELECT") return do_select(c); + if (kw == "UPDATE") return do_update(c); + if (kw == "DELETE") return do_delete(c); + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "unsupported statement: " + toks[0]); + } + + // Introspection for tests and the inspector. + bool has_table(const std::string& t) const { + std::lock_guard lock(mutex_); + return tables_.count(t) > 0; + } + Table table(const std::string& t) const { + std::lock_guard lock(mutex_); + auto it = tables_.find(t); + if (it == tables_.end()) + throw DbError(Backend::Sqlite, "SQLITE_NO_TABLE", "no such table: " + t); + return it->second; + } + std::vector table_names() const { + std::lock_guard lock(mutex_); + std::vector names; + for (const auto& [k, _] : tables_) names.push_back(k); + return names; + } + std::vector applied_pragmas() const { + std::lock_guard lock(mutex_); + return pragmas_; + } + + // Fault injection for healer tests. + void inject_busy(int n) { + std::lock_guard lock(mutex_); + busy_left_ = n; + } + void debug_drop_table(const std::string& t) { + std::lock_guard lock(mutex_); + tables_.erase(t); + } + void debug_drop_column(const std::string& t, const std::string& col) { + std::lock_guard lock(mutex_); + drop_column_locked(t, col); + } + + void drop_column_locked(const std::string& t, const std::string& col) { + Table& tab = need_table(t); + for (std::size_t i = 0; i < tab.cols.size(); ++i) { + if (tab.cols[i].name == col) { + tab.cols.erase(tab.cols.begin() + i); + for (auto& row : tab.rows) row.erase(row.begin() + i); + return; + } + } + } + void debug_shorten_varchar(const std::string& t, const std::string& col, + int len) { + std::lock_guard lock(mutex_); + Column& c = need_column(t, col); + c.type = "VARCHAR"; + c.length = len; + } + void debug_narrow_int(const std::string& t, const std::string& col) { + std::lock_guard lock(mutex_); + need_column(t, col).type = "SMALLINT"; + } + + std::int64_t page_estimate() const { + std::lock_guard lock(mutex_); + std::int64_t pages = 0; + for (const auto& [_, t] : tables_) pages += 1 + t.rows.size() / 16; + return pages; + } + + void save(const std::string& path) const { + std::lock_guard lock(mutex_); + std::ofstream f(path, std::ios::trunc); + if (!f) throw std::runtime_error("cannot write snapshot"); + f << "MOLO-STORE-1\n"; + for (const auto& [_, t] : tables_) { + f << "TABLE " << escape(t.name) << " " << t.next_id << "\n"; + for (const auto& c : t.cols) + f << "COL " << escape(c.name) << " " << c.type << " " << c.length + << " " << c.nullable << " " << c.pk << " " << c.unique << " " + << (c.has_def ? escape(c.def) : "-") << "\n"; + for (const auto& row : t.rows) { + f << "ROW " << row.size(); + for (const auto& v : row) f << " " << encode_value(v); + f << "\n"; + } + for (const auto& ix : t.indexes) f << "INDEX " << escape(ix) << "\n"; + f << "END\n"; + } + } + + void load(const std::string& path) { + std::lock_guard lock(mutex_); + std::ifstream f(path); + if (!f) return; // missing file means fresh database + std::string line; + std::getline(f, line); + if (line != "MOLO-STORE-1") throw std::runtime_error("bad snapshot"); + tables_.clear(); + Table* cur = nullptr; + while (std::getline(f, line)) { + std::istringstream ls(line); + std::string kind; + ls >> kind; + if (kind == "TABLE") { + std::string name; + ls >> name; + Table t; + t.name = unescape(name); + ls >> t.next_id; + tables_[t.name] = t; + cur = &tables_[t.name]; + } else if (kind == "COL" && cur) { + Column c; + std::string nm, df; + ls >> nm >> c.type >> c.length >> c.nullable >> c.pk >> c.unique >> df; + c.name = unescape(nm); + c.has_def = df != "-"; + if (c.has_def) c.def = unescape(df); + cur->cols.push_back(c); + } else if (kind == "ROW" && cur) { + std::size_t n = 0; + ls >> n; + std::vector row; + for (std::size_t i = 0; i < n; ++i) { + std::string cell; + ls >> cell; + row.push_back(decode_value(cell)); + } + cur->rows.push_back(std::move(row)); + } else if (kind == "INDEX" && cur) { + std::string ix; + ls >> ix; + cur->indexes.push_back(unescape(ix)); + } + } + } + + private: + struct Cursor { + const std::vector& toks; + std::size_t pos; + const std::vector& params; + std::size_t param_idx = 0; + + bool done() const { return pos >= toks.size(); } + std::string peek() const { return done() ? "" : toks[pos]; } + std::string next() { return done() ? "" : toks[pos++]; } + std::string expect() { + if (done()) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", "unexpected end"); + return toks[pos++]; + } + bool accept(const std::string& kw) { + if (!done() && store_detail::upper(toks[pos]) == kw) { + ++pos; + return true; + } + return false; + } + void want(const std::string& kw) { + if (store_detail::upper(expect()) != kw) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "expected " + kw); + } + Value next_param() { + if (param_idx >= params.size()) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "missing bound parameter"); + return params[param_idx++]; + } + }; + + bool has_locked(const std::string& t) const { + return tables_.count(t) > 0; + } + + const Table& table_locked(const std::string& t) const { + auto it = tables_.find(t); + if (it == tables_.end()) + throw DbError(Backend::Sqlite, "SQLITE_NO_TABLE", "no such table: " + t); + return it->second; + } + + Table& need_table(const std::string& t) { + auto it = tables_.find(t); + if (it == tables_.end()) + throw DbError(Backend::Sqlite, "SQLITE_NO_TABLE", "no such table: " + t); + return it->second; + } + + Column& need_column(const std::string& t, const std::string& col) { + Table& tab = need_table(t); + for (auto& c : tab.cols) + if (c.name == col) return c; + throw DbError(Backend::Sqlite, "SQLITE_NO_COLUMN", + "no such column: " + col); + } + + int col_index(const Table& tab, const std::string& col) { + for (std::size_t i = 0; i < tab.cols.size(); ++i) + if (tab.cols[i].name == col) return static_cast(i); + throw DbError(Backend::Sqlite, "SQLITE_NO_COLUMN", + "no such column: " + col); + } + + Value coerce(const Table& tab, const Column& col, Value v) { + const std::string& table = tab.name; + if (is_null(v)) { + if (!col.nullable && !col.has_def && !col.pk) + throw DbError(Backend::Sqlite, "SQLITE_CONSTRAINT", + "NOT NULL constraint failed: " + table + "." + col.name, + 1299); + return v; + } + const std::string ty = store_detail::upper(col.type); + if (ty == "SMALLINT") { + const auto* i = std::get_if(&v); + if (!i) + throw DbError(Backend::Sqlite, "SQLITE_MISMATCH", "datatype mismatch", + 20); + if (*i < -32768 || *i > 32767) + throw DbError(Backend::Sqlite, "SQLITE_MISMATCH", + "datatype mismatch: out of smallint range", 20); + return v; + } + if (ty == "INTEGER" || ty == "INT" || ty == "BIGINT") { + if (const auto* b = std::get_if(&v)) return make_int(*b ? 1 : 0); + if (!std::get_if(&v)) + throw DbError(Backend::Sqlite, "SQLITE_MISMATCH", "datatype mismatch", + 20); + return v; + } + if (ty == "BOOLEAN" || ty == "BOOL") { + if (const auto* b = std::get_if(&v)) return make_int(*b ? 1 : 0); + if (const auto* i = std::get_if(&v)) { + if (*i == 0 || *i == 1) return v; + } + throw DbError(Backend::Sqlite, "SQLITE_MISMATCH", "datatype mismatch", + 20); + } + if (ty == "DOUBLE" || ty == "REAL" || ty == "FLOAT") { + if (const auto* i = std::get_if(&v)) + return make_double(static_cast(*i)); + if (!std::get_if(&v)) + throw DbError(Backend::Sqlite, "SQLITE_MISMATCH", "datatype mismatch", + 20); + return v; + } + if (ty == "VARCHAR" || ty == "CHAR") { + const auto* s = std::get_if(&v); + if (!s) + throw DbError(Backend::Sqlite, "SQLITE_MISMATCH", "datatype mismatch", + 20); + if (col.length > 0 && static_cast(s->size()) > col.length) + throw DbError(Backend::Sqlite, "SQLITE_TOOBIG", + "string or blob too big (limit " + + std::to_string(col.length) + ")", + 18); + return v; + } + if (ty == "BLOB" || ty == "BYTEA") { + if (std::get_if(&v)) return v; + if (const auto* s = std::get_if(&v)) + return make_bytes(Bytes(s->begin(), s->end())); + throw DbError(Backend::Sqlite, "SQLITE_MISMATCH", "datatype mismatch", + 20); + } + // TEXT and TIME accept scalars, coerced to text except time passthrough. + if (ty == "TIME") return v; + if (std::get_if(&v)) return v; + if (const auto* i = std::get_if(&v)) + return make_text(std::to_string(*i)); + if (const auto* d = std::get_if(&v)) + return make_text(std::to_string(*d)); + if (const auto* b = std::get_if(&v)) + return make_text(*b ? "1" : "0"); + throw DbError(Backend::Sqlite, "SQLITE_MISMATCH", "datatype mismatch", 20); + } + + static Value default_value(const Column& col) { + if (!col.has_def || col.def == "NULL") return make_null(); + const std::string ty = store_detail::upper(col.type); + if (ty == "INTEGER" || ty == "INT" || ty == "BIGINT" || + ty == "SMALLINT" || ty == "BOOLEAN" || ty == "BOOL") { + try { + return make_int(std::stoll(col.def)); + } catch (...) { + return make_text(col.def); + } + } + if (ty == "DOUBLE" || ty == "REAL" || ty == "FLOAT") { + try { + return make_double(std::stod(col.def)); + } catch (...) { + return make_text(col.def); + } + } + return make_text(col.def); + } + + void check_unique(Table& tab, int idx, const Value& v, + std::int64_t skip_row = -1) { + if (is_null(v)) return; + for (std::size_t r = 0; r < tab.rows.size(); ++r) { + if (static_cast(r) == skip_row) continue; + if (tab.rows[r][idx] == v) + throw DbError(Backend::Sqlite, "SQLITE_CONSTRAINT", + "UNIQUE constraint failed: " + tab.name + "." + + tab.cols[idx].name, + 1555); + } + } + + Value parse_value_token(Cursor& c) { + const std::string t = c.expect(); + if (t == "?") return c.next_param(); + if (!t.empty() && t[0] == '$') return c.next_param(); + if (store_detail::upper(t) == "NULL") return make_null(); + if (store_detail::upper(t) == "TRUE") return make_bool(true); + if (store_detail::upper(t) == "FALSE") return make_bool(false); + if (store_detail::is_quoted(t)) return make_text(store_detail::unquote(t)); + try { + std::size_t n = 0; + const long long i = std::stoll(t, &n); + if (n == t.size()) return make_int(i); + } catch (...) { + } + try { + std::size_t n = 0; + const double d = std::stod(t, &n); + if (n == t.size()) return make_double(d); + } catch (...) { + } + return make_text(t); + } + + // ---- transactions ---- + Result do_txn_begin(Cursor&) { + if (!in_txn_) { + base_ = tables_; + in_txn_ = true; + } + return Result{}; + } + Result do_txn_end(bool commit) { + if (!in_txn_) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "no transaction active"); + if (!commit && base_) tables_ = *base_; + base_.reset(); + savepoints_.clear(); + in_txn_ = false; + return Result{}; + } + Result do_rollback(Cursor& c) { + if (c.accept("TO")) { + c.accept("SAVEPOINT"); + const std::string name = c.expect(); + for (auto it = savepoints_.rbegin(); it != savepoints_.rend(); ++it) { + if (it->first == name) { + tables_ = it->second; + return Result{}; + } + } + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "no such savepoint: " + name); + } + return do_txn_end(false); + } + Result do_savepoint(Cursor& c) { + const std::string name = c.expect(); + savepoints_.emplace_back(name, tables_); + return Result{}; + } + Result do_release(Cursor& c) { + c.accept("SAVEPOINT"); + const std::string name = c.expect(); + for (auto it = savepoints_.begin(); it != savepoints_.end(); ++it) { + if (it->first == name) { + savepoints_.erase(it); + return Result{}; + } + } + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "no such savepoint: " + name); + } + + // ---- pragma ---- + Result do_pragma(Cursor& c) { + std::string name = c.expect(); + std::string rest; + std::string table_arg; + if (c.accept("(")) { + table_arg = c.expect(); + if (c.peek() == ")") c.next(); + } + if (c.accept("=")) rest = c.expect(); + pragmas_.push_back(name + (rest.empty() ? "" : "=" + rest)); + if (store_detail::upper(name) == "TABLE_INFO" && !table_arg.empty()) { + if (store_detail::is_quoted(table_arg)) + table_arg = store_detail::unquote(table_arg); + const Table& tab = table_locked(table_arg); + std::vector> rows; + for (std::size_t i = 0; i < tab.cols.size(); ++i) { + const auto& col = tab.cols[i]; + std::string decl = col.type; + if (col.length > 0) decl += "(" + std::to_string(col.length) + ")"; + rows.push_back({make_int(i), make_text(col.name), make_text(decl), + make_int(col.nullable ? 0 : 1), + make_text(col.has_def ? col.def : ""), + make_int(col.pk ? 1 : 0)}); + } + return Result({"cid", "name", "type", "notnull", "dflt_value", "pk"}, + rows); + } + return Result{}; + } + + // ---- DDL ---- + Column parse_coldef(Cursor& c) { + Column col; + col.name = c.expect(); + col.type = store_detail::upper(c.expect()); + if (c.accept("(")) { + col.length = std::stoi(c.expect()); + c.want(")"); + } + while (!c.done() && c.peek() != "," && c.peek() != ")") { + const std::string k = store_detail::upper(c.next()); + if (k == "PRIMARY") { + c.want("KEY"); + col.pk = true; + col.nullable = false; + } else if (k == "NOT") { + c.want("NULL"); + col.nullable = false; + } else if (k == "NULL") { + col.nullable = true; + } else if (k == "UNIQUE") { + col.unique = true; + } else if (k == "DEFAULT") { + col.def = c.next(); + if (store_detail::is_quoted(col.def)) + col.def = store_detail::unquote(col.def); + col.has_def = true; + } else if (k == "CHECK") { + int depth = 0; // consume balanced parens + if (c.peek() == "(") { + do { + const std::string t = c.next(); + if (t == "(") ++depth; + if (t == ")") --depth; + } while (depth > 0 && !c.done()); + } + } else if (k == "REFERENCES") { + c.next(); // table, ignored for enforcement in v1 + } else { + break; + } + } + return col; + } + + Result do_create(Cursor& c) { + const bool unique = c.accept("UNIQUE"); + const std::string what = store_detail::upper(c.expect()); + if (what == "TABLE") { + if (c.accept("IF")) { + c.want("NOT"); + c.want("EXISTS"); + } + const std::string name = c.expect(); + if (tables_.count(name)) return Result{}; // IF NOT EXISTS path + Table tab; + tab.name = name; + c.want("("); + while (!c.done() && c.peek() != ")") { + const std::string pk = store_detail::upper(c.peek()); + // Table-level PRIMARY KEY (a, b) or UNIQUE (...) or FOREIGN KEY. + if (pk == "PRIMARY" || pk == "UNIQUE" || pk == "FOREIGN" || + pk == "CONSTRAINT" || pk == "CHECK") { + int depth = 0; + do { + const std::string t = c.next(); + if (t == "(") ++depth; + if (t == ")") { + if (depth == 0) break; + --depth; + } + if (t == "," && depth == 0) break; + } while (!c.done()); + // Mark single-col table PKs. + continue; + } + tab.cols.push_back(parse_coldef(c)); + c.accept(","); + } + if (!c.done()) c.next(); // consume ) + if (tab.cols.empty()) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "table needs columns"); + tables_[name] = std::move(tab); + (void)unique; + return Result{}; + } + if (what == "INDEX") { + if (c.accept("IF")) { + c.want("NOT"); + c.want("EXISTS"); + } + std::string first = c.expect(); + std::string on_table; + std::string ixname = first; + if (store_detail::upper(first) == "ON") { + on_table = c.expect(); + ixname = "ix_" + on_table; + } else { + c.want("ON"); + on_table = c.expect(); + } + Table& tab = need_table(on_table); + c.want("("); + while (!c.done() && c.peek() != ")") c.next(); + if (!c.done()) c.next(); + tab.indexes.push_back(ixname); + return Result{}; + } + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", "bad CREATE"); + } + + Result do_alter(Cursor& c) { + c.want("TABLE"); + const std::string name = c.expect(); + Table& tab = need_table(name); + const std::string op = store_detail::upper(c.expect()); + if (op == "ADD") { + c.accept("COLUMN"); + Column col = parse_coldef(c); + for (const auto& e : tab.cols) + if (e.name == col.name) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "duplicate column: " + col.name); + Value fill = default_value(col); + tab.cols.push_back(col); + for (auto& row : tab.rows) row.push_back(fill); + return Result{}; + } + if (op == "DROP") { + c.accept("COLUMN"); + const std::string col = c.expect(); + drop_column_locked(name, col); + return Result{}; + } + if (op == "RENAME") { + c.want("TO"); + const std::string target = c.expect(); + auto node = tables_.extract(name); + if (node.empty()) + throw DbError(Backend::Sqlite, "SQLITE_NO_TABLE", + "no such table: " + name); + node.key() = target; + node.mapped().name = target; + tables_.insert(std::move(node)); + return Result{}; + } + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "ALTER supports ADD/DROP COLUMN and RENAME TO only"); + } + + Result do_drop(Cursor& c) { + const std::string what = store_detail::upper(c.expect()); + if (what == "TABLE") { + if (c.accept("IF")) { + c.want("EXISTS"); + } + tables_.erase(c.expect()); + return Result{}; + } + if (what == "INDEX") { + if (c.accept("IF")) { + c.want("EXISTS"); + } + c.expect(); + return Result{}; + } + if (what == "COLUMN") return Result{}; + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", "bad DROP"); + } + + // ---- DML ---- + Result do_insert(Cursor& c) { + c.want("INTO"); + const std::string name = c.expect(); + Table& tab = need_table(name); + std::vector cols; + if (c.accept("(")) { + while (!c.done() && c.peek() != ")") { + cols.push_back(c.expect()); + c.accept(","); + } + c.want(")"); + } else { + for (const auto& col : tab.cols) cols.push_back(col.name); + } + for (auto& col : cols) col_index(tab, col); // validates names + c.want("VALUES"); + std::int64_t last_id = 0; + std::int64_t count = 0; + while (true) { + c.want("("); + std::vector vals; + while (!c.done() && c.peek() != ")") { + vals.push_back(parse_value_token(c)); + c.accept(","); + } + c.want(")"); + if (vals.size() != cols.size()) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "column/value count mismatch"); + std::vector row(tab.cols.size(), make_null()); + for (std::size_t i = 0; i < cols.size(); ++i) { + const int idx = col_index(tab, cols[i]); + row[idx] = coerce(tab, tab.cols[idx], vals[i]); + } + // Defaults and integer PK assignment. + for (std::size_t i = 0; i < tab.cols.size(); ++i) { + if (!is_null(row[i])) continue; + const auto& col = tab.cols[i]; + if (col.pk) { + row[i] = make_int(tab.next_id++); + last_id = std::get(row[i]); + } else if (col.has_def) { + row[i] = default_value(col); + } + } + // Re-coerce every cell: validates NOT NULL on omitted columns. + for (std::size_t i = 0; i < tab.cols.size(); ++i) + row[i] = coerce(tab, tab.cols[i], row[i]); + for (std::size_t i = 0; i < tab.cols.size(); ++i) { + if (tab.cols[i].pk || tab.cols[i].unique) + check_unique(tab, static_cast(i), row[i]); + } + // Track last id for explicit PK too. + for (std::size_t i = 0; i < tab.cols.size(); ++i) { + if (tab.cols[i].pk) { + if (const auto* id = std::get_if(&row[i])) { + last_id = *id; + if (*id >= tab.next_id) tab.next_id = *id + 1; + } + } + } + tab.rows.push_back(std::move(row)); + ++count; + if (!c.accept(",")) break; + } + return Result({}, {}, count, last_id); + } + + struct Cond { + std::string table; // may be empty + std::string column; + Value value; + std::vector in_values; + bool in_list = false; + }; + + struct ColRef { + std::string table; + std::string column; + bool star = false; + bool table_star = false; + bool count_star = false; + std::string alias; + }; + + ColRef parse_colref(Cursor& c) { + ColRef r; + std::string first = c.expect(); + if (first == "*") { + r.star = true; + return r; + } + if (store_detail::upper(first) == "COUNT" && c.peek() == "(") { + c.next(); + c.want("*"); + c.want(")"); + r.count_star = true; + r.alias = "count"; + return r; + } + if (c.peek() == ".") { + c.next(); + r.table = first; + const std::string second = c.expect(); + if (second == "*") { + r.table_star = true; + return r; + } + r.column = second; + } else { + r.column = first; + } + if (c.accept("AS")) r.alias = c.expect(); + return r; + } + + Cond parse_cond(Cursor& c) { + Cond cond; + // Left side: [table.]col + const std::string first = c.expect(); + if (c.peek() == ".") { + c.next(); + cond.table = first; + cond.column = c.expect(); + } else { + cond.column = first; + } + if (c.accept("IN")) { + cond.in_list = true; + c.want("("); + while (!c.done() && c.peek() != ")") { + cond.in_values.push_back(parse_value_token(c)); + c.accept(","); + } + c.want(")"); + return cond; + } + c.want("="); + cond.value = parse_value_token(c); + return cond; + } + + struct Source { + std::string table; + std::string alias; + const Table* ptr = nullptr; + }; + + bool row_matches(const Source& src, std::size_t row_idx, + const std::vector& conds, + const std::vector& /*params*/) { + for (const auto& cond : conds) { + if (!cond.table.empty() && cond.table != src.table && + cond.table != src.alias) + continue; // belongs to the other join side + int idx = -1; + for (std::size_t i = 0; i < src.ptr->cols.size(); ++i) { + if (src.ptr->cols[i].name == cond.column) { + idx = static_cast(i); + break; + } + } + if (idx < 0) + throw DbError(Backend::Sqlite, "SQLITE_NO_COLUMN", + "no such column: " + cond.column); + const Value& cell = src.ptr->rows[row_idx][idx]; + if (cond.in_list) { + bool any = false; + for (const auto& cand : cond.in_values) { + if (values_equal(cell, cand)) { + any = true; + break; + } + } + if (!any) return false; + } else if (is_null(cond.value)) { + if (!is_null(cell)) return false; + } else if (!values_equal(cell, cond.value)) { + return false; + } + } + return true; + } + + Result do_select(Cursor& c) { + // Bare SELECT without FROM: single row of literal expressions. + bool has_from = false; + for (const auto& t : c.toks) + if (store_detail::upper(t) == "FROM") has_from = true; + if (!has_from) { + std::vector names; + std::vector row; + while (!c.done()) { + const std::string t = c.peek(); + if (t == "?") { + c.next(); + names.push_back("?"); + row.push_back(c.next_param()); + } else if (!t.empty() && t[0] == '$') { + c.next(); + names.push_back(t); + row.push_back(c.next_param()); + } else if (store_detail::is_quoted(t)) { + names.push_back(t); + row.push_back(make_text(store_detail::unquote(c.next()))); + } else { + names.push_back(t); + const std::string tok = c.next(); + try { + std::size_t n = 0; + const long long i = std::stoll(tok, &n); + if (n == tok.size()) { + row.push_back(make_int(i)); + c.accept(","); + continue; + } + } catch (...) { + } + row.push_back(make_text(tok)); + } + c.accept(","); + } + return Result(names, {row}); + } + std::vector cols; + while (!c.done() && store_detail::upper(c.peek()) != "FROM") { + cols.push_back(parse_colref(c)); + c.accept(","); + } + c.want("FROM"); + Source left; + left.table = c.expect(); + if (!c.done()) { + const std::string pk = store_detail::upper(c.peek()); + if (pk != "WHERE" && pk != "JOIN" && pk != "ORDER" && pk != "LIMIT" && + pk != "OFFSET" && pk != "INNER" && pk != "LEFT" && pk != ",") { + if (c.accept("AS")) left.alias = c.expect(); + else if (c.peek() != ",") left.alias = c.next(); + } + } + if (left.alias.empty()) left.alias = left.table; + + // sqlite_master virtual table. + if (left.table == "sqlite_master") { + std::vector> rows; + for (const auto& [name, _] : tables_) rows.push_back({make_text(name)}); + // Consume optional WHERE type='table'. + return Result({"name"}, rows); + } + + // Bare SELECT 1 / SELECT COUNT(*) without real table. + if (!has_locked(left.table) && cols.size() == 1 && + (cols[0].count_star || cols[0].column == "1")) { + if (cols[0].count_star) return Result({"count"}, {{make_int(0)}}); + return Result({"1"}, {{make_int(1)}}); + } + left.ptr = &table_locked(left.table); + + bool has_join = false; + Source right; + std::vector> join_ons; + if (c.accept("INNER")) { /* inner is default */ } + if (c.accept("JOIN")) { + has_join = true; + right.table = c.expect(); + right.alias = right.table; + if (c.accept("AS")) right.alias = c.expect(); + right.ptr = &table_locked(right.table); + c.want("ON"); + while (true) { + ColRef l = parse_colref(c); + c.want("="); + ColRef r = parse_colref(c); + join_ons.emplace_back(l, r); + if (!c.accept("AND")) break; + } + } + + std::vector conds; + if (c.accept("WHERE")) { + while (true) { + conds.push_back(parse_cond(c)); + if (!c.accept("AND")) break; + } + } + // Validate cond columns exist (healable undefined_column path). + for (const auto& cond : conds) { + const Source* src = &left; + if (!cond.table.empty() && has_join && + (cond.table == right.table || cond.table == right.alias)) + src = &right; + bool found = false; + for (const auto& col : src->ptr->cols) + if (col.name == cond.column) found = true; + if (!found) + throw DbError(Backend::Sqlite, "SQLITE_NO_COLUMN", + "no such column: " + cond.column); + } + + std::vector> order; // col, desc + if (c.accept("ORDER")) { + c.want("BY"); + while (true) { + ColRef o = parse_colref(c); + bool desc = false; + if (c.accept("DESC")) desc = true; + else c.accept("ASC"); + order.emplace_back(o.column, desc); + if (!c.accept(",")) break; + } + } + std::optional limit; + std::optional offset; + if (c.accept("LIMIT")) { + Value v = parse_value_token(c); + if (const auto* i = std::get_if(&v)) limit = *i; + if (c.accept("OFFSET")) { + Value o = parse_value_token(c); + if (const auto* i = std::get_if(&o)) offset = *i; + } + } else if (c.accept("OFFSET")) { + Value o = parse_value_token(c); + if (const auto* i = std::get_if(&o)) offset = *i; + } + + // Gather candidate row pairs. + struct Pair { + std::size_t l; + std::size_t r; + bool has_r; + }; + std::vector pairs; + if (!has_join) { + for (std::size_t i = 0; i < left.ptr->rows.size(); ++i) + if (row_matches(left, i, conds, c.params)) pairs.push_back({i, 0, false}); + } else { + for (std::size_t i = 0; i < left.ptr->rows.size(); ++i) { + if (!row_matches(left, i, conds, c.params)) continue; + for (std::size_t j = 0; j < right.ptr->rows.size(); ++j) { + if (!row_matches(right, j, conds, c.params)) continue; + bool ok = true; + for (const auto& [a, b] : join_ons) { + const Value& va = join_cell(a, left, i, right, j); + const Value& vb = join_cell(b, left, i, right, j); + if (!(va == vb)) { + ok = false; + break; + } + } + if (ok) pairs.push_back({i, j, true}); + } + } + } + + // COUNT(*) fast path. + if (cols.size() == 1 && cols[0].count_star) + return Result({"count"}, {{make_int(pairs.size())}}); + + // Expand select list. + struct OutCol { + std::string name; + int side = 0; // 0 left, 1 right + int idx = 0; + }; + std::vector out; + auto expand = [&](const Source& s, int side, bool qualify) { + for (std::size_t i = 0; i < s.ptr->cols.size(); ++i) + out.push_back({qualify ? s.table + "." + s.ptr->cols[i].name + : s.ptr->cols[i].name, + side, static_cast(i)}); + }; + for (const auto& col : cols) { + if (col.star) { + expand(left, 0, has_join); + if (has_join) expand(right, 1, true); + } else if (col.table_star) { + if (col.table == left.table || col.table == left.alias) + expand(left, 0, true); + else if (has_join && + (col.table == right.table || col.table == right.alias)) + expand(right, 1, true); + else + throw DbError(Backend::Sqlite, "SQLITE_NO_TABLE", + "no such table: " + col.table); + } else if (col.count_star) { + // Only valid alone; handled above. Mixed COUNT is rejected. + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "mixed aggregate unsupported"); + } else { + int side = 0; + const Table* tab = left.ptr; + if (!col.table.empty()) { + if (col.table == left.table || col.table == left.alias) { + side = 0; + tab = left.ptr; + } else if (has_join && (col.table == right.table || + col.table == right.alias)) { + side = 1; + tab = right.ptr; + } else { + throw DbError(Backend::Sqlite, "SQLITE_NO_TABLE", + "no such table: " + col.table); + } + } else if (has_join) { + // Resolve unqualified against both sides. + int found = -1; + const Table* tabs[2] = {left.ptr, right.ptr}; + for (int s = 0; s < 2; ++s) { + for (const auto& cc : tabs[s]->cols) + if (cc.name == col.column) { + if (found >= 0) + throw DbError(Backend::Sqlite, "SQLITE_SYNTAX", + "ambiguous column: " + col.column); + found = s; + } + } + if (found < 0) + throw DbError(Backend::Sqlite, "SQLITE_NO_COLUMN", + "no such column: " + col.column); + side = found; + tab = tabs[found]; + } + const int idx = col_index(*tab, col.column); + out.push_back({col.alias.empty() + ? (col.table.empty() ? col.column + : col.table + "." + col.column) + : col.alias, + side, idx}); + } + } + + // Order. + std::vector seq(pairs.size()); + for (std::size_t i = 0; i < seq.size(); ++i) seq[i] = i; + for (auto it = order.rbegin(); it != order.rend(); ++it) { + const OutCol* oc = nullptr; + for (const auto& o : out) + if (o.name == it->first || o.name == left.table + "." + it->first || + (has_join && o.name == right.table + "." + it->first)) { + oc = &o; + break; + } + // ORDER BY column not in select list: resolve directly. + int oidx = -1; + const Source* osrc = &left; + if (!oc) { + for (std::size_t i = 0; i < left.ptr->cols.size(); ++i) + if (left.ptr->cols[i].name == it->first) oidx = i; + if (oidx < 0) + throw DbError(Backend::Sqlite, "SQLITE_NO_COLUMN", + "no such column: " + it->first); + } + const bool desc = it->second; + const Table* right_tab = has_join ? right.ptr : nullptr; + std::stable_sort(seq.begin(), seq.end(), [&](std::size_t a, std::size_t b) { + const Value& va = oc ? cell_of(oc->side, oc->idx, pairs[a].l, + pairs[a].r, left.ptr, right_tab) + : left.ptr->rows[pairs[a].l][oidx]; + const Value& vb = oc ? cell_of(oc->side, oc->idx, pairs[b].l, + pairs[b].r, left.ptr, right_tab) + : left.ptr->rows[pairs[b].l][oidx]; + const int cmp = compare_values(va, vb); + return desc ? cmp > 0 : cmp < 0; + }); + (void)osrc; + } + + // Offset/limit. + std::size_t start = 0; + if (offset && *offset > 0) start = static_cast(*offset); + std::size_t end = seq.size(); + if (limit && *limit >= 0 && + start + static_cast(*limit) < end) + end = start + static_cast(*limit); + + std::vector names; + for (const auto& o : out) names.push_back(o.name); + std::vector> rows; + const Table* right_tab = has_join ? right.ptr : nullptr; + std::size_t payload = 0; + for (std::size_t k = start; k < end; ++k) { + std::vector row; + for (const auto& o : out) { + const Value& cell = cell_of(o.side, o.idx, pairs[seq[k]].l, + pairs[seq[k]].r, left.ptr, right_tab); + payload += value_bytes(cell); + if (payload > caps::kMaxRowPayload) + throw DbError(Backend::Sqlite, "MOLO_LIMIT", + "row payload over 64 MiB cap"); + row.push_back(cell); + } + rows.push_back(std::move(row)); + } + return Result(std::move(names), std::move(rows)); + } + + Result do_update(Cursor& c) { + const std::string name = c.expect(); + Table& tab = need_table(name); + c.want("SET"); + std::vector> sets; + while (true) { + const std::string col = c.expect(); + c.want("="); + Value v = parse_value_token(c); + const int idx = col_index(tab, col); + sets.emplace_back(idx, coerce(tab, tab.cols[idx], v)); + if (!c.accept(",")) break; + } + std::vector conds; + if (c.accept("WHERE")) { + while (true) { + conds.push_back(parse_cond(c)); + if (!c.accept("AND")) break; + } + } + Source src{name, name, &tab}; + // Validate cond columns. + for (const auto& cond : conds) col_index(tab, cond.column); + std::int64_t affected = 0; + for (std::size_t i = 0; i < tab.rows.size(); ++i) { + if (!conds.empty() && !row_matches(src, i, conds, c.params)) continue; + for (const auto& [idx, v] : sets) { + if (tab.cols[idx].pk || tab.cols[idx].unique) + check_unique(tab, idx, v, i); + tab.rows[i][idx] = v; + } + ++affected; + } + return Result({}, {}, affected); + } + + Result do_delete(Cursor& c) { + c.want("FROM"); + const std::string name = c.expect(); + Table& tab = need_table(name); + std::vector conds; + if (c.accept("WHERE")) { + while (true) { + conds.push_back(parse_cond(c)); + if (!c.accept("AND")) break; + } + } + for (const auto& cond : conds) col_index(tab, cond.column); + Source src{name, name, &tab}; + std::vector> keep; + std::int64_t affected = 0; + for (std::size_t i = 0; i < tab.rows.size(); ++i) { + if (conds.empty() || row_matches(src, i, conds, c.params)) ++affected; + else keep.push_back(tab.rows[i]); + } + tab.rows = std::move(keep); + return Result({}, {}, affected); + } + + const Value& join_cell(const ColRef& ref, const Source& left, std::size_t li, + const Source& right, std::size_t ri) { + const Source* src = &left; + std::size_t idx = li; + if (!ref.table.empty() && + (ref.table == right.table || ref.table == right.alias)) { + src = &right; + idx = ri; + } + for (std::size_t i = 0; i < src->ptr->cols.size(); ++i) + if (src->ptr->cols[i].name == ref.column) + return src->ptr->rows[idx][i]; + throw DbError(Backend::Sqlite, "SQLITE_NO_COLUMN", + "no such column: " + ref.column); + } + + static const Value& cell_of(int side, int idx, std::size_t li, + std::size_t rj, const Table* left_tab, + const Table* right_tab) { + if (side == 1) { + if (!right_tab) throw std::runtime_error("bad join side"); + return right_tab->rows[rj][idx]; + } + return left_tab->rows[li][idx]; + } + + static std::size_t value_bytes(const Value& v) { + if (const auto* s = std::get_if(&v)) return s->size(); + if (const auto* b = std::get_if(&v)) return b->size(); + return 8; + } + + // Equality with bool/int cross compare for BOOLEAN-as-INTEGER columns. + static bool values_equal(const Value& cell, const Value& cand) { + if (cell == cand) return true; + const auto* bi = std::get_if(&cand); + const auto* ci = std::get_if(&cell); + if (bi && ci && (*bi ? 1 : 0) == *ci) return true; + const auto* bc = std::get_if(&cell); + const auto* ic = std::get_if(&cand); + return bc && ic && (*bc ? 1 : 0) == *ic; + } + + static int compare_values(const Value& a, const Value& b) { + if (is_null(a) && is_null(b)) return 0; + if (is_null(a)) return -1; + if (is_null(b)) return 1; + if (a.index() == b.index()) { + if (a < b) return -1; + if (b < a) return 1; + return 0; + } + const std::string sa = value_key(a); + const std::string sb = value_key(b); + if (sa < sb) return -1; + if (sb < sa) return 1; + return 0; + } + + static std::string value_key(const Value& v) { + if (const auto* i = std::get_if(&v)) + return "i" + std::to_string(*i); + if (const auto* d = std::get_if(&v)) + return "d" + std::to_string(*d); + if (const auto* s = std::get_if(&v)) return "s" + *s; + if (const auto* b = std::get_if(&v)) + return std::string("b") + (*b ? "1" : "0"); + return "n"; + } + + static std::string escape(const std::string& s) { + std::string out; + for (char c : s) { + if (c == '\\') out += "\\\\"; + else if (c == ' ') out += "\\s"; + else if (c == '\n') out += "\\n"; + else if (c == '\t') out += "\\t"; + else out += c; + } + return out; + } + + static std::string unescape(const std::string& s) { + std::string out; + for (std::size_t i = 0; i < s.size(); ++i) { + if (s[i] == '\\' && i + 1 < s.size()) { + const char n = s[++i]; + if (n == 's') out += ' '; + else if (n == 'n') out += '\n'; + else if (n == 't') out += '\t'; + else out += n; + } else { + out += s[i]; + } + } + return out; + } + + static std::string encode_value(const Value& v) { + if (is_null(v)) return "N"; + if (const auto* i = std::get_if(&v)) + return "I" + std::to_string(*i); + if (const auto* d = std::get_if(&v)) + return "F" + std::to_string(*d); + if (const auto* s = std::get_if(&v)) return "S" + escape(*s); + if (const auto* b = std::get_if(&v)) + return std::string("G") + (*b ? "1" : "0"); + if (const auto* bytes = std::get_if(&v)) { + static const char* hex = "0123456789abcdef"; + std::string out = "B"; + for (auto c : *bytes) { + out.push_back(hex[c >> 4]); + out.push_back(hex[c & 15]); + } + return out; + } + if (const auto* t = std::get_if(&v)) { + const auto micros = + std::chrono::duration_cast( + t->time_since_epoch()) + .count(); + return "T" + std::to_string(micros); + } + return "N"; + } + + static Value decode_value(const std::string& cell) { + if (cell.empty() || cell[0] == 'N') return make_null(); + const std::string body = cell.substr(1); + switch (cell[0]) { + case 'I': return make_int(std::stoll(body)); + case 'F': return make_double(std::stod(body)); + case 'S': return make_text(unescape(body)); + case 'G': return make_bool(body == "1"); + case 'B': { + Bytes b; + auto hex = [](char c) -> std::uint8_t { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + return c - 'A' + 10; + }; + for (std::size_t i = 0; i + 1 < body.size(); i += 2) + b.push_back((hex(body[i]) << 4) | hex(body[i + 1])); + return make_bytes(std::move(b)); + } + case 'T': + return make_time(TimePoint(std::chrono::microseconds(std::stoll(body)))); + default: return make_null(); + } + } + + mutable std::mutex mutex_; // serializes all store access; pooled + // connections share one store safely + std::map tables_; + std::vector pragmas_; + std::optional> base_; + std::vector>> savepoints_; + bool in_txn_ = false; + int busy_left_ = 0; +}; + +} // namespace molo::data::sqlite diff --git a/src/data/drivers/sqlite/wal.hpp b/src/data/drivers/sqlite/wal.hpp new file mode 100644 index 0000000..65e4187 --- /dev/null +++ b/src/data/drivers/sqlite/wal.hpp @@ -0,0 +1,61 @@ +#pragma once + +// WAL bookkeeping: page counting plus checkpoint scheduling. Every write +// reports its page footprint; every kSqliteCheckpointPages the scheduler +// runs a checkpoint through the owning database handle. + +#include +#include + +#include "../../caps.hpp" +#include "../../obs.hpp" + +namespace molo::data::sqlite { + +struct WalConfig { + int auto_checkpoint_pages = caps::kSqliteCheckpointPages; +}; + +class CheckpointScheduler { + public: + explicit CheckpointScheduler( + WalConfig config = WalConfig{}, + obs::MetricSink* metrics = &obs::null_sink()) + : config_(config), metrics_(metrics) {} + + void note_write(std::int64_t pages) { + pages_since_checkpoint_ += pages; + total_pages_ += pages; + } + + bool should_checkpoint() const { + return pages_since_checkpoint_ >= config_.auto_checkpoint_pages; + } + + // Runs fn exactly when a checkpoint is due. Returns true if it ran. + bool maybe_checkpoint(const std::function& fn) { + if (!should_checkpoint()) return false; + try { + fn(); + } catch (...) { + metrics_->counter("db.sqlite.checkpoint_failed", 1, ""); + throw; + } + pages_since_checkpoint_ = 0; + ++checkpoints_; + metrics_->counter("db.sqlite.checkpoint", 1, ""); + return true; + } + + std::int64_t total_pages() const { return total_pages_; } + std::int64_t checkpoints() const { return checkpoints_; } + + private: + WalConfig config_; + obs::MetricSink* metrics_; + std::int64_t pages_since_checkpoint_ = 0; + std::int64_t total_pages_ = 0; + std::int64_t checkpoints_ = 0; +}; + +} // namespace molo::data::sqlite diff --git a/src/data/error.hpp b/src/data/error.hpp new file mode 100644 index 0000000..6da7138 --- /dev/null +++ b/src/data/error.hpp @@ -0,0 +1,128 @@ +#pragma once + +// DbError plus pure error classification shared by the healer. +// Postgres codes are SQLSTATE strings; SQLite codes are "SQLITE_" +// with the numeric extended result code attached. + +#include +#include +#include + +namespace molo::data { + +enum class Backend { Postgres, Sqlite }; + +inline const char* backend_name(Backend b) { + return b == Backend::Postgres ? "postgres" : "sqlite"; +} + +enum class HealKind { + None, // fatal, no DDL attempted + CreateTable, // undefined table + AddColumn, // undefined column + ExtendVarchar, // string truncation, widen length x4 up to TEXT + WidenInt, // int vs bigint mismatch + RebuildTable, // SQLite table-rebuild procedure for widen ops +}; + +struct Classification { + HealKind kind = HealKind::None; + bool fatal = true; + bool retryable_txn = false; // serialization failure: caller may retry +}; + +// Pure function: (backend, code) -> plan or fatal. +inline Classification classify(Backend backend, const std::string& code) { + Classification c; + if (backend == Backend::Postgres) { + if (code == "42P01") { // undefined_table + c.kind = HealKind::CreateTable; + c.fatal = false; + } else if (code == "42703") { // undefined_column + c.kind = HealKind::AddColumn; + c.fatal = false; + } else if (code == "22001") { // string_data_right_truncation + c.kind = HealKind::ExtendVarchar; + c.fatal = false; + } else if (code == "42804" || code == "22P02" || code == "22P03") { + c.kind = HealKind::WidenInt; + c.fatal = false; + } else if (code == "40001") { // serialization_failure + c.kind = HealKind::None; + c.fatal = true; + c.retryable_txn = true; + } + return c; + } + // SQLite extended-code names produced by our handling layer. + if (code == "SQLITE_NO_TABLE") { + c.kind = HealKind::CreateTable; + c.fatal = false; + } else if (code == "SQLITE_NO_COLUMN") { + c.kind = HealKind::AddColumn; + c.fatal = false; + } else if (code == "SQLITE_TOOBIG") { + c.kind = HealKind::ExtendVarchar; + c.fatal = false; + } else if (code == "SQLITE_MISMATCH") { + c.kind = HealKind::WidenInt; + c.fatal = false; + } + return c; +} + +class DbError : public std::runtime_error { + public: + DbError(Backend backend, std::string code, std::string message, + std::int64_t native_code = 0) + : std::runtime_error("[" + std::string(backend_name(backend)) + ":" + + code + "] " + message), + backend_(backend), + code_(std::move(code)), + detail_(std::move(message)), + native_code_(native_code) { + const Classification c = classify(backend_, code_); + healable_ = !c.fatal; + retryable_txn_ = c.retryable_txn; + } + + Backend backend() const { return backend_; } + const std::string& code() const { return code_; } + const std::string& detail() const { return detail_; } + std::int64_t native_code() const { return native_code_; } + bool healable() const { return healable_; } + bool retryable_txn() const { return retryable_txn_; } + + // Attached by the healer when a retry was attempted. + const std::string& migration_record() const { return migration_record_; } + void set_migration_record(std::string r) { + migration_record_ = std::move(r); + } + + private: + Backend backend_; + std::string code_; + std::string detail_; + std::int64_t native_code_ = 0; + bool healable_ = false; + bool retryable_txn_ = false; + std::string migration_record_; +}; + +// Extract the missing table name from a SQLite-style "no such table: X". +inline std::string parse_missing_table(const std::string& message) { + const std::string prefix = "no such table: "; + const auto pos = message.find(prefix); + if (pos == std::string::npos) return ""; + return message.substr(pos + prefix.size()); +} + +// Extract the missing column name from a SQLite-style "no such column: X". +inline std::string parse_missing_column(const std::string& message) { + const std::string prefix = "no such column: "; + const auto pos = message.find(prefix); + if (pos == std::string::npos) return ""; + return message.substr(pos + prefix.size()); +} + +} // namespace molo::data diff --git a/src/data/kv/memory.hpp b/src/data/kv/memory.hpp new file mode 100644 index 0000000..414f2ef --- /dev/null +++ b/src/data/kv/memory.hpp @@ -0,0 +1,139 @@ +#pragma once + +// MemoryKv: in-memory KvStore with TTL, LRU eviction past the key cap, +// and paginated prefix scan. Expiry is lazy on access plus explicit purge. + +#include +#include +#include +#include +#include +#include +#include + +#include "../caps.hpp" +#include "store.hpp" + +namespace molo::data::kv { + +class MemoryKv final : public KvStore { + public: + explicit MemoryKv(int max_keys = caps::kKvMaxKeys) : max_keys_(max_keys) {} + + std::optional get(const std::string& key) override { + check_key(key); + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) return std::nullopt; + if (expired(it->second)) { + touch_erase(it->second); + entries_.erase(it); + return std::nullopt; + } + touch(it->second); + return it->second.value; + } + + void set(const std::string& key, Bytes value, int ttl_sec) override { + check_key(key); + check_value(value); + const int ttl = clamp_ttl(ttl_sec); + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it != entries_.end()) touch_erase(it->second); + Entry e; + e.value = std::move(value); + e.expires = std::chrono::steady_clock::now() + std::chrono::seconds(ttl); + e.lru = lru_.insert(lru_.end(), key); + entries_[key] = std::move(e); + ++sets_; + while (static_cast(entries_.size()) > max_keys_) { + entries_.erase(lru_.front()); + lru_.pop_front(); + ++evictions_; + } + } + + bool del(const std::string& key) override { + check_key(key); + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) return false; + touch_erase(it->second); + entries_.erase(it); + return true; + } + + bool exists(const std::string& key) override { + return get(key).has_value(); + } + + ScanPage scan(const std::string& prefix, const std::string& cursor, + int limit) override { + if (limit <= 0 || limit > caps::kKvScanPageMax) + throw std::invalid_argument("scan limit out of range"); + std::lock_guard lock(mutex_); + ScanPage page; + auto it = cursor.empty() ? entries_.begin() : entries_.upper_bound(cursor); + while (it != entries_.end() && + static_cast(page.keys.size()) < limit) { + if (it->first.rfind(prefix, 0) == 0 && !expired(it->second)) + page.keys.push_back(it->first); + ++it; + } + if (it != entries_.end()) { + page.done = false; + page.next_cursor = page.keys.empty() ? cursor : page.keys.back(); + } + return page; + } + + void purge_expired() { + std::lock_guard lock(mutex_); + for (auto it = entries_.begin(); it != entries_.end();) { + if (expired(it->second)) { + touch_erase(it->second); + it = entries_.erase(it); + } else { + ++it; + } + } + } + + std::size_t size() const { + std::lock_guard lock(mutex_); + return entries_.size(); + } + std::int64_t evictions() const { return evictions_; } + + private: + struct Entry { + Bytes value; + std::chrono::steady_clock::time_point expires; + std::list::iterator lru; + }; + + static bool expired(const Entry& e) { + return std::chrono::steady_clock::now() >= e.expires; + } + + void touch(Entry& e) { + lru_.erase(e.lru); + e.lru = lru_.insert(lru_.end(), entry_key(e)); + } + + void touch_erase(Entry& e) { lru_.erase(e.lru); } + + static std::string entry_key(const Entry& e) { + return *e.lru; + } + + int max_keys_; + std::map entries_; + std::list lru_; + mutable std::mutex mutex_; + std::int64_t sets_ = 0; + std::int64_t evictions_ = 0; +}; + +} // namespace molo::data::kv diff --git a/src/data/kv/store.hpp b/src/data/kv/store.hpp new file mode 100644 index 0000000..83e9543 --- /dev/null +++ b/src/data/kv/store.hpp @@ -0,0 +1,52 @@ +#pragma once + +// KvStore: the shared interface for KV and cache backends. Keys are UTF-8 +// up to 512 bytes, values up to 16 MiB, TTL up to 30 days. New backends +// implement this interface; callers never fork. + +#include +#include +#include +#include +#include + +#include "../caps.hpp" + +namespace molo::data::kv { + +using Bytes = std::vector; + +inline void check_key(const std::string& key) { + if (key.empty() || key.size() > caps::kKvMaxKeyBytes) + throw std::invalid_argument("KV key out of range"); +} + +inline void check_value(const Bytes& value) { + if (value.size() > caps::kKvMaxValueBytes) + throw std::invalid_argument("KV value over 16 MiB"); +} + +inline int clamp_ttl(int ttl_sec) { + if (ttl_sec <= 0) throw std::invalid_argument("KV needs positive TTL"); + if (ttl_sec > caps::kKvMaxTtlSec) return caps::kKvMaxTtlSec; + return ttl_sec; +} + +struct ScanPage { + std::vector keys; + std::string next_cursor; // empty means done + bool done = true; +}; + +class KvStore { + public: + virtual ~KvStore() = default; + virtual std::optional get(const std::string& key) = 0; + virtual void set(const std::string& key, Bytes value, int ttl_sec) = 0; + virtual bool del(const std::string& key) = 0; + virtual bool exists(const std::string& key) = 0; + virtual ScanPage scan(const std::string& prefix, const std::string& cursor, + int limit) = 0; +}; + +} // namespace molo::data::kv diff --git a/src/data/obs.hpp b/src/data/obs.hpp new file mode 100644 index 0000000..2686790 --- /dev/null +++ b/src/data/obs.hpp @@ -0,0 +1,55 @@ +#pragma once + +// Observability seam. The real sinks live in 02-observability.md; until that +// part lands, data code instruments through these interfaces with a NullSink +// default that compiles to near-zero overhead. Wiring is explicit: each +// component takes an optional sink pointer, never a global. + +#include +#include + +namespace molo::data::obs { + +class MetricSink { + public: + virtual ~MetricSink() = default; + virtual void counter(const std::string& name, std::int64_t delta, + const std::string& labels = "") = 0; + virtual void gauge(const std::string& name, double value, + const std::string& labels = "") = 0; + virtual void histogram(const std::string& name, double value_ms, + const std::string& labels = "") = 0; +}; + +class NullSink final : public MetricSink { + public: + void counter(const std::string&, std::int64_t, + const std::string&) override {} + void gauge(const std::string&, double, const std::string&) override {} + void histogram(const std::string&, double, + const std::string&) override {} +}; + +inline NullSink& null_sink() { + static NullSink s; + return s; +} + +// Structured event without secrets. Bound to molo::log / molo::events later. +class EventSink { + public: + virtual ~EventSink() = default; + virtual void emit(const std::string& name, const std::string& fields) = 0; +}; + +class NullEvents final : public EventSink { + public: + void emit(const std::string&, const std::string&) override {} +}; + +inline NullEvents& null_events() { + static NullEvents s; + return s; +} + +} // namespace molo::data::obs diff --git a/src/data/orm/field.hpp b/src/data/orm/field.hpp new file mode 100644 index 0000000..5939bb7 --- /dev/null +++ b/src/data/orm/field.hpp @@ -0,0 +1,163 @@ +#pragma once + +// Field descriptors: the single source of truth for column shape. +// DDL rendering per backend lives here so sync, healer and models agree. + +#include +#include +#include + +#include "../error.hpp" +#include "../value.hpp" + +#include "../../audit/pii/mark.h" + +namespace molo::data::orm { + +enum class FieldType { + BigInt, + Int, + Double, + Text, + Varchar, + Bytes, + Bool, + Time, +}; + +struct FieldDef { + std::string name; + FieldType type = FieldType::Text; + int length = 0; // Varchar only + bool nullable = true; + std::string default_value; + bool has_default = false; + bool unique = false; + bool indexed = false; + bool pk = false; + std::string check; + // PII classification. Default None keeps unclassified columns compiling + // until Registry::Build boot-fails them by name. + ::molo::audit::pii::FieldMark pii; + + static FieldDef primary_key(std::string name = "id") { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::BigInt; + f.nullable = false; + f.pk = true; + return f; + } + static FieldDef integer(std::string name) { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::Int; + return f; + } + static FieldDef bigint(std::string name) { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::BigInt; + return f; + } + static FieldDef number(std::string name) { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::Double; + return f; + } + static FieldDef text(std::string name) { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::Text; + return f; + } + static FieldDef varchar(std::string name, int len) { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::Varchar; + f.length = len; + return f; + } + static FieldDef bytes(std::string name) { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::Bytes; + return f; + } + static FieldDef boolean(std::string name) { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::Bool; + return f; + } + static FieldDef timestamp(std::string name) { + FieldDef f; + f.name = std::move(name); + f.type = FieldType::Time; + return f; + } +}; + +template +struct TypeMap; +template <> +struct TypeMap { + static constexpr FieldType type = FieldType::BigInt; +}; +template <> +struct TypeMap { + static constexpr FieldType type = FieldType::Int; +}; +template <> +struct TypeMap { + static constexpr FieldType type = FieldType::Double; +}; +template <> +struct TypeMap { + static constexpr FieldType type = FieldType::Text; +}; +template <> +struct TypeMap { + static constexpr FieldType type = FieldType::Bytes; +}; +template <> +struct TypeMap { + static constexpr FieldType type = FieldType::Bool; +}; +template <> +struct TypeMap { + static constexpr FieldType type = FieldType::Time; +}; + +// Base SQL type without constraints, per backend (see orm/types.md). +inline std::string ddl_base_type(const FieldDef& f, Backend backend) { + const bool pg = backend == Backend::Postgres; + switch (f.type) { + case FieldType::BigInt: return pg ? "BIGINT" : "INTEGER"; + case FieldType::Int: return pg ? "INTEGER" : "INTEGER"; + case FieldType::Double: return pg ? "DOUBLE PRECISION" : "REAL"; + case FieldType::Text: return "TEXT"; + case FieldType::Varchar: + return "VARCHAR(" + std::to_string(f.length) + ")"; + case FieldType::Bytes: return pg ? "BYTEA" : "BLOB"; + case FieldType::Bool: return pg ? "BOOLEAN" : "INTEGER"; + case FieldType::Time: return pg ? "TIMESTAMPTZ" : "TEXT"; + default: return "TEXT"; + } +} + +inline std::string ddl_column(const FieldDef& f, Backend backend) { + std::string sql = f.name + " " + ddl_base_type(f, backend); + if (f.pk) { + sql += backend == Backend::Postgres ? " GENERATED ALWAYS AS IDENTITY PRIMARY KEY" + : " PRIMARY KEY"; + } + if (!f.nullable && !f.pk) sql += " NOT NULL"; + if (f.has_default) sql += " DEFAULT " + f.default_value; + if (f.unique && !f.pk) sql += " UNIQUE"; + if (!f.check.empty()) sql += " CHECK (" + f.check + ")"; + return sql; +} + +} // namespace molo::data::orm diff --git a/src/data/orm/healer.hpp b/src/data/orm/healer.hpp new file mode 100644 index 0000000..33b22a5 --- /dev/null +++ b/src/data/orm/healer.hpp @@ -0,0 +1,396 @@ +#pragma once + +// Healer: classifies a failed statement, plans additive DDL, journals the +// migration record, applies, and retries exactly once. Widens only, never +// narrows: renames and drops need human-authored migrations. Postgres +// widening is a metadata-only ALTER; SQLite widening follows the +// documented table-rebuild procedure inside one transaction. + +#include +#include +#include + +#include "../caps.hpp" +#include "../connection.hpp" +#include "../context.hpp" +#include "../error.hpp" +#include "../obs.hpp" +#include "../result.hpp" +#include "../row.hpp" +#include "../statement.hpp" +#include "../transaction.hpp" +#include "../value.hpp" +#include "field.hpp" +#include "migration.hpp" +#include "migrator.hpp" +#include "schema.hpp" +#include "sync.hpp" + +namespace molo::data::orm { + +inline std::string parse_pg_missing_table(const std::string& message) { + const std::string prefix = "relation \""; + const auto pos = message.find(prefix); + if (pos == std::string::npos) return ""; + const auto end = message.find('"', pos + prefix.size()); + if (end == std::string::npos) return ""; + return message.substr(pos + prefix.size(), end - pos - prefix.size()); +} + +inline std::pair parse_pg_missing_column( + const std::string& message) { + // Either: column "c" of relation "t" does not exist + // Or: column "c" does not exist + const std::string prefix = "column \""; + const auto pos = message.find(prefix); + if (pos == std::string::npos) return {"", ""}; + const auto end = message.find('"', pos + prefix.size()); + if (end == std::string::npos) return {"", ""}; + const std::string col = message.substr(pos + prefix.size(), end - pos - prefix.size()); + std::string table; + const std::string rel = "of relation \""; + const auto rpos = message.find(rel, end); + if (rpos != std::string::npos) { + const auto rend = message.find('"', rpos + rel.size()); + if (rend != std::string::npos) + table = message.substr(rpos + rel.size(), rend - rpos - rel.size()); + } + return {table, col}; +} + +// Fallback: find the target table in INSERT INTO / UPDATE / DELETE FROM / +// SELECT ... FROM. Returns "" when nothing matches. +inline std::string table_from_sql(const std::string& sql) { + std::string upper; + for (char c : sql) upper += static_cast(std::toupper(c)); + for (const char* kw : {"INTO ", "UPDATE ", "FROM "}) { + const auto pos = upper.find(kw); + if (pos == std::string::npos) continue; + std::size_t start = pos + std::string(kw).size(); + while (start < sql.size() && sql[start] == ' ') ++start; + std::size_t end = start; + while (end < sql.size() && sql[end] != ' ' && sql[end] != '(' && + sql[end] != ';') + ++end; + if (end > start) return sql.substr(start, end - start); + } + return ""; +} + +struct HealPlan { + HealKind kind = HealKind::None; + std::string table; + std::string column; + std::vector ddl; + std::vector grown; // post-heal field set for widen kinds + std::string reason; + bool valid = false; +}; + +class Healer { + public: + Healer(std::vector models, std::string journal_dir, + obs::MetricSink* metrics = &obs::null_sink()) + : journal_dir_(std::move(journal_dir)), metrics_(metrics) { + for (auto& m : models) models_[m.table] = m; + } + + HealPlan plan(const DbError& e, const BoundStatement& stmt) const { + HealPlan p; + const Classification c = classify(e.backend(), e.code()); + if (c.fatal) return p; + p.kind = c.kind; + const bool pg = e.backend() == Backend::Postgres; + + std::string table, column; + if (!pg) { + table = parse_missing_table(e.detail()); + column = parse_missing_column(e.detail()); + } else { + table = parse_pg_missing_table(e.detail()); + auto [ct, cc] = parse_pg_missing_column(e.detail()); + if (!ct.empty()) table = ct; + column = cc; + } + if (table.empty()) table = table_from_sql(stmt.sql()); + // Error text is untrusted input: identifiers must be clean before DDL. + try { + check_identifier(table); + if (!column.empty()) check_identifier(column); + } catch (...) { + return p; + } + p.table = table; + p.column = column; + + const auto mit = models_.find(table); + if (mit == models_.end()) return p; // unknown model: cannot heal + const ModelInfo& model = mit->second; + + switch (c.kind) { + case HealKind::CreateTable: { + p.reason = "heal:undefined_table " + table; + p.ddl = {create_table_sql(model, e.backend())}; + for (const auto& f : model.fields) { + if (f.indexed && !f.pk) + p.ddl.push_back("CREATE INDEX IF NOT EXISTS ix_" + table + "_" + + f.name + " ON " + table + " (" + f.name + ")"); + } + p.valid = true; + return p; + } + case HealKind::AddColumn: { + if (column.empty()) return p; + const FieldDef* field = find_field(model, column); + if (!field) return p; + p.reason = "heal:undefined_column " + table + "." + column; + p.ddl = {"ALTER TABLE " + table + " ADD COLUMN " + + ddl_column(*field, e.backend())}; + p.valid = true; + return p; + } + case HealKind::ExtendVarchar: { + // Extend exactly the varchar columns a bound value exceeds. + // Growth multiplies by 4 until the bound values fit, so a + // second truncation on an already-healed column converges. + std::vector widened = model.fields; + bool any = false; + for (auto& f : widened) { + // Tenant guard (PATCHES 11.8): never alter tenant_id. + if (f.name == "tenant_id") continue; + if (f.type != FieldType::Varchar) continue; + std::size_t need = 0; + for (const auto& v : stmt.params()) { + const auto* s = std::get_if(&v); + if (s && s->size() > need) need = s->size(); + } + if (need > static_cast(f.length)) { + int grown = f.length * caps::kVarcharExtendFactor; + while (grown < static_cast(need) && + grown <= caps::kVarcharPromoteToTextAt) + grown *= caps::kVarcharExtendFactor; + if (grown > caps::kVarcharPromoteToTextAt) { + f.type = FieldType::Text; + f.length = 0; + } else { + f.length = grown; + } + p.column = f.name; + any = true; + } + } + if (!any) return p; + p.grown = widened; + p.reason = "heal:extend_varchar " + table + "." + p.column; + p.ddl = widen_ddl(model, widened, e.backend()); + p.valid = true; + return p; + } + case HealKind::WidenInt: { + std::vector widened = model.fields; + bool any = false; + for (auto& f : widened) { + // Tenant guard (PATCHES 11.8): never alter tenant_id. + if (f.name == "tenant_id") continue; + if (f.type == FieldType::Int || f.type == FieldType::BigInt) { + // Only widen when a bound int overflows int32, or any int + // field exists for a mismatch error. + for (const auto& v : stmt.params()) { + const auto* i = std::get_if(&v); + if (i && (*i > 2147483647LL || *i < -2147483648LL || + e.code() == "SQLITE_MISMATCH")) { + f.type = FieldType::BigInt; + p.column = f.name; + any = true; + break; + } + } + if (any) break; + } + } + if (!any) { + // No overflowing value found: widen the first narrow int field. + for (auto& f : widened) { + if (f.name == "tenant_id") continue; + if (f.type == FieldType::Int) { + f.type = FieldType::BigInt; + p.column = f.name; + any = true; + break; + } + } + } + if (!any) return p; + p.grown = widened; + p.reason = "heal:widen_int " + table + "." + p.column; + p.ddl = widen_ddl(model, widened, e.backend()); + p.valid = true; + return p; + } + default: + return p; + } + } + + Result execute_with_heal(Connection& conn, const Context& ctx, + const BoundStatement& stmt) { + try { + return conn.exec(ctx, stmt); + } catch (const DbError& e) { + HealPlan p = plan(e, stmt); + if (!p.valid) throw; + metrics_->counter("db.heal.planned", 1, + "kind=" + std::to_string(static_cast(p.kind))); + std::string combined; + for (const auto& s : p.ddl) combined += s + ";\n"; + Migration m = Migration::make(next_id(p), combined, p.reason, e.code()); + // Journal first: no journal write, no retry. + const std::string path = + Migrator::write_auto_journal(journal_dir_, m); + apply(conn, ctx, p, m); + metrics_->counter("db.heal.applied", 1, ""); + try { + Result r = conn.exec(ctx, stmt); + metrics_->counter("db.heal.succeeded", 1, ""); + return r; + } catch (DbError& retry_err) { + retry_err.set_migration_record(path); + metrics_->counter("db.heal.retry_failed", 1, ""); + throw; + } + } + } + + private: + static const FieldDef* find_field(const ModelInfo& m, + const std::string& col) { + for (const auto& f : m.fields) + if (f.name == col) return &f; + return nullptr; + } + + std::string next_id(const HealPlan& p) { + ++counter_; + char buf[16]; + std::snprintf(buf, sizeof(buf), "%04d", counter_ % 10000); + return "auto_" + std::string(buf) + "_" + p.table; + } + + // Postgres widening is metadata-only ALTER; SQLite widening is the + // table-rebuild procedure. p.ddl for SQLite holds only the descriptive + // DDL for the journal; apply() runs the rebuild steps instead. + static std::vector widen_ddl(const ModelInfo& old, + const std::vector& grown, + Backend backend) { + if (backend == Backend::Postgres) { + std::vector ddl; + for (std::size_t i = 0; i < old.fields.size(); ++i) { + const FieldDef& a = old.fields[i]; + const FieldDef& b = grown[i]; + if (a.type != b.type || a.length != b.length) + ddl.push_back("ALTER TABLE " + old.table + " ALTER COLUMN " + + b.name + " TYPE " + ddl_base_type(b, backend)); + } + return ddl; + } + // Journal records the end state as descriptive DDL. + ModelInfo tmp = old; + tmp.fields = grown; + return {"-- sqlite rebuild to: " + create_table_sql(tmp, backend)}; + } + + // Best-effort live schema read: non-target columns keep the wider of + // model and live widths, and the target keeps at least its live width. + static void preserve_live_widths(Connection& conn, const Context& ctx, + const HealPlan& p, + std::vector& grown) { + Result info; + try { + info = conn.exec( + ctx, BoundStatement::raw("PRAGMA table_info(" + p.table + ")")); + } catch (...) { + return; + } + for (std::size_t i = 0; i < info.size(); ++i) { + Row row = info.row(i); + const auto* name = std::get_if(&row.get("name")); + const auto* type = std::get_if(&row.get("type")); + if (!name || !type) continue; + const ColumnInfo live = normalize_column(*name, *type, true, false); + for (auto& f : grown) { + if (f.name != live.name) continue; + if (f.type == FieldType::Varchar) { + if (live.type == "TEXT") { + f.type = FieldType::Text; + f.length = 0; + } else if (live.type == "VARCHAR" && live.length > f.length) { + f.length = live.length; + } + } + } + } + } + + void apply(Connection& conn, const Context& ctx, const HealPlan& p, + const Migration& m) { + const ModelInfo& model = models_.at(p.table); + const bool rebuild = + !conn.is_postgres() && + (p.kind == HealKind::ExtendVarchar || p.kind == HealKind::WidenInt); + if (!rebuild) { + for (const auto& part : split_statements(m.sql_up)) { + if (part.rfind("--", 0) == 0) continue; + conn.exec(ctx, BoundStatement::raw(part)); + } + return; + } + // SQLite table-rebuild procedure inside one transaction. The plan's + // grown set is the base; live widths are preserved for every other + // column so a rebuild never narrows a previous heal. + std::vector grown = p.grown.empty() ? model.fields : p.grown; + preserve_live_widths(conn, ctx, p, grown); + Transaction txn(conn, ctx); + Result existing = + conn.exec(ctx, BoundStatement("SELECT * FROM " + p.table)); + ModelInfo tmp = model; + tmp.table = "molo_new_" + p.table; + tmp.fields = grown; + conn.exec(ctx, BoundStatement::raw(create_table_sql(tmp, conn.backend()))); + const bool pg = conn.is_postgres(); + for (std::size_t i = 0; i < existing.size(); ++i) { + Row row = existing.row(i); + std::string cols, holders; + std::vector params; + int n = 0; + for (const auto& c : row.columns()) { + if (!cols.empty()) { + cols += ", "; + holders += ", "; + } + cols += c; + holders += pg ? "$" + std::to_string(++n) : "?"; + params.push_back(row.get(c)); + } + conn.exec(ctx, BoundStatement("INSERT INTO " + tmp.table + " (" + cols + + ") VALUES (" + holders + ")", + params)); + } + conn.exec(ctx, BoundStatement::raw("DROP TABLE " + p.table)); + conn.exec(ctx, BoundStatement::raw("ALTER TABLE " + tmp.table + + " RENAME TO " + p.table)); + for (const auto& f : grown) { + if (f.indexed && !f.pk) + conn.exec(ctx, BoundStatement::raw("CREATE INDEX IF NOT EXISTS ix_" + + p.table + "_" + f.name + " ON " + + p.table + " (" + f.name + ")")); + } + txn.commit(); + } + + std::map models_; + std::string journal_dir_; + obs::MetricSink* metrics_; + int counter_ = 0; +}; + +} // namespace molo::data::orm diff --git a/src/data/orm/invalidation.hpp b/src/data/orm/invalidation.hpp new file mode 100644 index 0000000..499bdc4 --- /dev/null +++ b/src/data/orm/invalidation.hpp @@ -0,0 +1,54 @@ +#pragma once + +// Invalidation events: every ORM write emits one on the internal bus. +// Cache subscribes to drop stale entries; 07-supervisor-replication.md +// consumes the same stream for cross-instance sync. Ordering key is +// (tx_commit_seq, statement_seq); txid carries the commit sequence. + +#include +#include +#include +#include +#include + +namespace molo::data::orm { + +struct Invalidation { + std::string table; + std::string pk; + std::string op; // insert, update, delete + std::int64_t txid = 0; +}; + +class InvalidationBus { + public: + using Handler = std::function; + + int subscribe(Handler h) { + std::lock_guard lock(mutex_); + const int id = next_++; + handlers_[id] = std::move(h); + return id; + } + + void unsubscribe(int id) { + std::lock_guard lock(mutex_); + handlers_.erase(id); + } + + void publish(const Invalidation& inv) { + std::map copy; + { + std::lock_guard lock(mutex_); + copy = handlers_; + } + for (const auto& [_, h] : copy) h(inv); + } + + private: + std::mutex mutex_; + std::map handlers_; + int next_ = 1; +}; + +} // namespace molo::data::orm diff --git a/src/data/orm/migration.hpp b/src/data/orm/migration.hpp new file mode 100644 index 0000000..8396448 --- /dev/null +++ b/src/data/orm/migration.hpp @@ -0,0 +1,86 @@ +#pragma once + +// Migration record: append-only, checksummed, auditable. Human-authored +// migrations live in migrations/NNNN_name.sql; healer output lands in +// migrations/auto/ for review. Applied checksums are recorded in the +// schema_migrations table. + +#include +#include +#include +#include +#include +#include + +namespace molo::data::orm { + +inline std::string fnv1a_hex(const std::string& s) { + std::uint64_t h = 14695981039346656037ULL; + for (unsigned char c : s) { + h ^= c; + h *= 1099511628211ULL; + } + std::ostringstream os; + os << std::hex << std::setw(16) << std::setfill('0') << h; + return os.str(); +} + +struct Migration { + std::string id; // NNNN_name + std::string timestamp; // ISO-8601 UTC + std::string sql_up; + std::string reason; + std::string trigger_code; + std::string checksum; + + static Migration make(std::string id, std::string sql_up, std::string reason, + std::string trigger_code = "") { + Migration m; + m.id = std::move(id); + m.sql_up = sql_up; + m.reason = std::move(reason); + m.trigger_code = std::move(trigger_code); + const auto now = std::chrono::system_clock::now(); + const std::time_t t = std::chrono::system_clock::to_time_t(now); + std::tm tm{}; + gmtime_r(&t, &tm); + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &tm); + m.timestamp = buf; + m.checksum = fnv1a_hex(m.sql_up); + return m; + } + + std::string journal_text() const { + return "-- id: " + id + "\n-- at: " + timestamp + "\n-- reason: " + + reason + "\n-- trigger: " + trigger_code + "\n-- checksum: " + + checksum + "\n" + sql_up + "\n"; + } +}; + +// Split multi-statement SQL on semicolons outside quotes. Each part runs +// as its own statement since the extended protocol takes one at a time. +inline std::vector split_statements(const std::string& sql) { + std::vector parts; + std::string cur; + bool in_str = false; + for (std::size_t i = 0; i < sql.size(); ++i) { + const char c = sql[i]; + if (c == '\'' && (i == 0 || sql[i - 1] != '\\')) in_str = !in_str; + if (c == ';' && !in_str) { + if (!cur.empty()) parts.push_back(cur); + cur.clear(); + } else { + cur += c; + } + } + if (!cur.empty()) { + bool blank = true; + for (char c : cur) + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') blank = false; + if (!blank) parts.push_back(cur); + } + return parts; +} + +} // namespace molo::data::orm diff --git a/src/data/orm/migrator.hpp b/src/data/orm/migrator.hpp new file mode 100644 index 0000000..0e17cf4 --- /dev/null +++ b/src/data/orm/migrator.hpp @@ -0,0 +1,115 @@ +#pragma once + +// Migrator: applies migration records inside transactions, verifies +// checksums, and lists applied versus pending work. Checksum mismatch +// aborts loudly; the journal is append-only. + +#include +#include +#include +#include + +#include "../connection.hpp" +#include "../context.hpp" +#include "../error.hpp" +#include "../result.hpp" +#include "../statement.hpp" +#include "../transaction.hpp" +#include "../value.hpp" +#include "migration.hpp" + +namespace molo::data::orm { + +class Migrator { + public: + void ensure_journal_table(const Context& ctx, Connection& conn) { + conn.exec(ctx, BoundStatement::raw( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "id TEXT PRIMARY KEY, " + "applied_at TEXT NOT NULL, " + "checksum TEXT NOT NULL, " + "reason TEXT NOT NULL)")); + } + + void apply(const Context& ctx, Connection& conn, const Migration& m) { + ensure_journal_table(ctx, conn); + if (is_applied(ctx, conn, m.id)) { + verify_checksum(ctx, conn, m); + return; + } + Transaction txn(conn, ctx); + for (const auto& part : split_statements(m.sql_up)) + conn.exec(ctx, BoundStatement::raw(part)); + const bool pg = conn.is_postgres(); + BoundStatement rec( + "INSERT INTO schema_migrations (id, applied_at, checksum, reason) " + "VALUES (" + + std::string(pg ? "$1, $2, $3, $4" : "?, ?, ?, ?") + ")", + {make_text(m.id), make_text(m.timestamp), make_text(m.checksum), + make_text(m.reason)}); + conn.exec(ctx, rec); + txn.commit(); + } + + bool is_applied(const Context& ctx, Connection& conn, + const std::string& id) { + ensure_journal_table(ctx, conn); + const bool pg = conn.is_postgres(); + Result r = conn.exec( + ctx, BoundStatement("SELECT checksum FROM schema_migrations " + "WHERE id = " + std::string(pg ? "$1" : "?"), + {make_text(id)})); + return !r.empty(); + } + + std::vector> list_applied( + const Context& ctx, Connection& conn) { + ensure_journal_table(ctx, conn); + Result r = conn.exec( + ctx, BoundStatement("SELECT id, checksum FROM schema_migrations")); + std::vector> out; + for (std::size_t i = 0; i < r.size(); ++i) { + const auto& row = r.raw(i); + const auto* id = std::get_if(&row[0]); + const auto* sum = std::get_if(&row[1]); + if (id && sum) out.emplace_back(*id, *sum); + } + return out; + } + + // Reproducible pending list: known migrations minus applied ones. + std::vector pending(const Context& ctx, Connection& conn, + const std::vector& known) { + std::vector out; + for (const auto& m : known) + if (!is_applied(ctx, conn, m.id)) out.push_back(m); + return out; + } + + // Persist a healer record to migrations/auto/ for review. + static std::string write_auto_journal(const std::string& dir, + const Migration& m) { + std::filesystem::create_directories(dir); + const std::string path = dir + "/" + m.id + ".sql"; + std::ofstream f(path, std::ios::trunc); + if (!f) throw std::runtime_error("cannot write journal: " + path); + f << m.journal_text(); + return path; + } + + private: + void verify_checksum(const Context& ctx, Connection& conn, + const Migration& m) { + const bool pg = conn.is_postgres(); + Result r = conn.exec( + ctx, BoundStatement("SELECT checksum FROM schema_migrations " + "WHERE id = " + std::string(pg ? "$1" : "?"), + {make_text(m.id)})); + const auto* sum = std::get_if(&r.raw(0)[0]); + if (!sum || *sum != m.checksum) + throw DbError(conn.backend(), "MOLO_MIGRATION", + "checksum mismatch for applied migration " + m.id); + } +}; + +} // namespace molo::data::orm diff --git a/src/data/orm/model.hpp b/src/data/orm/model.hpp new file mode 100644 index 0000000..4a1a8fc --- /dev/null +++ b/src/data/orm/model.hpp @@ -0,0 +1,254 @@ +#pragma once + +// Model: CRTP base. Derived classes declare table_name(), primary_key() +// and fields(); instances carry values plus lifecycle hooks. Writes go +// through bound parameters and emit invalidation events for cache and +// replication consumers. + +#include +#include +#include + +#include "../connection.hpp" +#include "../context.hpp" +#include "../error.hpp" +#include "../result.hpp" +#include "../row.hpp" +#include "../statement.hpp" +#include "../value.hpp" +#include "field.hpp" +#include "invalidation.hpp" + +// Audit write hook (PATCHES 12, section 2). Active when the audit headers +// are on the include path (-Isrc builds); data-only test builds compile the +// hook out so they never link audit. +#if __has_include("audit/log/appender.h") +#include "audit/log/appender.h" +#elif !defined(AUDIT) +#define AUDIT(store, table, pk, op, before, after, ctx, actor) \ + ((void)(store), (void)(table), (void)(pk), (void)(op), (void)(before), (void)(after), \ + (void)(ctx), (void)(actor)) +#endif + +namespace molo::data::orm { + +template +class Model { + public: + Model() = default; + + static std::string table_name() { return Derived::table_name(); } + static std::string primary_key() { return Derived::primary_key(); } + static std::vector fields() { return Derived::fields(); } + + void set(const std::string& name, Value v) { values_[name] = std::move(v); } + + const Value& get(const std::string& name) const { + const auto it = values_.find(name); + if (it == values_.end()) + throw std::out_of_range("no such field: " + name); + return it->second; + } + + bool has(const std::string& name) const { + return values_.count(name) > 0; + } + + const std::map& values() const { return values_; } + + static Derived from_row(const Row& row) { + Derived m; + for (const auto& f : Derived::fields()) { + if (row.has(f.name)) m.set(f.name, row.get(f.name)); + } + // Also accept qualified table.column names from joins. + for (const auto& col : row.columns()) { + const std::string prefix = Derived::table_name() + "."; + if (col.rfind(prefix, 0) == 0) + m.set(col.substr(prefix.size()), row.get(col)); + } + m.set_persisted(true); + return m; + } + + bool persisted() const { return persisted_; } + void set_persisted(bool p) { persisted_ = p; } + + // Hooks for subclasses. Default: no-op. + virtual void on_before_save() {} + virtual void on_after_save() {} + virtual void on_before_delete() {} + virtual void on_after_delete() {} + virtual ~Model() = default; + + void save(Connection& conn, const Context& ctx, + InvalidationBus* bus = nullptr, std::int64_t txid = 0) { + on_before_save(); + const bool pg = conn.is_postgres(); + // Tenant stamp (PATCHES 11.5): stamp tenant_id on insert and scope + // update/delete to the context tenant, unless privileged, empty-tenant, + // or the model has no tenant_id column. + const bool scoped = + !ctx.privileged() && !ctx.tenant().empty() && ModelHasTenantColumn(); + if (!persisted_) { + if (scoped) set("tenant_id", make_text(ctx.tenant())); + std::string cols, holders; + std::vector params; + int n = 0; + for (const auto& f : Derived::fields()) { + if (f.pk && !has(f.name)) continue; // generated key + if (!cols.empty()) { + cols += ", "; + holders += ", "; + } + check_identifier(f.name); + cols += f.name; + holders += pg ? "$" + std::to_string(++n) : "?"; + auto it = values_.find(f.name); + params.push_back(it == values_.end() ? make_null() : it->second); + if (pg) (void)0; + } + if (!pg) { /* holders already ? */ } + BoundStatement stmt("INSERT INTO " + Derived::table_name() + " (" + + cols + ") VALUES (" + holders + ")", + params); + Result r = conn.exec(ctx, stmt); + if (!has(Derived::primary_key()) && r.last_insert_id() != 0) + set(Derived::primary_key(), make_int(r.last_insert_id())); + persisted_ = true; + if (bus) { + std::string pk; + if (has(Derived::primary_key())) { + const Value& v = get(Derived::primary_key()); + if (const auto* i = std::get_if(&v)) + pk = std::to_string(*i); + } + bus->publish( + Invalidation{Derived::table_name(), pk, "insert", txid}); + } + AUDIT(nullptr, Derived::table_name(), PkString(), "insert", "", ValuesJson(), ctx, + ""); + } else { + std::string sets; + std::vector params; + int n = 0; + for (const auto& f : Derived::fields()) { + if (f.pk) continue; + if (scoped && f.name == "tenant_id") continue; // rows never change tenants + auto it = values_.find(f.name); + if (it == values_.end()) continue; + if (!sets.empty()) sets += ", "; + check_identifier(f.name); + sets += f.name + " = " + (pg ? "$" + std::to_string(++n) : "?"); + params.push_back(it->second); + } + if (!sets.empty()) { + const Value& pkv = get(Derived::primary_key()); + params.push_back(pkv); + std::string where = " WHERE " + Derived::primary_key() + " = " + + (pg ? "$" + std::to_string(++n) : "?"); + if (scoped) { + // Never move a row across tenants: the SET list cannot change + // tenant_id because it is excluded below. + where += " AND tenant_id = " + (pg ? "$" + std::to_string(++n) : "?"); + params.push_back(make_text(ctx.tenant())); + } + BoundStatement stmt("UPDATE " + Derived::table_name() + " SET " + sets + where, + params); + conn.exec(ctx, stmt); + } + if (bus) { + std::string pk; + const Value& v = get(Derived::primary_key()); + if (const auto* i = std::get_if(&v)) + pk = std::to_string(*i); + bus->publish(Invalidation{Derived::table_name(), pk, "update", txid}); + } + AUDIT(nullptr, Derived::table_name(), PkString(), "update", "", ValuesJson(), ctx, + ""); + } + on_after_save(); + } + + void remove(Connection& conn, const Context& ctx, + InvalidationBus* bus = nullptr, std::int64_t txid = 0) { + on_before_delete(); + const bool pg = conn.is_postgres(); + const bool scoped = + !ctx.privileged() && !ctx.tenant().empty() && ModelHasTenantColumn(); + std::vector params{get(Derived::primary_key())}; + std::string sql = "DELETE FROM " + Derived::table_name() + " WHERE " + + Derived::primary_key() + " = " + (pg ? "$1" : "?"); + if (scoped) { + sql += " AND tenant_id = "; + sql += pg ? "$2" : "?"; + params.push_back(make_text(ctx.tenant())); + } + BoundStatement stmt(sql, params); + conn.exec(ctx, stmt); + persisted_ = false; + if (bus) { + std::string pk; + const Value& v = get(Derived::primary_key()); + if (const auto* i = std::get_if(&v)) + pk = std::to_string(*i); + bus->publish(Invalidation{Derived::table_name(), pk, "delete", txid}); + } + AUDIT(nullptr, Derived::table_name(), PkString(), "delete", ValuesJson(), "", ctx, ""); + on_after_delete(); + } + + private: + static bool ModelHasTenantColumn() { + for (const auto& f : Derived::fields()) + if (f.name == "tenant_id") return true; + return false; + } + + std::string PkString() const { + if (!has(Derived::primary_key())) return ""; + const Value& v = get(Derived::primary_key()); + if (const auto* i = std::get_if(&v)) return std::to_string(*i); + if (const auto* s = std::get_if(&v)) return *s; + return ""; + } + + static std::string JsonEscape(const std::string& s) { + std::string o; + for (char c : s) { + switch (c) { + case '"': o += "\\\""; break; + case '\\': o += "\\\\"; break; + case '\n': o += "\\n"; break; + case '\r': o += "\\r"; break; + case '\t': o += "\\t"; break; + default: o += c; break; + } + } + return o; + } + + static std::string ValueJson(const Value& v) { + if (const auto* i = std::get_if(&v)) return std::to_string(*i); + if (const auto* d = std::get_if(&v)) return std::to_string(*d); + if (const auto* s = std::get_if(&v)) return "\"" + JsonEscape(*s) + "\""; + if (const auto* b = std::get_if(&v)) return *b ? "true" : "false"; + return "null"; + } + + std::string ValuesJson() const { + std::string o = "{"; + bool first = true; + for (const auto& [k, v] : values_) { + if (!first) o += ","; + first = false; + o += "\"" + JsonEscape(k) + "\":" + ValueJson(v); + } + return o + "}"; + } + + std::map values_; + bool persisted_ = false; +}; + +} // namespace molo::data::orm diff --git a/src/data/orm/query.hpp b/src/data/orm/query.hpp new file mode 100644 index 0000000..e08e7b9 --- /dev/null +++ b/src/data/orm/query.hpp @@ -0,0 +1,218 @@ +#pragma once + +// Query: type-safe builder producing one BoundStatement. Filters, joins, +// ordering and pagination compose; all values bind as parameters. all() +// refuses result sets over 10,000 rows: use limit/offset pagination or a +// cursor beyond that. + +#include +#include +#include + +#include "../../tenancy/orm_scope.h" +#include "../caps.hpp" +#include "../connection.hpp" +#include "../context.hpp" +#include "../error.hpp" +#include "../result.hpp" +#include "../statement.hpp" +#include "../value.hpp" +#include "model.hpp" + +namespace molo::data::orm { + +template +class Query { + public: + explicit Query(Connection& conn) : conn_(conn), pg_(conn.is_postgres()) {} + + Query& filter(const std::string& column, Value value) { + check_identifier(column); + filters_.emplace_back(column, std::move(value)); + return *this; + } + + Query& filter_in(const std::string& column, std::vector values) { + check_identifier(column); + if (values.empty()) + throw std::invalid_argument("filter_in needs values"); + in_filters_.emplace_back(column, std::move(values)); + return *this; + } + + Query& order(const std::string& column, bool desc = false) { + check_identifier(column); + orders_.emplace_back(column, desc); + return *this; + } + + Query& limit(std::int64_t n) { + limit_ = n; + return *this; + } + + Query& offset(std::int64_t n) { + offset_ = n; + return *this; + } + + Query& join(const std::string& table, const std::string& left_col, + const std::string& right_col) { + check_identifier(table); + check_identifier(left_col); + check_identifier(right_col); + joins_.push_back({table, left_col, right_col}); + return *this; + } + + Query& select(std::vector columns) { + for (const auto& c : columns) check_identifier(c); + select_ = std::move(columns); + return *this; + } + + BoundStatement build() const { + std::string sql = "SELECT "; + if (select_.empty()) { + sql += joins_.empty() ? "*" : M::table_name() + ".*"; + } else { + for (std::size_t i = 0; i < select_.size(); ++i) { + if (i > 0) sql += ", "; + sql += select_[i]; + } + } + sql += " FROM " + M::table_name(); + for (const auto& j : joins_) + sql += " JOIN " + j.table + " ON " + j.left + " = " + j.right; + std::vector params; + int n = 0; + auto holder = [&] { + ++n; + return pg_ ? "$" + std::to_string(n) : "?"; + }; + if (!filters_.empty() || !in_filters_.empty()) { + sql += " WHERE "; + bool first = true; + for (const auto& [col, v] : filters_) { + if (!first) sql += " AND "; + first = false; + sql += col + " = " + holder(); + params.push_back(v); + } + for (const auto& [col, vs] : in_filters_) { + if (!first) sql += " AND "; + first = false; + sql += col + " IN ("; + for (std::size_t i = 0; i < vs.size(); ++i) { + if (i > 0) sql += ", "; + sql += holder(); + params.push_back(vs[i]); + } + sql += ")"; + } + } + if (!orders_.empty()) { + sql += " ORDER BY "; + for (std::size_t i = 0; i < orders_.size(); ++i) { + if (i > 0) sql += ", "; + sql += orders_[i].first + (orders_[i].second ? " DESC" : " ASC"); + } + } + if (limit_ >= 0) { + sql += " LIMIT " + holder(); + params.push_back(make_int(limit_)); + } + if (offset_ > 0) { + sql += " OFFSET " + holder(); + params.push_back(make_int(offset_)); + } + return BoundStatement(sql, params); + } + + // Tenant-scoped build (PATCHES 11.4): injects the tenant_id filter + // unless the context is privileged, empty-tenant, already scoped, or the + // model has no tenant_id column. Join queries keep working but the + // injected predicate is unqualified, so joined tenant tables must scope + // themselves explicitly. + BoundStatement build(const Context& ctx) const { + BoundStatement s = build(); + if (!ctx.privileged() && !ctx.tenant().empty() && ModelHasTenantColumn()) { + return BoundStatement(::molo::tenancy::OrmScope::ApplySelect( + s.sql(), ctx.tenant(), false), + s.params()); + } + return s; + } + + BoundStatement build_count() const { + return build_count_impl(nullptr); + } + + BoundStatement build_count(const Context& ctx) const { + return build_count_impl(&ctx); + } + + std::vector all(const Context& ctx) { + Result r = conn_.exec(ctx, build(ctx)); + if (static_cast(r.size()) > caps::kMaxRowsAll) + throw DbError(conn_.backend(), "MOLO_LIMIT", + "Query::all over 10,000 rows, paginate instead"); + std::vector out; + for (std::size_t i = 0; i < r.size(); ++i) + out.push_back(M::from_row(r.row(i))); + return out; + } + + bool exists(const Context& ctx) { + Query copy = *this; + copy.limit(1); + return !conn_.exec(ctx, copy.build(ctx)).empty(); + } + + std::int64_t count(const Context& ctx) { + Result r = conn_.exec(ctx, build_count(ctx)); + if (r.empty()) return 0; + const Value& v = r.raw(0)[0]; + if (const auto* i = std::get_if(&v)) return *i; + return 0; + } + + private: + static bool ModelHasTenantColumn() { + for (const auto& f : M::fields()) + if (f.name == "tenant_id") return true; + return false; + } + + BoundStatement build_count_impl(const Context* ctx) const { + // Same filters, COUNT(*) projection, no ordering/pagination. + Query copy = *this; + copy.orders_.clear(); + copy.limit_ = -1; + copy.offset_ = 0; + BoundStatement inner = (ctx != nullptr) ? copy.build(*ctx) : copy.build(); + std::string sql = "SELECT COUNT(*) FROM " + M::table_name(); + const std::string& full = inner.sql(); + const auto where = full.find(" WHERE "); + if (where != std::string::npos) sql += full.substr(where); + return BoundStatement(sql, inner.params()); + } + + struct JoinSpec { + std::string table; + std::string left; + std::string right; + }; + + Connection& conn_; + bool pg_; + std::vector> filters_; + std::vector>> in_filters_; + std::vector> orders_; + std::vector joins_; + std::vector select_; + std::int64_t limit_ = -1; + std::int64_t offset_ = 0; +}; + +} // namespace molo::data::orm diff --git a/src/data/orm/relations.hpp b/src/data/orm/relations.hpp new file mode 100644 index 0000000..5092f27 --- /dev/null +++ b/src/data/orm/relations.hpp @@ -0,0 +1,80 @@ +#pragma once + +// Relations: BelongsTo, HasMany, ManyToMany with lazy loading and eager +// preload. Preload issues one IN query per relation instead of N+1. + +#include +#include +#include + +#include "../connection.hpp" +#include "../context.hpp" +#include "../value.hpp" +#include "model.hpp" +#include "query.hpp" + +namespace molo::data::orm { + +template +struct BelongsTo { + std::string fk; // column on Child holding Parent pk + std::string ref; // pk column on Parent + + std::vector load(Connection& conn, const Context& ctx, + const Child& child) const { + return Query(conn).filter(ref, child.get(fk)).all(ctx); + } +}; + +template +struct HasMany { + std::string fk; // column on Child holding Parent pk + std::string ref; // pk column on Parent + + std::vector load(Connection& conn, const Context& ctx, + const Parent& parent) const { + return Query(conn).filter(fk, parent.get(ref)).all(ctx); + } + + // Eager: one IN query for all parents. Returns children per parent key. + std::map> preload( + Connection& conn, const Context& ctx, + const std::vector& parents) const { + std::map> grouped; + if (parents.empty()) return grouped; + std::vector keys; + for (const auto& p : parents) keys.push_back(p.get(ref)); + std::vector all = + Query(conn).filter_in(fk, keys).all(ctx); + for (auto& child : all) { + const Value& v = child.get(fk); + std::string key; + if (const auto* i = std::get_if(&v)) + key = std::to_string(*i); + else if (const auto* s = std::get_if(&v)) + key = *s; + grouped[key].push_back(std::move(child)); + } + return grouped; + } +}; + +template +struct ManyToMany { + std::string through; // join table + std::string left_fk; // join column holding A pk + std::string right_fk; // join column holding B pk + std::string left_ref; // pk column on A + std::string right_ref; // pk column on B + + std::vector load(Connection& conn, const Context& ctx, + const A& a) const { + Query q(conn); + q.join(through, B::table_name() + "." + right_ref, + through + "." + right_fk); + q.filter(through + "." + left_fk, a.get(left_ref)); + return q.all(ctx); + } +}; + +} // namespace molo::data::orm diff --git a/src/data/orm/schema.hpp b/src/data/orm/schema.hpp new file mode 100644 index 0000000..0dbeff8 --- /dev/null +++ b/src/data/orm/schema.hpp @@ -0,0 +1,159 @@ +#pragma once + +// Schema introspection per backend. SQLite reads PRAGMA table_info plus +// sqlite_master; Postgres reads information_schema. The syncer diffs this +// against model fields. + +#include +#include + +#include "../connection.hpp" +#include "../context.hpp" +#include "../error.hpp" +#include "../result.hpp" +#include "../row.hpp" +#include "../statement.hpp" +#include "../value.hpp" + +namespace molo::data::orm { + +struct ColumnInfo { + std::string name; + std::string type; // normalized upper base type + int length = 0; + bool nullable = true; + bool pk = false; +}; + +struct TableInfo { + std::string name; + std::vector columns; + std::vector indexes; + + const ColumnInfo* find(const std::string& col) const { + for (const auto& c : columns) + if (c.name == col) return &c; + return nullptr; + } +}; + +class SchemaInspector { + public: + virtual ~SchemaInspector() = default; + virtual std::vector list_tables(const Context& ctx, + Connection& conn) = 0; + virtual TableInfo inspect_table(const Context& ctx, Connection& conn, + const std::string& table) = 0; +}; + +inline ColumnInfo normalize_column(std::string name, std::string decl, + bool nullable, bool pk) { + ColumnInfo c; + c.name = std::move(name); + c.nullable = nullable; + c.pk = pk; + std::string upper; + for (char ch : decl) upper += static_cast(std::toupper(ch)); + const auto paren = upper.find('('); + if (paren != std::string::npos) { + c.length = std::stoi(upper.substr(paren + 1)); + upper = upper.substr(0, paren); + } + // Trim trailing spaces. + while (!upper.empty() && upper.back() == ' ') upper.pop_back(); + c.type = upper; + return c; +} + +class SqliteInspector final : public SchemaInspector { + public: + std::vector list_tables(const Context& ctx, + Connection& conn) override { + Result r = conn.exec( + ctx, BoundStatement( + "SELECT name FROM sqlite_master WHERE type = 'table'")); + std::vector names; + for (std::size_t i = 0; i < r.size(); ++i) { + const Value& v = r.raw(i)[0]; + if (const auto* s = std::get_if(&v)) names.push_back(*s); + } + return names; + } + + TableInfo inspect_table(const Context& ctx, Connection& conn, + const std::string& table) override { + check_identifier(table); + Result r = conn.exec( + ctx, BoundStatement::raw("PRAGMA table_info(" + table + ")")); + TableInfo info; + info.name = table; + for (std::size_t i = 0; i < r.size(); ++i) { + Row row = r.row(i); + const auto* name = std::get_if(&row.get("name")); + const auto* type = std::get_if(&row.get("type")); + const auto* notnull = std::get_if(&row.get("notnull")); + const auto* pk = std::get_if(&row.get("pk")); + if (!name || !type) continue; + info.columns.push_back(normalize_column( + *name, *type, !notnull || *notnull == 0, pk && *pk == 1)); + } + return info; + } +}; + +class PgInspector final : public SchemaInspector { + public: + static BoundStatement list_tables_sql() { + return BoundStatement( + "SELECT tablename FROM pg_tables WHERE schemaname = $1", + {make_text("public")}); + } + + static BoundStatement inspect_sql(const std::string& table) { + check_identifier(table); + return BoundStatement( + "SELECT column_name, data_type, character_maximum_length, " + "is_nullable FROM information_schema.columns WHERE table_name = $1 " + "ORDER BY ordinal_position", + {make_text(table)}); + } + + std::vector list_tables(const Context& ctx, + Connection& conn) override { + Result r = conn.exec(ctx, list_tables_sql()); + std::vector names; + for (std::size_t i = 0; i < r.size(); ++i) { + const Value& v = r.raw(i)[0]; + if (const auto* s = std::get_if(&v)) names.push_back(*s); + } + return names; + } + + TableInfo inspect_table(const Context& ctx, Connection& conn, + const std::string& table) override { + Result r = conn.exec(ctx, inspect_sql(table)); + TableInfo info; + info.name = table; + for (std::size_t i = 0; i < r.size(); ++i) { + Row row = r.row(i); + const auto* name = std::get_if(&row.get("column_name")); + const auto* type = std::get_if(&row.get("data_type")); + if (!name || !type) continue; + int len = 0; + const Value& ml = row.get("character_maximum_length"); + if (const auto* li = std::get_if(&ml)) + len = static_cast(*li); + const auto* nullable = std::get_if(&row.get("is_nullable")); + ColumnInfo c; + c.name = *name; + c.type = *type; + for (char& ch : c.type) ch = static_cast(std::toupper(ch)); + c.length = len; + c.nullable = !nullable || *nullable == "YES"; + info.columns.push_back(c); + } + return info; + } +}; + +} // namespace molo::data::orm diff --git a/src/data/orm/sync.hpp b/src/data/orm/sync.hpp new file mode 100644 index 0000000..7aa27bb --- /dev/null +++ b/src/data/orm/sync.hpp @@ -0,0 +1,122 @@ +#pragma once + +// Syncer: diffs registered models against introspected schema and applies +// additive DDL inside one transaction per table. Runs at startup (and on +// demand). Failures abort startup unless the caller passes sync_on_boot +// false, in which case the caller downgrades health instead. + +#include +#include +#include + +#include "../../tenancy/rls_policy.h" +#include "../connection.hpp" +#include "../context.hpp" +#include "../error.hpp" +#include "../statement.hpp" +#include "../transaction.hpp" +#include "field.hpp" +#include "schema.hpp" + +namespace molo::data::orm { + +struct ModelInfo { + std::string table; + std::string pk; + std::vector fields; +}; + +template +ModelInfo describe_model() { + return ModelInfo{M::table_name(), M::primary_key(), M::fields()}; +} + +struct SyncReport { + int tables_created = 0; + int columns_added = 0; + int indexes_added = 0; +}; + +inline std::string create_table_sql(const ModelInfo& m, Backend backend) { + std::string sql = "CREATE TABLE IF NOT EXISTS " + m.table + " ("; + for (std::size_t i = 0; i < m.fields.size(); ++i) { + if (i > 0) sql += ", "; + sql += ddl_column(m.fields[i], backend); + } + sql += ")"; + return sql; +} + +inline bool ModelHasTenant(const ModelInfo& m) { + for (const auto& f : m.fields) + if (f.name == "tenant_id") return true; + return false; +} + +class Syncer { + public: + SyncReport sync(const Context& ctx, Connection& conn, + const std::vector& models) { + SyncReport report; + std::unique_ptr inspector = conn.is_postgres() + ? std::unique_ptr(new PgInspector()) + : std::unique_ptr(new SqliteInspector()); + std::vector existing; + try { + existing = inspector->list_tables(ctx, conn); + } catch (const DbError&) { + existing = {}; + } + const Backend backend = conn.backend(); + for (const auto& m : models) { + check_identifier(m.table); + const bool known = + std::find(existing.begin(), existing.end(), m.table) != + existing.end(); + Transaction txn(conn, ctx); + if (!known) { + conn.exec(ctx, BoundStatement::raw(create_table_sql(m, backend))); + for (const auto& f : m.fields) { + if (f.indexed && !f.pk) { + conn.exec(ctx, BoundStatement::raw( + "CREATE INDEX IF NOT EXISTS ix_" + m.table + + "_" + f.name + " ON " + m.table + " (" + + f.name + ")")); + ++report.indexes_added; + } + } + ++report.tables_created; + } else { + TableInfo info = inspector->inspect_table(ctx, conn, m.table); + for (const auto& f : m.fields) { + if (!info.find(f.name)) { + conn.exec(ctx, BoundStatement::raw("ALTER TABLE " + m.table + + " ADD COLUMN " + + ddl_column(f, backend))); + ++report.columns_added; + } + } + } + // Tenant scope (PATCHES 11.6): composite index plus RLS policy for + // models declaring tenant_id. DropPolicy first keeps re-sync + // idempotent; RLS is Postgres-only. + if (ModelHasTenant(m)) { + conn.exec(ctx, BoundStatement::raw( + ::molo::tenancy::RlsPolicy::TenantIndexDdl(m.table))); + ++report.indexes_added; + if (conn.is_postgres()) { + conn.exec(ctx, BoundStatement::raw( + ::molo::tenancy::RlsPolicy::EnableRls(m.table))); + conn.exec(ctx, BoundStatement::raw( + ::molo::tenancy::RlsPolicy::DropPolicy(m.table))); + conn.exec(ctx, BoundStatement::raw( + ::molo::tenancy::RlsPolicy::PolicyDdl(m.table))); + } + } + txn.commit(); + } + return report; + } +}; + +} // namespace molo::data::orm diff --git a/src/data/orm/types.md b/src/data/orm/types.md new file mode 100644 index 0000000..46088c2 --- /dev/null +++ b/src/data/orm/types.md @@ -0,0 +1,24 @@ +# ORM type mapping (single source of truth) + +C++ `Value` variant to backend column types. Field descriptors in +`orm/field.hpp` (`ddl_base_type`) implement exactly this table. + +| C++ Value | Postgres | SQLite | Notes | +|--------------|---------------------|---------------|-----------------------------| +| int64 | BIGINT | INTEGER | bool stored as 0/1 in SQLite| +| int (Field) | INTEGER | INTEGER | widen target is BIGINT | +| double | DOUBLE PRECISION | REAL | | +| string TEXT | TEXT | TEXT | unlimited length | +| string V(n) | VARCHAR(n) | VARCHAR(n) | enforced, heal extends x4 | +| Bytes | BYTEA | BLOB | hex text format on PG text | +| bool | BOOLEAN | INTEGER 0/1 | cross-compared in filters | +| TimePoint | TIMESTAMPTZ | TEXT ISO-8601 | micros since Unix epoch | + +Rules: + +- Extension is one way: INTEGER to BIGINT, VARCHAR(n) to VARCHAR(4n), + VARCHAR past 16384 to TEXT. The healer never narrows. +- Primary keys render `GENERATED ALWAYS AS IDENTITY` on Postgres and + plain `PRIMARY KEY` with store-assigned ids on SQLite. +- `SMALLINT` appears only as a fault-injection state in tests; it is not + a modeled field type. diff --git a/src/data/pool.hpp b/src/data/pool.hpp new file mode 100644 index 0000000..06640e0 --- /dev/null +++ b/src/data/pool.hpp @@ -0,0 +1,268 @@ +#pragma once + +// Pool: generic connection pool over any Connection implementation. +// Default mode is transaction pooling: a checkout serves one transaction +// (or one statement batch) and returns. Long-lived session state uses +// acquire_pinned(), which reserves the connection until release. +// Exhaustion fails fast with DbError plus metrics, never waits forever. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../tenancy/rls_policy.h" +#include "connection.hpp" +#include "context.hpp" +#include "error.hpp" +#include "obs.hpp" +#include "pool_config.hpp" + +namespace molo::data { + +namespace detail { +// Single-quote escaping for tenant literals in SET LOCAL. +inline std::string EscapeTenantLiteral(const std::string& slug) { + std::string o; + for (char c : slug) { + if (c == '\'') o += '\''; + o += c; + } + return o; +} +} // namespace detail + +template +class Pool { + public: + using Factory = std::function()>; + + explicit Pool(PoolConfig config, Factory factory, + obs::MetricSink* metrics = &obs::null_sink()) + : config_(config), factory_(std::move(factory)), metrics_(metrics) { + config_.validate(); + } + + Pool(const Pool&) = delete; + Pool& operator=(const Pool&) = delete; + + // RAII checkout. Returning to the pool happens on destruction unless + // release() was called for a pinned handle. + class Checkout { + public: + Checkout() = default; + Checkout(Pool* pool, std::unique_ptr conn, bool pinned) + : pool_(pool), conn_(std::move(conn)), pinned_(pinned) {} + Checkout(Checkout&& other) noexcept + : pool_(other.pool_), + conn_(std::move(other.conn_)), + pinned_(other.pinned_), + scope_(std::move(other.scope_)) { + other.pool_ = nullptr; + } + Checkout& operator=(Checkout&& other) noexcept { + if (this != &other) { + if (conn_ && pool_) pool_->release(std::move(conn_), pinned_, std::move(scope_)); + pool_ = other.pool_; + conn_ = std::move(other.conn_); + pinned_ = other.pinned_; + scope_ = std::move(other.scope_); + other.pool_ = nullptr; + } + return *this; + } + Checkout(const Checkout&) = delete; + Checkout& operator=(const Checkout&) = delete; + + ~Checkout() { + if (conn_ && pool_) pool_->release(std::move(conn_), pinned_, std::move(scope_)); + } + + Conn& operator*() { return *conn_; } + Conn* operator->() { return conn_.get(); } + explicit operator bool() const { return conn_ != nullptr; } + const std::string& scope_tag() const { return scope_; } + void set_scope_tag(std::string s) { scope_ = std::move(s); } + + private: + Pool* pool_ = nullptr; + std::unique_ptr conn_; + bool pinned_ = false; + std::string scope_; + }; + + Checkout acquire(const Context& ctx) { return acquire_impl(ctx, false); } + + // Session-pinned checkout for transactions and session state. Binds the + // request tenant as session scope (Postgres SET LOCAL); a reused slot + // found dirty throws instead of leaking across tenants (PATCHES 11.2). + Checkout acquire_pinned(const Context& ctx) { + Checkout out = acquire_impl(ctx, true); + if (!ctx.tenant().empty() && out->backend() == Backend::Postgres) { + out->exec(ctx, BoundStatement::raw(::molo::tenancy::RlsPolicy::SetLocal( + detail::EscapeTenantLiteral(ctx.tenant())))); + out.set_scope_tag(ctx.tenant()); + } + return out; + } + + void evict_idle() { + std::lock_guard lock(mutex_); + evict_idle_locked(std::chrono::steady_clock::now()); + } + + int idle_count() const { + std::lock_guard lock(mutex_); + return static_cast(idle_.size()); + } + + int total_count() const { + std::lock_guard lock(mutex_); + return total_; + } + + std::int64_t exhausted_count() const { return exhausted_; } + + private: + struct IdleEntry { + std::unique_ptr conn; + std::chrono::steady_clock::time_point idle_since; + std::chrono::steady_clock::time_point born; + std::string scope_tag; // "" means clean; anything else is a leak bug + }; + + Checkout acquire_impl(const Context& ctx, bool pinned) { + std::unique_lock lock(mutex_); + if (waiters_ >= config_.max_waiters) { + ++exhausted_; + metrics_->counter("db.pool.exhausted", 1, ""); + throw DbError(first_backend(), "MOLO_POOL", + "pool waiter queue full, failing fast"); + } + ++waiters_; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(config_.acquire_timeout_sec); + std::unique_ptr conn; + while (true) { + evict_idle_locked(std::chrono::steady_clock::now()); + if (!idle_.empty()) { + if (!idle_.back().scope_tag.empty()) { + --waiters_; + throw DbError(first_backend(), "scope_dirty", + "pooled connection scope not reset"); + } + conn = std::move(idle_.back().conn); + idle_.pop_back(); + break; + } + if (total_ < config_.max_size) { + lock.unlock(); + try { + conn = factory_(); + } catch (...) { + lock.lock(); + --waiters_; + throw; + } + lock.lock(); + ++total_; + break; + } + if (ctx.expired() || std::chrono::steady_clock::now() >= deadline) { + --waiters_; + ++exhausted_; + metrics_->counter("db.pool.exhausted", 1, ""); + throw DbError(first_backend(), "MOLO_POOL", + "pool acquire timed out, failing fast"); + } + waiter_cv_.wait_until(lock, deadline); + } + --waiters_; + metrics_->gauge("db.pool.in_use", static_cast(total_ - idle_.size()), + ""); + const bool probe = config_.health_probe_on_acquire; + lock.unlock(); + if (probe && conn) { + try { + conn->ping(ctx); + } catch (...) { + discard(); + metrics_->counter("db.pool.probe_failed", 1, ""); + throw; + } + } + return Checkout(this, std::move(conn), pinned); + } + + void release(std::unique_ptr conn, bool /*pinned*/, std::string scope) { + std::lock_guard lock(mutex_); + // Tenant scope reset (PATCHES 11.2): one bounded RESET under the lock so + // a checkout can never leak across tenants. A failed RESET drops the + // slot instead of parking a poisoned connection. + if (!scope.empty() && conn->backend() == Backend::Postgres) { + try { + conn->exec(Context("pool", "", 5), + BoundStatement::raw(::molo::tenancy::RlsPolicy::ResetLocal())); + } catch (...) { + if (total_ > 0) --total_; + metrics_->counter("db.pool.scope_reset_failed", 1, ""); + waiter_cv_.notify_one(); + return; + } + } + const auto now = std::chrono::steady_clock::now(); + idle_.push_back( + IdleEntry{std::move(conn), now, now - std::chrono::seconds(0), ""}); + // Born time is unknown for returned connections; lifetime is enforced + // at eviction using idle_since as a conservative approximation plus a + // per-entry generation counter via total_ churn. See evict below. + waiter_cv_.notify_one(); + } + + void evict_idle_locked(std::chrono::steady_clock::time_point now) { + auto it = idle_.begin(); + while (it != idle_.end()) { + const auto idle_for = now - it->idle_since; + if (idle_for >= std::chrono::seconds(config_.idle_evict_sec) || + static_cast(idle_.size()) > + config_.max_size) { // defensive, never above max + it = idle_.erase(it); + --total_; + metrics_->counter("db.pool.evicted", 1, ""); + } else { + ++it; + } + } + // Lifetime enforcement: drop oldest idle entries past max lifetime. + while (!idle_.empty() && + now - idle_.front().idle_since >= + std::chrono::seconds(config_.max_lifetime_sec)) { + idle_.erase(idle_.begin()); + --total_; + metrics_->counter("db.pool.expired", 1, ""); + } + } + + void discard() { + std::lock_guard lock(mutex_); + if (total_ > 0) --total_; + } + + Backend first_backend() const { return Backend::Postgres; } + + PoolConfig config_; + Factory factory_; + obs::MetricSink* metrics_; + mutable std::mutex mutex_; + std::condition_variable waiter_cv_; + std::vector idle_; + int total_ = 0; + int waiters_ = 0; + std::int64_t exhausted_ = 0; +}; + +} // namespace molo::data diff --git a/src/data/pool_config.hpp b/src/data/pool_config.hpp new file mode 100644 index 0000000..5dd614b --- /dev/null +++ b/src/data/pool_config.hpp @@ -0,0 +1,41 @@ +#pragma once + +// PoolConfig: static ceilings for pooling, tunable only through validated +// config that clamps into these ranges. + +#include + +#include "caps.hpp" + +namespace molo::data { + +struct PoolConfig { + int min_size = caps::kPgPoolMin; + int max_size = caps::kPgPoolMax; + int acquire_timeout_sec = caps::kPoolAcquireTimeoutSec; + int idle_evict_sec = caps::kPoolIdleEvictSec; + int max_lifetime_sec = caps::kPoolMaxLifetimeSec; + int max_waiters = caps::kPoolMaxWaiters; + bool health_probe_on_acquire = true; + + static PoolConfig postgres() { return PoolConfig{}; } + + static PoolConfig sqlite() { + PoolConfig c; + c.min_size = 1; + c.max_size = caps::kSqliteMaxReaders; + return c; + } + + void validate() const { + if (min_size < 0 || max_size < 1 || min_size > max_size) + throw std::invalid_argument("bad pool size range"); + if (acquire_timeout_sec < 1 || + acquire_timeout_sec > caps::kStatementTimeoutHardSec) + throw std::invalid_argument("bad pool acquire timeout"); + if (max_waiters < 1 || max_waiters > caps::kPoolMaxWaiters) + throw std::invalid_argument("bad pool waiter cap"); + } +}; + +} // namespace molo::data diff --git a/src/data/pubsub/broker.hpp b/src/data/pubsub/broker.hpp new file mode 100644 index 0000000..7ddfd31 --- /dev/null +++ b/src/data/pubsub/broker.hpp @@ -0,0 +1,140 @@ +#pragma once + +// Broker: at-most-once, non-persistent fan-out. Publish appends to every +// matching subscription and returns; it never awaits subscribers. A publish +// with no subscribers is a counted no-op. Delivery order holds per +// publisher to channel. Anything needing durability uses the durable +// channel from 07-supervisor-replication.md, never this broker. + +#include +#include +#include +#include +#include + +#include "../caps.hpp" +#include "../context.hpp" +#include "../obs.hpp" +#include "channel.hpp" +#include "subscription.hpp" + +namespace molo::data::pubsub { + +struct DeliveryReport { + int callbacks = 0; + int queued = 0; + int dropped = 0; + bool no_subscribers = true; +}; + +class Broker { + public: + explicit Broker(obs::MetricSink* metrics = &obs::null_sink()) + : metrics_(metrics) {} + + // Tenant scoping (PATCHES 11.11): channel names are prefixed per + // tenant at publish and subscribe time. unsubscribe() is id-keyed and + // needs no scoping. + static std::string scoped_channel(const Context& ctx, const std::string& ch) { + return ctx.tenant().empty() ? ch : "tenant:" + ctx.tenant() + ":" + ch; + } + + int subscribe(const Context& ctx, const std::string& pattern, + std::function handler) { + return subscribe(scoped_channel(ctx, pattern), std::move(handler)); + } + + int subscribe(const std::string& pattern, + std::function handler) { + check_channel(pattern); + std::lock_guard lock(mutex_); + if (static_cast(callbacks_.size() + queued_.size()) >= + caps::kBrokerPatternMax) + throw std::runtime_error("broker pattern cap reached"); + const int id = next_++; + callbacks_[id] = CallbackSubscription{id, pattern, std::move(handler)}; + return id; + } + + std::shared_ptr subscribe_queued( + const Context& ctx, const std::string& pattern) { + return subscribe_queued(scoped_channel(ctx, pattern)); + } + + std::shared_ptr subscribe_queued( + const std::string& pattern) { + check_channel(pattern); + std::lock_guard lock(mutex_); + if (static_cast(callbacks_.size() + queued_.size()) >= + caps::kBrokerPatternMax) + throw std::runtime_error("broker pattern cap reached"); + const int id = next_++; + auto sub = std::make_shared(id, pattern); + queued_[id] = sub; + return sub; + } + + void unsubscribe(int id) { + std::lock_guard lock(mutex_); + callbacks_.erase(id); + queued_.erase(id); + } + + DeliveryReport publish(const Context& ctx, const std::string& channel, Payload payload, + const std::string& content_type = "") { + return publish(scoped_channel(ctx, channel), std::move(payload), content_type); + } + + DeliveryReport publish(const std::string& channel, Payload payload, + const std::string& content_type = "") { + check_channel(channel); + if (payload.size() > caps::kPayloadMaxBytes) + throw std::invalid_argument("pubsub payload over 1 MiB"); + std::map cbs; + std::map> qs; + { + std::lock_guard lock(mutex_); + cbs = callbacks_; + qs = queued_; + } + DeliveryReport report; + Message m{channel, payload, content_type}; + for (const auto& [_, sub] : cbs) { + if (!pattern_match(channel, sub.pattern)) continue; + report.no_subscribers = false; + sub.handler(m); // inline: handlers must not block + ++report.callbacks; + } + for (const auto& [_, sub] : qs) { + if (!pattern_match(channel, sub->pattern())) continue; + report.no_subscribers = false; + if (sub->push(m)) { + ++report.dropped; + metrics_->counter("pubsub.dropped", 1, ""); + } else { + ++report.queued; + } + } + ++published_; + metrics_->counter("pubsub.published", 1, ""); + if (report.no_subscribers) { + ++no_subscriber_; + metrics_->counter("pubsub.no_subscriber", 1, ""); + } + return report; + } + + std::int64_t published() const { return published_; } + std::int64_t no_subscriber() const { return no_subscriber_; } + + private: + obs::MetricSink* metrics_; + std::mutex mutex_; + std::map callbacks_; + std::map> queued_; + int next_ = 1; + std::int64_t published_ = 0; + std::int64_t no_subscriber_ = 0; +}; + +} // namespace molo::data::pubsub diff --git a/src/data/pubsub/channel.hpp b/src/data/pubsub/channel.hpp new file mode 100644 index 0000000..10f41b4 --- /dev/null +++ b/src/data/pubsub/channel.hpp @@ -0,0 +1,53 @@ +#pragma once + +// Channel names and pattern matching. Channels are plain strings shaped +// like topic:sub; patterns use * as a single-segment wildcard. Payload is +// bytes plus a content-type hint; JSON is a caller helper, not the wire. + +#include +#include +#include + +#include "../caps.hpp" + +namespace molo::data::pubsub { + +inline void check_channel(const std::string& name) { + if (name.empty() || name.size() > caps::kChannelMaxLen) + throw std::invalid_argument("channel name out of range"); + for (char c : name) { + const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == ':' || c == '_' || + c == '-' || c == '.' || c == '/' || c == '*'; + if (!ok) throw std::invalid_argument("bad channel name"); + } +} + +inline std::vector split_segments(const std::string& s) { + std::vector segs; + std::string cur; + for (char c : s) { + if (c == ':') { + segs.push_back(cur); + cur.clear(); + } else { + cur += c; + } + } + segs.push_back(cur); + return segs; +} + +// Strict match: same segment count, each segment equal or *. +inline bool pattern_match(const std::string& topic, + const std::string& pattern) { + const auto t = split_segments(topic); + const auto p = split_segments(pattern); + if (t.size() != p.size()) return false; + for (std::size_t i = 0; i < t.size(); ++i) { + if (p[i] != "*" && p[i] != t[i]) return false; + } + return true; +} + +} // namespace molo::data::pubsub diff --git a/src/data/pubsub/subscription.hpp b/src/data/pubsub/subscription.hpp new file mode 100644 index 0000000..d348c64 --- /dev/null +++ b/src/data/pubsub/subscription.hpp @@ -0,0 +1,77 @@ +#pragma once + +// Subscriptions: callback (inline, must not block) and queued (bounded at +// 1024 messages, oldest drop counted) flavors for slow consumers. + +#include +#include +#include +#include +#include +#include + +#include "../caps.hpp" + +namespace molo::data::pubsub { + +using Payload = std::vector; + +struct Message { + std::string channel; + Payload payload; + std::string content_type; +}; + +class QueuedSubscription { + public: + QueuedSubscription(int id, std::string pattern) + : id_(id), pattern_(std::move(pattern)) {} + + int id() const { return id_; } + const std::string& pattern() const { return pattern_; } + + // Returns true when an older message was dropped to make room. + bool push(Message m) { + std::lock_guard lock(mutex_); + bool dropped = false; + if (static_cast(queue_.size()) >= caps::kSubscriberQueueMax) { + queue_.pop_front(); + ++dropped_; + dropped = true; + } + queue_.push_back(std::move(m)); + return dropped; + } + + std::vector poll(int max = caps::kSubscriberQueueMax) { + std::lock_guard lock(mutex_); + std::vector out; + while (!queue_.empty() && static_cast(out.size()) < max) { + out.push_back(std::move(queue_.front())); + queue_.pop_front(); + } + return out; + } + + int depth() const { + std::lock_guard lock(mutex_); + return static_cast(queue_.size()); + } + + std::int64_t dropped() const { return dropped_; } + + private: + int id_; + std::string pattern_; + mutable std::mutex mutex_; + std::deque queue_; + std::int64_t dropped_ = 0; +}; + +struct CallbackSubscription { + int id = 0; + std::string pattern; + std::function handler; +}; + +} // namespace molo::data::pubsub diff --git a/src/data/result.hpp b/src/data/result.hpp new file mode 100644 index 0000000..0d868eb --- /dev/null +++ b/src/data/result.hpp @@ -0,0 +1,50 @@ +#pragma once + +// Result: rows plus affected count plus last insert id. + +#include +#include +#include +#include + +#include "row.hpp" +#include "value.hpp" + +namespace molo::data { + +class Result { + public: + Result() = default; + + Result(std::vector columns, + std::vector> rows, std::int64_t affected = 0, + std::int64_t last_insert_id = 0) + : columns_(std::move(columns)), + rows_(std::move(rows)), + affected_(affected), + last_insert_id_(last_insert_id) {} + + static Result command(std::int64_t affected, + std::int64_t last_insert_id = 0) { + return Result{{}, {}, affected, last_insert_id}; + } + + const std::vector& columns() const { return columns_; } + std::size_t size() const { return rows_.size(); } + bool empty() const { return rows_.empty(); } + + Row row(std::size_t i) const { return Row{columns_, rows_.at(i)}; } + + const std::vector& raw(std::size_t i) const { return rows_.at(i); } + + std::int64_t affected() const { return affected_; } + std::int64_t last_insert_id() const { return last_insert_id_; } + + private: + std::vector columns_; + std::vector> rows_; + std::int64_t affected_ = 0; + std::int64_t last_insert_id_ = 0; +}; + +} // namespace molo::data diff --git a/src/data/row.hpp b/src/data/row.hpp new file mode 100644 index 0000000..fdb71d0 --- /dev/null +++ b/src/data/row.hpp @@ -0,0 +1,45 @@ +#pragma once + +// Row: one named result row. Lightweight view-friendly value object. + +#include +#include +#include +#include + +#include "value.hpp" + +namespace molo::data { + +class Row { + public: + Row() = default; + Row(std::vector columns, std::vector values) + : columns_(std::move(columns)), values_(std::move(values)) { + if (columns_.size() != values_.size()) + throw std::invalid_argument("row columns/values size mismatch"); + } + + std::size_t size() const { return values_.size(); } + const std::vector& columns() const { return columns_; } + + const Value& at(std::size_t i) const { return values_.at(i); } + + const Value& get(const std::string& name) const { + for (std::size_t i = 0; i < columns_.size(); ++i) + if (columns_[i] == name) return values_[i]; + throw std::out_of_range("no such column: " + name); + } + + bool has(const std::string& name) const { + for (const auto& c : columns_) + if (c == name) return true; + return false; + } + + private: + std::vector columns_; + std::vector values_; +}; + +} // namespace molo::data diff --git a/src/data/statement.hpp b/src/data/statement.hpp new file mode 100644 index 0000000..4da3dde --- /dev/null +++ b/src/data/statement.hpp @@ -0,0 +1,62 @@ +#pragma once + +// BoundStatement: constant SQL text plus bound values. This is the only way +// values reach a driver, which is the structural SQL injection defense. +// Constant fragments needing dynamic SQL use raw() and are flagged for +// review; user input must never flow through raw(). + +#include +#include +#include + +#include "caps.hpp" +#include "error.hpp" +#include "value.hpp" + +namespace molo::data { + +inline void check_identifier(const std::string& name) { + if (name.empty() || name.size() > caps::kMaxIdentifierLen) + throw std::invalid_argument("identifier length out of range"); + for (char c : name) { + const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '.'; + if (!ok) throw std::invalid_argument("bad identifier: " + name); + } +} + +class BoundStatement { + public: + BoundStatement() = default; + + BoundStatement(std::string sql, std::vector params = {}) + : sql_(std::move(sql)), params_(std::move(params)), has_raw_(false) { + if (params_.size() > caps::kMaxBoundParams) + throw std::invalid_argument("too many bound parameters"); + } + + // Constant fragment escape hatch. Callers must keep user input out and + // mark the call site for review. + static BoundStatement raw(std::string sql, std::vector params = {}) { + BoundStatement s(std::move(sql), std::move(params)); + s.has_raw_ = true; + return s; + } + + const std::string& sql() const { return sql_; } + const std::vector& params() const { return params_; } + bool has_raw() const { return has_raw_; } + + void bind(Value v) { + if (params_.size() >= caps::kMaxBoundParams) + throw std::invalid_argument("too many bound parameters"); + params_.push_back(std::move(v)); + } + + private: + std::string sql_; + std::vector params_; + bool has_raw_ = false; +}; + +} // namespace molo::data diff --git a/src/data/transaction.hpp b/src/data/transaction.hpp new file mode 100644 index 0000000..6b91023 --- /dev/null +++ b/src/data/transaction.hpp @@ -0,0 +1,116 @@ +#pragma once + +// Transaction: RAII guard. Defaults to READ COMMITTED, SERIALIZABLE opt-in +// per transaction. Nested scopes use savepoints named molo_sp__. +// Going out of scope without commit rolls back. The connection stays pinned +// for the transaction lifetime; callers hold a pinned pool checkout. +// Serialization failures (40001) surface as retryable DbError; the layer +// never auto-retries user transactions. + +#include +#include + +#include "../tenancy/rls_policy.h" +#include "connection.hpp" +#include "context.hpp" +#include "error.hpp" +#include "statement.hpp" + +namespace molo::data { + +class Transaction { + public: + Transaction(Connection& conn, const Context& ctx, + Isolation iso = Isolation::ReadCommitted) + : conn_(conn), ctx_(ctx) { + std::string sql = "BEGIN"; + if (conn_.is_postgres()) { + sql += " ISOLATION LEVEL "; + sql += isolation_name(iso); + } + conn_.exec(ctx_, BoundStatement::raw(sql)); + // Tenant scope (PATCHES 11.3): SET LOCAL after BEGIN on Postgres so the + // transaction carries the request tenant even when the pool checkout + // predates resolution. + if (conn_.is_postgres() && !ctx_.tenant().empty()) { + std::string esc; + for (char c : ctx_.tenant()) { + if (c == '\'') esc += '\''; + esc += c; + } + conn_.exec(ctx_, BoundStatement::raw(::molo::tenancy::RlsPolicy::SetLocal(esc))); + } + active_ = true; + } + + Transaction(const Transaction&) = delete; + Transaction& operator=(const Transaction&) = delete; + + ~Transaction() { + if (active_) { + try { + conn_.exec(ctx_, BoundStatement::raw("ROLLBACK")); + } catch (...) { + } + } + } + + void commit() { + ensure_active(); + conn_.exec(ctx_, BoundStatement::raw("COMMIT")); + active_ = false; + } + + void rollback() { + ensure_active(); + conn_.exec(ctx_, BoundStatement::raw("ROLLBACK")); + active_ = false; + } + + // Nested scope via savepoint. Returns the savepoint name. + std::string savepoint() { + ensure_active(); + ++depth_; + std::string name = + "molo_sp_" + std::to_string(depth_) + "_" + std::to_string(++counter_); + conn_.exec(ctx_, BoundStatement::raw("SAVEPOINT " + name)); + return name; + } + + void release_savepoint(const std::string& name) { + ensure_active(); + check_identifier(depth_of(name)); + conn_.exec(ctx_, BoundStatement::raw("RELEASE SAVEPOINT " + name)); + if (depth_ > 0) --depth_; + } + + void rollback_to(const std::string& name) { + ensure_active(); + check_identifier(depth_of(name)); + conn_.exec(ctx_, BoundStatement::raw("ROLLBACK TO SAVEPOINT " + name)); + } + + bool active() const { return active_; } + int depth() const { return depth_; } + Connection& connection() { return conn_; } + + private: + void ensure_active() const { + if (!active_) throw DbError(conn_.backend(), "MOLO_TXN", "transaction not active"); + } + + static std::string depth_of(const std::string& name) { + // Savepoint names are generated above; validate shape before use. + if (name.rfind("molo_sp_", 0) != 0) + throw std::invalid_argument("bad savepoint name"); + return name; + } + + Connection& conn_; + Context ctx_; + bool active_ = false; + int depth_ = 0; + int counter_ = 0; +}; + +} // namespace molo::data diff --git a/src/data/value.hpp b/src/data/value.hpp new file mode 100644 index 0000000..c38ed07 --- /dev/null +++ b/src/data/value.hpp @@ -0,0 +1,46 @@ +#pragma once + +// Value: the single cell type for rows, bound parameters and codecs. +// Use the make_* factories: bare ints and bools are ambiguous in the raw +// variant, so implicit construction from them is intentionally unsupported. + +#include +#include +#include +#include +#include + +namespace molo::data { + +using Bytes = std::vector; +using TimePoint = std::chrono::system_clock::time_point; +using Value = + std::variant; + +inline Value make_null() { return Value{std::monostate{}}; } +inline Value make_int(std::int64_t v) { return Value{v}; } +inline Value make_double(double v) { return Value{v}; } +inline Value make_text(std::string v) { return Value{std::move(v)}; } +inline Value make_bytes(Bytes v) { return Value{std::move(v)}; } +inline Value make_bool(bool v) { return Value{v}; } +inline Value make_time(TimePoint v) { return Value{v}; } + +inline bool is_null(const Value& v) { + return std::holds_alternative(v); +} + +inline const char* value_type_name(const Value& v) { + switch (v.index()) { + case 0: return "null"; + case 1: return "int"; + case 2: return "double"; + case 3: return "text"; + case 4: return "bytes"; + case 5: return "bool"; + case 6: return "time"; + default: return "unknown"; + } +} + +} // namespace molo::data diff --git a/src/governance.mk b/src/governance.mk new file mode 100644 index 0000000..52760ec --- /dev/null +++ b/src/governance.mk @@ -0,0 +1,39 @@ +# src/governance.mk: build fragment for the MoloApp governance part. +# Included by the root Makefile: include src/governance.mk +# Standalone use: make -f src/governance.mk govern-test + +GOVERN_CXX ?= g++ +GOVERN_CXXFLAGS ?= -std=c++23 -O2 -Wall -Wextra -pthread +GOVERN_INC ?= -Isrc +GOVERN_BUILD ?= build/governance + +GOVERN_LIB_SRCS := src/governance/diff/GovernGate.cpp +GOVERN_LIB_OBJS := $(GOVERN_BUILD)/GovernGate.o + +GOVERN_TESTS := keys_scope keys_issuer keyauth_http versions \ + deprecation_headers tiers differ_vectors gate surface_lint +GOVERN_TEST_BINS := $(addprefix $(GOVERN_BUILD)/,$(addsuffix _test,$(GOVERN_TESTS))) + +GOVERN_APIDIFF := $(GOVERN_BUILD)/apidiff + +.PHONY: govern-build govern-test govern-clean + +govern-build: $(GOVERN_TEST_BINS) $(GOVERN_APIDIFF) + +govern-test: govern-build + @for t in $(GOVERN_TEST_BINS); do ./$$t || exit 1; done + +govern-clean: + rm -rf $(GOVERN_BUILD) + +$(GOVERN_BUILD): + mkdir -p $(GOVERN_BUILD) + +$(GOVERN_BUILD)/GovernGate.o: src/governance/diff/GovernGate.cpp | $(GOVERN_BUILD) + $(GOVERN_CXX) $(GOVERN_CXXFLAGS) $(GOVERN_INC) -c $< -o $@ + +$(GOVERN_BUILD)/%_test: tests/src/governance/%_test.cpp $(GOVERN_LIB_OBJS) | $(GOVERN_BUILD) + $(GOVERN_CXX) $(GOVERN_CXXFLAGS) $(GOVERN_INC) $< $(GOVERN_LIB_OBJS) -o $@ + +$(GOVERN_APIDIFF): src/governance/diff/apidiff_main.cpp $(GOVERN_LIB_OBJS) | $(GOVERN_BUILD) + $(GOVERN_CXX) $(GOVERN_CXXFLAGS) $(GOVERN_INC) $< $(GOVERN_LIB_OBJS) -o $@ diff --git a/src/governance/PATCHES.md b/src/governance/PATCHES.md new file mode 100644 index 0000000..d67ac09 --- /dev/null +++ b/src/governance/PATCHES.md @@ -0,0 +1,248 @@ +# Governance base-tree patches + +Exact repo-relative paths plus unified diff hunks for every edit the +`src/governance/` part needs in the base tree. Context lines follow the seam +spec (router, middleware, errors, openapi, guard APIs); the integrator applies +each hunk to the matching declaration in the base revision and keeps headers +compiling with `-std=c++23 -O2 -Wall -Wextra -pthread`. + +Rule for all hunks: additive only, no behavior change for routes without +governance fields set. + +--- + +## 1. `src/app/routing/route.h`: governance fields on RouteOptions + +```diff +--- a/src/app/routing/route.h ++++ b/src/app/routing/route.h +@@ struct RouteOptions + struct RouteOptions { + std::string name; + std::vector tags; + std::string produces; + std::string consumes; + std::size_t max_body = 0; ++ // Governance (molo::govern): auth and lifecycle for this route. ++ enum class GovAuth { kNone, kKey, kKeyScopes, kSession }; ++ GovAuth gov_auth = GovAuth::kNone; ++ std::vector gov_required_scopes; ++ int gov_api_version = 0; // 0 = derived from the /v{n} prefix ++ long gov_deprecation_unix = 0; ++ std::string gov_sunset_httpdate; ++ std::string gov_tier_override; // "", "free", "pro", "enterprise" ++ bool gov_has_summary = false; ++ bool gov_has_example = false; + }; +``` + +`Route::with_prefix` must copy every `gov_*` field to the derived route: + +```diff +--- a/src/app/routing/route.h ++++ b/src/app/routing/route.h +@@ Route Route::with_prefix + Route Route::with_prefix(std::string_view prefix) const { + Route out = *this; + out.pattern_ = JoinPrefix(prefix, pattern_); ++ out.options_.gov_api_version = options_.gov_api_version; ++ out.options_.gov_required_scopes = options_.gov_required_scopes; ++ out.options_.gov_deprecation_unix = options_.gov_deprecation_unix; ++ out.options_.gov_sunset_httpdate = options_.gov_sunset_httpdate; ++ out.options_.gov_tier_override = options_.gov_tier_override; + return out; + } +``` + +--- + +## 2. `src/app/routing/router.h`: startup validation hook + +```diff +--- a/src/app/routing/router.h ++++ b/src/app/routing/router.h +@@ class Router + const std::vector& routes() const; + void freeze(); ++ // Governance hook: validates /v{n} prefixes, at most 3 live majors, ++ // Sunset >= Deprecation, and declared auth on public routes. ++ // Throws StartupError on violation. Called by Application::startup(). ++ void validate_governance() const; +``` + +--- + +## 3. `src/app/app/application.h`: startup validation plus sunset 410 + +```diff +--- a/src/app/app/application.h ++++ b/src/app/app/application.h +@@ Application::startup + void startup() { ++ router_.validate_governance(); + router_.freeze(); + // ... existing startup work ... + } +``` + +Request handling maps a sunset-version match to 410 instead of 404, using +`molo::govern::versions::ApiVersion` and `molo::govern::Problem::VersionGone`: + +```diff +--- a/src/app/app/application.h ++++ b/src/app/app/application.h +@@ Application::handle + Task handle(Request& req) { ++ if (IsSunsetVersion(req.path())) { ++ const auto problem = ::molo::govern::Problem::VersionGone(); ++ Response res(410); ++ res.set_content_type(std::string(::molo::govern::Problem::kContentType)); ++ res.body() = problem.to_json_string(); ++ co_return res; ++ } + co_return co_await chain_.run(req, terminal_); + } +``` + +`mount()` and `url_for()` keep the `/v{n}` prefix of the mounted router; +`KeyAuth` middleware is registered in the Security/SessionAuth slot and stores +the key via `req.set_ext("gov_key", key)`. + +--- + +## 4. `src/app/errors/http_error.h`: Gone (410) + +```diff +--- a/src/app/errors/http_error.h ++++ b/src/app/errors/http_error.h +@@ class TooManyRequests + class TooManyRequests : public HttpError { + public: + explicit TooManyRequests(std::string_view msg = "Too Many Requests") + : HttpError(429, "too_many_requests", msg) {} + }; ++class Gone : public HttpError { ++ public: ++ explicit Gone(std::string_view msg = "Gone") : HttpError(410, "gone", msg) {} ++}; +``` + +--- + +## 5. `src/app/errors/handler.h`: problem+json plus header passthrough + +```diff +--- a/src/app/errors/handler.h ++++ b/src/app/errors/handler.h +@@ class ErrorHandler + Response render(const HttpError& err, const Request& req, std::string_view rid); + Response render_unknown(const std::exception& ex, const Request& req, + std::string_view rid); ++ // Governance: RFC 9457 renderer for problem bodies. Copies Retry-After ++ // and Link headers from the source headers map when present. ++ Response render_problem(int status, std::string_view type, std::string_view title, ++ std::string_view detail, const Headers& extra); +``` + +--- + +## 6. `src/clients/openapi/builder.h`: honor governance declarations + +```diff +--- a/src/clients/openapi/builder.h ++++ b/src/clients/openapi/builder.h +@@ class Builder + Document build(const BuilderInput& input); + void extend_hook(Hook hook); ++ // Governance: registers the hook that copies RouteOptions gov_* fields ++ // into RouteDecl::deprecated and RouteDecl::security so the OpenAPI ++ // document and the runtime headers share one source. ++ void use_governance_declarations(); +``` + +```diff +--- a/src/clients/openapi/builder.cpp ++++ b/src/clients/openapi/builder.cpp +@@ Builder::build + Document Builder::build(const BuilderInput& input) { ++ ApplyGovernanceDeclarations(input); + // ... existing build work ... + } +``` + +--- + +## 7. `src/security/guard/RateLimiter.h`: tier/key budget dimension + +```diff +--- a/src/security/guard/RateLimiter.h ++++ b/src/security/guard/RateLimiter.h +@@ struct Budget + struct Budget { + long per_min = 0; + long burst_mult = 0; + long burst_sec = 0; + }; ++// Governance: per-(tier, key, route-prefix) budget lookup with ++// downward-only override. Counting algorithm unchanged; governance ++// supplies the limits and the response headers. ++struct TierBudgetKey { ++ std::string tier; ++ std::string key_id; ++ std::string route_prefix; ++}; +@@ class RateLimiter + Verdict check(const HttpRequestView& req, std::string_view user_id); + Verdict enforce(const HttpRequestView& req, HttpResponseView& res, + std::string_view user_id); ++ Verdict check_tier(const HttpRequestView& req, const TierBudgetKey& gov); ++ Verdict enforce_tier(const HttpRequestView& req, HttpResponseView& res, ++ const TierBudgetKey& gov); +``` + +```diff +--- a/src/security/guard/RateLimiter.cpp ++++ b/src/security/guard/RateLimiter.cpp +@@ RateLimiter::enforce ++RateLimiter::Verdict RateLimiter::check_tier(const HttpRequestView& req, ++ const TierBudgetKey& gov) { ++ // Resolve limits via molo::govern::tiers::TierPolicy::Resolve with the ++ // route tier override, then run the existing counting path. ++ return check_with_budget(req, LookupTierBudget(gov)); ++} ++ ++RateLimiter::Verdict RateLimiter::enforce_tier(const HttpRequestView& req, ++ HttpResponseView& res, ++ const TierBudgetKey& gov) { ++ Verdict v = check_tier(req, gov); ++ if (!v.allowed) ApplyTierHeaders(res, gov, v); ++ return v; ++} +``` + +--- + +## 8. `Makefile`: include the governance fragment + +```diff +--- a/Makefile ++++ b/Makefile +@@ includes + include src/app/app.mk ++include src/governance.mk +``` + +--- + +## Base files touched + +- `src/app/routing/route.h` +- `src/app/routing/router.h` +- `src/app/app/application.h` +- `src/app/errors/http_error.h` +- `src/app/errors/handler.h` +- `src/clients/openapi/builder.h` +- `src/clients/openapi/builder.cpp` +- `src/security/guard/RateLimiter.h` +- `src/security/guard/RateLimiter.cpp` +- `Makefile` diff --git a/src/governance/adapters/BaseAdapters.h b/src/governance/adapters/BaseAdapters.h new file mode 100644 index 0000000..dce8563 --- /dev/null +++ b/src/governance/adapters/BaseAdapters.h @@ -0,0 +1,40 @@ +#pragma once + +// ADAPTER: minimal local stand-ins for base-tree HTTP types so this part +// compiles standalone in an isolated worktree. The real base headers +// (src/app/http_msg/...) replace these at integration time; production code +// paths use the map/string overloads, not these structs. + +#include +#include +#include + +namespace molo::govern::adapters { + +struct HttpHeaders { + std::map fields; + + void set(std::string name, std::string value) { fields[std::move(name)] = std::move(value); } + [[nodiscard]] std::string get(std::string_view name) const { + auto it = fields.find(std::string(name)); + return it == fields.end() ? std::string() : it->second; + } + [[nodiscard]] bool contains(std::string_view name) const { + return fields.find(std::string(name)) != fields.end(); + } +}; + +struct HttpRequest { + std::string method = "GET"; + std::string path = "/"; + HttpHeaders headers; + std::string peer_ip = "127.0.0.1"; +}; + +struct HttpResponse { + int status = 200; + HttpHeaders headers; + std::string body; +}; + +} // namespace molo::govern::adapters diff --git a/src/governance/caps.h b/src/governance/caps.h new file mode 100644 index 0000000..fae5eae --- /dev/null +++ b/src/governance/caps.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +namespace molo::govern { + +// Static ceilings for the governance part. No autotuning; every knob has a +// readable constant. Mirrors src/governance/caps.h in the base tree layout. +struct Caps { + static constexpr int kMinMajor = 1; + static constexpr int kMaxMajor = 99; + static constexpr int kMaxLiveMajors = 3; + + static constexpr std::size_t kMaxScopesPerKey = 64; + static constexpr std::size_t kMaxScopeLen = 64; + static constexpr std::size_t kMaxKeys = 100000; + static constexpr std::size_t kKeyIdLen = 16; + static constexpr std::size_t kKeySecretBytes = 32; + static constexpr std::size_t kKeySecretB64Len = 43; + static constexpr long kRotateOverlapSec = 86400; + static constexpr long kDefaultKeyTtlSec = 31536000; + + static constexpr long kRetryAfterMaxSec = 300; + static constexpr std::size_t kMaxQuotaRows = 100000; + + static constexpr std::size_t kMaxPaths = 5000; + static constexpr std::size_t kMaxSpecBytes = 10u * 1024u * 1024u; + static constexpr std::size_t kMaxRoutes = 5000; + static constexpr std::size_t kMaxFindings = 4096; + static constexpr std::size_t kMaxChanges = 65536; + static constexpr std::size_t kMaxDeprecationRules = 1024; +}; + +} // namespace molo::govern diff --git a/src/governance/diff/ApiDiffCli.h b/src/governance/diff/ApiDiffCli.h new file mode 100644 index 0000000..807c23d --- /dev/null +++ b/src/governance/diff/ApiDiffCli.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "GovernGate.h" +#include "SpecDiffer.h" +#include "SpecSnapshot.h" + +namespace molo::govern::diff { + +// CLI: apidiff breaking|changelog|diff +// Exit 0 on success (breaking exits 1 when breaking changes exist). +class ApiDiffCli { + public: + using Reader = std::function; + + static int Run(int argc, char** argv) { + std::vector args; + for (int i = 0; i < argc; ++i) args.emplace_back(argv[i]); + return Execute(args, std::cout, std::cerr, ReadFile); + } + + static int Execute(const std::vector& args, std::ostream& out, + std::ostream& err, Reader reader) { + if (args.size() != 4) { + err << "usage: apidiff breaking|changelog|diff \n"; + return 2; + } + const std::string& cmd = args[1]; + if (cmd != "breaking" && cmd != "changelog" && cmd != "diff") { + err << "unknown command: " << cmd << "\n"; + return 2; + } + std::string old_text; + std::string new_text; + std::string io_error; + if (!reader(args[2], old_text, io_error)) { + err << "cannot read " << args[2] << ": " << io_error << "\n"; + return 2; + } + if (!reader(args[3], new_text, io_error)) { + err << "cannot read " << args[3] << ": " << io_error << "\n"; + return 2; + } + SpecSnapshot old_snap; + SpecSnapshot new_snap; + std::string error; + if (!SpecSnapshot::Parse(old_text, old_snap, error)) { + err << "bad old spec: " << error << "\n"; + return 2; + } + if (!SpecSnapshot::Parse(new_text, new_snap, error)) { + err << "bad new spec: " << error << "\n"; + return 2; + } + const std::vector changes = SpecDiffer::Diff(old_snap, new_snap); + if (cmd == "breaking") { + out << GovernGate::Summary(changes) << "\n"; + if (!changes.empty()) out << GovernGate::DiffText(old_snap, new_snap); + return GovernGate::Evaluate(changes) == GovernGate::Verdict::kPass ? 0 : 1; + } + if (cmd == "changelog") { + out << GovernGate::Changelog(changes); + return 0; + } + out << GovernGate::DiffText(old_snap, new_snap); + return 0; + } + + private: + static bool ReadFile(const std::string& path, std::string& text, std::string& error) { + std::ifstream f(path, std::ios::binary); + if (!f) { + error = "open failed"; + return false; + } + std::ostringstream ss; + ss << f.rdbuf(); + text = ss.str(); + if (text.size() > Caps::kMaxSpecBytes) { + error = "file exceeds kMaxSpecBytes"; + return false; + } + return true; + } +}; + +} // namespace molo::govern::diff diff --git a/src/governance/diff/BreakingRules.h b/src/governance/diff/BreakingRules.h new file mode 100644 index 0000000..4f5e17c --- /dev/null +++ b/src/governance/diff/BreakingRules.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include + +namespace molo::govern::diff { + +// Breaking-change rule vocabulary for spec diffs. ERR rules fail the govern +// gate, WARN rules need human review, INFO rules are changelog entries. +class BreakingRules { + public: + enum class Severity { kInfo, kWarning, kError }; + + struct Change { + Severity severity = Severity::kInfo; + std::string rule; + std::string target; + std::string message; + }; + + [[nodiscard]] static bool IsBreaking(Severity s) { return s == Severity::kError; } + + [[nodiscard]] static std::string_view SeverityName(Severity s) { + switch (s) { + case Severity::kError: return "ERR"; + case Severity::kWarning: return "WARN"; + case Severity::kInfo: return "INFO"; + } + return "INFO"; + } + + // Rule codes (stable, referenced by golden vectors): + // ERR: removed-endpoint, removed-required-param, new-required-param, + // param-type-changed, param-now-required, removed-response, auth-added + // WARN: removed-optional-param, deprecated, operation-id-changed, + // auth-removed, response-type-changed + // INFO: added-endpoint, added-optional-param, undeprecated, + // summary-changed, added-response +}; + +} // namespace molo::govern::diff diff --git a/src/governance/diff/GovernGate.cpp b/src/governance/diff/GovernGate.cpp new file mode 100644 index 0000000..111d146 --- /dev/null +++ b/src/governance/diff/GovernGate.cpp @@ -0,0 +1,76 @@ +#include "governance/diff/GovernGate.h" + +#include "governance/diff/SpecDiffer.h" + +namespace molo::govern::diff { + +GovernGate::Verdict GovernGate::Evaluate( + const std::vector& changes) { + for (const auto& c : changes) { + if (BreakingRules::IsBreaking(c.severity)) return Verdict::kFail; + } + return Verdict::kPass; +} + +std::string GovernGate::Changelog(const std::vector& changes) { + std::string out; + const char* sections[3] = {"Breaking", "Warnings", "Changes"}; + const BreakingRules::Severity kinds[3] = {BreakingRules::Severity::kError, + BreakingRules::Severity::kWarning, + BreakingRules::Severity::kInfo}; + for (int s = 0; s < 3; ++s) { + bool header = false; + for (const auto& c : changes) { + if (c.severity != kinds[s]) continue; + if (!header) { + out += "## "; + out += sections[s]; + out += "\n"; + header = true; + } + out += "- ["; + out += c.rule; + out += "] "; + out += c.target; + out += ": "; + out += c.message; + out += "\n"; + } + } + if (out.empty()) out = "No changes.\n"; + return out; +} + +std::string GovernGate::DiffText(const SpecSnapshot& old_snap, const SpecSnapshot& new_snap) { + const std::vector changes = SpecDiffer::Diff(old_snap, new_snap); + std::string out; + for (const auto& c : changes) { + out += std::string(BreakingRules::SeverityName(c.severity)); + out += " "; + out += c.rule; + out += " "; + out += c.target; + out += " :: "; + out += c.message; + out += "\n"; + } + if (out.empty()) out = "No changes.\n"; + return out; +} + +std::string GovernGate::Summary(const std::vector& changes) { + long err = 0; + long warn = 0; + long info = 0; + for (const auto& c : changes) { + if (c.severity == BreakingRules::Severity::kError) ++err; + if (c.severity == BreakingRules::Severity::kWarning) ++warn; + if (c.severity == BreakingRules::Severity::kInfo) ++info; + } + std::string out = "ERR=" + std::to_string(err) + " WARN=" + std::to_string(warn) + + " INFO=" + std::to_string(info) + + (err == 0 ? " PASS" : " FAIL"); + return out; +} + +} // namespace molo::govern::diff diff --git a/src/governance/diff/GovernGate.h b/src/governance/diff/GovernGate.h new file mode 100644 index 0000000..b006244 --- /dev/null +++ b/src/governance/diff/GovernGate.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +#include "BreakingRules.h" +#include "SpecSnapshot.h" + +namespace molo::govern::diff { + +// Release gate over spec changes: any ERR change fails the gate. +class GovernGate { + public: + enum class Verdict { kPass, kFail }; + + [[nodiscard]] static Verdict Evaluate(const std::vector& changes); + [[nodiscard]] static std::string Changelog(const std::vector& changes); + [[nodiscard]] static std::string DiffText(const SpecSnapshot& old_snap, + const SpecSnapshot& new_snap); + [[nodiscard]] static std::string Summary(const std::vector& changes); +}; + +} // namespace molo::govern::diff diff --git a/src/governance/diff/SpecDiffer.h b/src/governance/diff/SpecDiffer.h new file mode 100644 index 0000000..56dbb39 --- /dev/null +++ b/src/governance/diff/SpecDiffer.h @@ -0,0 +1,159 @@ +#pragma once + +#include +#include +#include + +#include "../caps.h" +#include "BreakingRules.h" +#include "SpecSnapshot.h" + +namespace molo::govern::diff { + +// Compares two spec snapshots and emits ordered rule changes. Output order +// is deterministic: endpoints sorted by target, rules in fixed order. +class SpecDiffer { + public: + [[nodiscard]] static std::vector Diff(const SpecSnapshot& old_snap, + const SpecSnapshot& new_snap) { + std::vector out; + const auto push = [&](BreakingRules::Severity sev, const std::string& rule, + const std::string& target, const std::string& msg) { + if (out.size() >= Caps::kMaxChanges) return; + BreakingRules::Change c; + c.severity = sev; + c.rule = rule; + c.target = target; + c.message = msg; + out.push_back(std::move(c)); + }; + + std::vector olds; + std::vector news; + for (const auto& e : old_snap.endpoints) olds.push_back(&e); + for (const auto& e : new_snap.endpoints) news.push_back(&e); + const auto by_target = [](const SpecSnapshot::Endpoint* a, + const SpecSnapshot::Endpoint* b) { + if (a->path != b->path) return a->path < b->path; + return a->method < b->method; + }; + std::sort(olds.begin(), olds.end(), by_target); + std::sort(news.begin(), news.end(), by_target); + + for (const auto* o : olds) { + const SpecSnapshot::Endpoint* n = new_snap.Find(o->method, o->path); + const std::string target = o->method + " " + o->path; + if (n == nullptr) { + push(BreakingRules::Severity::kError, "removed-endpoint", target, + "endpoint removed"); + continue; + } + if (o->operation_id != n->operation_id) { + push(BreakingRules::Severity::kWarning, "operation-id-changed", target, + "operation_id " + o->operation_id + " -> " + n->operation_id); + } + if (!o->deprecated && n->deprecated) { + push(BreakingRules::Severity::kWarning, "deprecated", target, "endpoint deprecated"); + } + if (o->deprecated && !n->deprecated) { + push(BreakingRules::Severity::kInfo, "undeprecated", target, + "endpoint undeprecated"); + } + if (o->summary != n->summary) { + push(BreakingRules::Severity::kInfo, "summary-changed", target, "summary changed"); + } + if (AuthLevel(o->security) < AuthLevel(n->security)) { + push(BreakingRules::Severity::kError, "auth-added", target, + "auth requirement added: " + n->security); + } else if (AuthLevel(o->security) > AuthLevel(n->security)) { + push(BreakingRules::Severity::kWarning, "auth-removed", target, + "auth requirement removed: " + n->security); + } + DiffParams(*o, *n, target, push); + DiffResponses(*o, *n, target, push); + } + for (const auto* n : news) { + if (old_snap.Find(n->method, n->path) == nullptr) { + push(BreakingRules::Severity::kInfo, "added-endpoint", n->method + " " + n->path, + "endpoint added"); + } + } + return out; + } + + private: + template + static void DiffParams(const SpecSnapshot::Endpoint& o, const SpecSnapshot::Endpoint& n, + const std::string& target, Push& push) { + const auto find = [](const std::vector& ps, + const std::string& name) -> const SpecSnapshot::Param* { + for (const auto& p : ps) { + if (p.name == name) return &p; + } + return nullptr; + }; + for (const auto& op : o.params) { + const SpecSnapshot::Param* np = find(n.params, op.name); + if (np == nullptr) { + if (op.required) { + push(BreakingRules::Severity::kError, "removed-required-param", target, + "required param removed: " + op.name); + } else { + push(BreakingRules::Severity::kWarning, "removed-optional-param", target, + "optional param removed: " + op.name); + } + continue; + } + if (op.type != np->type) { + push(BreakingRules::Severity::kError, "param-type-changed", target, + "param " + op.name + " type " + op.type + " -> " + np->type); + } + if (!op.required && np->required) { + push(BreakingRules::Severity::kError, "param-now-required", target, + "param " + op.name + " is now required"); + } + } + for (const auto& np : n.params) { + if (find(o.params, np.name) == nullptr) { + if (np.required) { + push(BreakingRules::Severity::kError, "new-required-param", target, + "new required param: " + np.name); + } else { + push(BreakingRules::Severity::kInfo, "added-optional-param", target, + "new optional param: " + np.name); + } + } + } + } + + template + static void DiffResponses(const SpecSnapshot::Endpoint& o, const SpecSnapshot::Endpoint& n, + const std::string& target, Push& push) { + for (const auto& [code, model] : o.responses) { + auto it = n.responses.find(code); + if (it == n.responses.end()) { + push(BreakingRules::Severity::kError, "removed-response", target, + "response removed: " + std::to_string(code)); + } else if (it->second != model) { + push(BreakingRules::Severity::kWarning, "response-type-changed", target, + "response " + std::to_string(code) + " model " + model + " -> " + it->second); + } + } + for (const auto& [code, model] : n.responses) { + if (o.responses.find(code) == o.responses.end()) { + push(BreakingRules::Severity::kInfo, "added-response", target, + "response added: " + std::to_string(code)); + } + } + } + + [[nodiscard]] static int AuthLevel(const std::string& security) { + if (security.empty() || security == "none") return 0; + if (security == "key") return 1; + if (security == "key+scopes") return 2; + if (security == "session") return 3; + return 1; + } +}; + +} // namespace molo::govern::diff diff --git a/src/governance/diff/SpecSnapshot.h b/src/governance/diff/SpecSnapshot.h new file mode 100644 index 0000000..1334b02 --- /dev/null +++ b/src/governance/diff/SpecSnapshot.h @@ -0,0 +1,392 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../caps.h" + +namespace molo::govern::diff { + +// Canonical API spec snapshot, stored at specs/api/v{n}.json in the base +// tree. Minimal in-house JSON reader, stdlib only, capped at kMaxSpecBytes. +class SpecSnapshot { + public: + struct Param { + std::string name; + bool required = false; + std::string type; + }; + + struct Endpoint { + std::string method; + std::string path; + std::string operation_id; + std::string summary; + std::string example; + bool deprecated = false; + std::string security; // "", "none", "key", "key+scopes", "session" + std::vector params; + std::map responses; // status -> model name + }; + + std::vector endpoints; + + [[nodiscard]] const Endpoint* Find(std::string_view method, std::string_view path) const { + for (const auto& e : endpoints) { + if (e.method == method && e.path == path) return &e; + } + return nullptr; + } + + [[nodiscard]] std::string Target(const Endpoint& e) const { + (void)this; + return e.method + " " + e.path; + } + + // Canonical JSON: endpoints sorted by (path, method), stable field order. + [[nodiscard]] std::string CanonicalJson() const { + std::vector order; + for (const auto& e : endpoints) order.push_back(&e); + std::sort(order.begin(), order.end(), [](const Endpoint* a, const Endpoint* b) { + if (a->path != b->path) return a->path < b->path; + return a->method < b->method; + }); + std::string out = "{\"version\":1,\"endpoints\":["; + bool first = true; + for (const Endpoint* e : order) { + if (!first) out += ","; + first = false; + out += "{\"method\":\"" + Escape(e->method) + "\",\"path\":\"" + Escape(e->path) + + "\",\"operation_id\":\"" + Escape(e->operation_id) + "\",\"summary\":\"" + + Escape(e->summary) + "\",\"example\":\"" + Escape(e->example) + + "\",\"deprecated\":" + (e->deprecated ? "true" : "false") + ",\"security\":\"" + + Escape(e->security) + "\",\"params\":["; + bool pf = true; + std::vector ps = e->params; + std::sort(ps.begin(), ps.end(), + [](const Param& a, const Param& b) { return a.name < b.name; }); + for (const auto& p : ps) { + if (!pf) out += ","; + pf = false; + out += "{\"name\":\"" + Escape(p.name) + "\",\"required\":" + + (p.required ? "true" : "false") + ",\"type\":\"" + Escape(p.type) + "\"}"; + } + out += "],\"responses\":{"; + bool rf = true; + for (const auto& [code, model] : e->responses) { + if (!rf) out += ","; + rf = false; + out += "\"" + std::to_string(code) + "\":\"" + Escape(model) + "\""; + } + out += "}}"; + } + out += "]}"; + return out; + } + + // Parse a snapshot document. Returns true on success; error holds the + // reason on failure. + static bool Parse(std::string_view text, SpecSnapshot& out, std::string& error) { + if (text.size() > Caps::kMaxSpecBytes) { + error = "spec exceeds kMaxSpecBytes"; + return false; + } + Parser p(text); + Value root; + if (!p.ParseValue(root) || !p.End()) { + error = "invalid JSON at offset " + std::to_string(p.pos); + return false; + } + if (root.kind != Value::Kind::kObject) { + error = "spec root must be an object"; + return false; + } + const Value* eps = ObjGet(root, "endpoints"); + if (eps == nullptr || eps->kind != Value::Kind::kArray) { + error = "spec.endpoints must be an array"; + return false; + } + SpecSnapshot snap; + for (const auto& ev : eps->items) { + if (ev.kind != Value::Kind::kObject) { + error = "endpoint must be an object"; + return false; + } + if (snap.endpoints.size() >= Caps::kMaxPaths) { + error = "spec exceeds kMaxPaths"; + return false; + } + Endpoint e; + e.method = StrField(ev, "method"); + e.path = StrField(ev, "path"); + e.operation_id = StrField(ev, "operation_id"); + e.summary = StrField(ev, "summary"); + e.example = StrField(ev, "example"); + e.deprecated = BoolField(ev, "deprecated"); + e.security = StrField(ev, "security"); + if (e.method.empty() || e.path.empty()) { + error = "endpoint needs method and path"; + return false; + } + const Value* params = ObjGet(ev, "params"); + if (params != nullptr) { + if (params->kind != Value::Kind::kArray) { + error = "endpoint.params must be an array"; + return false; + } + for (const auto& pv : params->items) { + if (pv.kind != Value::Kind::kObject) { + error = "param must be an object"; + return false; + } + Param pr; + pr.name = StrField(pv, "name"); + pr.required = BoolField(pv, "required"); + pr.type = StrField(pv, "type"); + if (pr.name.empty()) { + error = "param needs a name"; + return false; + } + e.params.push_back(std::move(pr)); + } + } + const Value* resp = ObjGet(ev, "responses"); + if (resp != nullptr) { + if (resp->kind != Value::Kind::kObject) { + error = "endpoint.responses must be an object"; + return false; + } + for (std::size_t i = 0; i < resp->keys.size(); ++i) { + int code = 0; + try { + code = std::stoi(resp->keys[i]); + } catch (...) { + error = "response key must be a status code"; + return false; + } + e.responses[code] = resp->items[i].str; + } + } + snap.endpoints.push_back(std::move(e)); + } + out = std::move(snap); + return true; + } + + private: + struct Value { + enum class Kind { kNull, kBool, kNum, kStr, kArray, kObject }; + Kind kind = Kind::kNull; + bool boolean = false; + std::string str; + std::vector items; + std::vector keys; // parallel to items for objects + }; + + struct Parser { + std::string_view s; + std::size_t pos = 0; + explicit Parser(std::string_view v) : s(v) {} + + bool End() { + SkipWs(); + return pos == s.size(); + } + void SkipWs() { + while (pos < s.size() && + (s[pos] == ' ' || s[pos] == '\t' || s[pos] == '\n' || s[pos] == '\r')) { + ++pos; + } + } + bool ParseValue(Value& out) { + SkipWs(); + if (pos >= s.size()) return false; + const char c = s[pos]; + if (c == '{') return ParseObject(out); + if (c == '[') return ParseArray(out); + if (c == '"') { + out.kind = Value::Kind::kStr; + return ParseString(out.str); + } + if (c == 't' && s.substr(pos, 4) == "true") { + pos += 4; + out.kind = Value::Kind::kBool; + out.boolean = true; + return true; + } + if (c == 'f' && s.substr(pos, 5) == "false") { + pos += 5; + out.kind = Value::Kind::kBool; + out.boolean = false; + return true; + } + if (c == 'n' && s.substr(pos, 4) == "null") { + pos += 4; + out.kind = Value::Kind::kNull; + return true; + } + if (c == '-' || (c >= '0' && c <= '9')) { + out.kind = Value::Kind::kNum; + out.str.clear(); + while (pos < s.size() && + (s[pos] == '-' || s[pos] == '+' || s[pos] == '.' || s[pos] == 'e' || + s[pos] == 'E' || (s[pos] >= '0' && s[pos] <= '9'))) { + out.str += s[pos++]; + } + return !out.str.empty(); + } + return false; + } + bool ParseObject(Value& out) { + ++pos; // { + out.kind = Value::Kind::kObject; + SkipWs(); + if (pos < s.size() && s[pos] == '}') { + ++pos; + return true; + } + while (true) { + SkipWs(); + if (pos >= s.size() || s[pos] != '"') return false; + std::string key; + if (!ParseString(key)) return false; + SkipWs(); + if (pos >= s.size() || s[pos] != ':') return false; + ++pos; + Value v; + if (!ParseValue(v)) return false; + out.keys.push_back(std::move(key)); + out.items.push_back(std::move(v)); + SkipWs(); + if (pos >= s.size()) return false; + if (s[pos] == ',') { + ++pos; + continue; + } + if (s[pos] == '}') { + ++pos; + return true; + } + return false; + } + } + bool ParseArray(Value& out) { + ++pos; // [ + out.kind = Value::Kind::kArray; + SkipWs(); + if (pos < s.size() && s[pos] == ']') { + ++pos; + return true; + } + while (true) { + Value v; + if (!ParseValue(v)) return false; + out.items.push_back(std::move(v)); + SkipWs(); + if (pos >= s.size()) return false; + if (s[pos] == ',') { + ++pos; + continue; + } + if (s[pos] == ']') { + ++pos; + return true; + } + return false; + } + } + bool ParseString(std::string& out) { + ++pos; // opening quote + out.clear(); + while (pos < s.size()) { + const char c = s[pos++]; + if (c == '"') return true; + if (c == '\\') { + if (pos >= s.size()) return false; + const char e = s[pos++]; + switch (e) { + case '"': out += '"'; break; + case '\\': out += '\\'; break; + case '/': out += '/'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'u': { + if (pos + 4 > s.size()) return false; + unsigned cp = 0; + for (int i = 0; i < 4; ++i) { + const char h = s[pos++]; + cp <<= 4; + if (h >= '0' && h <= '9') cp |= static_cast(h - '0'); + else if (h >= 'a' && h <= 'f') + cp |= static_cast(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') + cp |= static_cast(h - 'A' + 10); + else + return false; + } + if (cp < 0x80) { + out += static_cast(cp); + } else if (cp < 0x800) { + out += static_cast(0xC0 | (cp >> 6)); + out += static_cast(0x80 | (cp & 0x3F)); + } else { + out += static_cast(0xE0 | (cp >> 12)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } + break; + } + default: return false; + } + } else { + out += c; + } + } + return false; + } + }; + + [[nodiscard]] static const Value* ObjGet(const Value& obj, std::string_view key) { + for (std::size_t i = 0; i < obj.keys.size(); ++i) { + if (obj.keys[i] == key) return &obj.items[i]; + } + return nullptr; + } + [[nodiscard]] static std::string StrField(const Value& obj, std::string_view key) { + const Value* v = ObjGet(obj, key); + if (v == nullptr) return {}; + if (v->kind == Value::Kind::kStr || v->kind == Value::Kind::kNum) return v->str; + return {}; + } + [[nodiscard]] static bool BoolField(const Value& obj, std::string_view key) { + const Value* v = ObjGet(obj, key); + return v != nullptr && v->kind == Value::Kind::kBool && v->boolean; + } + + [[nodiscard]] static std::string Escape(std::string_view in) { + std::string out; + for (char c : in) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (static_cast(c) < 0x20) { + out += "?"; + } else { + out += c; + } + } + } + return out; + } +}; + +} // namespace molo::govern::diff diff --git a/src/governance/diff/apidiff_main.cpp b/src/governance/diff/apidiff_main.cpp new file mode 100644 index 0000000..76907ef --- /dev/null +++ b/src/governance/diff/apidiff_main.cpp @@ -0,0 +1,4 @@ +// apidiff CLI entry point. See ApiDiffCli for commands. +#include "governance/diff/ApiDiffCli.h" + +int main(int argc, char** argv) { return molo::govern::diff::ApiDiffCli::Run(argc, argv); } diff --git a/src/governance/keys/ApiKey.h b/src/governance/keys/ApiKey.h new file mode 100644 index 0000000..d12f7a0 --- /dev/null +++ b/src/governance/keys/ApiKey.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include "Scope.h" + +namespace molo::govern::keys { + +// Stored API key record. Only the hash of the secret token is stored; the +// secret itself is shown once at mint time. +class ApiKey { + public: + std::string id; + std::string prefix; + std::string key_hash; + std::vector scopes; + std::string tier = "free"; + long created_unix = 0; + long expires_unix = 0; // 0 means never expires + bool revoked = false; + + [[nodiscard]] bool Live(long now_unix) const { + if (revoked) return false; + if (expires_unix != 0 && now_unix >= expires_unix) return false; + return true; + } + + [[nodiscard]] bool HasScope(const std::string& required) const { + return Scope::AllowsAny(scopes, required); + } +}; + +} // namespace molo::govern::keys diff --git a/src/governance/keys/KeyAuth.h b/src/governance/keys/KeyAuth.h new file mode 100644 index 0000000..8adc0fe --- /dev/null +++ b/src/governance/keys/KeyAuth.h @@ -0,0 +1,109 @@ +#pragma once + +#include +#include +#include +#include + +#include "../adapters/BaseAdapters.h" +#include "../problem.h" +#include "KeyIssuer.h" +#include "KeyStore.h" + +namespace molo::govern::keys { + +// Authenticates requests by Bearer token or X-Api-Key header. At integration +// time this runs as a Security/SessionAuth middleware and stores the key in +// request extensions under "gov_key" (see PATCHES.md). +class KeyAuth { + public: + struct AuthResult { + bool ok = false; + int http_status = 401; + std::string code; + std::optional key; + }; + + explicit KeyAuth(KeyStore& store, long now_unix) : store_(store), now_(now_unix) {} + void set_now(long v) { now_ = v; } + + [[nodiscard]] AuthResult Authenticate(std::string_view authorization, + std::string_view api_key_header) const { + std::string token = ExtractToken(authorization, api_key_header); + if (token.empty()) { + return Fail(401, "unauthorized"); + } + std::string id; + std::string secret; + if (!KeyIssuer::ParseToken(token, id, secret)) { + return Fail(401, "unauthorized"); + } + auto found = store_.Find(id); + if (!found.has_value()) { + return Fail(401, "unauthorized"); + } + if (found->key_hash != KeyIssuer::HashToken(token)) { + return Fail(401, "unauthorized"); + } + if (!found->Live(now_)) { + return Fail(401, "expired_key"); + } + AuthResult ok; + ok.ok = true; + ok.http_status = 200; + ok.code = "ok"; + ok.key = *found; + return ok; + } + + [[nodiscard]] AuthResult AuthenticateRequest(const adapters::HttpRequest& req) const { + return Authenticate(req.headers.get("Authorization"), req.headers.get("X-Api-Key")); + } + + [[nodiscard]] static AuthResult RequireScopes(const AuthResult& auth, + const std::vector& required) { + if (!auth.ok || !auth.key.has_value()) return auth; + for (const auto& need : required) { + if (!auth.key->HasScope(need)) { + return Fail(403, "insufficient_scope"); + } + } + return auth; + } + + [[nodiscard]] static Problem ToProblem(const AuthResult& auth) { + if (auth.code == "expired_key") return Problem::ExpiredKey(); + if (auth.code == "insufficient_scope") return Problem::InsufficientScope(); + return Problem::Unauthorized(); + } + + private: + [[nodiscard]] static AuthResult Fail(int status, std::string_view code) { + AuthResult r; + r.ok = false; + r.http_status = status; + r.code = std::string(code); + return r; + } + + [[nodiscard]] static std::string ExtractToken(std::string_view authorization, + std::string_view api_key_header) { + constexpr std::string_view kBearer = "Bearer "; + if (authorization.size() > kBearer.size() && + authorization.substr(0, kBearer.size()) == kBearer) { + std::string t(authorization.substr(kBearer.size())); + while (!t.empty() && (t.back() == ' ' || t.back() == '\t')) t.pop_back(); + while (!t.empty() && (t.front() == ' ' || t.front() == '\t')) t.erase(t.begin()); + return t; + } + std::string t(api_key_header); + while (!t.empty() && (t.back() == ' ' || t.back() == '\t' || t.back() == '\n')) t.pop_back(); + while (!t.empty() && (t.front() == ' ' || t.front() == '\t')) t.erase(t.begin()); + return t; + } + + KeyStore& store_; + long now_; +}; + +} // namespace molo::govern::keys diff --git a/src/governance/keys/KeyIssuer.h b/src/governance/keys/KeyIssuer.h new file mode 100644 index 0000000..4be83f0 --- /dev/null +++ b/src/governance/keys/KeyIssuer.h @@ -0,0 +1,226 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "ApiKey.h" +#include "KeyStore.h" +#include "Scope.h" + +namespace molo::govern::keys { + +// Mints, rotates and revokes API keys. +// Token format: molo__ where secret43 is 32 random bytes in +// unpadded base64url. Only SHA-256(token) is stored. +class KeyIssuer { + public: + struct Issued { + ApiKey record; + std::string token; + }; + + explicit KeyIssuer(KeyStore& store, long now_unix) : store_(store), now_(now_unix) {} + void set_now(long v) { now_ = v; } + + // Mint a new key. ttl_sec 0 means never expires. + [[nodiscard]] Issued Mint(std::string_view tier, const std::vector& scopes, + long ttl_sec = Caps::kDefaultKeyTtlSec) { + Issued out; + out.token = MakeToken(RandomId(), RandomSecret()); + out.record.id = TokenId(out.token); + out.record.prefix = out.token.substr(0, 5 + Caps::kKeyIdLen); + out.record.key_hash = HashToken(out.token); + out.record.tier = std::string(tier); + for (const auto& s : scopes) { + if (Scope::Valid(s) && out.record.scopes.size() < Caps::kMaxScopesPerKey) { + out.record.scopes.push_back(s); + } + } + out.record.created_unix = now_; + out.record.expires_unix = ttl_sec == 0 ? 0 : now_ + ttl_sec; + out.record.revoked = false; + store_.Put(out.record); + return out; + } + + // Rotate: mint a replacement key and revoke the old one after the overlap + // window by setting its expiry to now + overlap. + [[nodiscard]] Issued Rotate(const std::string& id, + long overlap_sec = Caps::kRotateOverlapSec) { + Issued out; + auto old = store_.Find(id); + if (!old.has_value()) return out; + out = Mint(old->tier, old->scopes, 0); + ApiKey retired = *old; + retired.expires_unix = now_ + overlap_sec; + store_.Put(retired); + return out; + } + + bool Revoke(const std::string& id) { return store_.Revoke(id); } + + [[nodiscard]] static bool ParseToken(std::string_view token, std::string& id_out, + std::string& secret_out) { + constexpr std::string_view kPrefix = "molo_"; + if (token.size() != kPrefix.size() + Caps::kKeyIdLen + 1 + Caps::kKeySecretB64Len) { + return false; + } + if (token.substr(0, kPrefix.size()) != kPrefix) return false; + if (token[kPrefix.size() + Caps::kKeyIdLen] != '_') return false; + id_out = std::string(token.substr(kPrefix.size(), Caps::kKeyIdLen)); + secret_out = + std::string(token.substr(kPrefix.size() + Caps::kKeyIdLen + 1, Caps::kKeySecretB64Len)); + for (char c : id_out) { + const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9'); + if (!ok) return false; + } + for (char c : secret_out) { + const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '-' || c == '_'; + if (!ok) return false; + } + return true; + } + + [[nodiscard]] static std::string TokenId(std::string_view token) { + std::string id; + std::string secret; + if (!ParseToken(token, id, secret)) return {}; + return id; + } + + [[nodiscard]] static std::string HashToken(std::string_view token) { + return Sha256Hex(token); + } + + [[nodiscard]] static std::string MakeToken(std::string_view id, std::string_view secret) { + return "molo_" + std::string(id) + "_" + std::string(secret); + } + + private: + [[nodiscard]] static std::string RandomId() { + static constexpr char kAlphabet[] = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + std::string out; + out.reserve(Caps::kKeyIdLen); + for (unsigned char b : RandomBytes(Caps::kKeyIdLen)) { + out += kAlphabet[b % 62]; + } + return out; + } + + [[nodiscard]] static std::string RandomSecret() { + return Base64UrlNoPad(RandomBytes(Caps::kKeySecretBytes)); + } + + [[nodiscard]] static std::vector RandomBytes(std::size_t n) { + std::vector out(n, 0); + bool filled = false; + if (FILE* f = std::fopen("/dev/urandom", "rb")) { + const std::size_t got = std::fread(out.data(), 1, n, f); + std::fclose(f); + filled = got == n; + } + if (!filled) { + std::random_device rd; + for (auto& b : out) b = static_cast(rd() & 0xFF); + } + return out; + } + + [[nodiscard]] static std::string Base64UrlNoPad(const std::vector& in) { + static constexpr char kB64[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + std::string out; + out.reserve((in.size() * 4 + 2) / 3); + for (std::size_t i = 0; i < in.size(); i += 3) { + const unsigned a = in[i]; + const unsigned b = i + 1 < in.size() ? in[i + 1] : 0; + const unsigned c = i + 2 < in.size() ? in[i + 2] : 0; + const unsigned triple = (a << 16) | (b << 8) | c; + out += kB64[(triple >> 18) & 63]; + out += kB64[(triple >> 12) & 63]; + if (i + 1 < in.size()) out += kB64[(triple >> 6) & 63]; + if (i + 2 < in.size()) out += kB64[triple & 63]; + } + return out; + } + + // Minimal in-house SHA-256 (stdlib only, no third-party crypto). + [[nodiscard]] static std::string Sha256Hex(std::string_view msg) { + static constexpr std::uint32_t kK[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + std::uint32_t h[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + std::vector data(msg.begin(), msg.end()); + const std::uint64_t bit_len = static_cast(data.size()) * 8; + data.push_back(0x80); + while (data.size() % 64 != 56) data.push_back(0x00); + for (int i = 7; i >= 0; --i) { + data.push_back(static_cast((bit_len >> (i * 8)) & 0xFF)); + } + const auto rotr = [](std::uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); }; + for (std::size_t off = 0; off < data.size(); off += 64) { + std::uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(data[off + i * 4]) << 24) | + (static_cast(data[off + i * 4 + 1]) << 16) | + (static_cast(data[off + i * 4 + 2]) << 8) | + static_cast(data[off + i * 4 + 3]); + } + for (int i = 16; i < 64; ++i) { + const std::uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + const std::uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + std::uint32_t v[8]; + for (int i = 0; i < 8; ++i) v[i] = h[i]; + for (int i = 0; i < 64; ++i) { + const std::uint32_t s1 = rotr(v[4], 6) ^ rotr(v[4], 11) ^ rotr(v[4], 25); + const std::uint32_t ch = (v[4] & v[5]) ^ (~v[4] & v[6]); + const std::uint32_t t1 = v[7] + s1 + ch + kK[i] + w[i]; + const std::uint32_t s0 = rotr(v[0], 2) ^ rotr(v[0], 13) ^ rotr(v[0], 22); + const std::uint32_t maj = (v[0] & v[1]) ^ (v[0] & v[2]) ^ (v[1] & v[2]); + const std::uint32_t t2 = s0 + maj; + v[7] = v[6]; + v[6] = v[5]; + v[5] = v[4]; + v[4] = v[3] + t1; + v[3] = v[2]; + v[2] = v[1]; + v[1] = v[0]; + v[0] = t1 + t2; + } + for (int i = 0; i < 8; ++i) h[i] += v[i]; + } + static constexpr char kHex[] = "0123456789abcdef"; + std::string out; + out.reserve(64); + for (std::uint32_t x : h) { + for (int i = 7; i >= 0; --i) out += kHex[(x >> (i * 4)) & 0xF]; + } + return out; + } + + KeyStore& store_; + long now_; +}; + +} // namespace molo::govern::keys diff --git a/src/governance/keys/KeyStore.h b/src/governance/keys/KeyStore.h new file mode 100644 index 0000000..81bb576 --- /dev/null +++ b/src/governance/keys/KeyStore.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "ApiKey.h" + +namespace molo::govern::keys { + +// Abstract key store. MemoryKeyStore is the in-process implementation; a +// DB-backed store plugs in here at integration time. +class KeyStore { + public: + virtual ~KeyStore() = default; + + virtual bool Put(const ApiKey& key) = 0; + [[nodiscard]] virtual std::optional Find(const std::string& id) const = 0; + virtual bool Revoke(const std::string& id) = 0; + [[nodiscard]] virtual std::size_t Size() const = 0; +}; + +} // namespace molo::govern::keys diff --git a/src/governance/keys/MemoryKeyStore.h b/src/governance/keys/MemoryKeyStore.h new file mode 100644 index 0000000..dcbee4f --- /dev/null +++ b/src/governance/keys/MemoryKeyStore.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +#include "../caps.h" +#include "KeyStore.h" + +namespace molo::govern::keys { + +// Thread-safe in-memory key store with a static capacity cap. +class MemoryKeyStore : public KeyStore { + public: + bool Put(const ApiKey& key) override { + std::lock_guard lock(mu_); + if (rows_.find(key.id) == rows_.end() && rows_.size() >= Caps::kMaxKeys) return false; + rows_[key.id] = key; + return true; + } + + [[nodiscard]] std::optional Find(const std::string& id) const override { + std::lock_guard lock(mu_); + auto it = rows_.find(id); + if (it == rows_.end()) return std::nullopt; + return it->second; + } + + bool Revoke(const std::string& id) override { + std::lock_guard lock(mu_); + auto it = rows_.find(id); + if (it == rows_.end()) return false; + it->second.revoked = true; + return true; + } + + [[nodiscard]] std::size_t Size() const override { + std::lock_guard lock(mu_); + return rows_.size(); + } + + private: + mutable std::mutex mu_; + std::unordered_map rows_; +}; + +} // namespace molo::govern::keys diff --git a/src/governance/keys/Scope.h b/src/governance/keys/Scope.h new file mode 100644 index 0000000..352b019 --- /dev/null +++ b/src/governance/keys/Scope.h @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include + +#include "../caps.h" + +namespace molo::govern::keys { + +// Scope names are colon-separated segments, e.g. "read:users". +// A granted scope authorizes a required scope when both have the same segment +// count and every granted segment equals the required one or is a full "*". +class Scope { + public: + [[nodiscard]] static bool Valid(std::string_view name) { + if (name.empty() || name.size() > Caps::kMaxScopeLen) return false; + if (name.front() == ':' || name.back() == ':') return false; + bool prev_colon = false; + for (char c : name) { + if (c == ':') { + if (prev_colon) return false; + prev_colon = true; + continue; + } + prev_colon = false; + const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '*' || c == '.'; + if (!ok) return false; + } + // A '*' must occupy a whole segment. + for (const std::string& seg : Split(name)) { + if (seg.find('*') != std::string::npos && seg != "*") return false; + } + return true; + } + + // Parse a comma or space separated scope list. Invalid entries are skipped. + [[nodiscard]] static std::vector Parse(std::string_view list) { + std::vector out; + std::string cur; + const auto flush = [&] { + if (!cur.empty()) { + if (Valid(cur) && out.size() < Caps::kMaxScopesPerKey) out.push_back(cur); + cur.clear(); + } + }; + for (char c : list) { + if (c == ',' || c == ' ' || c == '\t' || c == '\n') { + flush(); + } else { + cur += c; + } + } + flush(); + return out; + } + + [[nodiscard]] static bool Allows(std::string_view granted, std::string_view required) { + if (!Valid(granted) || !Valid(required)) return false; + const std::vector g = Split(granted); + const std::vector r = Split(required); + if (g.size() != r.size()) return false; + for (std::size_t i = 0; i < g.size(); ++i) { + if (g[i] != "*" && g[i] != r[i]) return false; + } + return true; + } + + [[nodiscard]] static bool AllowsAny(const std::vector& granted, + std::string_view required) { + for (const auto& g : granted) { + if (Allows(g, required)) return true; + } + return false; + } + + private: + [[nodiscard]] static std::vector Split(std::string_view v) { + std::vector parts; + std::string cur; + for (char c : v) { + if (c == ':') { + parts.push_back(cur); + cur.clear(); + } else { + cur += c; + } + } + parts.push_back(cur); + return parts; + } +}; + +} // namespace molo::govern::keys diff --git a/src/governance/problem.h b/src/governance/problem.h new file mode 100644 index 0000000..00d3f9d --- /dev/null +++ b/src/governance/problem.h @@ -0,0 +1,105 @@ +#pragma once + +#include +#include + +namespace molo::govern { + +// RFC 9457 problem-details body. The base ErrorHandler renders +// {error:{code,message,request_id}}; this helper renders problem+json for +// governance failures and bypasses the base handler (see PATCHES.md). +class Problem { + public: + static constexpr std::string_view kContentType = "application/problem+json"; + static constexpr std::string_view kBaseUri = "https://molo.dev/problems/"; + + Problem(int status, std::string_view type, std::string_view title) + : status_(status), type_(type), title_(title) {} + Problem(int status, std::string_view type, std::string_view title, std::string_view detail) + : status_(status), type_(type), title_(title), detail_(std::string(detail)) {} + + [[nodiscard]] int status() const { return status_; } + [[nodiscard]] const std::string& type() const { return type_; } + [[nodiscard]] const std::string& title() const { return title_; } + [[nodiscard]] const std::string& detail() const { return detail_; } + [[nodiscard]] const std::string& instance() const { return instance_; } + void set_instance(std::string v) { instance_ = std::move(v); } + + [[nodiscard]] std::string to_json_string() const { + std::string out = "{\"type\":\""; + out += Escape(type_); + out += "\",\"title\":\""; + out += Escape(title_); + out += "\",\"status\":"; + out += std::to_string(status_); + if (!detail_.empty()) { + out += ",\"detail\":\""; + out += Escape(detail_); + out += "\""; + } + if (!instance_.empty()) { + out += ",\"instance\":\""; + out += Escape(instance_); + out += "\""; + } + out += "}"; + return out; + } + + static Problem ExpiredKey(std::string_view detail = "API key is expired or revoked") { + return {401, Join("expired_key"), "Expired API key", detail}; + } + static Problem Unauthorized(std::string_view detail = "Missing or invalid API key") { + return {401, Join("unauthorized"), "Unauthorized", detail}; + } + static Problem InsufficientScope(std::string_view detail = "API key lacks required scope") { + return {403, Join("insufficient_scope"), "Forbidden", detail}; + } + static Problem QuotaExceeded(std::string_view detail = "Quota exceeded for this key") { + return {429, Join("quota-exceeded"), "Too Many Requests", detail}; + } + static Problem VersionGone(std::string_view detail = "This API version is sunset") { + return {410, Join("version_gone"), "Gone", detail}; + } + + static std::string Escape(std::string_view in) { + std::string out; + out.reserve(in.size() + 8); + for (char c : in) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (static_cast(c) < 0x20) { + char buf[8]; + const char* hex = "0123456789abcdef"; + buf[0] = '\\'; + buf[1] = 'u'; + buf[2] = '0'; + buf[3] = '0'; + buf[4] = hex[(static_cast(c) >> 4) & 0xF]; + buf[5] = hex[static_cast(c) & 0xF]; + out.append(buf, 6); + } else { + out += c; + } + } + } + return out; + } + + private: + static std::string Join(std::string_view leaf) { + return std::string(kBaseUri) + std::string(leaf); + } + int status_; + std::string type_; + std::string title_; + std::string detail_; + std::string instance_; +}; + +} // namespace molo::govern diff --git a/src/governance/surface/RouteLinter.h b/src/governance/surface/RouteLinter.h new file mode 100644 index 0000000..0fd471f --- /dev/null +++ b/src/governance/surface/RouteLinter.h @@ -0,0 +1,215 @@ +#pragma once + +#include +#include +#include + +#include "../caps.h" +#include "../versions/ApiVersion.h" +#include "SurfaceRules.h" + +namespace molo::govern::surface { + +// Lints route declarations against S1-S10. Reads route patterns, not views. +class RouteLinter { + public: + struct Route { + std::string method; + std::string path; + std::vector statuses; + std::string auth; // none|key|key+scopes|session + std::vector scopes; + bool paginated = false; + bool has_page_params = false; + bool has_envelope = false; + bool has_filters = false; + bool filter_in_query = true; + bool has_body = false; + bool idempotency_key = false; + bool has_problem_body = false; + bool has_summary = false; + bool has_example = false; + }; + + [[nodiscard]] static std::vector Lint(std::span routes) { + std::vector out; + for (const auto& r : routes) { + const std::string target = r.method + " " + r.path; + CheckS1(r, target, out); + CheckS2(r, target, out); + CheckS3(r, target, out); + CheckS4(r, target, out); + CheckS5(r, target, out); + CheckS6(r, target, out); + CheckS7(r, target, out); + CheckS8(r, target, out); + CheckS9(r, target, out); + CheckS10(r, target, out); + if (out.size() >= Caps::kMaxFindings) break; + } + return out; + } + + private: + using Findings = std::vector; + + static void Add(Findings& out, const std::string& rule, const std::string& target, + const std::string& msg) { + if (out.size() >= Caps::kMaxFindings) return; + SurfaceRules::Finding f; + f.rule = rule; + f.severity = std::string(SurfaceRules::SeverityOf(rule)); + f.target = target; + f.message = msg; + out.push_back(std::move(f)); + } + + // S1: literal collection segments are plural nouns. + static void CheckS1(const Route& r, const std::string& target, Findings& out) { + for (const std::string& seg : Split(r.path)) { + if (seg.empty() || seg[0] == ':') continue; + if (seg[0] == 'v' && versions::ApiVersion::ParseMajor("/" + seg) != 0) continue; + if (seg == "status" || seg == "health" || seg == "version" || seg == "openapi" || + seg == "auth" || seg == "metrics" || seg == "docs") { + continue; + } + if (seg.back() != 's') { + Add(out, "S1", target, "segment '" + seg + "' is not a plural noun"); + } + } + } + + // S2: known method; GET/HEAD/DELETE carry no body. + static void CheckS2(const Route& r, const std::string& target, Findings& out) { + const bool known = r.method == "GET" || r.method == "POST" || r.method == "PUT" || + r.method == "PATCH" || r.method == "DELETE" || r.method == "HEAD" || + r.method == "OPTIONS"; + if (!known) { + Add(out, "S2", target, "unknown method '" + r.method + "'"); + return; + } + if (r.has_body && (r.method == "GET" || r.method == "HEAD" || r.method == "DELETE")) { + Add(out, "S2", target, r.method + " must not carry a body"); + } + } + + // S3: at least one 2xx; only known status codes. + static void CheckS3(const Route& r, const std::string& target, Findings& out) { + if (r.statuses.empty()) { + Add(out, "S3", target, "no statuses declared"); + return; + } + bool ok2xx = false; + for (int s : r.statuses) { + if (s >= 200 && s < 300) ok2xx = true; + if (!KnownStatus(s)) Add(out, "S3", target, "unknown status " + std::to_string(s)); + } + if (!ok2xx) Add(out, "S3", target, "no 2xx status declared"); + } + + // S4: routes with 4xx/5xx declare a problem body. + static void CheckS4(const Route& r, const std::string& target, Findings& out) { + bool has_err = false; + for (int s : r.statuses) { + if (s >= 400) has_err = true; + } + if (has_err && !r.has_problem_body) { + Add(out, "S4", target, "error statuses without problem body"); + } + } + + // S5: paginated routes use page/per_page and the {data,meta,links} envelope. + static void CheckS5(const Route& r, const std::string& target, Findings& out) { + if (!r.paginated) return; + if (!r.has_page_params) Add(out, "S5", target, "paginated route without page/per_page"); + if (!r.has_envelope) Add(out, "S5", target, "paginated route without {data,meta,links}"); + } + + // S6: filters travel in query params only. + static void CheckS6(const Route& r, const std::string& target, Findings& out) { + if (r.has_filters && !r.filter_in_query) { + Add(out, "S6", target, "filters must travel in query params"); + } + } + + // S7: auth declared. + static void CheckS7(const Route& r, const std::string& target, Findings& out) { + const bool known = r.auth == "none" || r.auth == "key" || r.auth == "key+scopes" || + r.auth == "session"; + if (!known) { + Add(out, "S7", target, "auth not declared"); + return; + } + if (r.auth == "key+scopes" && r.scopes.empty()) { + Add(out, "S7", target, "key+scopes without scopes"); + } + } + + // S8: mutating methods accept Idempotency-Key. + static void CheckS8(const Route& r, const std::string& target, Findings& out) { + if ((r.method == "POST" || r.method == "PUT" || r.method == "PATCH") && + !r.idempotency_key) { + Add(out, "S8", target, "missing Idempotency-Key support"); + } + } + + // S9: summary plus example. + static void CheckS9(const Route& r, const std::string& target, Findings& out) { + if (!r.has_summary) Add(out, "S9", target, "missing summary"); + if (!r.has_example) Add(out, "S9", target, "missing example"); + } + + // S10: /v{n} prefix. + static void CheckS10(const Route& r, const std::string& target, Findings& out) { + if (versions::ApiVersion::ParseMajor(r.path) == 0) { + Add(out, "S10", target, "missing /v{n} prefix"); + } + } + + [[nodiscard]] static bool KnownStatus(int s) { + switch (s) { + case 200: + case 201: + case 202: + case 204: + case 206: + case 301: + case 302: + case 303: + case 307: + case 308: + case 400: + case 401: + case 403: + case 404: + case 405: + case 409: + case 410: + case 415: + case 422: + case 429: + case 500: + case 503: + return true; + default: + return false; + } + } + + [[nodiscard]] static std::vector Split(const std::string& path) { + std::vector parts; + std::string cur; + for (char c : path) { + if (c == '/') { + parts.push_back(cur); + cur.clear(); + } else { + cur += c; + } + } + parts.push_back(cur); + return parts; + } +}; + +} // namespace molo::govern::surface diff --git a/src/governance/surface/SurfaceRules.h b/src/governance/surface/SurfaceRules.h new file mode 100644 index 0000000..d31b5f4 --- /dev/null +++ b/src/governance/surface/SurfaceRules.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include + +namespace molo::govern::surface { + +// Route surface rules S1-S10. Metadata only; RouteLinter enforces them. +// S1 plural nouns, S2 method semantics, S3 status set, S4 problem body, +// S5 pagination envelope, S6 query-only filters, S7 auth declared, +// S8 Idempotency-Key, S9 summary plus example, S10 version prefix. +class SurfaceRules { + public: + struct Finding { + std::string rule; + std::string severity; // "ERR" or "WARN" + std::string target; + std::string message; + }; + + [[nodiscard]] static std::vector RuleIds() { + return {"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10"}; + } + + [[nodiscard]] static std::string_view Describe(std::string_view id) { + if (id == "S1") return "collection segments use plural nouns"; + if (id == "S2") return "method semantics: known method, no body on GET/HEAD/DELETE"; + if (id == "S3") return "status set: at least one 2xx, only known codes"; + if (id == "S4") return "error statuses declare a problem body"; + if (id == "S5") return "paginated routes use page/per_page plus {data,meta,links}"; + if (id == "S6") return "filters travel in query params only"; + if (id == "S7") return "auth declared: none|key|key+scopes|session"; + if (id == "S8") return "POST/PUT/PATCH accept Idempotency-Key"; + if (id == "S9") return "route has summary and example"; + if (id == "S10") return "path carries a /v{n} prefix"; + return "unknown rule"; + } + + [[nodiscard]] static std::string_view SeverityOf(std::string_view id) { + if (id == "S2" || id == "S3" || id == "S5" || id == "S7" || id == "S10") return "ERR"; + return "WARN"; + } +}; + +} // namespace molo::govern::surface diff --git a/src/governance/tiers/QuotaLedger.h b/src/governance/tiers/QuotaLedger.h new file mode 100644 index 0000000..c962d54 --- /dev/null +++ b/src/governance/tiers/QuotaLedger.h @@ -0,0 +1,105 @@ +#pragma once + +#include +#include +#include +#include + +#include "../caps.h" +#include "TierTable.h" + +namespace molo::govern::tiers { + +// In-memory quota ledger. Ledger keys follow gov:quota:{id}:{window} so a +// KV-backed ledger can take over without changing callers. +class QuotaLedger { + public: + struct Usage { + long minute_count = 0; + long day_count = 0; + }; + + [[nodiscard]] static std::string LedgerKey(std::string_view id, std::string_view window) { + return "gov:quota:" + std::string(id) + ":" + std::string(window); + } + + Usage Record(std::string_view id, long now_unix) { + std::lock_guard lock(mu_); + PruneLocked(now_unix); + const long minute = now_unix / 60; + const long day = now_unix / 86400; + Row& row = rows_[std::string(id)]; + if (row.minute != minute) { + row.minute = minute; + row.minute_count = 0; + } + if (row.day != day) { + row.day = day; + row.day_count = 0; + } + ++row.minute_count; + ++row.day_count; + return Usage{row.minute_count, row.day_count}; + } + + [[nodiscard]] Usage Peek(std::string_view id, long now_unix) { + std::lock_guard lock(mu_); + const long minute = now_unix / 60; + const long day = now_unix / 86400; + auto it = rows_.find(std::string(id)); + if (it == rows_.end()) return {}; + Usage out; + if (it->second.minute == minute) out.minute_count = it->second.minute_count; + if (it->second.day == day) out.day_count = it->second.day_count; + return out; + } + + // Check-then-record. Returns true when the request is within quota. + // retry_after_sec is set when over quota. + bool Check(std::string_view id, const TierTable::Limits& limits, long now_unix, + long& retry_after_sec) { + Usage u = Record(id, now_unix); + if (u.minute_count <= limits.per_min && u.day_count <= limits.per_day) { + retry_after_sec = 0; + return true; + } + if (u.minute_count > limits.per_min) { + retry_after_sec = 60 - (now_unix % 60); + if (retry_after_sec <= 0) retry_after_sec = 1; + } else { + retry_after_sec = 86400 - (now_unix % 86400); + } + if (retry_after_sec > Caps::kRetryAfterMaxSec) retry_after_sec = Caps::kRetryAfterMaxSec; + return false; + } + + [[nodiscard]] std::size_t Size() const { + std::lock_guard lock(mu_); + return rows_.size(); + } + + private: + struct Row { + long minute = -1; + long day = -1; + long minute_count = 0; + long day_count = 0; + }; + + void PruneLocked(long now_unix) { + if (rows_.size() < Caps::kMaxQuotaRows) return; + const long day = now_unix / 86400; + for (auto it = rows_.begin(); it != rows_.end();) { + if (it->second.day < day - 1) { + it = rows_.erase(it); + } else { + ++it; + } + } + } + + mutable std::mutex mu_; + std::unordered_map rows_; +}; + +} // namespace molo::govern::tiers diff --git a/src/governance/tiers/TierHeaders.h b/src/governance/tiers/TierHeaders.h new file mode 100644 index 0000000..b74306a --- /dev/null +++ b/src/governance/tiers/TierHeaders.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +#include "../adapters/BaseAdapters.h" +#include "../caps.h" +#include "TierTable.h" + +namespace molo::govern::tiers { + +// Rate-limit response headers: RateLimit-Policy plus RateLimit (rev11 draft) +// plus X-RateLimit-* compat plus Retry-After capped at 300. +class TierHeaders { + public: + static void Apply(std::map& headers, + const TierTable::Limits& policy, long minute_remaining, + long day_remaining, long reset_sec, long retry_after_sec) { + if (minute_remaining < 0) minute_remaining = 0; + if (day_remaining < 0) day_remaining = 0; + if (reset_sec < 0) reset_sec = 0; + headers["RateLimit-Policy"] = "\"min\";q=" + std::to_string(policy.per_min) + + ";w=60, \"day\";q=" + std::to_string(policy.per_day) + + ";w=86400"; + headers["RateLimit"] = "limit=" + std::to_string(policy.per_min) + ", remaining=" + + std::to_string(minute_remaining) + ", reset=" + + std::to_string(reset_sec); + headers["X-RateLimit-Limit"] = std::to_string(policy.per_min); + headers["X-RateLimit-Remaining"] = std::to_string(minute_remaining); + headers["X-RateLimit-Reset"] = std::to_string(reset_sec); + headers["X-RateLimit-Day-Remaining"] = std::to_string(day_remaining); + if (retry_after_sec > 0) { + if (retry_after_sec > Caps::kRetryAfterMaxSec) retry_after_sec = Caps::kRetryAfterMaxSec; + headers["Retry-After"] = std::to_string(retry_after_sec); + } + } + + static void Apply(adapters::HttpResponse& res, const TierTable::Limits& policy, + long minute_remaining, long day_remaining, long reset_sec, + long retry_after_sec) { + std::map tmp(res.headers.fields.begin(), res.headers.fields.end()); + Apply(tmp, policy, minute_remaining, day_remaining, reset_sec, retry_after_sec); + for (const auto& [k, v] : tmp) res.headers.set(k, v); + } +}; + +} // namespace molo::govern::tiers diff --git a/src/governance/tiers/TierPolicy.h b/src/governance/tiers/TierPolicy.h new file mode 100644 index 0000000..2edfd43 --- /dev/null +++ b/src/governance/tiers/TierPolicy.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +#include "TierTable.h" + +namespace molo::govern::tiers { + +// Resolves the effective limits for a key on a route. A route-level tier +// override may only lower limits, never raise them above the key tier. +class TierPolicy { + public: + [[nodiscard]] static TierTable::Limits Resolve(std::string_view key_tier, + std::string_view route_override) { + const TierTable::Limits base = TierTable::ForTier(key_tier); + if (route_override.empty() || !TierTable::Known(route_override)) return base; + const TierTable::Limits over = TierTable::ForTier(route_override); + TierTable::Limits out = base; + if (over.per_min < out.per_min) out.per_min = over.per_min; + if (over.per_day < out.per_day) out.per_day = over.per_day; + if (over.concurrency < out.concurrency) out.concurrency = over.concurrency; + if (over.burst < out.burst) out.burst = over.burst; + return out; + } +}; + +} // namespace molo::govern::tiers diff --git a/src/governance/tiers/TierTable.h b/src/governance/tiers/TierTable.h new file mode 100644 index 0000000..989d8a4 --- /dev/null +++ b/src/governance/tiers/TierTable.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +namespace molo::govern::tiers { + +// Static per-tier rate limits. Burst is the short-window allowance on top of +// the per-minute budget. Tiers: free, pro, enterprise. +class TierTable { + public: + struct Limits { + long per_min = 60; + long per_day = 10000; + int concurrency = 4; + int burst = 10; + }; + + [[nodiscard]] static Limits ForTier(std::string_view tier) { + if (tier == "pro") return Limits{600, 1000000, 32, 100}; + if (tier == "enterprise" || tier == "ent") return Limits{6000, 50000000, 256, 1000}; + return Limits{60, 10000, 4, 10}; + } + + [[nodiscard]] static bool Known(std::string_view tier) { + return tier == "free" || tier == "pro" || tier == "enterprise" || tier == "ent"; + } + + [[nodiscard]] static std::string Canonical(std::string_view tier) { + if (tier == "ent") return "enterprise"; + if (Known(tier)) return std::string(tier); + return "free"; + } +}; + +} // namespace molo::govern::tiers diff --git a/src/governance/versions/ApiVersion.h b/src/governance/versions/ApiVersion.h new file mode 100644 index 0000000..9047579 --- /dev/null +++ b/src/governance/versions/ApiVersion.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +#include "../caps.h" + +namespace molo::govern::versions { + +// One API major version with its lifecycle dates. All public routes carry a +// /v{n} prefix with n in [kMinMajor, kMaxMajor]. +class ApiVersion { + public: + enum class Status { kCurrent, kDeprecated, kSunset }; + + int major = 1; + long deprecation_unix = 0; // 0 means not deprecated + long sunset_unix = 0; // 0 means no sunset scheduled + int successor = 0; // 0 means no successor + + [[nodiscard]] Status At(long now_unix) const { + if (sunset_unix != 0 && now_unix >= sunset_unix) return Status::kSunset; + if (deprecation_unix != 0 && now_unix >= deprecation_unix) return Status::kDeprecated; + return Status::kCurrent; + } + + [[nodiscard]] bool DatesConsistent() const { + if (deprecation_unix != 0 && sunset_unix != 0) return sunset_unix >= deprecation_unix; + return true; + } + + // Parse the /v{n} prefix of a path. Returns 0 when absent or out of range. + [[nodiscard]] static int ParseMajor(std::string_view path) { + if (path.size() < 3 || path[0] != '/' || path[1] != 'v') return 0; + std::size_t i = 2; + int n = 0; + int digits = 0; + while (i < path.size() && path[i] >= '0' && path[i] <= '9') { + n = n * 10 + (path[i] - '0'); + ++i; + ++digits; + } + if (digits == 0 || digits > 2) return 0; + if (i < path.size() && path[i] != '/') return 0; + if (n < Caps::kMinMajor || n > Caps::kMaxMajor) return 0; + // Reject leading zeros: canonical form is /v1 .. /v99. + if (digits == 2 && path[2] == '0') return 0; + return n; + } + + [[nodiscard]] static std::string Prefix(int major) { return "/v" + std::to_string(major); } +}; + +} // namespace molo::govern::versions diff --git a/src/governance/versions/DeprecationMiddleware.h b/src/governance/versions/DeprecationMiddleware.h new file mode 100644 index 0000000..24bb64a --- /dev/null +++ b/src/governance/versions/DeprecationMiddleware.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "../adapters/BaseAdapters.h" +#include "../caps.h" + +namespace molo::govern::versions { + +// Emits RFC 8594 Deprecation plus Sunset plus Link headers for deprecated +// version prefixes. Faces RouterAdjacent at integration time. +class DeprecationMiddleware { + public: + struct Rule { + std::string prefix; + long deprecation_unix = 0; + std::string sunset_httpdate; + std::string successor_prefix; + }; + + bool AddRule(Rule rule) { + if (rules_.size() >= Caps::kMaxDeprecationRules) return false; + if (rule.prefix.empty() || rule.prefix[0] != '/') return false; + rules_.push_back(std::move(rule)); + return true; + } + + [[nodiscard]] std::size_t RuleCount() const { return rules_.size(); } + + // Headers for a request path at now. Empty map when no rule matches. + [[nodiscard]] std::map HeadersFor(std::string_view path, + long now_unix) const { + std::map out; + const Rule* best = nullptr; + for (const auto& r : rules_) { + if (path.size() < r.prefix.size()) continue; + if (path.substr(0, r.prefix.size()) != r.prefix) continue; + if (best == nullptr || r.prefix.size() > best->prefix.size()) best = &r; + } + if (best == nullptr) return out; + if (best->deprecation_unix != 0 && now_unix >= best->deprecation_unix) { + out["Deprecation"] = "@" + std::to_string(best->deprecation_unix); + } else if (best->deprecation_unix != 0) { + out["Deprecation"] = "@" + std::to_string(best->deprecation_unix); + } else { + out["Deprecation"] = "true"; + } + if (!best->sunset_httpdate.empty()) out["Sunset"] = best->sunset_httpdate; + std::string link; + if (!best->sunset_httpdate.empty()) { + link += "; rel=\"deprecation\""; + } + if (!best->successor_prefix.empty()) { + if (!link.empty()) link += ", "; + link += "<" + best->successor_prefix + ">; rel=\"successor-version\""; + } + if (!link.empty()) out["Link"] = link; + return out; + } + + void Apply(adapters::HttpResponse& res, std::string_view path, long now_unix) const { + for (const auto& [k, v] : HeadersFor(path, now_unix)) res.headers.set(k, v); + } + + [[nodiscard]] static std::string FormatImf(long unix_time) { + std::time_t t = static_cast(unix_time); + std::tm tm{}; +#if defined(_WIN32) + gmtime_s(&tm, &t); +#else + gmtime_r(&t, &tm); +#endif + char buf[64]; + std::strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", &tm); + return {buf}; + } + + private: + std::vector rules_; +}; + +} // namespace molo::govern::versions diff --git a/src/governance/versions/VersionedRouter.h b/src/governance/versions/VersionedRouter.h new file mode 100644 index 0000000..f028d18 --- /dev/null +++ b/src/governance/versions/VersionedRouter.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "../caps.h" +#include "ApiVersion.h" + +namespace molo::govern::versions { + +// Standalone version-aware route table used for tests and as the reference +// for the base Router/Application startup validation (see PATCHES.md). +// Patterns support ":name" segments. A match against a sunset major reports +// gone so the caller can answer 410 instead of 404. +class VersionedRouter { + public: + struct Match { + bool found = false; + bool gone = false; + std::string name; + std::map params; + int major = 0; + }; + + bool Add(std::string_view method, std::string_view pattern, std::string_view name) { + if (frozen_) return false; + if (entries_.size() >= Caps::kMaxRoutes) return false; + const int major = ApiVersion::ParseMajor(pattern); + if (major == 0) return false; + for (const auto& e : entries_) { + if (e.method == method && e.pattern == pattern) return false; + if (e.name == name) return false; + } + if (!live_.contains(major)) { + if (static_cast(live_.size()) >= Caps::kMaxLiveMajors) return false; + live_.insert(major); + } + entries_.push_back({std::string(method), std::string(pattern), std::string(name), major}); + return true; + } + + void SetSunset(int major, long sunset_unix) { sunsets_[major] = sunset_unix; } + void Freeze() { frozen_ = true; } + + [[nodiscard]] std::size_t Size() const { return entries_.size(); } + + [[nodiscard]] Match Lookup(std::string_view method, std::string_view path, + long now_unix) const { + Match miss; + for (const auto& e : entries_) { + std::map params; + if (!MatchPattern(e.pattern, path, params)) continue; + if (e.method != method) continue; + auto it = sunsets_.find(e.major); + if (it != sunsets_.end() && it->second != 0 && now_unix >= it->second) { + Match gone; + gone.gone = true; + gone.major = e.major; + gone.params = params; + return gone; + } + Match hit; + hit.found = true; + hit.name = e.name; + hit.params = params; + hit.major = e.major; + return hit; + } + return miss; + } + + private: + struct Entry { + std::string method; + std::string pattern; + std::string name; + int major = 0; + }; + + [[nodiscard]] static std::vector Split(std::string_view v) { + std::vector parts; + std::string cur; + for (char c : v) { + if (c == '/') { + parts.push_back(cur); + cur.clear(); + } else { + cur += c; + } + } + parts.push_back(cur); + return parts; + } + + [[nodiscard]] static bool MatchPattern(std::string_view pattern, std::string_view path, + std::map& params) { + const std::vector want = Split(pattern); + const std::vector got = Split(path); + if (want.size() != got.size()) return false; + for (std::size_t i = 0; i < want.size(); ++i) { + if (!want[i].empty() && want[i][0] == ':') { + if (got[i].empty()) return false; + params[want[i].substr(1)] = got[i]; + } else if (want[i] != got[i]) { + return false; + } + } + return true; + } + + std::vector entries_; + std::map sunsets_; + std::set live_; + bool frozen_ = false; +}; + +} // namespace molo::govern::versions diff --git a/src/observability/README.md b/src/observability/README.md new file mode 100644 index 0000000..012eaf3 --- /dev/null +++ b/src/observability/README.md @@ -0,0 +1,58 @@ +# Observability backbone (02-observability.md) + +Four pillars plus correlation: logging (`molo::log`), typed events +(`molo::events`), metrics (`molo::metrics`), distributed tracing +(`molo::trace`), correlation (`molo::corr`), shared config and lifecycle +hooks (`molo::obs`). Spec: `plans/02-observability.md`. + +## Layout + +- `config.h`: compile flags, runtime toggles, static ceilings. +- `hooks.h`: runtime lifecycle hook points, all optional no-ops until wired. +- `correlation/`: request IDs, `RequestContext`, shared redaction helper. +- `logging/`: levels, records, loggers, sinks (console, file with rotation, + null, ring, async), JSON format, registry, `MOLO_LOG_*` macros. +- `events/`: event base with clone, sync/async listeners, dispatcher with + pubsub bridge hook, lifecycle event set. +- `metrics/`: counter, gauge, histogram, RAII timer, labels, registry with + cardinality caps, Prometheus text renderer. +- `tracing/`: IDs, span context, spans, tracer, samplers, W3C propagator, + batch processor, exporters (memory, OTLP/HTTP JSON). +- `observability.h`: umbrella header. `observability.mk`: root Makefile fragment. + +## Rules that matter here + +- Disabled pillars cost one branch at runtime and compile to nothing at + compile time (`MOLO_OBSERVABILITY`, `MOLO_ACTIVE_LEVEL`, + `MOLO_ENABLE_METRICS`, `MOLO_ENABLE_TRACING`, `MOLO_ENABLE_EVENTS`). +- No pillar blocks a handler: bounded queues, drop-on-full, visible counters. +- Static ceilings live in `config.h` (`Caps`). Overflows drop and count. +- Secrets are redacted by `molo::corr::is_sensitive_key` in loggers and spans. +- Label values must be bounded sets; the registry maps the 65th distinct + value per key to `"other"`. Raw paths and user IDs are forbidden. +- Counters end in `_total` (auto-appended), histograms export cumulative + `_bucket` plus `_sum` and `_count`. +- Server span names are `{METHOD} {route_template}`; raw paths forbidden. + +## Wiring points owned elsewhere + +- Runtime threads and request lifecycle call `molo::obs::fire_*` (01, 05). +- Pubsub core registers via `Dispatcher::set_publisher` (01 or 06). +- `GET /metrics` renders `molo::metrics::render` (09). +- OTLP sender becomes the framework HTTP client via `HttpSender` (08); + `NullHttpSender` is the marked stub until then. +- gzip for OTLP needs the codec from 01 via `set_compressor`. + +## Build and test + +Included by the root Makefile through `observability.mk` (`obs-test` target). +Standalone: + +``` +g++ -std=c++23 -O2 -Wall -Wextra -pthread -Isrc \ + -Itests/src/observability tests/src/observability/_test.cpp \ + -o build/obs_tests/ +``` + +Tests assert on `RingSink`, registry snapshots and `MemoryExporter`, never +on files or network, except `FileSink` rotation which uses `build/`. diff --git a/src/observability/config.h b/src/observability/config.h new file mode 100644 index 0000000..b25e632 --- /dev/null +++ b/src/observability/config.h @@ -0,0 +1,100 @@ +#pragma once + +// Compile-time feature flags, runtime toggles and static ceilings for every +// observability pillar (logging, events, metrics, tracing, correlation). +// Every knob has a static ceiling. No autotuning, no unbounded growth. + +#ifndef MOLO_OBSERVABILITY +#define MOLO_OBSERVABILITY 1 +#endif + +#ifndef MOLO_ENABLE_METRICS +#define MOLO_ENABLE_METRICS 1 +#endif + +#ifndef MOLO_ENABLE_TRACING +#define MOLO_ENABLE_TRACING 1 +#endif + +#ifndef MOLO_ENABLE_EVENTS +#define MOLO_ENABLE_EVENTS 1 +#endif + +#ifndef MOLO_ACTIVE_LEVEL +#define MOLO_ACTIVE_LEVEL 1 +#endif + +#include +#include +#include +#include +#include +#include + +namespace molo::obs { + +// Shortest decimal that round trips a finite double (tries 15, 16, then +// 17 significant digits). Shared by the Prometheus renderer, the JSON log +// format and the OTLP exporter so 0.1 renders as "0.1", not as +// "0.10000000000000001". Callers handle non-finite values themselves. +inline std::string format_double_finite(double value) { + for (int precision : {15, 16, 17}) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "%.*g", precision, value); + if (std::strtod(buf, nullptr) == value) return buf; + } + char buf[32]; + std::snprintf(buf, sizeof(buf), "%.17g", value); + return buf; +} + +// Master switch: 0 removes every pillar at compile time. +inline constexpr bool kEnabled = (MOLO_OBSERVABILITY != 0); +inline constexpr bool kMetricsOn = kEnabled && (MOLO_ENABLE_METRICS != 0); +inline constexpr bool kTracingOn = kEnabled && (MOLO_ENABLE_TRACING != 0); +inline constexpr bool kEventsOn = kEnabled && (MOLO_ENABLE_EVENTS != 0); + +// Static ceilings. Overridable via typed config only, never grown at runtime. +// Exceeding a ceiling drops data and counts the drop. +struct Caps { + // Logging. + static constexpr std::size_t kLogQueue = 8192; + static constexpr std::size_t kMaxLogLine = 8192; + static constexpr std::size_t kFileRotateFiles = 10; + static constexpr std::size_t kFileRotateBytes = 10u * 1024u * 1024u; + static constexpr std::size_t kRingRecords = 1024; + // Events. + static constexpr std::size_t kEventQueue = 4096; + // Metrics. + static constexpr std::size_t kMaxSeries = 65536; + static constexpr std::size_t kMaxLabelValues = 64; + static constexpr std::size_t kMaxLabels = 16; + static constexpr std::size_t kMaxLabelValueLen = 128; + // Tracing. + static constexpr std::size_t kSpanBuffer = 4096; + static constexpr std::size_t kSpanBatch = 512; + static constexpr int kSpanBatchIntervalSec = 5; + static constexpr std::size_t kMaxSpanAttrs = 64; + static constexpr std::size_t kMaxSpanEvents = 32; + static constexpr std::size_t kMaxAttrValue = 1024; + static constexpr std::size_t kMaxTracestate = 512; + // OTLP export. + static constexpr int kOtlpTimeoutSec = 10; + static constexpr int kOtlpRetries = 3; +}; + +// Runtime toggles. Read once per call site through a cached atomic, so a +// disabled pillar costs one predictable branch plus argument skip. +struct RuntimeToggles { + std::atomic logging{true}; + std::atomic metrics{true}; + std::atomic tracing{true}; + std::atomic events{true}; + + static RuntimeToggles& instance() { + static RuntimeToggles toggles; + return toggles; + } +}; + +} // namespace molo::obs diff --git a/src/observability/correlation/correlation.h b/src/observability/correlation/correlation.h new file mode 100644 index 0000000..641990e --- /dev/null +++ b/src/observability/correlation/correlation.h @@ -0,0 +1,5 @@ +#pragma once + +#include "observability/correlation/redaction.h" +#include "observability/correlation/request_id.h" +#include "observability/correlation/request_context.h" diff --git a/src/observability/correlation/redaction.h b/src/observability/correlation/redaction.h new file mode 100644 index 0000000..3366128 --- /dev/null +++ b/src/observability/correlation/redaction.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include +#include + +namespace molo::corr { + +// Shared secret guard for rule 12: secrets, passwords, tokens, session +// contents and request bodies are never logged, never set as span +// attributes, and never appear in metric labels. Matching fails closed: +// any key containing a sensitive needle is redacted. +inline constexpr std::array kSensitiveKeys = { + "password", "passwd", "secret", "token", "session", "cookie", + "auth", "api_key", "apikey", "key", "set-cookie", "body", +}; + +namespace detail { + +inline bool is_word_char(char ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_'; +} + +// The short needle "key" only matches as a word of its own: both sides +// must be a string edge, a separator, or a camelCase transition. This +// keeps "api_key", "key-id", "key" and "cacheKey" redacted while innocent +// words such as "monkey" or "keyboard" pass through. +inline bool key_matches(std::string_view lowered, std::string_view original) { + std::string_view needle = "key"; + std::size_t pos = lowered.find(needle); + while (pos != std::string_view::npos) { + bool left_ok = (pos == 0) || !is_word_char(lowered[pos - 1]); + if (!left_ok && pos < original.size()) { + unsigned char cur = static_cast(original[pos]); + unsigned char prev = static_cast(original[pos - 1]); + if (std::isupper(cur) && std::islower(prev)) left_ok = true; + } + std::size_t end = pos + needle.size(); + bool right_ok = (end >= lowered.size()) || !is_word_char(lowered[end]); + if (left_ok && right_ok) return true; + pos = lowered.find(needle, pos + 1); + } + return false; +} + +} // namespace detail + +inline bool is_sensitive_key(std::string_view key) { + std::string lowered; + lowered.reserve(key.size()); + for (char ch : key) lowered += static_cast(std::tolower(static_cast(ch))); + for (std::string_view needle : kSensitiveKeys) { + if (needle == "key") { + if (detail::key_matches(lowered, key)) return true; + } else if (lowered.find(needle) != std::string::npos) { + return true; + } + } + return false; +} + +inline constexpr std::string_view kRedacted = "***"; + +inline std::string redact_value(std::string_view key, std::string_view value) { + if (is_sensitive_key(key)) return std::string(kRedacted); + return std::string(value); +} + +} // namespace molo::corr diff --git a/src/observability/correlation/request_context.cpp b/src/observability/correlation/request_context.cpp new file mode 100644 index 0000000..21bdc48 --- /dev/null +++ b/src/observability/correlation/request_context.cpp @@ -0,0 +1,20 @@ +#include "observability/correlation/request_context.h" + +#include "observability/correlation/request_id.h" + +namespace molo::corr { + +RequestContext::RequestContext() : request_id_(generate_request_id()), span_() {} + +RequestContext::RequestContext(std::string request_id) + : request_id_(std::move(request_id)), span_() { + if (!is_valid_request_id(request_id_)) request_id_ = generate_request_id(); +} + +RequestContext RequestContext::with_span(molo::trace::SpanContext context) const { + RequestContext copy(*this); + copy.set_span_context(context); + return copy; +} + +} // namespace molo::corr diff --git a/src/observability/correlation/request_context.h b/src/observability/correlation/request_context.h new file mode 100644 index 0000000..eec6762 --- /dev/null +++ b/src/observability/correlation/request_context.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "observability/tracing/span_context.h" + +namespace molo::corr { + +// Per-request correlation state: one request ID plus the current span +// context. Created per request and passed explicitly (never a global) +// through middleware, handlers, ORM and replication. +class RequestContext { + public: + RequestContext(); + explicit RequestContext(std::string request_id); + + const std::string& request_id() const { return request_id_; } + const molo::trace::SpanContext& span_context() const { return span_; } + + void set_span_context(molo::trace::SpanContext context) { span_ = context; } + RequestContext with_span(molo::trace::SpanContext context) const; + + private: + std::string request_id_; + molo::trace::SpanContext span_; +}; + +} // namespace molo::corr diff --git a/src/observability/correlation/request_id.cpp b/src/observability/correlation/request_id.cpp new file mode 100644 index 0000000..ce0e050 --- /dev/null +++ b/src/observability/correlation/request_id.cpp @@ -0,0 +1,39 @@ +#include "observability/correlation/request_id.h" + +#include +#include +#include +#include + +namespace molo::corr { + +std::string generate_request_id() { + static std::atomic counter{0}; + std::size_t thread_salt = std::hash{}(std::this_thread::get_id()); + std::uint64_t seq = counter.fetch_add(1, std::memory_order_relaxed); + thread_local std::mt19937_64 rng{std::random_device{}() ^ thread_salt ^ (seq + 0x9e3779b97f4a7c15ULL)}; + std::uint64_t hi = rng(); + std::uint64_t lo = (rng() << 1) ^ seq; + if (hi == 0 && lo == 0) lo = 1; + char buf[33]; + std::snprintf(buf, sizeof(buf), "%016llx%016llx", static_cast(hi), + static_cast(lo)); + return buf; +} + +bool is_valid_request_id(std::string_view value) { + if (value.empty() || value.size() > 64) return false; + for (char ch : value) { + bool ok = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || + ch == '-' || ch == '_'; + if (!ok) return false; + } + return true; +} + +std::string normalize_request_id(std::string_view inbound) { + if (is_valid_request_id(inbound)) return std::string(inbound); + return generate_request_id(); +} + +} // namespace molo::corr diff --git a/src/observability/correlation/request_id.h b/src/observability/correlation/request_id.h new file mode 100644 index 0000000..3598d25 --- /dev/null +++ b/src/observability/correlation/request_id.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +namespace molo::corr { + +// Request IDs are 128-bit hex strings (32 chars). Inbound X-Request-ID +// values are reused when valid, otherwise a fresh ID is generated. +std::string generate_request_id(); +bool is_valid_request_id(std::string_view value); +std::string normalize_request_id(std::string_view inbound); + +} // namespace molo::corr diff --git a/src/observability/events/dispatcher.cpp b/src/observability/events/dispatcher.cpp new file mode 100644 index 0000000..6819a0c --- /dev/null +++ b/src/observability/events/dispatcher.cpp @@ -0,0 +1,161 @@ +#include "observability/events/dispatcher.h" + +#include "observability/events/event.h" + +namespace molo::events { + +Dispatcher::Dispatcher() = default; + +Dispatcher::~Dispatcher() { stop(); } + +std::uint64_t Dispatcher::on(std::string type, SyncListener listener) { + std::lock_guard lock(mutex_); + Entry entry; + entry.id = next_id_++; + entry.type = std::move(type); + entry.sync = std::move(listener); + listeners_.push_back(std::move(entry)); + return next_id_ - 1; +} + +std::uint64_t Dispatcher::on_async(std::string type, AsyncListener listener) { + std::lock_guard lock(mutex_); + Entry entry; + entry.id = next_id_++; + entry.type = std::move(type); + entry.async = std::move(listener); + entry.is_async = true; + listeners_.push_back(std::move(entry)); + return next_id_ - 1; +} + +bool Dispatcher::off(std::uint64_t id) { + std::lock_guard lock(mutex_); + for (auto it = listeners_.begin(); it != listeners_.end(); ++it) { + if (it->id == id) { + listeners_.erase(it); + return true; + } + } + return false; +} + +bool Dispatcher::has_listeners(const std::string& type) const { + std::lock_guard lock(mutex_); + for (const Entry& entry : listeners_) { + if (entry.type == type) return true; + } + return false; +} + +void Dispatcher::emit(const Event& event) { + if constexpr (!molo::obs::kEventsOn) return; + if (!molo::obs::RuntimeToggles::instance().events.load(std::memory_order_relaxed)) return; + + std::vector sync_fns; + bool want_async = false; + EventPublisher publisher; + { + std::lock_guard lock(mutex_); + for (const Entry& entry : listeners_) { + if (entry.type != event.type()) continue; + if (entry.is_async) { + want_async = true; + } else if (entry.sync) { + sync_fns.push_back(entry.sync); + } + } + publisher = publisher_; + } + emitted_.fetch_add(1, std::memory_order_relaxed); + + if (publisher) { + try { + publisher(event); + } catch (...) { + } + } + for (const auto& fn : sync_fns) { + try { + fn(event); + } catch (...) { + } + } + if (want_async) { + std::shared_ptr owned = event.clone(); + std::lock_guard lock(mutex_); + if (!started_ || queue_.size() >= queue_capacity_) { + dropped_.fetch_add(1, std::memory_order_relaxed); + } else { + queue_.push_back(std::move(owned)); + cv_.notify_one(); + } + } +} + +void Dispatcher::set_publisher(EventPublisher publisher) { + std::lock_guard lock(mutex_); + publisher_ = std::move(publisher); +} + +void Dispatcher::start() { + std::lock_guard lock(mutex_); + if (started_) return; + started_ = true; + running_ = true; + worker_ = std::thread([this] { run(); }); +} + +void Dispatcher::stop() { + { + std::lock_guard lock(mutex_); + running_ = false; + } + cv_.notify_all(); + if (worker_.joinable()) worker_.join(); + std::lock_guard lock(mutex_); + started_ = false; +} + +std::uint64_t Dispatcher::dropped_async() const { return dropped_.load(std::memory_order_relaxed); } + +std::uint64_t Dispatcher::failed_async() const { return failed_.load(std::memory_order_relaxed); } + +std::uint64_t Dispatcher::emitted_total() const { return emitted_.load(std::memory_order_relaxed); } + +Dispatcher& Dispatcher::global() { + static Dispatcher instance; + return instance; +} + +void Dispatcher::run() { + for (;;) { + std::shared_ptr event; + { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return !queue_.empty() || !running_; }); + if (queue_.empty() && !running_) return; + if (queue_.empty()) continue; + event = queue_.front(); + queue_.pop_front(); + } + std::vector fns; + { + std::lock_guard lock(mutex_); + for (const Entry& entry : listeners_) { + if (entry.is_async && entry.type == event->type() && entry.async) { + fns.push_back(entry.async); + } + } + } + for (const auto& fn : fns) { + try { + fn(event); + } catch (...) { + failed_.fetch_add(1, std::memory_order_relaxed); + } + } + } +} + +} // namespace molo::events diff --git a/src/observability/events/dispatcher.h b/src/observability/events/dispatcher.h new file mode 100644 index 0000000..0ac5fd7 --- /dev/null +++ b/src/observability/events/dispatcher.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "observability/config.h" +#include "observability/events/listener.h" + +namespace molo::events { + +class Event; + +// Typed event dispatcher. Sync listeners run inline with a zero-listener +// fast path; async listeners run on a background worker through a bounded +// queue with drop-on-full. A publisher bridge forwards every event to the +// pubsub core once that part wires it via set_publisher. +class Dispatcher { + public: + Dispatcher(); + ~Dispatcher(); + + Dispatcher(const Dispatcher&) = delete; + Dispatcher& operator=(const Dispatcher&) = delete; + + std::uint64_t on(std::string type, SyncListener listener); + std::uint64_t on_async(std::string type, AsyncListener listener); + bool off(std::uint64_t id); + bool has_listeners(const std::string& type) const; + + void emit(const Event& event); + + void set_publisher(EventPublisher publisher); + + void start(); + void stop(); + + std::uint64_t dropped_async() const; + std::uint64_t failed_async() const; + std::uint64_t emitted_total() const; + + static Dispatcher& global(); + + private: + struct Entry { + std::uint64_t id = 0; + std::string type; + SyncListener sync; + AsyncListener async; + bool is_async = false; + }; + + void run(); + + mutable std::mutex mutex_; + std::vector listeners_; + std::uint64_t next_id_ = 1; + EventPublisher publisher_; + std::atomic emitted_{0}; + + std::size_t queue_capacity_ = molo::obs::Caps::kEventQueue; + std::deque> queue_; + std::condition_variable cv_; + std::thread worker_; + bool running_ = false; + bool started_ = false; + std::atomic dropped_{0}; + std::atomic failed_{0}; +}; + +} // namespace molo::events diff --git a/src/observability/events/event.h b/src/observability/events/event.h new file mode 100644 index 0000000..fdf6977 --- /dev/null +++ b/src/observability/events/event.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include + +#include "observability/correlation/request_context.h" + +namespace molo::events { + +// Base class for every typed event. Carries a type ID, a timestamp and the +// request/trace linkage of the context that raised it. +class Event { + public: + explicit Event(std::string type) + : type_(std::move(type)), timestamp_(std::chrono::system_clock::now()) {} + + Event(std::string type, std::string request_id, std::string trace_id, std::string span_id) + : type_(std::move(type)), + timestamp_(std::chrono::system_clock::now()), + request_id_(std::move(request_id)), + trace_id_(std::move(trace_id)), + span_id_(std::move(span_id)) {} + + virtual ~Event() = default; + + Event(const Event&) = default; + Event& operator=(const Event&) = default; + Event(Event&&) noexcept = default; + Event& operator=(Event&&) noexcept = default; + + virtual std::shared_ptr clone() const { return std::make_shared(*this); } + + const std::string& type() const { return type_; } + std::chrono::system_clock::time_point timestamp() const { return timestamp_; } + const std::string& request_id() const { return request_id_; } + const std::string& trace_id() const { return trace_id_; } + const std::string& span_id() const { return span_id_; } + + void set_request_id(std::string id) { request_id_ = std::move(id); } + + static Event from_context(std::string type, const molo::corr::RequestContext& context) { + return Event(std::move(type), context.request_id(), context.span_context().trace_id.to_hex(), + context.span_context().span_id.to_hex()); + } + + private: + std::string type_; + std::chrono::system_clock::time_point timestamp_; + std::string request_id_; + std::string trace_id_; + std::string span_id_; +}; + +// CRTP helper so derived events clone without slicing. +template +class ClonedEvent : public Event { + protected: + using Event::Event; + + public: + std::shared_ptr clone() const override { + return std::make_shared(static_cast(*this)); + } +}; + +} // namespace molo::events diff --git a/src/observability/events/lifecycle_events.h b/src/observability/events/lifecycle_events.h new file mode 100644 index 0000000..ebc336a --- /dev/null +++ b/src/observability/events/lifecycle_events.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include + +#include "observability/correlation/request_context.h" +#include "observability/events/event.h" + +namespace molo::events { + +// Framework lifecycle event set. Every event carries the request/trace +// linkage of the context that raised it. +class ServerStartedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "server.started"; + ServerStartedEvent() : ClonedEvent(kType) {} +}; + +class ServerStoppedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "server.stopped"; + ServerStoppedEvent() : ClonedEvent(kType) {} +}; + +class ConnectionOpenedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "connection.opened"; + explicit ConnectionOpenedEvent(std::string conn_id) + : ClonedEvent(kType), conn_id(std::move(conn_id)) {} + std::string conn_id; +}; + +class ConnectionClosedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "connection.closed"; + explicit ConnectionClosedEvent(std::string conn_id) + : ClonedEvent(kType), conn_id(std::move(conn_id)) {} + std::string conn_id; +}; + +class RequestStartedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "request.started"; + RequestStartedEvent(std::string method, std::string route, + const molo::corr::RequestContext& context) + : ClonedEvent(kType, context.request_id(), context.span_context().trace_id.to_hex(), + context.span_context().span_id.to_hex()), + method(std::move(method)), + route(std::move(route)) {} + std::string method; + std::string route; +}; + +class RequestFinishedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "request.finished"; + RequestFinishedEvent(std::string method, std::string route, int status, + std::uint64_t duration_ms, const molo::corr::RequestContext& context) + : ClonedEvent(kType, context.request_id(), context.span_context().trace_id.to_hex(), + context.span_context().span_id.to_hex()), + method(std::move(method)), + route(std::move(route)), + status(status), + duration_ms(duration_ms) {} + std::string method; + std::string route; + int status = 0; + std::uint64_t duration_ms = 0; +}; + +class TaskEnqueuedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "task.enqueued"; + TaskEnqueuedEvent(std::string task, std::string task_id) + : ClonedEvent(kType), task(std::move(task)), task_id(std::move(task_id)) {} + std::string task; + std::string task_id; +}; + +class TaskStartedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "task.started"; + TaskStartedEvent(std::string task, std::string task_id) + : ClonedEvent(kType), task(std::move(task)), task_id(std::move(task_id)) {} + std::string task; + std::string task_id; +}; + +class TaskFinishedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "task.finished"; + TaskFinishedEvent(std::string task, std::string task_id, std::uint64_t duration_ms) + : ClonedEvent(kType), + task(std::move(task)), + task_id(std::move(task_id)), + duration_ms(duration_ms) {} + std::string task; + std::string task_id; + std::uint64_t duration_ms = 0; +}; + +class TaskFailedEvent : public ClonedEvent { + public: + static constexpr const char* kType = "task.failed"; + TaskFailedEvent(std::string task, std::string task_id, std::string error_kind) + : ClonedEvent(kType), + task(std::move(task)), + task_id(std::move(task_id)), + error_kind(std::move(error_kind)) {} + std::string task; + std::string task_id; + std::string error_kind; +}; + +} // namespace molo::events diff --git a/src/observability/events/listener.h b/src/observability/events/listener.h new file mode 100644 index 0000000..141ef6b --- /dev/null +++ b/src/observability/events/listener.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace molo::events { + +class Event; + +// Runs inline on the emitting thread. Must be fast and must not block. +using SyncListener = std::function; + +// Runs on the dispatcher background worker. Receives a cloned event. +using AsyncListener = std::function)>; + +// Bridge into the pubsub core (owned outside this part). Called for every +// emitted event when set; null by default. +using EventPublisher = std::function; + +} // namespace molo::events diff --git a/src/observability/hooks.h b/src/observability/hooks.h new file mode 100644 index 0000000..390ed8f --- /dev/null +++ b/src/observability/hooks.h @@ -0,0 +1,64 @@ +#pragma once + +#include + +#include "observability/correlation/request_context.h" + +namespace molo::obs { + +// Runtime lifecycle hook points (wired by 01-runtime.md and +// 05-request-routing.md). Every hook is optional: unset hooks are plain +// no-ops, so all call sites stay safe before and after wiring. +struct LifecycleHooks { + std::function on_server_start; + std::function on_server_stop; + std::function on_request_start; + std::function on_request_finish; + + static LifecycleHooks& global() { + static LifecycleHooks hooks; + return hooks; + } +}; + +inline void fire_server_start() { + const auto& hook = LifecycleHooks::global().on_server_start; + if (hook) { + try { + hook(); + } catch (...) { + } + } +} + +inline void fire_server_stop() { + const auto& hook = LifecycleHooks::global().on_server_stop; + if (hook) { + try { + hook(); + } catch (...) { + } + } +} + +inline void fire_request_start(const molo::corr::RequestContext& context) { + const auto& hook = LifecycleHooks::global().on_request_start; + if (hook) { + try { + hook(context); + } catch (...) { + } + } +} + +inline void fire_request_finish(const molo::corr::RequestContext& context) { + const auto& hook = LifecycleHooks::global().on_request_finish; + if (hook) { + try { + hook(context); + } catch (...) { + } + } +} + +} // namespace molo::obs diff --git a/src/observability/logging/async_queue.cpp b/src/observability/logging/async_queue.cpp new file mode 100644 index 0000000..50343bc --- /dev/null +++ b/src/observability/logging/async_queue.cpp @@ -0,0 +1,69 @@ +#include "observability/logging/async_queue.h" + +#include + +namespace molo::log { + +AsyncQueue::AsyncQueue(Callback callback, std::size_t capacity) + : callback_(std::move(callback)), capacity_(capacity) { + if (capacity_ < 1) capacity_ = 1; + worker_ = std::thread([this] { run(); }); +} + +AsyncQueue::~AsyncQueue() { stop(); } + +bool AsyncQueue::try_push(Record record) { + { + std::lock_guard lock(mutex_); + if (!running_ || queue_.size() >= capacity_) { + dropped_.fetch_add(1, std::memory_order_relaxed); + return false; + } + queue_.push_back(std::move(record)); + } + cv_.notify_one(); + return true; +} + +void AsyncQueue::stop() { + { + std::lock_guard lock(mutex_); + running_ = false; + } + cv_.notify_all(); + if (worker_.joinable()) worker_.join(); +} + +std::uint64_t AsyncQueue::dropped() const { return dropped_.load(std::memory_order_relaxed); } + +std::uint64_t AsyncQueue::failed() const { return failed_.load(std::memory_order_relaxed); } + +std::size_t AsyncQueue::size() const { + std::lock_guard lock(mutex_); + return queue_.size(); +} + +void AsyncQueue::run() { + for (;;) { + std::vector batch; + { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return !queue_.empty() || !running_; }); + if (queue_.empty() && !running_) return; + batch.reserve(queue_.size()); + while (!queue_.empty()) { + batch.push_back(std::move(queue_.front())); + queue_.pop_front(); + } + } + for (const Record& record : batch) { + try { + callback_(record); + } catch (...) { + failed_.fetch_add(1, std::memory_order_relaxed); + } + } + } +} + +} // namespace molo::log diff --git a/src/observability/logging/async_queue.h b/src/observability/logging/async_queue.h new file mode 100644 index 0000000..c9fa96a --- /dev/null +++ b/src/observability/logging/async_queue.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "observability/config.h" +#include "observability/logging/record.h" + +namespace molo::log { + +// Bounded MPSC queue plus a dedicated writer thread. Producers never block: +// a full queue drops the record and counts the drop. The worker drains the +// queue on stop. +class AsyncQueue { + public: + using Callback = std::function; + + explicit AsyncQueue(Callback callback, std::size_t capacity = molo::obs::Caps::kLogQueue); + ~AsyncQueue(); + + AsyncQueue(const AsyncQueue&) = delete; + AsyncQueue& operator=(const AsyncQueue&) = delete; + + // Returns false and counts a drop when the queue is full or stopped. + bool try_push(Record record); + void stop(); + + std::uint64_t dropped() const; + std::uint64_t failed() const; + std::size_t size() const; + + private: + void run(); + + Callback callback_; + std::size_t capacity_; + mutable std::mutex mutex_; + std::condition_variable cv_; + std::deque queue_; + std::thread worker_; + bool running_ = true; + std::atomic dropped_{0}; + std::atomic failed_{0}; +}; + +} // namespace molo::log diff --git a/src/observability/logging/async_sink.h b/src/observability/logging/async_sink.h new file mode 100644 index 0000000..39ef294 --- /dev/null +++ b/src/observability/logging/async_sink.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include + +#include "observability/config.h" +#include "observability/logging/async_queue.h" +#include "observability/logging/sink.h" + +namespace molo::log { + +// Sink decorator that moves writes to a background thread through a bounded +// queue. Drops (never blocks) under saturation. +class AsyncSink : public Sink { + public: + explicit AsyncSink(std::shared_ptr inner, + std::size_t capacity = molo::obs::Caps::kLogQueue) + : inner_(std::move(inner)), + queue_([owned = inner_](const Record& record) { owned->write(record); }, capacity) {} + + void write(const Record& record) override { queue_.try_push(record); } + + void flush() override { + queue_.stop(); + inner_->flush(); + } + + std::uint64_t dropped() const { return queue_.dropped() + queue_.failed(); } + + private: + std::shared_ptr inner_; + AsyncQueue queue_; +}; + +} // namespace molo::log diff --git a/src/observability/logging/console_sink.h b/src/observability/logging/console_sink.h new file mode 100644 index 0000000..6bf7ef4 --- /dev/null +++ b/src/observability/logging/console_sink.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +#include "observability/logging/json_format.h" +#include "observability/logging/sink.h" + +namespace molo::log { + +// Writes records to stderr, either as JSON lines or as short text lines. +class ConsoleSink : public Sink { + public: + explicit ConsoleSink(bool json = false) : json_(json) {} + + void write(const Record& record) override { + std::lock_guard lock(mutex_); + if (json_) { + std::cerr << to_json(record) << "\n"; + return; + } + std::cerr << "[" << format_ts(record.timestamp) << "] [" << to_string(record.level) << "] [" + << record.logger << "] " << record.message; + if (!record.request_id.empty()) std::cerr << " request_id=" << record.request_id; + std::cerr << "\n"; + } + + void flush() override { + std::lock_guard lock(mutex_); + std::cerr << std::flush; + } + + private: + bool json_; + std::mutex mutex_; +}; + +} // namespace molo::log diff --git a/src/observability/logging/file_sink.cpp b/src/observability/logging/file_sink.cpp new file mode 100644 index 0000000..83b5301 --- /dev/null +++ b/src/observability/logging/file_sink.cpp @@ -0,0 +1,60 @@ +#include "observability/logging/file_sink.h" + +#include + +#include "observability/logging/json_format.h" + +namespace molo::log { + +FileSink::FileSink(std::string path, std::size_t max_bytes, std::size_t max_files) + : path_(std::move(path)), max_bytes_(max_bytes), max_files_(max_files) { + if (max_files_ < 1) max_files_ = 1; + if (max_bytes_ < 1024) max_bytes_ = 1024; + open_locked(); +} + +FileSink::~FileSink() { + std::lock_guard lock(mutex_); + if (out_.is_open()) out_.close(); +} + +void FileSink::open_locked() { + out_.open(path_, std::ios::out | std::ios::app | std::ios::binary); + current_bytes_ = 0; + if (out_.is_open()) { + out_.seekp(0, std::ios::end); + auto pos = out_.tellp(); + if (pos >= 0) current_bytes_ = static_cast(pos); + } +} + +void FileSink::rotate_locked() { + if (out_.is_open()) out_.close(); + // Keeps exactly max_files_ files: path, path.1, ..., path.(max_files-1). + std::remove((path_ + "." + std::to_string(max_files_ - 1)).c_str()); + for (std::size_t i = max_files_ - 1; i > 0; --i) { + std::string from = (i == 1) ? path_ : path_ + "." + std::to_string(i - 1); + std::string to = path_ + "." + std::to_string(i); + std::rename(from.c_str(), to.c_str()); + } + open_locked(); +} + +void FileSink::write(const Record& record) { + std::string line = to_json(record); + line += "\n"; + std::lock_guard lock(mutex_); + if (!out_.is_open()) open_locked(); + if (!out_.is_open()) return; + if (current_bytes_ + line.size() > max_bytes_) rotate_locked(); + if (!out_.is_open()) return; + out_ << line; + current_bytes_ += line.size(); +} + +void FileSink::flush() { + std::lock_guard lock(mutex_); + if (out_.is_open()) out_.flush(); +} + +} // namespace molo::log diff --git a/src/observability/logging/file_sink.h b/src/observability/logging/file_sink.h new file mode 100644 index 0000000..0e9cbd6 --- /dev/null +++ b/src/observability/logging/file_sink.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include + +#include "observability/config.h" +#include "observability/logging/sink.h" + +namespace molo::log { + +// Appends JSON lines to a file with size-based rotation. Keeps at most +// max_files files (path, path.1, ...), each capped at max_bytes. +class FileSink : public Sink { + public: + FileSink(std::string path, std::size_t max_bytes = molo::obs::Caps::kFileRotateBytes, + std::size_t max_files = molo::obs::Caps::kFileRotateFiles); + ~FileSink() override; + + void write(const Record& record) override; + void flush() override; + + private: + void open_locked(); + void rotate_locked(); + + std::string path_; + std::size_t max_bytes_; + std::size_t max_files_; + std::size_t current_bytes_ = 0; + std::mutex mutex_; + std::ofstream out_; +}; + +} // namespace molo::log diff --git a/src/observability/logging/json_format.cpp b/src/observability/logging/json_format.cpp new file mode 100644 index 0000000..667eb8d --- /dev/null +++ b/src/observability/logging/json_format.cpp @@ -0,0 +1,126 @@ +#include "observability/logging/json_format.h" + +#include +#include +#include + +#include "observability/config.h" + +namespace molo::log { +namespace { + +void append_escaped(std::string& out, std::string_view text) { + for (char ch : text) { + switch (ch) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + if (static_cast(ch) < 0x20) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", + static_cast(static_cast(ch))); + out += buf; + } else { + out += ch; + } + } + } +} + +std::string format_double(double value) { + if (!std::isfinite(value)) return "null"; + return molo::obs::format_double_finite(value); +} + +} // namespace + +std::string escape_json(std::string_view text) { + std::string out; + out.reserve(text.size() + 8); + append_escaped(out, text); + return out; +} + +std::string format_ts(std::chrono::system_clock::time_point time) { + using namespace std::chrono; + auto total_ms = duration_cast(time.time_since_epoch()).count(); + int millis = static_cast(total_ms % 1000); + if (millis < 0) millis += 1000; + std::time_t raw = system_clock::to_time_t(time); + std::tm broken{}; +#if defined(_WIN32) + gmtime_s(&broken, &raw); +#else + gmtime_r(&raw, &broken); +#endif + char date[32]; + std::strftime(date, sizeof(date), "%Y-%m-%dT%H:%M:%S", &broken); + char out[48]; + std::snprintf(out, sizeof(out), "%s.%03dZ", date, millis); + return out; +} + +std::string field_value_json(const FieldValue& value) { + return std::visit( + [](auto&& raw) -> std::string { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return "\"" + escape_json(raw) + "\""; + } else if constexpr (std::is_same_v) { + return raw ? "true" : "false"; + } else if constexpr (std::is_same_v) { + return format_double(raw); + } else { + return std::to_string(raw); + } + }, + value); +} + +std::string to_json(const Record& record) { + std::string out; + out.reserve(256 + record.message.size()); + out += "{\"ts\":\""; + out += format_ts(record.timestamp); + out += "\",\"level\":\""; + out += to_string(record.level); + out += "\",\"logger\":\""; + append_escaped(out, record.logger); + out += "\",\"msg\":\""; + append_escaped(out, record.message); + out += "\",\"request_id\":\""; + append_escaped(out, record.request_id); + out += "\",\"trace_id\":\""; + append_escaped(out, record.trace_id); + out += "\",\"span_id\":\""; + append_escaped(out, record.span_id); + out += "\",\"fields\":{"; + bool first = true; + for (const Field& field : record.fields) { + if (!first) out += ","; + first = false; + out += "\""; + append_escaped(out, field.key); + out += "\":"; + out += field_value_json(field.value); + } + out += "}"; + if (record.truncated) out += ",\"truncated\":true"; + out += "}"; + return out; +} + +} // namespace molo::log diff --git a/src/observability/logging/json_format.h b/src/observability/logging/json_format.h new file mode 100644 index 0000000..9b08087 --- /dev/null +++ b/src/observability/logging/json_format.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include + +#include "observability/logging/record.h" + +namespace molo::log { + +// Fixed JSON schema renderer for log records: +// {"ts":"...","level":"...","logger":"...","msg":"...", +// "request_id":"...","trace_id":"...","span_id":"...","fields":{...}} +std::string to_json(const Record& record); +std::string field_value_json(const FieldValue& value); +std::string escape_json(std::string_view text); + +// UTC RFC 3339 with millis, for example 2026-10-23T14:30:00.123Z. +std::string format_ts(std::chrono::system_clock::time_point time); + +} // namespace molo::log diff --git a/src/observability/logging/level.h b/src/observability/logging/level.h new file mode 100644 index 0000000..ed660c8 --- /dev/null +++ b/src/observability/logging/level.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include + +// Numeric level ids used by the compile-time gating in macros.h. +#define MOLO_LEVEL_TRACE 0 +#define MOLO_LEVEL_DEBUG 1 +#define MOLO_LEVEL_INFO 2 +#define MOLO_LEVEL_WARN 3 +#define MOLO_LEVEL_ERROR 4 +#define MOLO_LEVEL_CRITICAL 5 +#define MOLO_LEVEL_OFF 6 + +namespace molo::log { + +enum class Level : std::uint8_t { + Trace = MOLO_LEVEL_TRACE, + Debug = MOLO_LEVEL_DEBUG, + Info = MOLO_LEVEL_INFO, + Warn = MOLO_LEVEL_WARN, + Error = MOLO_LEVEL_ERROR, + Critical = MOLO_LEVEL_CRITICAL, +}; + +constexpr int to_int(Level level) { return static_cast(level); } + +constexpr std::string_view to_string(Level level) { + switch (level) { + case Level::Trace: + return "trace"; + case Level::Debug: + return "debug"; + case Level::Info: + return "info"; + case Level::Warn: + return "warn"; + case Level::Error: + return "error"; + case Level::Critical: + return "critical"; + } + return "info"; +} + +constexpr std::optional from_string(std::string_view name) { + if (name == "trace") return Level::Trace; + if (name == "debug") return Level::Debug; + if (name == "info") return Level::Info; + if (name == "warn" || name == "warning") return Level::Warn; + if (name == "error") return Level::Error; + if (name == "critical" || name == "fatal") return Level::Critical; + return std::nullopt; +} + +} // namespace molo::log diff --git a/src/observability/logging/logger.cpp b/src/observability/logging/logger.cpp new file mode 100644 index 0000000..6e7ca08 --- /dev/null +++ b/src/observability/logging/logger.cpp @@ -0,0 +1,94 @@ +#include "observability/logging/logger.h" + +#include "observability/config.h" +#include "observability/correlation/redaction.h" +#include "observability/logging/registry.h" + +namespace molo::log { + +Logger::Logger(std::string name, Registry* registry) + : name_(std::move(name)), registry_(registry), level_(Level::Debug) {} + +Level Logger::level() const { return level_.load(std::memory_order_relaxed); } + +void Logger::set_level(Level level) { level_.store(level, std::memory_order_relaxed); } + +bool Logger::is_enabled(Level level) const { + if (to_int(level) < to_int(this->level())) return false; + if (registry_ != nullptr && to_int(level) < to_int(registry_->global_level())) return false; + return true; +} + +void Logger::log(Level level, std::string_view message, std::source_location location, + std::vector fields) { + if constexpr (!molo::obs::kEnabled) return; + if (!molo::obs::RuntimeToggles::instance().logging.load(std::memory_order_relaxed)) return; + if (!is_enabled(level)) return; + + for (Field& field : fields) { + if (molo::corr::is_sensitive_key(field.key)) field.value = std::string(molo::corr::kRedacted); + } + + Record record; + record.level = level; + record.logger = name_; + if (message.size() > molo::obs::Caps::kMaxLogLine) { + record.message = std::string(message.substr(0, molo::obs::Caps::kMaxLogLine)); + record.truncated = true; + } else { + record.message = std::string(message); + } + record.file = location.file_name(); + record.line = location.line(); + record.func = location.function_name(); + { + std::lock_guard lock(context_mutex_); + record.request_id = request_id_; + record.trace_id = trace_id_; + record.span_id = span_id_; + } + record.fields = std::move(fields); + if (registry_ != nullptr) registry_->emit(record); +} + +std::shared_ptr Logger::child(std::string_view suffix) { + std::shared_ptr kid; + if (registry_ != nullptr) { + kid = registry_->get(name_ + "." + std::string(suffix)); + } else { + kid = std::make_shared(name_ + "." + std::string(suffix), nullptr); + } + kid->set_level(level()); + std::lock_guard lock(context_mutex_); + kid->set_request_id(request_id_); + kid->set_span(trace_id_, span_id_); + return kid; +} + +void Logger::set_request_id(std::string request_id) { + std::lock_guard lock(context_mutex_); + request_id_ = std::move(request_id); +} + +void Logger::set_span(std::string trace_id, std::string span_id) { + std::lock_guard lock(context_mutex_); + trace_id_ = std::move(trace_id); + span_id_ = std::move(span_id); +} + +std::string Logger::request_id() const { + std::lock_guard lock(context_mutex_); + return request_id_; +} + +std::string Logger::trace_id() const { + std::lock_guard lock(context_mutex_); + return trace_id_; +} + +std::string Logger::span_id() const { + std::lock_guard lock(context_mutex_); + return span_id_; +} + +} // namespace molo::log diff --git a/src/observability/logging/logger.h b/src/observability/logging/logger.h new file mode 100644 index 0000000..c02b1ce --- /dev/null +++ b/src/observability/logging/logger.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "observability/logging/level.h" +#include "observability/logging/record.h" + +namespace molo::log { + +class Registry; + +// Named logger with a per-logger level and request correlation context. +// Child loggers inherit the context of their parent. +class Logger { + public: + Logger(std::string name, Registry* registry); + + const std::string& name() const { return name_; } + + Level level() const; + void set_level(Level level); + bool is_enabled(Level level) const; + + void log(Level level, std::string_view message, + std::source_location location = std::source_location::current(), + std::vector fields = {}); + + std::shared_ptr child(std::string_view suffix); + + void set_request_id(std::string request_id); + void set_span(std::string trace_id, std::string span_id); + std::string request_id() const; + std::string trace_id() const; + std::string span_id() const; + + private: + std::string name_; + Registry* registry_; + std::atomic level_; + mutable std::mutex context_mutex_; + std::string request_id_; + std::string trace_id_; + std::string span_id_; +}; + +} // namespace molo::log diff --git a/src/observability/logging/macros.h b/src/observability/logging/macros.h new file mode 100644 index 0000000..3621d1e --- /dev/null +++ b/src/observability/logging/macros.h @@ -0,0 +1,107 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "observability/config.h" +#include "observability/logging/level.h" +#include "observability/logging/record.h" + +namespace molo::log::detail { + +template +struct is_shared_ptr : std::false_type {}; +template +struct is_shared_ptr> : std::true_type {}; +template +inline constexpr bool is_shared_ptr_v = is_shared_ptr>::value; + +template +inline auto& resolve_logger(L& logger) { + if constexpr (is_shared_ptr_v) { + return *logger; + } else { + return logger; + } +} + +template +inline void log_if(L& logger, Level level, std::string_view message, std::source_location location, + std::vector fields = {}) { + if constexpr (!molo::obs::kEnabled) return; + if (!molo::obs::RuntimeToggles::instance().logging.load(std::memory_order_relaxed)) return; + auto& resolved = resolve_logger(logger); + if (!resolved.is_enabled(level)) return; + resolved.log(level, message, location, std::move(fields)); +} + +} // namespace molo::log::detail + +// Log macros with compile-time level gating (spdlog SPDLOG_ACTIVE_LEVEL +// pattern): calls below MOLO_ACTIVE_LEVEL compile to nothing, so debug and +// trace logging can stay in hot paths. Usage: +// MOLO_LOG_INFO(logger, "request.finished"); +// MOLO_LOG_INFO(logger, "db.query", {{"table", t}, {"rows", n}}); +#if MOLO_OBSERVABILITY && (MOLO_LEVEL_TRACE >= MOLO_ACTIVE_LEVEL) +#define MOLO_LOG_TRACE(logger, message, ...) \ + do { \ + ::molo::log::detail::log_if((logger), ::molo::log::Level::Trace, (message), \ + ::std::source_location::current() __VA_OPT__(, ) __VA_ARGS__); \ + } while (0) +#else +#define MOLO_LOG_TRACE(logger, message, ...) ((void)0) +#endif + +#if MOLO_OBSERVABILITY && (MOLO_LEVEL_DEBUG >= MOLO_ACTIVE_LEVEL) +#define MOLO_LOG_DEBUG(logger, message, ...) \ + do { \ + ::molo::log::detail::log_if((logger), ::molo::log::Level::Debug, (message), \ + ::std::source_location::current() __VA_OPT__(, ) __VA_ARGS__); \ + } while (0) +#else +#define MOLO_LOG_DEBUG(logger, message, ...) ((void)0) +#endif + +#if MOLO_OBSERVABILITY && (MOLO_LEVEL_INFO >= MOLO_ACTIVE_LEVEL) +#define MOLO_LOG_INFO(logger, message, ...) \ + do { \ + ::molo::log::detail::log_if((logger), ::molo::log::Level::Info, (message), \ + ::std::source_location::current() __VA_OPT__(, ) __VA_ARGS__); \ + } while (0) +#else +#define MOLO_LOG_INFO(logger, message, ...) ((void)0) +#endif + +#if MOLO_OBSERVABILITY && (MOLO_LEVEL_WARN >= MOLO_ACTIVE_LEVEL) +#define MOLO_LOG_WARN(logger, message, ...) \ + do { \ + ::molo::log::detail::log_if((logger), ::molo::log::Level::Warn, (message), \ + ::std::source_location::current() __VA_OPT__(, ) __VA_ARGS__); \ + } while (0) +#else +#define MOLO_LOG_WARN(logger, message, ...) ((void)0) +#endif + +#if MOLO_OBSERVABILITY && (MOLO_LEVEL_ERROR >= MOLO_ACTIVE_LEVEL) +#define MOLO_LOG_ERROR(logger, message, ...) \ + do { \ + ::molo::log::detail::log_if((logger), ::molo::log::Level::Error, (message), \ + ::std::source_location::current() __VA_OPT__(, ) __VA_ARGS__); \ + } while (0) +#else +#define MOLO_LOG_ERROR(logger, message, ...) ((void)0) +#endif + +#if MOLO_OBSERVABILITY && (MOLO_LEVEL_CRITICAL >= MOLO_ACTIVE_LEVEL) +#define MOLO_LOG_CRITICAL(logger, message, ...) \ + do { \ + ::molo::log::detail::log_if((logger), ::molo::log::Level::Critical, (message), \ + ::std::source_location::current() __VA_OPT__(, ) __VA_ARGS__); \ + } while (0) +#else +#define MOLO_LOG_CRITICAL(logger, message, ...) ((void)0) +#endif diff --git a/src/observability/logging/null_sink.h b/src/observability/logging/null_sink.h new file mode 100644 index 0000000..5b1b678 --- /dev/null +++ b/src/observability/logging/null_sink.h @@ -0,0 +1,13 @@ +#pragma once + +#include "observability/logging/sink.h" + +namespace molo::log { + +// Sink that drops every record. Used when logging output is disabled. +class NullSink : public Sink { + public: + void write(const Record& /*record*/) override {} +}; + +} // namespace molo::log diff --git a/src/observability/logging/record.h b/src/observability/logging/record.h new file mode 100644 index 0000000..be2b216 --- /dev/null +++ b/src/observability/logging/record.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "observability/logging/level.h" + +namespace molo::log { + +// One structured field on a log record. Prefer fields over string building. +using FieldValue = std::variant; + +struct Field { + std::string key; + FieldValue value; +}; + +// A single log record. Every record emitted inside a request carries +// request_id; every record inside a trace also carries trace_id and span_id. +struct Record { + std::chrono::system_clock::time_point timestamp = std::chrono::system_clock::now(); + Level level = Level::Info; + std::string logger; + std::string message; + std::string file; + std::uint32_t line = 0; + std::string func; + std::string request_id; + std::string trace_id; + std::string span_id; + std::vector fields; + bool truncated = false; +}; + +} // namespace molo::log diff --git a/src/observability/logging/registry.cpp b/src/observability/logging/registry.cpp new file mode 100644 index 0000000..54b5df3 --- /dev/null +++ b/src/observability/logging/registry.cpp @@ -0,0 +1,88 @@ +#include "observability/logging/registry.h" + +#include "audit/redact/redactor.h" +#include "observability/logging/logger.h" +#include "observability/logging/sink.h" + +namespace molo::log { + +Registry::Registry() : global_level_(Level::Debug), dropped_(0) { + for (auto& counter : records_) counter.store(0, std::memory_order_relaxed); +} + +std::shared_ptr Registry::get(const std::string& name) { + std::lock_guard lock(mutex_); + auto it = loggers_.find(name); + if (it != loggers_.end()) { + if (auto existing = it->second.lock()) return existing; + } + auto logger = std::make_shared(name, this); + loggers_[name] = logger; + return logger; +} + +void Registry::add_sink(std::shared_ptr sink) { + if (!sink) return; + std::lock_guard lock(mutex_); + sinks_.push_back(std::move(sink)); +} + +void Registry::clear_sinks() { + std::lock_guard lock(mutex_); + sinks_.clear(); +} + +std::size_t Registry::sink_count() const { + std::lock_guard lock(mutex_); + return sinks_.size(); +} + +void Registry::set_global_level(Level level) { global_level_.store(level, std::memory_order_relaxed); } + +Level Registry::global_level() const { return global_level_.load(std::memory_order_relaxed); } + +void Registry::emit(const Record& record) { + // Audit patch: every record passes through the redactor before fan-out, + // so PII can never reach a sink through any logger. + Record scrubbed = record; + ::molo::audit::redact::Redactor::RedactRecord(scrubbed); + std::vector> sinks; + std::function hook; + { + std::lock_guard lock(mutex_); + sinks = sinks_; + hook = hook_; + } + records_[static_cast(to_int(scrubbed.level))].fetch_add(1, std::memory_order_relaxed); + for (const auto& sink : sinks) { + try { + sink->write(scrubbed); + } catch (...) { + dropped_.fetch_add(1, std::memory_order_relaxed); + } + } + if (hook) { + try { + hook(scrubbed); + } catch (...) { + } + } +} + +std::uint64_t Registry::records_total(Level level) const { + return records_[static_cast(to_int(level))].load(std::memory_order_relaxed); +} + +std::uint64_t Registry::dropped_total() const { return dropped_.load(std::memory_order_relaxed); } + +void Registry::set_record_hook(std::function hook) { + std::lock_guard lock(mutex_); + hook_ = std::move(hook); +} + +Registry& Registry::global() { + static Registry instance; + return instance; +} + +} // namespace molo::log diff --git a/src/observability/logging/registry.h b/src/observability/logging/registry.h new file mode 100644 index 0000000..fa8cf54 --- /dev/null +++ b/src/observability/logging/registry.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "observability/logging/level.h" +#include "observability/logging/record.h" + +namespace molo::log { + +class Logger; +class Sink; + +// Owns named loggers, the global level and the sink list. Internal record +// and drop counters feed molo_log_records_total and molo_log_dropped_total; +// wire them to metrics through set_record_hook / dropped_total. +class Registry { + public: + Registry(); + + std::shared_ptr get(const std::string& name); + void add_sink(std::shared_ptr sink); + void clear_sinks(); + std::size_t sink_count() const; + + void set_global_level(Level level); + Level global_level() const; + + void emit(const Record& record); + + std::uint64_t records_total(Level level) const; + std::uint64_t dropped_total() const; + + // Called for every accepted record. Used to wire metrics without a + // logging -> metrics code dependency. + void set_record_hook(std::function hook); + + static Registry& global(); + + private: + mutable std::mutex mutex_; + std::map> loggers_; + std::vector> sinks_; + std::atomic global_level_; + std::array, 6> records_; + std::atomic dropped_; + std::function hook_; +}; + +} // namespace molo::log diff --git a/src/observability/logging/ring_sink.h b/src/observability/logging/ring_sink.h new file mode 100644 index 0000000..16709b6 --- /dev/null +++ b/src/observability/logging/ring_sink.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +#include "observability/config.h" +#include "observability/logging/sink.h" + +namespace molo::log { + +// In-memory tail of the most recent records. Used by tests and by the +// `logs` inspection command. Bounded, oldest records are evicted first. +class RingSink : public Sink { + public: + explicit RingSink(std::size_t capacity = molo::obs::Caps::kRingRecords) : capacity_(capacity) {} + + void write(const Record& record) override { + std::lock_guard lock(mutex_); + if (buffer_.size() >= capacity_ && !buffer_.empty()) buffer_.pop_front(); + if (capacity_ > 0) buffer_.push_back(record); + } + + std::vector snapshot() const { + std::lock_guard lock(mutex_); + return std::vector(buffer_.begin(), buffer_.end()); + } + + std::size_t size() const { + std::lock_guard lock(mutex_); + return buffer_.size(); + } + + void clear() { + std::lock_guard lock(mutex_); + buffer_.clear(); + } + + private: + std::size_t capacity_; + mutable std::mutex mutex_; + std::deque buffer_; +}; + +} // namespace molo::log diff --git a/src/observability/logging/sink.h b/src/observability/logging/sink.h new file mode 100644 index 0000000..6d6cc5f --- /dev/null +++ b/src/observability/logging/sink.h @@ -0,0 +1,16 @@ +#pragma once + +#include "observability/logging/record.h" + +namespace molo::log { + +// Output interface for log records. Sinks and exporters are the only +// components allowed to touch I/O; pillars never call network or disk. +class Sink { + public: + virtual ~Sink() = default; + virtual void write(const Record& record) = 0; + virtual void flush() {} +}; + +} // namespace molo::log diff --git a/src/observability/metrics/counter.h b/src/observability/metrics/counter.h new file mode 100644 index 0000000..8bf3e12 --- /dev/null +++ b/src/observability/metrics/counter.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +#include "observability/config.h" +#include "observability/metrics/labels.h" + +namespace molo::metrics { + +// Monotonic counter. Names end in _total (appended automatically when +// created through the registry). Thread safe, lock free on the hot path. +class Counter { + public: + Counter(std::string name, std::string help, Labels labels = {}) + : name_(std::move(name)), help_(std::move(help)), labels_(std::move(labels)), value_(0.0) {} + + void inc(double amount = 1.0) { + if constexpr (!molo::obs::kMetricsOn) return; + if (!molo::obs::RuntimeToggles::instance().metrics.load(std::memory_order_relaxed)) return; + if (!(amount > 0.0)) return; + value_.fetch_add(amount, std::memory_order_relaxed); + } + + double value() const { return value_.load(std::memory_order_relaxed); } + + const std::string& name() const { return name_; } + const std::string& help() const { return help_; } + const Labels& labels() const { return labels_; } + + private: + std::string name_; + std::string help_; + Labels labels_; + std::atomic value_; +}; + +} // namespace molo::metrics diff --git a/src/observability/metrics/gauge.h b/src/observability/metrics/gauge.h new file mode 100644 index 0000000..42d5fc5 --- /dev/null +++ b/src/observability/metrics/gauge.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +#include "observability/config.h" +#include "observability/metrics/labels.h" + +namespace molo::metrics { + +// Gauge for values that go up and down. Thread safe on the hot path. +class Gauge { + public: + Gauge(std::string name, std::string help, Labels labels = {}) + : name_(std::move(name)), help_(std::move(help)), labels_(std::move(labels)), value_(0.0) {} + + void inc(double amount = 1.0) { + if constexpr (!molo::obs::kMetricsOn) return; + if (!molo::obs::RuntimeToggles::instance().metrics.load(std::memory_order_relaxed)) return; + value_.fetch_add(amount, std::memory_order_relaxed); + } + + void dec(double amount = 1.0) { inc(-amount); } + + void set(double value) { + if constexpr (!molo::obs::kMetricsOn) return; + if (!molo::obs::RuntimeToggles::instance().metrics.load(std::memory_order_relaxed)) return; + value_.store(value, std::memory_order_relaxed); + } + + double value() const { return value_.load(std::memory_order_relaxed); } + + const std::string& name() const { return name_; } + const std::string& help() const { return help_; } + const Labels& labels() const { return labels_; } + + private: + std::string name_; + std::string help_; + Labels labels_; + std::atomic value_; +}; + +} // namespace molo::metrics diff --git a/src/observability/metrics/histogram.h b/src/observability/metrics/histogram.h new file mode 100644 index 0000000..c255956 --- /dev/null +++ b/src/observability/metrics/histogram.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "observability/config.h" +#include "observability/metrics/labels.h" + +namespace molo::metrics { + +// Histogram with cumulative buckets. Always exports _bucket, _sum and +// _count series. Thread safe. +class Histogram { + public: + struct Snapshot { + std::vector cumulative; + double sum = 0.0; + std::uint64_t count = 0; + }; + + Histogram(std::string name, std::string help, Labels labels, std::vector bounds) + : name_(std::move(name)), + help_(std::move(help)), + labels_(std::move(labels)), + bounds_(std::move(bounds)), + buckets_(bounds_.size(), 0), + sum_(0.0), + count_(0) {} + + void observe(double value) { + if constexpr (!molo::obs::kMetricsOn) return; + if (!molo::obs::RuntimeToggles::instance().metrics.load(std::memory_order_relaxed)) return; + if (std::isnan(value)) return; + std::lock_guard lock(mutex_); + auto it = std::upper_bound(bounds_.begin(), bounds_.end(), value); + for (auto jt = it; jt != bounds_.end(); ++jt) { + buckets_[static_cast(jt - bounds_.begin())]++; + } + sum_ += value; + count_++; + } + + Snapshot snapshot() const { + std::lock_guard lock(mutex_); + Snapshot out; + out.cumulative = buckets_; + out.sum = sum_; + out.count = count_; + return out; + } + + const std::string& name() const { return name_; } + const std::string& help() const { return help_; } + const Labels& labels() const { return labels_; } + const std::vector& bounds() const { return bounds_; } + + // Default HTTP duration buckets in seconds. + static const std::vector& default_http_buckets() { + static const std::vector buckets = {0.005, 0.01, 0.025, 0.05, 0.1, 0.25, + 0.5, 1.0, 2.5, 5.0, 10.0}; + return buckets; + } + + private: + std::string name_; + std::string help_; + Labels labels_; + std::vector bounds_; + std::vector buckets_; + double sum_; + std::uint64_t count_; + mutable std::mutex mutex_; +}; + +} // namespace molo::metrics diff --git a/src/observability/metrics/labels.h b/src/observability/metrics/labels.h new file mode 100644 index 0000000..565748d --- /dev/null +++ b/src/observability/metrics/labels.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "observability/config.h" + +namespace molo::metrics { + +// Label set for one metric series. At most 16 labels; values are truncated +// to 128 chars. Label values must come from bounded sets only (route +// templates, methods, status classes, error kinds). Raw paths, user IDs, +// query strings, full error text and secrets are forbidden here. +class Labels { + public: + Labels() = default; + + Labels& add(std::string key, std::string value) { + if (key.empty()) throw std::invalid_argument("metric label key must not be empty"); + if (items_.size() >= molo::obs::Caps::kMaxLabels) { + throw std::invalid_argument("too many labels on one metric"); + } + if (value.size() > molo::obs::Caps::kMaxLabelValueLen) { + value.resize(molo::obs::Caps::kMaxLabelValueLen); + } + items_.emplace_back(std::move(key), std::move(value)); + return *this; + } + + std::size_t size() const { return items_.size(); } + bool empty() const { return items_.empty(); } + + const std::vector>& items() const { return items_; } + + bool operator==(const Labels& other) const { return items_ == other.items_; } + + // Canonical identity used as the series map key. + std::string key() const { + std::string out; + for (const auto& item : items_) { + out += item.first; + out += "=\""; + out += item.second; + out += "\","; + } + return out; + } + + private: + std::vector> items_; +}; + +} // namespace molo::metrics diff --git a/src/observability/metrics/prometheus.cpp b/src/observability/metrics/prometheus.cpp new file mode 100644 index 0000000..76c444f --- /dev/null +++ b/src/observability/metrics/prometheus.cpp @@ -0,0 +1,141 @@ +#include "observability/metrics/prometheus.h" + +#include +#include + +#include "observability/metrics/labels.h" +#include "observability/metrics/registry.h" + +namespace molo::metrics { + +std::string escape_label_value(std::string_view value) { + std::string out; + out.reserve(value.size() + 4); + for (char ch : value) { + if (ch == '\\') { + out += "\\\\"; + } else if (ch == '"') { + out += "\\\""; + } else if (ch == '\n') { + out += "\\n"; + } else { + out += ch; + } + } + return out; +} + +std::string escape_help(std::string_view help) { + std::string out; + out.reserve(help.size() + 4); + for (char ch : help) { + if (ch == '\\') { + out += "\\\\"; + } else if (ch == '\n') { + out += "\\n"; + } else { + out += ch; + } + } + return out; +} + +std::string format_sample_double(double value) { + if (std::isnan(value)) return "Nan"; + if (std::isinf(value)) return value > 0 ? "+Inf" : "-Inf"; + return molo::obs::format_double_finite(value); +} + +std::string label_string(const Labels& labels) { + if (labels.empty()) return ""; + std::string out = "{"; + bool first = true; + for (const auto& [key, value] : labels.items()) { + if (!first) out += ","; + first = false; + out += key; + out += "=\""; + out += escape_label_value(value); + out += "\""; + } + out += "}"; + return out; +} + +std::string render(const Registry& registry) { + Registry::Snapshot snap = registry.snapshot(); + std::string out; + out.reserve(1024 + 128 * (snap.counters.size() + snap.gauges.size() + snap.histograms.size())); + for (const auto& metric : snap.counters) { + out += "# HELP "; + out += metric.name; + out += " "; + out += escape_help(metric.help); + out += "\n# TYPE "; + out += metric.name; + out += " counter\n"; + out += metric.name; + out += label_string(metric.labels); + out += " "; + out += format_sample_double(metric.value); + out += "\n"; + } + for (const auto& metric : snap.gauges) { + out += "# HELP "; + out += metric.name; + out += " "; + out += escape_help(metric.help); + out += "\n# TYPE "; + out += metric.name; + out += " gauge\n"; + out += metric.name; + out += label_string(metric.labels); + out += " "; + out += format_sample_double(metric.value); + out += "\n"; + } + for (const auto& metric : snap.histograms) { + out += "# HELP "; + out += metric.name; + out += " "; + out += escape_help(metric.help); + out += "\n# TYPE "; + out += metric.name; + out += " histogram\n"; + for (std::size_t i = 0; i < metric.bounds.size(); ++i) { + out += metric.name; + out += "_bucket"; + Labels with_le = metric.labels; + with_le.add("le", format_sample_double(metric.bounds[i])); + out += label_string(with_le); + out += " "; + out += std::to_string(metric.cumulative[i]); + out += "\n"; + } + { + out += metric.name; + out += "_bucket"; + Labels with_le = metric.labels; + with_le.add("le", "+Inf"); + out += label_string(with_le); + out += " "; + out += std::to_string(metric.count); + out += "\n"; + } + out += metric.name; + out += "_sum"; + out += label_string(metric.labels); + out += " "; + out += format_sample_double(metric.sum); + out += "\n"; + out += metric.name; + out += "_count"; + out += label_string(metric.labels); + out += " "; + out += std::to_string(metric.count); + out += "\n"; + } + return out; +} + +} // namespace molo::metrics diff --git a/src/observability/metrics/prometheus.h b/src/observability/metrics/prometheus.h new file mode 100644 index 0000000..b841f97 --- /dev/null +++ b/src/observability/metrics/prometheus.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace molo::metrics { + +class Labels; +class Registry; + +// Prometheus text exposition renderer (version 0.0.4): # HELP and # TYPE +// comment lines followed by name{label="value"} value samples. Served by +// GET /metrics (wired in 09-cli-ops.md). +std::string render(const Registry& registry); +std::string escape_label_value(std::string_view value); +std::string escape_help(std::string_view help); +std::string format_sample_double(double value); +std::string label_string(const Labels& labels); + +} // namespace molo::metrics diff --git a/src/observability/metrics/registry.cpp b/src/observability/metrics/registry.cpp new file mode 100644 index 0000000..516d1ae --- /dev/null +++ b/src/observability/metrics/registry.cpp @@ -0,0 +1,193 @@ +#include "observability/metrics/registry.h" + +#include +#include + +namespace molo::metrics { +namespace { + +bool valid_name_char(char ch, bool first) { + if (ch >= 'a' && ch <= 'z') return true; + if (ch >= 'A' && ch <= 'Z') return true; + if (ch == '_' || ch == ':') return true; + if (!first && ch >= '0' && ch <= '9') return true; + return false; +} + +bool valid_metric_name(std::string_view name) { + if (name.empty()) return false; + for (std::size_t i = 0; i < name.size(); ++i) { + if (!valid_name_char(name[i], i == 0)) return false; + } + return true; +} + +bool ends_with_total(const std::string& name) { + return name.size() >= 6 && name.compare(name.size() - 6, 6, "_total") == 0; +} + +} // namespace + +Registry::Registry() + : overflow_counter_("molo_metrics_cardinality_overflows_total", + "Series whose label value overflowed into other", Labels{}), + blackhole_counter_("molo_blackhole_total", "Overflow sink, never exported", Labels{}), + blackhole_gauge_("molo_blackhole", "Overflow sink, never exported", Labels{}), + blackhole_histogram_("molo_blackhole_hist", "Overflow sink, never exported", Labels{}, + Histogram::default_http_buckets()), + overflows_(0) {} + +Counter& Registry::counter(const std::string& name, const std::string& help, Labels labels) { + std::string full = name; + if (!ends_with_total(full)) full += "_total"; + if (!valid_metric_name(full)) throw std::invalid_argument("invalid counter name: " + full); + std::lock_guard lock(mutex_); + Labels mapped = apply_cardinality(full, std::move(labels)); + std::string key = "c\x01" + full + "\x01" + mapped.key(); + auto it = counters_.find(key); + if (it != counters_.end()) return *it->second; + if (total_series_locked() >= molo::obs::Caps::kMaxSeries) { + overflows_.fetch_add(1, std::memory_order_relaxed); + overflow_counter_.inc(); + return blackhole_counter_; + } + auto inserted = + counters_.emplace(key, std::make_unique(full, help, std::move(mapped))); + return *inserted.first->second; +} + +Gauge& Registry::gauge(const std::string& name, const std::string& help, Labels labels) { + if (!valid_metric_name(name)) throw std::invalid_argument("invalid gauge name: " + name); + std::lock_guard lock(mutex_); + Labels mapped = apply_cardinality(name, std::move(labels)); + std::string key = "g\x01" + name + "\x01" + mapped.key(); + auto it = gauges_.find(key); + if (it != gauges_.end()) return *it->second; + if (total_series_locked() >= molo::obs::Caps::kMaxSeries) { + overflows_.fetch_add(1, std::memory_order_relaxed); + overflow_counter_.inc(); + return blackhole_gauge_; + } + auto inserted = gauges_.emplace(key, std::make_unique(name, help, std::move(mapped))); + return *inserted.first->second; +} + +Histogram& Registry::histogram(const std::string& name, const std::string& help, Labels labels, + std::vector bounds) { + if (!valid_metric_name(name)) throw std::invalid_argument("invalid histogram name: " + name); + std::lock_guard lock(mutex_); + Labels mapped = apply_cardinality(name, std::move(labels)); + std::string key = "h\x01" + name + "\x01" + mapped.key(); + auto it = histograms_.find(key); + if (it != histograms_.end()) return *it->second; + if (total_series_locked() >= molo::obs::Caps::kMaxSeries) { + overflows_.fetch_add(1, std::memory_order_relaxed); + overflow_counter_.inc(); + return blackhole_histogram_; + } + auto inserted = histograms_.emplace( + key, std::make_unique(name, help, std::move(mapped), std::move(bounds))); + return *inserted.first->second; +} + +void Registry::ensure_framework_metrics() { + gauge("molo_http_requests_in_flight", "HTTP requests currently in flight"); + counter("molo_log_dropped_total", "Log records dropped under saturation"); + counter("molo_spans_dropped_total", "Spans dropped under saturation"); + counter("molo_trace_propagator_errors_total", "Malformed traceparent headers"); + for (const char* level : {"trace", "debug", "info", "warn", "error", "critical"}) { + Labels labels; + labels.add("level", level); + counter("molo_log_records_total", "Log records emitted", labels); + } + for (const char* kind : {"server", "client", "internal", "producer", "consumer"}) { + Labels labels; + labels.add("kind", kind); + counter("molo_spans_total", "Spans started", labels); + } +} + +std::size_t Registry::series_count() const { + std::lock_guard lock(mutex_); + return total_series_locked(); +} + +std::uint64_t Registry::cardinality_overflows() const { + return overflows_.load(std::memory_order_relaxed); +} + +Registry::Snapshot Registry::snapshot() const { + Snapshot out; + std::lock_guard lock(mutex_); + for (const auto& entry : counters_) { + const Counter& metric = *entry.second; + CounterSnap snap; + snap.name = metric.name(); + snap.help = metric.help(); + snap.labels = metric.labels(); + snap.value = metric.value(); + out.counters.push_back(std::move(snap)); + } + CounterSnap overflow; + overflow.name = overflow_counter_.name(); + overflow.help = overflow_counter_.help(); + overflow.labels = overflow_counter_.labels(); + overflow.value = overflow_counter_.value(); + out.counters.push_back(std::move(overflow)); + for (const auto& entry : gauges_) { + const Gauge& metric = *entry.second; + GaugeSnap snap; + snap.name = metric.name(); + snap.help = metric.help(); + snap.labels = metric.labels(); + snap.value = metric.value(); + out.gauges.push_back(std::move(snap)); + } + for (const auto& entry : histograms_) { + const Histogram& metric = *entry.second; + HistogramSnap snap; + snap.name = metric.name(); + snap.help = metric.help(); + snap.labels = metric.labels(); + snap.bounds = metric.bounds(); + Histogram::Snapshot inner = metric.snapshot(); + snap.cumulative = std::move(inner.cumulative); + snap.sum = inner.sum; + snap.count = inner.count; + out.histograms.push_back(std::move(snap)); + } + return out; +} + +Registry& Registry::global() { + static Registry instance; + return instance; +} + +Labels Registry::apply_cardinality(const std::string& metric, Labels labels) { + Labels mapped; + for (const auto& [key, value] : labels.items()) { + std::string track = metric + "\x00" + key; + auto& seen = distinct_[track]; + if (value == "other" || seen.count(value) > 0) { + mapped.add(key, value); + continue; + } + if (seen.size() >= molo::obs::Caps::kMaxLabelValues) { + seen.insert("other"); + mapped.add(key, "other"); + overflows_.fetch_add(1, std::memory_order_relaxed); + overflow_counter_.inc(); + } else { + seen.insert(value); + mapped.add(key, value); + } + } + return mapped; +} + +std::size_t Registry::total_series_locked() const { + return counters_.size() + gauges_.size() + histograms_.size(); +} + +} // namespace molo::metrics diff --git a/src/observability/metrics/registry.h b/src/observability/metrics/registry.h new file mode 100644 index 0000000..75188f8 --- /dev/null +++ b/src/observability/metrics/registry.h @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "observability/metrics/counter.h" +#include "observability/metrics/gauge.h" +#include "observability/metrics/histogram.h" +#include "observability/metrics/labels.h" + +namespace molo::metrics { + +// Owns every metric series. Per-app instances are supported; global() +// is the process default. Enforces the static ceilings: at most 65536 +// series, at most 64 distinct values per label key (the 65th maps to +// "other" and counts molo_metrics_cardinality_overflows_total). +class Registry { + public: + struct CounterSnap { + std::string name; + std::string help; + Labels labels; + double value = 0.0; + }; + + struct GaugeSnap { + std::string name; + std::string help; + Labels labels; + double value = 0.0; + }; + + struct HistogramSnap { + std::string name; + std::string help; + Labels labels; + std::vector bounds; + std::vector cumulative; + double sum = 0.0; + std::uint64_t count = 0; + }; + + struct Snapshot { + std::vector counters; + std::vector gauges; + std::vector histograms; + }; + + Registry(); + + Counter& counter(const std::string& name, const std::string& help, Labels labels = Labels{}); + Gauge& gauge(const std::string& name, const std::string& help, Labels labels = Labels{}); + Histogram& histogram(const std::string& name, const std::string& help, + Labels labels = Labels{}, + std::vector bounds = Histogram::default_http_buckets()); + + // Pre-registers the fixed-cardinality framework metrics. Series with + // open label sets (routes, methods) are created lazily on first use. + void ensure_framework_metrics(); + + std::size_t series_count() const; + std::uint64_t cardinality_overflows() const; + Snapshot snapshot() const; + + static Registry& global(); + + private: + Labels apply_cardinality(const std::string& metric, Labels labels); + std::size_t total_series_locked() const; + + mutable std::mutex mutex_; + std::map> counters_; + std::map> gauges_; + std::map> histograms_; + std::map> distinct_; + Counter overflow_counter_; + Counter blackhole_counter_; + Gauge blackhole_gauge_; + Histogram blackhole_histogram_; + std::atomic overflows_; +}; + +} // namespace molo::metrics diff --git a/src/observability/metrics/timer.h b/src/observability/metrics/timer.h new file mode 100644 index 0000000..2307030 --- /dev/null +++ b/src/observability/metrics/timer.h @@ -0,0 +1,48 @@ +#pragma once + +#include + +#include "observability/config.h" +#include "observability/metrics/histogram.h" + +namespace molo::metrics { + +// RAII timer that observes its lifetime into a histogram on destruction. +// Compiles to a no-op when metrics are disabled at compile time. +class Timer { + public: + explicit Timer(Histogram& histogram) + : histogram_(&histogram), start_(std::chrono::steady_clock::now()) {} + + ~Timer() { + if constexpr (!molo::obs::kMetricsOn) return; + if (cancelled_) return; + histogram_->observe(elapsed_seconds()); + } + + Timer(const Timer&) = delete; + Timer& operator=(const Timer&) = delete; + + double elapsed_seconds() const { + auto now = std::chrono::steady_clock::now(); + return std::chrono::duration(now - start_).count(); + } + + void cancel() { cancelled_ = true; } + + private: + Histogram* histogram_; + std::chrono::steady_clock::time_point start_; + bool cancelled_ = false; +}; + +} // namespace molo::metrics + +#define MOLO_TIMER_CONCAT_IMPL(a, b) a##b +#define MOLO_TIMER_CONCAT(a, b) MOLO_TIMER_CONCAT_IMPL(a, b) +#if MOLO_OBSERVABILITY && MOLO_ENABLE_METRICS +#define MOLO_TIMER(histogram) \ + ::molo::metrics::Timer MOLO_TIMER_CONCAT(molo_timer_, __LINE__)(histogram) +#else +#define MOLO_TIMER(histogram) ((void)0) +#endif diff --git a/src/observability/observability.h b/src/observability/observability.h new file mode 100644 index 0000000..6119faa --- /dev/null +++ b/src/observability/observability.h @@ -0,0 +1,48 @@ +#pragma once + +// Umbrella header for the observability backbone. + +#include "observability/config.h" +#include "observability/hooks.h" + +#include "observability/correlation/redaction.h" +#include "observability/correlation/request_context.h" +#include "observability/correlation/request_id.h" + +#include "observability/logging/async_queue.h" +#include "observability/logging/async_sink.h" +#include "observability/logging/console_sink.h" +#include "observability/logging/file_sink.h" +#include "observability/logging/json_format.h" +#include "observability/logging/level.h" +#include "observability/logging/logger.h" +#include "observability/logging/macros.h" +#include "observability/logging/null_sink.h" +#include "observability/logging/record.h" +#include "observability/logging/registry.h" +#include "observability/logging/ring_sink.h" + +#include "observability/events/dispatcher.h" +#include "observability/events/event.h" +#include "observability/events/lifecycle_events.h" +#include "observability/events/listener.h" + +#include "observability/metrics/counter.h" +#include "observability/metrics/gauge.h" +#include "observability/metrics/histogram.h" +#include "observability/metrics/labels.h" +#include "observability/metrics/prometheus.h" +#include "observability/metrics/registry.h" +#include "observability/metrics/timer.h" + +#include "observability/tracing/batch_processor.h" +#include "observability/tracing/exporter.h" +#include "observability/tracing/http_sender.h" +#include "observability/tracing/ids.h" +#include "observability/tracing/memory_exporter.h" +#include "observability/tracing/otlp_http.h" +#include "observability/tracing/propagator.h" +#include "observability/tracing/sampler.h" +#include "observability/tracing/span.h" +#include "observability/tracing/span_context.h" +#include "observability/tracing/tracer.h" diff --git a/src/observability/observability.mk b/src/observability/observability.mk new file mode 100644 index 0000000..36b6831 --- /dev/null +++ b/src/observability/observability.mk @@ -0,0 +1,59 @@ +# Observability backbone (plan 02-observability.md) +# +# Fragment for the root Makefile. Include it with: +# include src/observability/observability.mk +# Requires: g++ 15 with -std=c++23, -Isrc on the compile line. + +ifndef OBS_MK_INCLUDED +OBS_MK_INCLUDED := 1 + +OBS_CXX ?= g++ +OBS_CXXFLAGS ?= -std=c++23 -O2 -Wall -Wextra -pthread -Isrc + +OBS_SRCS := \ + src/observability/correlation/request_context.cpp \ + src/observability/correlation/request_id.cpp \ + src/observability/events/dispatcher.cpp \ + src/observability/logging/async_queue.cpp \ + src/observability/logging/file_sink.cpp \ + src/observability/logging/json_format.cpp \ + src/observability/logging/logger.cpp \ + src/observability/logging/registry.cpp \ + src/observability/metrics/prometheus.cpp \ + src/observability/metrics/registry.cpp \ + src/observability/tracing/batch_processor.cpp \ + src/observability/tracing/otlp_http.cpp \ + src/observability/tracing/propagator.cpp \ + src/observability/tracing/span.cpp \ + src/observability/tracing/tracer.cpp + +OBS_OBJS := $(OBS_SRCS:src/%.cpp=build/obj/%.o) +OBS_LIB := build/obs/libobservability.a + +OBS_TEST_SRCS := \ + tests/src/observability/correlation_test.cpp \ + tests/src/observability/events_test.cpp \ + tests/src/observability/logging_test.cpp \ + tests/src/observability/metrics_endpoint_test.cpp \ + tests/src/observability/otlp_export_test.cpp \ + tests/src/observability/propagation_test.cpp \ + tests/src/observability/tracing_test.cpp + +OBS_TEST_BINS := $(OBS_TEST_SRCS:tests/src/observability/%.cpp=build/obs_tests/%) + +$(OBS_LIB): $(OBS_OBJS) + mkdir -p $(dir $@) + ar rcs $@ $(OBS_OBJS) + +build/obj/%.o: src/%.cpp + mkdir -p $(dir $@) + $(OBS_CXX) $(OBS_CXXFLAGS) -c $< -o $@ + +build/obs_tests/%: tests/src/observability/%.cpp $(OBS_LIB) + mkdir -p $(dir $@) + $(OBS_CXX) $(OBS_CXXFLAGS) -Itests/src/observability $< $(OBS_LIB) -o $@ + +obs-test: $(OBS_TEST_BINS) + for t in $(OBS_TEST_BINS); do ./$$t || exit 1; done + +endif diff --git a/src/observability/tracing/batch_processor.cpp b/src/observability/tracing/batch_processor.cpp new file mode 100644 index 0000000..9402a25 --- /dev/null +++ b/src/observability/tracing/batch_processor.cpp @@ -0,0 +1,96 @@ +#include "observability/tracing/batch_processor.h" + +namespace molo::trace { + +BatchProcessor::BatchProcessor(std::shared_ptr exporter, std::size_t max_buffer, + std::size_t batch_size, std::chrono::seconds interval) + : exporter_(std::move(exporter)), + max_buffer_(max_buffer), + batch_size_(batch_size), + interval_(interval) { + if (max_buffer_ < 1) max_buffer_ = 1; + if (batch_size_ < 1) batch_size_ = 1; + worker_ = std::thread([this] { run(); }); +} + +BatchProcessor::~BatchProcessor() { + { + std::lock_guard lock(mutex_); + running_ = false; + } + cv_.notify_all(); + if (worker_.joinable()) worker_.join(); + flush(); +} + +void BatchProcessor::add(FinishedSpan span) { + bool ready = false; + { + std::lock_guard lock(mutex_); + if (queue_.size() >= max_buffer_) { + dropped_.fetch_add(1, std::memory_order_relaxed); + return; + } + queue_.push_back(std::move(span)); + ready = queue_.size() >= batch_size_; + } + if (ready) cv_.notify_one(); +} + +void BatchProcessor::flush() { + for (;;) { + std::vector batch; + { + std::unique_lock lock(mutex_); + if (queue_.empty()) { + flushed_cv_.wait(lock, [this] { return in_flight_ == 0; }); + return; + } + batch.reserve(batch_size_); + for (auto it = queue_.begin(); it != queue_.end() && batch.size() < batch_size_;) { + batch.push_back(std::move(*it)); + it = queue_.erase(it); + } + ++in_flight_; + } + bool ok = false; + try { + ok = exporter_->export_spans(std::move(batch)); + } catch (...) { + ok = false; + } + { + std::lock_guard lock(mutex_); + --in_flight_; + if (ok) { + exported_.fetch_add(1, std::memory_order_relaxed); + } else { + failed_.fetch_add(1, std::memory_order_relaxed); + } + } + flushed_cv_.notify_all(); + } +} + +std::uint64_t BatchProcessor::dropped() const { return dropped_.load(std::memory_order_relaxed); } + +std::uint64_t BatchProcessor::exported_total() const { + return exported_.load(std::memory_order_relaxed); +} + +std::uint64_t BatchProcessor::failed_batches() const { + return failed_.load(std::memory_order_relaxed); +} + +void BatchProcessor::run() { + for (;;) { + { + std::unique_lock lock(mutex_); + cv_.wait_for(lock, interval_, [this] { return !running_ || queue_.size() >= batch_size_; }); + if (!running_) return; + } + flush(); + } +} + +} // namespace molo::trace diff --git a/src/observability/tracing/batch_processor.h b/src/observability/tracing/batch_processor.h new file mode 100644 index 0000000..895fd4a --- /dev/null +++ b/src/observability/tracing/batch_processor.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "observability/config.h" +#include "observability/tracing/exporter.h" + +namespace molo::trace { + +// Bounded span buffer plus background batch export. Exports when the batch +// size is reached or the interval elapses, whichever comes first. A full +// buffer drops spans and counts the drop, never grows memory. +class BatchProcessor { + public: + explicit BatchProcessor(std::shared_ptr exporter, + std::size_t max_buffer = molo::obs::Caps::kSpanBuffer, + std::size_t batch_size = molo::obs::Caps::kSpanBatch, + std::chrono::seconds interval = std::chrono::seconds( + molo::obs::Caps::kSpanBatchIntervalSec)); + ~BatchProcessor(); + + BatchProcessor(const BatchProcessor&) = delete; + BatchProcessor& operator=(const BatchProcessor&) = delete; + + void add(FinishedSpan span); + // Exports everything queued and waits for in-flight exports, so every + // span added before the call is exported when it returns. + void flush(); + + std::uint64_t dropped() const; + std::uint64_t exported_total() const; + std::uint64_t failed_batches() const; + + private: + void run(); + + std::shared_ptr exporter_; + std::size_t max_buffer_; + std::size_t batch_size_; + std::chrono::seconds interval_; + mutable std::mutex mutex_; + std::condition_variable cv_; + std::condition_variable flushed_cv_; + std::deque queue_; + std::size_t in_flight_ = 0; + std::thread worker_; + bool running_ = true; + std::atomic dropped_{0}; + std::atomic exported_{0}; + std::atomic failed_{0}; +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/exporter.h b/src/observability/tracing/exporter.h new file mode 100644 index 0000000..754b5ed --- /dev/null +++ b/src/observability/tracing/exporter.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include "observability/tracing/span.h" + +namespace molo::trace { + +// Sink interface for finished spans. Exporters are the only tracing +// components allowed to touch I/O. +class Exporter { + public: + virtual ~Exporter() = default; + virtual bool export_spans(std::vector batch) = 0; + virtual std::string name() const = 0; +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/http_sender.h b/src/observability/tracing/http_sender.h new file mode 100644 index 0000000..99750b8 --- /dev/null +++ b/src/observability/tracing/http_sender.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::trace { + +// Minimal HTTP POST interface used by the OTLP exporter. The production +// sender is the framework HTTP client (wired by 08-clients-openapi.md); +// until then NullHttpSender is the marked stub. +class HttpSender { + public: + using HeaderList = std::vector>; + virtual ~HttpSender() = default; + virtual bool post(const std::string& url, const HeaderList& headers, const std::string& body, + std::chrono::seconds timeout) = 0; +}; + +// STUB: drops every request and reports failure. Replace with the +// framework HTTP client sender once 08-clients-openapi.md lands. +class NullHttpSender : public HttpSender { + public: + bool post(const std::string& /*url*/, const HeaderList& /*headers*/, + const std::string& /*body*/, std::chrono::seconds /*timeout*/) override { + attempts_.fetch_add(1, std::memory_order_relaxed); + return false; + } + + std::uint64_t attempts() const { return attempts_.load(std::memory_order_relaxed); } + + private: + std::atomic attempts_{0}; +}; + +// Test and wiring double: delegates to a callback. +class FunctionHttpSender : public HttpSender { + public: + using PostFn = std::function; + explicit FunctionHttpSender(PostFn fn) : fn_(std::move(fn)) {} + + bool post(const std::string& url, const HeaderList& headers, const std::string& body, + std::chrono::seconds timeout) override { + return fn_(url, headers, body, timeout); + } + + private: + PostFn fn_; +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/ids.h b/src/observability/tracing/ids.h new file mode 100644 index 0000000..dcab4c0 --- /dev/null +++ b/src/observability/tracing/ids.h @@ -0,0 +1,170 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace molo::trace { +namespace detail { + +inline int hex_value(char ch) { + if (ch >= '0' && ch <= '9') return ch - '0'; + if (ch >= 'a' && ch <= 'f') return ch - 'a' + 10; + if (ch >= 'A' && ch <= 'F') return ch - 'A' + 10; + return -1; +} + +inline void append_hex(std::string& out, const std::uint8_t* bytes, std::size_t count) { + static constexpr char kDigits[] = "0123456789abcdef"; + for (std::size_t i = 0; i < count; ++i) { + out += kDigits[bytes[i] >> 4]; + out += kDigits[bytes[i] & 0x0f]; + } +} + +inline std::mt19937_64& id_rng() { + static std::atomic salt{0}; + std::size_t thread_salt = std::hash{}(std::this_thread::get_id()); + thread_local std::mt19937_64 rng{std::random_device{}() ^ thread_salt ^ + salt.fetch_add(0x9e3779b97f4a7c15ULL, std::memory_order_relaxed)}; + return rng; +} + +} // namespace detail + +// 128-bit trace ID, W3C hex encoded (32 chars). +class TraceId { + public: + TraceId() = default; + + static TraceId generate() { + TraceId id; + do { + std::uint64_t hi = detail::id_rng()(); + std::uint64_t lo = detail::id_rng()(); + for (int i = 0; i < 8; ++i) { + id.bytes_[i] = static_cast(hi >> (56 - 8 * i)); + id.bytes_[8 + i] = static_cast(lo >> (56 - 8 * i)); + } + } while (!id.valid()); + return id; + } + + static std::optional from_hex(std::string_view text) { + if (text.size() != 32) return std::nullopt; + TraceId id; + for (std::size_t i = 0; i < 16; ++i) { + int hi = detail::hex_value(text[2 * i]); + int lo = detail::hex_value(text[2 * i + 1]); + if (hi < 0 || lo < 0) return std::nullopt; + id.bytes_[i] = static_cast((hi << 4) | lo); + } + return id; + } + + std::string to_hex() const { + std::string out; + out.reserve(32); + detail::append_hex(out, bytes_.data(), bytes_.size()); + return out; + } + + bool valid() const { + for (std::uint8_t byte : bytes_) { + if (byte != 0) return true; + } + return false; + } + + std::uint64_t prefix_u64() const { + std::uint64_t value = 0; + for (int i = 0; i < 8; ++i) value = (value << 8) | bytes_[i]; + return value; + } + + bool operator==(const TraceId& other) const { return bytes_ == other.bytes_; } + bool operator!=(const TraceId& other) const { return !(*this == other); } + + private: + std::array bytes_{}; +}; + +// 64-bit span ID, W3C hex encoded (16 chars). +class SpanId { + public: + SpanId() = default; + + static SpanId generate() { + SpanId id; + do { + std::uint64_t raw = detail::id_rng()(); + for (int i = 0; i < 8; ++i) id.bytes_[i] = static_cast(raw >> (56 - 8 * i)); + } while (!id.valid()); + return id; + } + + static std::optional from_hex(std::string_view text) { + if (text.size() != 16) return std::nullopt; + SpanId id; + for (std::size_t i = 0; i < 8; ++i) { + int hi = detail::hex_value(text[2 * i]); + int lo = detail::hex_value(text[2 * i + 1]); + if (hi < 0 || lo < 0) return std::nullopt; + id.bytes_[i] = static_cast((hi << 4) | lo); + } + return id; + } + + std::string to_hex() const { + std::string out; + out.reserve(16); + detail::append_hex(out, bytes_.data(), bytes_.size()); + return out; + } + + bool valid() const { + for (std::uint8_t byte : bytes_) { + if (byte != 0) return true; + } + return false; + } + + bool operator==(const SpanId& other) const { return bytes_ == other.bytes_; } + bool operator!=(const SpanId& other) const { return !(*this == other); } + + private: + std::array bytes_{}; +}; + +// One-byte W3C trace flags. Bit 0 is the sampled flag. +struct TraceFlags { + std::uint8_t value = 0; + + static constexpr std::uint8_t kSampled = 0x01; + + bool is_sampled() const { return (value & kSampled) != 0; } + + std::string to_hex() const { + std::string out; + out.reserve(2); + detail::append_hex(out, &value, 1); + return out; + } + + static std::optional from_hex(std::string_view text) { + if (text.size() != 2) return std::nullopt; + int hi = detail::hex_value(text[0]); + int lo = detail::hex_value(text[1]); + if (hi < 0 || lo < 0) return std::nullopt; + TraceFlags flags; + flags.value = static_cast((hi << 4) | lo); + return flags; + } +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/memory_exporter.h b/src/observability/tracing/memory_exporter.h new file mode 100644 index 0000000..f6ce826 --- /dev/null +++ b/src/observability/tracing/memory_exporter.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +#include "observability/tracing/exporter.h" + +namespace molo::trace { + +// In-memory exporter for tests and inspection. Unbounded by design for +// test use only; production paths use the bounded batch processor. +class MemoryExporter : public Exporter { + public: + bool export_spans(std::vector batch) override { + std::lock_guard lock(mutex_); + for (auto& span : batch) spans_.push_back(std::move(span)); + return true; + } + + std::string name() const override { return "memory"; } + + std::vector spans() const { + std::lock_guard lock(mutex_); + return spans_; + } + + void clear() { + std::lock_guard lock(mutex_); + spans_.clear(); + } + + private: + mutable std::mutex mutex_; + std::vector spans_; +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/otlp_http.cpp b/src/observability/tracing/otlp_http.cpp new file mode 100644 index 0000000..ba5399c --- /dev/null +++ b/src/observability/tracing/otlp_http.cpp @@ -0,0 +1,192 @@ +#include "observability/tracing/otlp_http.h" + +#include +#include +#include + +#include "observability/logging/json_format.h" + +namespace molo::trace { +namespace { + +int kind_number(SpanKind kind) { + switch (kind) { + case SpanKind::Internal: + return 1; + case SpanKind::Server: + return 2; + case SpanKind::Client: + return 3; + case SpanKind::Producer: + return 4; + case SpanKind::Consumer: + return 5; + } + return 1; +} + +int status_number(StatusCode code) { + switch (code) { + case StatusCode::Unset: + return 0; + case StatusCode::Ok: + return 1; + case StatusCode::Error: + return 2; + } + return 0; +} + +std::string nanos_json(std::chrono::system_clock::time_point time) { + auto nanos = std::chrono::duration_cast(time.time_since_epoch()).count(); + if (nanos < 0) nanos = 0; + return "\"" + std::to_string(static_cast(nanos)) + "\""; +} + +std::string attribute_value_json(const AttributeValue& value) { + return std::visit( + [](auto&& raw) -> std::string { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return "{\"stringValue\":\"" + molo::log::escape_json(raw) + "\"}"; + } else if constexpr (std::is_same_v) { + return std::string("{\"boolValue\":") + (raw ? "true}" : "false}"); + } else if constexpr (std::is_same_v) { + if (!std::isfinite(raw)) return "{\"doubleValue\":0}"; + return std::string("{\"doubleValue\":") + molo::obs::format_double_finite(raw) + "}"; + } else { + return "{\"intValue\":\"" + std::to_string(raw) + "\"}"; + } + }, + value); +} + +void append_attributes(std::string& out, const std::vector& attributes) { + out += "\"attributes\":["; + bool first = true; + for (const Attribute& attr : attributes) { + if (!first) out += ","; + first = false; + out += "{\"key\":\""; + out += molo::log::escape_json(attr.key); + out += "\",\"value\":"; + out += attribute_value_json(attr.value); + out += "}"; + } + out += "]"; +} + +void append_span(std::string& out, const FinishedSpan& span) { + out += "{\"traceId\":\""; + out += span.context.trace_id.to_hex(); + out += "\",\"spanId\":\""; + out += span.context.span_id.to_hex(); + out += "\""; + if (span.parent_span_id.valid()) { + out += ",\"parentSpanId\":\""; + out += span.parent_span_id.to_hex(); + out += "\""; + } + out += ",\"name\":\""; + out += molo::log::escape_json(span.name); + out += "\",\"kind\":"; + out += std::to_string(kind_number(span.kind)); + out += ",\"startTimeUnixNano\":"; + out += nanos_json(span.start); + out += ",\"endTimeUnixNano\":"; + out += nanos_json(span.end); + out += ","; + append_attributes(out, span.attributes); + out += ",\"events\":["; + bool first = true; + for (const SpanEvent& event : span.events) { + if (!first) out += ","; + first = false; + out += "{\"timeUnixNano\":"; + out += nanos_json(event.timestamp); + out += ",\"name\":\""; + out += molo::log::escape_json(event.name); + out += "\",\"attributes\":[]}"; + } + out += "],\"status\":{\"code\":"; + out += std::to_string(status_number(span.status)); + if (!span.status_desc.empty()) { + out += ",\"message\":\""; + out += molo::log::escape_json(span.status_desc); + out += "\""; + } + out += "}}"; +} + +} // namespace + +OtlpHttpExporter::OtlpHttpExporter(std::string endpoint, std::shared_ptr sender, + std::vector backoff, + std::chrono::seconds timeout, bool gzip) + : endpoint_(std::move(endpoint)), + sender_(std::move(sender)), + backoff_(std::move(backoff)), + timeout_(timeout), + gzip_(gzip) { + if (!sender_) sender_ = std::make_shared(); +} + +bool OtlpHttpExporter::export_spans(std::vector batch) { + if (batch.empty()) return true; + std::string body = build_request_json(batch, "molo"); + HttpSender::HeaderList headers = {{"Content-Type", "application/json"}}; + if (gzip_ && compressor_) { + std::string compressed; + if (compressor_(body, compressed)) { + body = std::move(compressed); + headers.emplace_back("Content-Encoding", "gzip"); + } + } + // First attempt plus at most kOtlpRetries retries with backoff. + const int max_attempts = 1 + molo::obs::Caps::kOtlpRetries; + for (int attempt = 0; attempt < max_attempts; ++attempt) { + requests_.fetch_add(1, std::memory_order_relaxed); + bool ok = false; + try { + ok = sender_->post(endpoint_, headers, body, timeout_); + } catch (...) { + ok = false; + } + if (ok) return true; + failed_.fetch_add(1, std::memory_order_relaxed); + if (attempt + 1 < max_attempts && !backoff_.empty()) { + std::size_t index = static_cast(attempt); + if (index >= backoff_.size()) index = backoff_.size() - 1; + std::this_thread::sleep_for(backoff_[index]); + } + } + return false; +} + +void OtlpHttpExporter::set_compressor(Compressor compressor) { compressor_ = std::move(compressor); } + +std::string OtlpHttpExporter::build_request_json(const std::vector& batch, + const std::string& scope_name) { + std::string out = "{\"resourceSpans\":[{\"resource\":{\"attributes\":[]},\"scopeSpans\":[{"; + out += "\"scope\":{\"name\":\""; + out += molo::log::escape_json(scope_name); + out += "\"},\"spans\":["; + bool first = true; + for (const FinishedSpan& span : batch) { + if (!first) out += ","; + first = false; + append_span(out, span); + } + out += "]}]}]}"; + return out; +} + +std::uint64_t OtlpHttpExporter::requests_total() const { + return requests_.load(std::memory_order_relaxed); +} + +std::uint64_t OtlpHttpExporter::requests_failed() const { + return failed_.load(std::memory_order_relaxed); +} + +} // namespace molo::trace diff --git a/src/observability/tracing/otlp_http.h b/src/observability/tracing/otlp_http.h new file mode 100644 index 0000000..7992abe --- /dev/null +++ b/src/observability/tracing/otlp_http.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "observability/config.h" +#include "observability/tracing/exporter.h" +#include "observability/tracing/http_sender.h" + +namespace molo::trace { + +// OTLP/HTTP JSON exporter: POSTs ExportTraceServiceRequest documents to a +// collector with a 10 s timeout and at most 3 retries with backoff. +// gzip is only applied when a compressor is injected (the gzip codec is +// owned by 01-runtime.md); otherwise identity is sent and documented. +class OtlpHttpExporter : public Exporter { + public: + using Compressor = std::function; + + explicit OtlpHttpExporter( + std::string endpoint, std::shared_ptr sender, + std::vector backoff = {std::chrono::milliseconds(100), + std::chrono::milliseconds(500), + std::chrono::milliseconds(2000)}, + std::chrono::seconds timeout = std::chrono::seconds(molo::obs::Caps::kOtlpTimeoutSec), + bool gzip = false); + + bool export_spans(std::vector batch) override; + std::string name() const override { return "otlp-http"; } + + void set_compressor(Compressor compressor); + + static std::string build_request_json(const std::vector& batch, + const std::string& scope_name); + + std::uint64_t requests_total() const; + std::uint64_t requests_failed() const; + + private: + std::string endpoint_; + std::shared_ptr sender_; + std::vector backoff_; + std::chrono::seconds timeout_; + bool gzip_; + Compressor compressor_; + std::atomic requests_{0}; + std::atomic failed_{0}; +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/propagator.cpp b/src/observability/tracing/propagator.cpp new file mode 100644 index 0000000..2f2fde7 --- /dev/null +++ b/src/observability/tracing/propagator.cpp @@ -0,0 +1,100 @@ +#include "observability/tracing/propagator.h" + +#include +#include + +#include "observability/config.h" + +namespace molo::trace { +namespace { + +Headers::const_iterator find_header_ci(const Headers& headers, const char* name) { + std::string wanted(name); + for (auto it = headers.begin(); it != headers.end(); ++it) { + const std::string& key = it->first; + if (key.size() != wanted.size()) continue; + bool same = true; + for (std::size_t i = 0; i < key.size(); ++i) { + if (std::tolower(static_cast(key[i])) != wanted[i]) { + same = false; + break; + } + } + if (same) return it; + } + return headers.end(); +} + +std::vector split_dash(std::string_view text) { + std::vector parts; + std::string current; + for (char ch : text) { + if (ch == '-') { + parts.push_back(current); + current.clear(); + } else { + current += ch; + } + } + parts.push_back(current); + return parts; +} + +} // namespace + +std::optional W3CPropagator::extract(const Headers& headers, + std::function on_error) { + auto fail = [&]() -> std::optional { + if (on_error) { + try { + on_error(); + } catch (...) { + } + } + return std::nullopt; + }; + auto it = find_header_ci(headers, "traceparent"); + if (it == headers.end()) return std::nullopt; + + std::vector parts = split_dash(it->second); + if (parts.size() < 4) return fail(); + if (parts[0].size() != 2 || parts[0] == "ff") return fail(); + bool version_hex = true; + for (char ch : parts[0]) { + if (!std::isxdigit(static_cast(ch))) version_hex = false; + } + if (!version_hex) return fail(); + + auto trace_id = TraceId::from_hex(parts[1]); + auto parent_id = SpanId::from_hex(parts[2]); + auto flags = TraceFlags::from_hex(parts[3]); + if (!trace_id.has_value() || !trace_id->valid()) return fail(); + if (!parent_id.has_value() || !parent_id->valid()) return fail(); + if (!flags.has_value()) return fail(); + + SpanContext context; + context.trace_id = *trace_id; + context.span_id = *parent_id; + context.flags = *flags; + auto state = find_header_ci(headers, "tracestate"); + if (state != headers.end()) { + context.tracestate = state->second; + if (context.tracestate.size() > molo::obs::Caps::kMaxTracestate) { + context.tracestate.resize(molo::obs::Caps::kMaxTracestate); + } + } + return context; +} + +void W3CPropagator::inject(const SpanContext& context, Headers& headers) { + if (!context.is_valid()) return; + headers["traceparent"] = render_traceparent(context); + if (!context.tracestate.empty()) headers["tracestate"] = context.tracestate; +} + +std::string W3CPropagator::render_traceparent(const SpanContext& context) { + return "00-" + context.trace_id.to_hex() + "-" + context.span_id.to_hex() + "-" + + context.flags.to_hex(); +} + +} // namespace molo::trace diff --git a/src/observability/tracing/propagator.h b/src/observability/tracing/propagator.h new file mode 100644 index 0000000..a4ac80b --- /dev/null +++ b/src/observability/tracing/propagator.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include +#include + +#include "observability/tracing/span_context.h" + +namespace molo::trace { + +using Headers = std::map; + +// W3C Trace Context propagation: parses inbound traceparent/tracestate +// headers and injects the current context into outbound headers. +// Malformed headers start a new trace and fire on_error (wired to +// molo_trace_propagator_errors_total). +class W3CPropagator { + public: + static std::optional extract(const Headers& headers, + std::function on_error = nullptr); + static void inject(const SpanContext& context, Headers& headers); + static std::string render_traceparent(const SpanContext& context); +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/sampler.h b/src/observability/tracing/sampler.h new file mode 100644 index 0000000..333af04 --- /dev/null +++ b/src/observability/tracing/sampler.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include + +#include "observability/tracing/ids.h" + +namespace molo::trace { + +// Head-based sampler: the decision is made once per trace from the trace +// ID and stays immutable for the lifetime of that trace. +class Sampler { + public: + virtual ~Sampler() = default; + virtual bool should_sample(const TraceId& trace_id) = 0; + virtual std::string name() const = 0; +}; + +class AlwaysOnSampler : public Sampler { + public: + bool should_sample(const TraceId& /*trace_id*/) override { return true; } + std::string name() const override { return "always_on"; } +}; + +class AlwaysOffSampler : public Sampler { + public: + bool should_sample(const TraceId& /*trace_id*/) override { return false; } + std::string name() const override { return "always_off"; } +}; + +// Samples a deterministic fraction of traces. The same trace ID always +// gets the same decision. +class RatioSampler : public Sampler { + public: + explicit RatioSampler(double ratio) : ratio_(ratio) { + if (ratio_ < 0.0) ratio_ = 0.0; + if (ratio_ > 1.0) ratio_ = 1.0; + } + + bool should_sample(const TraceId& trace_id) override { + if (ratio_ <= 0.0) return false; + if (ratio_ >= 1.0) return true; + const double threshold = + ratio_ * static_cast(std::numeric_limits::max()); + return static_cast(trace_id.prefix_u64()) < threshold; + } + + std::string name() const override { return "ratio"; } + double ratio() const { return ratio_; } + + private: + double ratio_; +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/span.cpp b/src/observability/tracing/span.cpp new file mode 100644 index 0000000..d06e69a --- /dev/null +++ b/src/observability/tracing/span.cpp @@ -0,0 +1,120 @@ +#include "observability/tracing/span.h" + +#include "observability/config.h" +#include "observability/correlation/redaction.h" + +namespace molo::trace { + +std::string_view span_kind_name(SpanKind kind) { + switch (kind) { + case SpanKind::Internal: + return "internal"; + case SpanKind::Server: + return "server"; + case SpanKind::Client: + return "client"; + case SpanKind::Producer: + return "producer"; + case SpanKind::Consumer: + return "consumer"; + } + return "internal"; +} + +struct Span::State { + std::string name; + SpanKind kind = SpanKind::Internal; + SpanId parent_span_id; + std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); + std::vector attributes; + std::vector events; + StatusCode status = StatusCode::Unset; + std::string status_desc; + bool attributes_truncated = false; + bool events_truncated = false; + bool ended = false; + EndHandler on_end; +}; + +Span::Span() = default; + +Span Span::null_with_context(SpanContext context) { + Span span; + span.context_ = context; + return span; +} + +Span::Span(std::string name, SpanKind kind, SpanContext context, SpanId parent_span_id, + EndHandler on_end) + : context_(context), state_(std::make_unique()) { + state_->name = std::move(name); + state_->kind = kind; + state_->parent_span_id = parent_span_id; + state_->on_end = std::move(on_end); +} + +Span::~Span() { end(); } + +Span::Span(Span&& other) noexcept = default; +Span& Span::operator=(Span&& other) noexcept = default; + +void Span::set_attribute(std::string key, AttributeValue value) { + if (state_ == nullptr || state_->ended) return; + if (molo::corr::is_sensitive_key(key)) { + value = std::string(molo::corr::kRedacted); + } + if (auto* text = std::get_if(&value)) { + if (text->size() > molo::obs::Caps::kMaxAttrValue) { + text->resize(molo::obs::Caps::kMaxAttrValue); + state_->attributes_truncated = true; + } + } + if (state_->attributes.size() >= molo::obs::Caps::kMaxSpanAttrs) { + state_->attributes_truncated = true; + return; + } + state_->attributes.push_back(Attribute{std::move(key), std::move(value)}); +} + +void Span::add_event(std::string name) { + if (state_ == nullptr || state_->ended) return; + if (state_->events.size() >= molo::obs::Caps::kMaxSpanEvents) { + state_->events_truncated = true; + return; + } + SpanEvent event; + event.name = std::move(name); + state_->events.push_back(std::move(event)); +} + +void Span::set_status(StatusCode code, std::string description) { + if (state_ == nullptr || state_->ended) return; + state_->status = code; + state_->status_desc = std::move(description); +} + +void Span::end() { + if (state_ == nullptr || state_->ended) return; + state_->ended = true; + FinishedSpan finished; + finished.name = std::move(state_->name); + finished.kind = state_->kind; + finished.context = context_; + finished.parent_span_id = state_->parent_span_id; + finished.start = state_->start; + finished.end = std::chrono::system_clock::now(); + finished.attributes = std::move(state_->attributes); + finished.events = std::move(state_->events); + finished.status = state_->status; + finished.status_desc = std::move(state_->status_desc); + finished.attributes_truncated = state_->attributes_truncated; + finished.events_truncated = state_->events_truncated; + if (state_->on_end) { + try { + state_->on_end(std::move(finished)); + } catch (...) { + } + } +} + +} // namespace molo::trace diff --git a/src/observability/tracing/span.h b/src/observability/tracing/span.h new file mode 100644 index 0000000..aaff937 --- /dev/null +++ b/src/observability/tracing/span.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "observability/tracing/ids.h" +#include "observability/tracing/span_context.h" + +namespace molo::trace { + +enum class SpanKind : std::uint8_t { Internal, Server, Client, Producer, Consumer }; + +std::string_view span_kind_name(SpanKind kind); + +enum class StatusCode : std::uint8_t { Unset, Ok, Error }; + +using AttributeValue = std::variant; + +struct Attribute { + std::string key; + AttributeValue value; +}; + +struct SpanEvent { + std::chrono::system_clock::time_point timestamp = std::chrono::system_clock::now(); + std::string name; +}; + +struct FinishedSpan { + std::string name; + SpanKind kind = SpanKind::Internal; + SpanContext context; + SpanId parent_span_id; + std::chrono::system_clock::time_point start; + std::chrono::system_clock::time_point end; + std::vector attributes; + std::vector events; + StatusCode status = StatusCode::Unset; + std::string status_desc; + bool attributes_truncated = false; + bool events_truncated = false; +}; + +// One distributed tracing span. RAII: ending happens on end() or on +// destruction. A null span (valid() == false) records nothing but still +// carries a usable context for propagation. Single-owner, thread-affine: +// a span is used from the thread that created it. +class Span { + public: + using EndHandler = std::function; + + Span(); + static Span null_with_context(SpanContext context); + Span(std::string name, SpanKind kind, SpanContext context, SpanId parent_span_id, + EndHandler on_end); + + ~Span(); + + Span(const Span&) = delete; + Span& operator=(const Span&) = delete; + Span(Span&& other) noexcept; + Span& operator=(Span&& other) noexcept; + + bool valid() const { return state_ != nullptr; } + const SpanContext& context() const { return context_; } + + void set_attribute(std::string key, AttributeValue value); + void add_event(std::string name); + void set_status(StatusCode code, std::string description = ""); + void end(); + + private: + struct State; + + SpanContext context_; + std::unique_ptr state_; +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/span_context.h b/src/observability/tracing/span_context.h new file mode 100644 index 0000000..88c9b08 --- /dev/null +++ b/src/observability/tracing/span_context.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +#include "observability/tracing/ids.h" + +namespace molo::trace { + +// Propagatable trace identity: trace ID, span ID, flags, vendor state. +struct SpanContext { + TraceId trace_id; + SpanId span_id; + TraceFlags flags; + std::string tracestate; + + bool is_valid() const { return trace_id.valid() && span_id.valid(); } + bool is_sampled() const { return is_valid() && flags.is_sampled(); } + + static SpanContext invalid() { return SpanContext{}; } +}; + +} // namespace molo::trace diff --git a/src/observability/tracing/tracer.cpp b/src/observability/tracing/tracer.cpp new file mode 100644 index 0000000..361d560 --- /dev/null +++ b/src/observability/tracing/tracer.cpp @@ -0,0 +1,75 @@ +#include "observability/tracing/tracer.h" + +#include "observability/config.h" +#include "observability/tracing/batch_processor.h" + +namespace molo::trace { + +Tracer::Tracer(std::string name, std::shared_ptr sampler, BatchProcessor* processor) + : name_(std::move(name)), + sampler_(std::move(sampler)), + processor_(processor), + started_(0), + dropped_(0) { + if (!sampler_) sampler_ = std::make_shared(); +} + +Tracer::Tracer(std::string name) + : Tracer(std::move(name), std::make_shared(), nullptr) {} + +Span Tracer::start_span(std::string_view name, SpanKind kind, const SpanContext& parent) { + started_.fetch_add(1, std::memory_order_relaxed); + + const bool has_parent = parent.is_valid(); + TraceId trace_id = has_parent ? parent.trace_id : TraceId::generate(); + SpanId span_id = SpanId::generate(); + // Head-based and immutable: children inherit the parent decision, + // roots ask the sampler exactly once. + const bool sampled = has_parent ? parent.is_sampled() : sampler_->should_sample(trace_id); + + SpanContext context; + context.trace_id = trace_id; + context.span_id = span_id; + context.flags.value = sampled ? TraceFlags::kSampled : 0; + if (has_parent) context.tracestate = parent.tracestate; + + if constexpr (!molo::obs::kTracingOn) { + dropped_.fetch_add(1, std::memory_order_relaxed); + return Span::null_with_context(context); + } + if (!molo::obs::RuntimeToggles::instance().tracing.load(std::memory_order_relaxed)) { + dropped_.fetch_add(1, std::memory_order_relaxed); + return Span::null_with_context(context); + } + if (!sampled) return Span::null_with_context(context); + BatchProcessor* processor = processor_.load(std::memory_order_relaxed); + if (processor == nullptr) { + dropped_.fetch_add(1, std::memory_order_relaxed); + return Span::null_with_context(context); + } + SpanId parent_span = has_parent ? parent.span_id : SpanId{}; + return Span(std::string(name), kind, context, parent_span, + [this](FinishedSpan finished) { + BatchProcessor* target = processor_.load(std::memory_order_relaxed); + if (target != nullptr) { + target->add(std::move(finished)); + } else { + dropped_.fetch_add(1, std::memory_order_relaxed); + } + }); +} + +void Tracer::set_processor(BatchProcessor* processor) { + processor_.store(processor, std::memory_order_relaxed); +} + +std::uint64_t Tracer::spans_started() const { return started_.load(std::memory_order_relaxed); } + +std::uint64_t Tracer::spans_dropped() const { return dropped_.load(std::memory_order_relaxed); } + +Tracer& Tracer::global() { + static Tracer instance("molo-global", std::make_shared(), nullptr); + return instance; +} + +} // namespace molo::trace diff --git a/src/observability/tracing/tracer.h b/src/observability/tracing/tracer.h new file mode 100644 index 0000000..9bada2a --- /dev/null +++ b/src/observability/tracing/tracer.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "observability/tracing/sampler.h" +#include "observability/tracing/span.h" +#include "observability/tracing/span_context.h" + +namespace molo::trace { + +class BatchProcessor; + +// Creates spans. When tracing is disabled (compile time or runtime), when +// the sampler drops the trace, or when no processor is wired, start_span +// returns a null span: one branch, no allocation, no recording, but still +// carrying a valid context for propagation. Lifetime rule: every span must +// end before its tracer and batch processor are destroyed (tracers and +// processors are normally process-lifetime singletons). +class Tracer { + public: + Tracer(std::string name, std::shared_ptr sampler, BatchProcessor* processor); + explicit Tracer(std::string name); + + Span start_span(std::string_view name, SpanKind kind = SpanKind::Internal, + const SpanContext& parent = SpanContext::invalid()); + + void set_processor(BatchProcessor* processor); + const std::string& name() const { return name_; } + + std::uint64_t spans_started() const; + std::uint64_t spans_dropped() const; + + static Tracer& global(); + + private: + std::string name_; + std::shared_ptr sampler_; + std::atomic processor_; + std::atomic started_; + std::atomic dropped_; +}; + +} // namespace molo::trace diff --git a/src/operability.mk b/src/operability.mk new file mode 100644 index 0000000..4984911 --- /dev/null +++ b/src/operability.mk @@ -0,0 +1,71 @@ +# Part 13 operability build fragment. Included from the root Makefile with: +# -include src/operability.mk +# Production uses -Isrc only (real base headers). Standalone verification +# (this worktree has no base tree) sets OPERABILITY_STANDALONE=1 to add the +# ADAPTER shims under src/operability/adapters/include. + +CXX ?= g++ +AR ?= ar +OPERABILITY_STANDALONE ?= 0 + +OPERABILITY_SRCS := \ + src/operability/deploy/rolling.cpp \ + src/operability/deploy/blue_green.cpp \ + src/operability/deploy/lifecycle.cpp \ + src/operability/coord/lease.cpp \ + src/operability/coord/dist_lock.cpp \ + src/operability/coord/election.cpp \ + src/operability/resilience/breaker.cpp \ + src/operability/resilience/bulkhead.cpp \ + src/operability/resilience/shedder.cpp \ + src/operability/resilience/deadline.cpp \ + src/operability/resilience/slo.cpp \ + src/operability/chaos/injector.cpp \ + src/operability/chaos/experiment.cpp \ + src/operability/debug/endpoint.cpp \ + src/operability/debug/snapshot.cpp + +OPERABILITY_OBJS := $(patsubst src/%.cpp,build/obj/%.o,$(OPERABILITY_SRCS)) +OPERABILITY_LIB := build/liboperability.a + +OPERABILITY_TEST_SRCS := \ + tests/src/operability/test_coord.cpp \ + tests/src/operability/test_lifecycle.cpp \ + tests/src/operability/test_breaker_bulkhead.cpp \ + tests/src/operability/test_shed_deadline.cpp \ + tests/src/operability/test_deploy.cpp \ + tests/src/operability/test_slo.cpp \ + tests/src/operability/test_chaos.cpp \ + tests/src/operability/test_debug.cpp \ + tests/src/operability/main.cpp + +OPERABILITY_CXXFLAGS := -std=c++23 -O2 -Wall -Wextra -Wpedantic -Werror -pthread -Isrc +ifeq ($(OPERABILITY_STANDALONE),1) +OPERABILITY_CXXFLAGS += -Isrc/operability/adapters/include +endif + +$(OPERABILITY_LIB): $(OPERABILITY_OBJS) + mkdir -p $(dir $@) + $(AR) rcs $@ $^ + +build/obj/%.o: src/%.cpp + mkdir -p $(dir $@) + $(CXX) $(OPERABILITY_CXXFLAGS) -c $< -o $@ + +build/operability_tests: $(OPERABILITY_LIB) $(OPERABILITY_TEST_SRCS) + mkdir -p build + $(CXX) $(OPERABILITY_CXXFLAGS) -DMOLO_ENABLE_DEBUG $(OPERABILITY_TEST_SRCS) $(OPERABILITY_LIB) -o $@ + +build/operability_tests_nodebug: $(OPERABILITY_LIB) $(OPERABILITY_TEST_SRCS) + mkdir -p build + $(CXX) $(OPERABILITY_CXXFLAGS) $(OPERABILITY_TEST_SRCS) $(OPERABILITY_LIB) -o $@ + +.PHONY: operability operability-tests operability-check + +operability: $(OPERABILITY_LIB) + +operability-tests: build/operability_tests build/operability_tests_nodebug + +operability-check: operability-tests + ./build/operability_tests + ./build/operability_tests_nodebug diff --git a/src/operability/PATCHES.md b/src/operability/PATCHES.md new file mode 100644 index 0000000..4440725 --- /dev/null +++ b/src/operability/PATCHES.md @@ -0,0 +1,334 @@ +# Part 13 operability: required base-tree edits + +This file lists every edit the base tree needs to integrate the +`src/operability/` part. Paths are repo-relative. Hunks are unified diffs +anchored on the seam symbols; this worktree does not contain the base `src/` +tree, so context line numbers are symbolic and the integrator anchors each +hunk on the named symbol. No hunk changes existing behavior when the new +code paths are not taken. + +Conventions used by the new code: + +- Includes use `-Isrc`: `"operability/..."`, `"supervisor/..."`, etc. +- Namespaces are the leaf names from the seam spec: `molo::deploy`, + `molo::coord`, `molo::resilience`, `molo::chaos`, `molo::debug`, with shared + caps in `molo::operability::Caps` (`src/operability/caps.h`). +- Production builds use `-Isrc` only. The `ADAPTER` shims under + `src/operability/adapters/include/` are standalone-only (see + `src/operability.mk`, `OPERABILITY_STANDALONE=1`) and must never be on the + production include path. +- All new logic takes `molo::common::Clock` and `molo::common::Observer` by + reference for frozen-clock tests. +- Static caps live in `src/operability/caps.h`. Nothing autotunes. + +## 1. `Makefile`: include the operability fragment + +Anchor: the per-part `.mk` include block (next to the other `-include` +lines such as the supervisor fragment). + +```diff +--- a/Makefile ++++ b/Makefile +@@ -1,0 +1,1 @@ + -include src/cli.mk ++-include src/operability.mk +``` + +Effect: builds `build/liboperability.a` and the `operability`, +`operability-tests`, `operability-check` targets. + +## 2. `src/supervisor/common/kv_store.hpp`: fencing CAS extension + +`molo::coord::Fence` and `molo::coord::DistLock` need compare-and-swap on +(key, expected token). This adds one virtual plus the in-memory +implementation; existing `get/set/erase/acquire/renew` signatures are +untouched. + +```diff +--- a/src/supervisor/common/kv_store.hpp ++++ b/src/supervisor/common/kv_store.hpp +@@ class KvStore + virtual bool acquire(const std::string& key, const std::string& value, std::int64_t ttl_ms) = 0; + virtual bool renew(const std::string& key, const std::string& expected, std::int64_t ttl_ms) = 0; ++ // Fencing extension: set key to next only when its current value equals ++ // expected. Empty expected matches a missing key. ++ virtual bool cas(const std::string& key, const std::string& expected, const std::string& next, ++ std::int64_t ttl_ms) = 0; +``` + +```diff +--- a/src/supervisor/common/kv_store.hpp ++++ b/src/supervisor/common/kv_store.hpp +@@ class InMemoryKvStore + bool renew(const std::string& key, const std::string& expected, std::int64_t ttl_ms) override; ++ bool cas(const std::string& key, const std::string& expected, const std::string& next, ++ std::int64_t ttl_ms) override; +``` + +The `InMemoryKvStore::cas` body follows `renew` but stores `next` on match +and inserts when the key is missing and `expected` is empty (mirror the +ADAPTER shim at +`src/operability/adapters/include/supervisor/common/kv_store.hpp`, which is +the reference implementation). Every other `KvStore` backend must implement +`cas` with the same semantics; a backend that cannot do atomic CAS must +return false so callers treat fencing as unavailable. + +## 3. `src/cli/ops/ops_routes.hpp`: debug handler declaration + +Anchor: `molo::ops::OpsRoutes`, next to `healthz/readyz/startupz/metrics`. + +```diff +--- a/src/cli/ops/ops_routes.hpp ++++ b/src/cli/ops/ops_routes.hpp +@@ class OpsRoutes + OpsResponse healthz() const; + OpsResponse readyz() const; + OpsResponse startupz() const; + OpsResponse metrics(bool gzip) const; ++ // Admin-only debug surface. Returns 404 unless the process was built with ++ // MOLO_ENABLE_DEBUG and the caller is the admin listener (see the .cpp). ++ OpsResponse debug(const std::string& path, const std::string& token, bool admin_listener) const; +``` + +`OpsResponse` needs no change (`{status, content_type, body, gzip}` covers +the debug responses; debug bodies are never gzipped by this handler). + +## 4. `src/cli/ops/ops_routes.cpp`: debug handler definition + +Anchor: after `OpsRoutes::metrics`. Delegates to `molo::debug::Endpoint`; +public listeners always get 404, and builds without `MOLO_ENABLE_DEBUG` +always get 404. + +```diff +--- a/src/cli/ops/ops_routes.cpp ++++ b/src/cli/ops/ops_routes.cpp +@@ + #include "cli/ops/ops_routes.hpp" ++#include "operability/debug/endpoint.h" ++#include "operability/debug/snapshot.h" +@@ OpsResponse OpsRoutes::metrics(bool gzip) const + ... ++OpsResponse OpsRoutes::debug(const std::string& path, const std::string& token, ++ bool admin_listener) const { ++ OpsResponse r; ++ r.status = 404; ++ r.content_type = "text/plain"; ++ r.body = "not found"; ++ r.gzip = false; ++#ifdef MOLO_ENABLE_DEBUG ++ if (!admin_listener) return r; ++ // Auth gate: reuse the ops auth check used by the admin listener. The ++ // predicate below is the integration point, see note in section 7. ++ molo::debug::Endpoint ep([this] { return this->debug_snapshot(); }, ++ [](const std::string& t) { return ops_admin_auth(t); }); ++ const auto out = ep.handle(path, token); ++ r.status = out.status; ++ r.content_type = out.content_type; ++ r.body = out.body; ++#else ++ (void)path; ++ (void)token; ++ (void)admin_listener; ++#endif ++ return r; ++} +``` + +Notes: + +- `debug_snapshot()` is a small new private helper on `OpsRoutes` (declare + in the hpp next to `debug`) that builds a `molo::debug::Snapshot`, + registers sections, and returns `capture()`. One suggested section per + subsystem; keep each section small, the snapshot caps at 1 MB. +- `ops_admin_auth` stands for the existing admin listener auth check (03 + security gate owner). Wire the real predicate; never accept an empty + token. +- Routing rule, to be enforced at the listener setup call site (exact mount + file unconfirmed in the seam spec, see section 7): `GET /debug/*` routes + to `OpsRoutes::debug` with `admin_listener=true` on the admin listener + only. The public listener must not route `/debug/*` at all (404). + +## 5. `src/supervisor/tasks/scheduler.hpp`: elect via coord + +Anchor: `molo::tasks::Scheduler` private lease members. The scheduler keeps +its public surface (`tick_once()`, `is_leader()`); only the mechanism moves +onto `molo::coord::LeaderElection`. + +```diff +--- a/src/supervisor/tasks/scheduler.hpp ++++ b/src/supervisor/tasks/scheduler.hpp +@@ + #include "supervisor/tasks/scheduler.hpp" ++#include "operability/coord/election.h" +@@ class Scheduler + private: +- // old internal lease fields (leader key, ttl, renew deadline) ++ molo::coord::LeaderElection election_; +``` + +Construct `election_` with the existing queue/kv/clock/observer members: +key `"molo/scheduler/leader"` (keep the current `Options::leader_key` +default), ttl 10000, renew 3000, node id from the scheduler options or +worker id. + +## 6. `src/supervisor/tasks/scheduler.cpp`: delegate to the election + +Anchor: `Scheduler::tick_once` and `Scheduler::is_leader`. + +```diff +--- a/src/supervisor/tasks/scheduler.cpp ++++ b/src/supervisor/tasks/scheduler.cpp +@@ bool Scheduler::tick_once() +- // old elect_locked lease handling +- ... ++ const bool leader = election_.tick(); ++ if (!leader) return false; // follower: no dispatch this tick + ... // existing dispatch work, unchanged +@@ bool Scheduler::is_leader() const +- return lease_held_locked(); ++ return election_.is_leader(); +``` + +Delete the replaced private lease helpers. Observer metrics keep their +existing names; the election additionally emits `election.won/lost` events +and `election_renewed_total`. + +## 7. `src/clients/request.h`: deadline header helper + +Anchor: `molo::clients::ClientRequest`, next to `header()`. + +```diff +--- a/src/clients/request.h ++++ b/src/clients/request.h +@@ class ClientRequest + // existing header/timeouts accessors ++ // Absolute X-Deadline-Ms value when present and well formed. ++ std::optional deadline_ms() const; +``` + +Implement in the existing request source file with +`molo::resilience::Deadline::parse_header` on the `X-Deadline-Ms` header: + +```cpp +std::optional ClientRequest::deadline_ms() const { + const auto v = header("X-Deadline-Ms"); // existing accessor + if (!v) return std::nullopt; + return molo::resilience::Deadline::parse_header(*v); +} +``` + +(Adjust to the real `header()` return type: string, optional, or out-param.) + +## 8. `src/clients/client.cpp`: breaker plus deadline propagation + +Anchor: `molo::clients::Client::send`, before `send_inner`. Order: +deadline outermost, then shedder/bulkhead/breaker, per the resilience +wiring (exact middleware ordering slot in `src/app` is unconfirmed, see +section 9; the client-side half below is exact). + +```diff +--- a/src/clients/client.cpp ++++ b/src/clients/client.cpp +@@ + #include "clients/client.h" ++#include "operability/resilience/breaker.h" ++#include "operability/resilience/deadline.h" +@@ Task Client::send(ClientRequest req) ++ // 1. Breaker gate, innermost stage: never retry into an open breaker. ++ // `breaker_` is a per-host molo::resilience::Breaker owned by Client. ++ if (!breaker_.allow()) { ++ co_return ClientResponse::failure(503, "breaker open"); ++ } ++ // 2. Deadline propagation: convert the absolute inbound X-Deadline-Ms to ++ // remaining budget minus the 50 ms floor margin for the outbound hop. ++ if (const auto abs = req.deadline_ms()) { ++ molo::resilience::Deadline dl(clock_); ++ const std::int64_t budget = dl.remaining_ms(*abs); ++ if (budget == 0) { ++ breaker_.on_failure(); ++ co_return ClientResponse::failure(504, "deadline exceeded"); ++ } ++ req.set_header("X-Deadline-Ms", ++ molo::resilience::Deadline::format_header(clock_.wall_ms() + budget)); ++ reqmutable_timeouts_for(budget); // clamp hop timeouts to budget ++ } + auto resp = co_await send_inner(std::move(req)); ++ if (resp.ok()) { ++ breaker_.on_success(); ++ } else if (resp.status() >= 500 || resp.transport_error()) { ++ breaker_.on_failure(); ++ } + co_return resp; +``` + +Notes: + +- `breaker_` needs a `molo::common::Clock` and `molo::common::Observer` at + `Client` construction; add them to the existing client config or + constructor the same way timeouts are threaded today. +- `ClientResponse::failure`, `resp.ok()`, `resp.status()`, + `resp.transport_error()` are placeholders for the real response API: + count transport errors and 5xx as breaker failures, everything else as + success. Never report a shed 503 from downstream as a local failure + twice; the downstream already counted it. +- Retry policy (`RetryPolicy::should_retry_status/backoff_for`) must skip + retries while `breaker_.allow()` is false and must honor downstream + `Retry-After` via `RetryPolicy::parse_retry_after`. +- `reqmutable_timeouts_for` stands for clamping the outbound hop timeouts + (`Timeouts{connect,read,write,pool_acquire,total}`) to the propagated + budget. Keep `Deadline::budget_for(phase)` shares proportional. + +## 9. Unconfirmed sites (from the seam spec, kept as gaps) + +These need the base tree to pin the exact file; the hunks above are written +so they apply once the site is known: + +- Middleware ordering slot for deadline outermost then + shedder/bulkhead/breaker in `src/app` (05 request routing). The + per-request wiring is: `Deadline` check, `Shedder::should_shed` (503 plus + `Retry-After` on shed), `Bulkhead::try_acquire` (shed on `kRejected`), + then dispatch; `Breaker` stays innermost at the client (section 8). +- Admin listener bind site that mounts `OpsRoutes` (section 4 routing + rule). Public listener must 404 `/debug/*`. +- Realtime close path (06, connection close 1001) and auth gate owner + (03 security): `Lifecycle::run_drain` phase 2 should close realtime + connections with 1001; the debug auth predicate must be the 03 gate. + +## 10. CLI wiring (rollout, rollback, experiment, debug) + +`molo::cli::Command(name, brief)` with `flag()/child()/handler()` is the +seam (`src/cli/cli/command.hpp`). Register four commands at the existing +command registry (same site that registers supervisor/task commands): + +- `rollout start --strategy rolling|blue-green --batch N --max-unavailable M` + driving `molo::deploy::Rolling` / `molo::deploy::BlueGreen` step loops + with readiness probes polling `readyz/startupz`. +- `rollout abort` and `rollback` driving `Strategy::abort` and + `BlueGreen::rollback`. +- `experiment start|abort --blast N --duration MS --max-burn X` driving + `molo::chaos::Experiment` plus `molo::chaos::Injector` hook setup. +- `debug snapshot` printing `molo::debug::Snapshot::capture()`. + +Handlers return `ExitCode` and write through `Output&`; no new CLI +framework code is needed. Exact registry file to be confirmed against the +base tree. + +## 11. Verification + +Standalone (this worktree, no base tree): + +```sh +make -f src/operability.mk OPERABILITY_STANDALONE=1 operability-check +``` + +This builds `build/liboperability.a` with +`g++ -std=c++23 -O2 -Wall -Wextra -Wpedantic -Werror -pthread` and runs +`build/operability_tests` (with `MOLO_ENABLE_DEBUG`) plus +`build/operability_tests_nodebug` (without it). All eight suites +(coord, lifecycle, breaker/bulkhead, shed/deadline, deploy, slo, chaos, +debug) run on frozen clocks with no real waiting. + +Production (base tree present): drop `OPERABILITY_STANDALONE=1` so only +`-Isrc` is used and the real +`supervisor/common/{clock,kv_store,observer}.hpp` headers satisfy the +includes. The ADAPTER shims must not be on the include path. diff --git a/src/operability/adapters/include/supervisor/common/clock.hpp b/src/operability/adapters/include/supervisor/common/clock.hpp new file mode 100644 index 0000000..408b843 --- /dev/null +++ b/src/operability/adapters/include/supervisor/common/clock.hpp @@ -0,0 +1,59 @@ +// ADAPTER: standalone shim for missing base header "supervisor/common/clock.hpp". +// Production build uses the real base header from the base tree; this file is +// only on the include path for standalone verification (see operability.mk, +// OPERABILITY_STANDALONE=1). Signatures mirror the seam spec. +#pragma once + +#include +#include +#include + +namespace molo::common { + +// Minimal clock abstraction. Production Clock has identical member names. +class Clock { + public: + virtual ~Clock() = default; + virtual std::int64_t wall_ms() const = 0; + virtual std::int64_t steady_ms() const = 0; + virtual void sleep_ms(std::int64_t ms) = 0; +}; + +// Frozen clock for deterministic tests. +class FrozenClock : public Clock { + public: + explicit FrozenClock(std::int64_t start_ms = 0) : now_(start_ms) {} + + std::int64_t wall_ms() const override { return now_; } + std::int64_t steady_ms() const override { return now_; } + void sleep_ms(std::int64_t ms) override { + if (ms > 0) now_ += ms; + } + void advance(std::int64_t ms) { + if (ms > 0) now_ += ms; + } + void set(std::int64_t ms) { now_ = ms; } + + private: + std::int64_t now_; +}; + +// System clock for non-test standalone runs. +class SystemClock : public Clock { + public: + std::int64_t wall_ms() const override { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + std::int64_t steady_ms() const override { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + } + void sleep_ms(std::int64_t ms) override { + if (ms > 0) std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } +}; + +} // namespace molo::common diff --git a/src/operability/adapters/include/supervisor/common/kv_store.hpp b/src/operability/adapters/include/supervisor/common/kv_store.hpp new file mode 100644 index 0000000..d1a7cd9 --- /dev/null +++ b/src/operability/adapters/include/supervisor/common/kv_store.hpp @@ -0,0 +1,124 @@ +// ADAPTER: standalone shim for missing base header "supervisor/common/kv_store.hpp". +// Production build uses the real base header from the base tree; this file is +// only on the include path for standalone verification (see operability.mk, +// OPERABILITY_STANDALONE=1). Signatures mirror the seam spec. The cas() method +// is the documented fencing extension (CAS on key plus expected token). +#pragma once + +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" + +namespace molo::common { + +// Minimal KV abstraction. TTLs are in milliseconds; ttl_ms <= 0 means no expiry. +class KvStore { + public: + virtual ~KvStore() = default; + virtual bool get(const std::string& key, std::string& value) = 0; + virtual void set(const std::string& key, const std::string& value, std::int64_t ttl_ms) = 0; + virtual void erase(const std::string& key) = 0; + virtual bool acquire(const std::string& key, const std::string& value, std::int64_t ttl_ms) = 0; + virtual bool renew(const std::string& key, const std::string& expected, std::int64_t ttl_ms) = 0; + // Fencing extension: set only when current value equals expected. + virtual bool cas(const std::string& key, const std::string& expected, const std::string& next, + std::int64_t ttl_ms) = 0; +}; + +// In-memory store with clock-driven expiry. Mirrors InMemoryKvStore(Clock&). +class InMemoryKvStore : public KvStore { + public: + explicit InMemoryKvStore(Clock& clock) : clock_(clock) {} + + bool get(const std::string& key, std::string& value) override { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + auto it = map_.find(key); + if (it == map_.end() || is_expired(it->second, now)) { + if (it != map_.end()) map_.erase(it); + return false; + } + value = it->second.value; + return true; + } + + void set(const std::string& key, const std::string& value, std::int64_t ttl_ms) override { + std::lock_guard lock(mutex_); + Entry e; + e.value = value; + e.expires_ms = ttl_ms > 0 ? clock_.steady_ms() + ttl_ms : 0; + map_[key] = e; + if (map_.size() > kMaxKeys) map_.erase(map_.begin()); + } + + void erase(const std::string& key) override { + std::lock_guard lock(mutex_); + map_.erase(key); + } + + bool acquire(const std::string& key, const std::string& value, std::int64_t ttl_ms) override { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + auto it = map_.find(key); + if (it != map_.end() && !is_expired(it->second, now)) return false; + Entry e; + e.value = value; + e.expires_ms = ttl_ms > 0 ? now + ttl_ms : 0; + map_[key] = e; + return true; + } + + bool renew(const std::string& key, const std::string& expected, std::int64_t ttl_ms) override { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + auto it = map_.find(key); + if (it == map_.end() || is_expired(it->second, now)) { + if (it != map_.end()) map_.erase(it); + return false; + } + if (it->second.value != expected) return false; + it->second.expires_ms = ttl_ms > 0 ? now + ttl_ms : 0; + return true; + } + + bool cas(const std::string& key, const std::string& expected, const std::string& next, + std::int64_t ttl_ms) override { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + auto it = map_.find(key); + if (it == map_.end() || is_expired(it->second, now)) { + if (it != map_.end()) map_.erase(it); + if (!expected.empty()) return false; + Entry e; + e.value = next; + e.expires_ms = ttl_ms > 0 ? now + ttl_ms : 0; + map_[key] = e; + return true; + } + if (it->second.value != expected) return false; + it->second.value = next; + it->second.expires_ms = ttl_ms > 0 ? now + ttl_ms : 0; + return true; + } + + private: + struct Entry { + std::string value; + std::int64_t expires_ms = 0; // 0 means no expiry + }; + + static constexpr std::size_t kMaxKeys = 65536; + + static bool is_expired(const Entry& e, std::int64_t now) { + return e.expires_ms > 0 && e.expires_ms <= now; + } + + Clock& clock_; + std::mutex mutex_; + std::map map_; +}; + +} // namespace molo::common diff --git a/src/operability/adapters/include/supervisor/common/observer.hpp b/src/operability/adapters/include/supervisor/common/observer.hpp new file mode 100644 index 0000000..b83a011 --- /dev/null +++ b/src/operability/adapters/include/supervisor/common/observer.hpp @@ -0,0 +1,80 @@ +// ADAPTER: standalone shim for missing base header "supervisor/common/observer.hpp". +// Production build uses the real base header from the base tree; this file is +// only on the include path for standalone verification (see operability.mk, +// OPERABILITY_STANDALONE=1). Signatures mirror the seam spec. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace molo::common { + +// Minimal observer abstraction for transition and audit events. +class Observer { + public: + virtual ~Observer() = default; + virtual void event(const std::string& type, const std::string& detail) = 0; + virtual void counter(const std::string& name, std::int64_t delta) = 0; + virtual void gauge(const std::string& name, double value) = 0; +}; + +// Recording observer for tests. Keeps every event up to a static cap. +class RecordingObserver : public Observer { + public: + struct Record { + std::string type; + std::string detail; + }; + + void event(const std::string& type, const std::string& detail) override { + std::lock_guard lock(mutex_); + if (events_.size() >= kMaxEvents) return; + events_.push_back(Record{type, detail}); + } + + void counter(const std::string& name, std::int64_t delta) override { + std::lock_guard lock(mutex_); + for (auto& c : counters_) { + if (c.first == name) { + c.second += delta; + return; + } + } + if (counters_.size() < kMaxEvents) counters_.emplace_back(name, delta); + } + + void gauge(const std::string&, double) override {} + + std::vector events() const { + std::lock_guard lock(mutex_); + return events_; + } + + std::int64_t counter_value(const std::string& name) const { + std::lock_guard lock(mutex_); + for (const auto& c : counters_) { + if (c.first == name) return c.second; + } + return 0; + } + + private: + static constexpr std::size_t kMaxEvents = 4096; + mutable std::mutex mutex_; + std::vector events_; + std::vector> counters_; +}; + +// Null observer for production wiring where no listener is attached. +class NullObserver : public Observer { + public: + void event(const std::string&, const std::string&) override {} + void counter(const std::string&, std::int64_t) override {} + void gauge(const std::string&, double) override {} +}; + +} // namespace molo::common diff --git a/src/operability/caps.h b/src/operability/caps.h new file mode 100644 index 0000000..6978819 --- /dev/null +++ b/src/operability/caps.h @@ -0,0 +1,57 @@ +// Static ceilings for part 13 operability. Every limit is a compile-time +// constant; nothing here autotunes. Production may only lower these values +// through explicit config that clamps to the ceiling. +#pragma once + +#include +#include + +namespace molo::operability { + +// Single home for all operability caps. Never instantiated. +class Caps { + public: + Caps() = delete; + + // Coordination. + static constexpr std::size_t kMaxLeases = 1024; + static constexpr std::size_t kMaxLockWaiters = 256; + static constexpr std::int64_t kLeaderTtlMs = 10000; + static constexpr std::int64_t kLeaderRenewMs = 3000; + static constexpr std::int64_t kMaxLeaseTtlMs = 3600000; + + // Deployments. + static constexpr std::size_t kMaxDeployInstances = 4096; + static constexpr std::size_t kMaxDeployBatch = 256; + static constexpr std::int64_t kDeployReadyTimeoutMs = 300000; + static constexpr std::int64_t kBlueGreenRollbackWindowMs = 1800000; + + // Graceful shutdown drain. + static constexpr std::int64_t kDrainDeadlineMs = 30000; + static constexpr std::int64_t kDrainPollMs = 50; + + // Resilience. + static constexpr int kBreakerConsecutiveFailures = 20; + static constexpr int kBreakerWindowMinSamples = 10; + static constexpr std::int64_t kBreakerWindowMs = 10000; + static constexpr int kBreakerWindowFailurePct = 50; + static constexpr int kBreakerHalfOpenProbes = 3; + static constexpr std::int64_t kBreakerOpenHoldMs = 30000; + static constexpr std::size_t kBulkheadMaxPartitions = 32; + static constexpr std::size_t kBulkheadMaxConcurrent = 64; + static constexpr std::size_t kBulkheadMaxQueue = 128; + static constexpr std::int64_t kDeadlineFloorMarginMs = 50; + static constexpr std::int64_t kShedRetryAfterMaxS = 120; + static constexpr std::size_t kSloMaxRoutes = 1024; + static constexpr std::size_t kSloMaxSamplesPerRoute = 8192; + + // Chaos. + static constexpr std::size_t kChaosMaxHooks = 64; + static constexpr int kChaosMaxBlastPct = 100; + + // Debug endpoints. + static constexpr std::size_t kDebugMaxResponseBytes = 1048576; + static constexpr std::size_t kDebugMaxPathBytes = 512; +}; + +} // namespace molo::operability diff --git a/src/operability/chaos/experiment.cpp b/src/operability/chaos/experiment.cpp new file mode 100644 index 0000000..f6a9176 --- /dev/null +++ b/src/operability/chaos/experiment.cpp @@ -0,0 +1,87 @@ +#include "operability/chaos/experiment.h" + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::chaos { + +Experiment::Experiment(::molo::common::Clock& clock, ::molo::common::Observer& observer, + std::string name, int blast_pct, std::int64_t duration_ms, double max_burn) + : clock_(clock), + observer_(observer), + name_(std::move(name)), + blast_pct_(blast_pct), + duration_ms_(duration_ms), + max_burn_(max_burn) { + if (blast_pct_ < 0) blast_pct_ = 0; + if (blast_pct_ > ::molo::operability::Caps::kChaosMaxBlastPct) { + blast_pct_ = ::molo::operability::Caps::kChaosMaxBlastPct; + } + if (duration_ms_ < 0) duration_ms_ = 0; + if (max_burn_ < 0.0) max_burn_ = 0.0; +} + +bool Experiment::start() { + std::lock_guard lock(mutex_); + if (state_ != State::kIdle || name_.empty()) return false; + state_ = State::kRunning; + start_ms_ = clock_.steady_ms(); + observer_.event("chaos.experiment.start", name_); + return true; +} + +void Experiment::finish() { + std::lock_guard lock(mutex_); + if (state_ != State::kRunning) return; + state_ = State::kFinished; + observer_.event("chaos.experiment.finish", name_); +} + +void Experiment::abort(const std::string& reason) { + std::lock_guard lock(mutex_); + if (state_ != State::kRunning) return; + state_ = State::kAborted; + observer_.event("chaos.experiment.abort", name_ + " " + reason); +} + +bool Experiment::should_apply(std::uint64_t seq) { + std::lock_guard lock(mutex_); + if (state_ != State::kRunning) return false; + if (duration_ms_ > 0 && clock_.steady_ms() - start_ms_ >= duration_ms_) { + state_ = State::kFinished; + observer_.event("chaos.experiment.finish", name_); + return false; + } + if (blast_pct_ <= 0) return false; + if (blast_pct_ >= 100) return true; + std::uint64_t x = seq + 0x9e3779b97f4a7c15ULL; + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9ULL; + x ^= x >> 27; + return (x % 100) < static_cast(blast_pct_); +} + +bool Experiment::abort_if_breached(const BurnProbe& probe) { + if (!probe) return false; + double burn = 0.0; + try { + burn = probe(); + } catch (...) { + return false; + } + if (burn <= max_burn_) return false; + abort("burn ceiling breached"); + return true; +} + +Experiment::State Experiment::state() const { + std::lock_guard lock(mutex_); + return state_; +} + +const std::string& Experiment::name() const { return name_; } + +int Experiment::blast_pct() const { return blast_pct_; } + +} // namespace molo::chaos diff --git a/src/operability/chaos/experiment.h b/src/operability/chaos/experiment.h new file mode 100644 index 0000000..50f064f --- /dev/null +++ b/src/operability/chaos/experiment.h @@ -0,0 +1,56 @@ +// Guardrailed chaos experiment: bounds a set of hooks by blast radius, +// duration and abort conditions. should_apply gates each operation by a +// deterministic traffic sample; abort_if_breached stops the experiment +// when the observed burn rate crosses the ceiling. +#pragma once + +#include +#include +#include +#include + +namespace molo::common { +class Clock; +class Observer; +} // namespace molo::common + +namespace molo::chaos { + +class Experiment { + public: + enum class State : int { kIdle = 0, kRunning = 1, kFinished = 2, kAborted = 3 }; + + using BurnProbe = std::function; + + Experiment(::molo::common::Clock& clock, ::molo::common::Observer& observer, std::string name, + int blast_pct, std::int64_t duration_ms, double max_burn); + + Experiment(const Experiment&) = delete; + Experiment& operator=(const Experiment&) = delete; + + bool start(); + void finish(); + void abort(const std::string& reason); + + // True when seq falls inside the blast radius while running. + bool should_apply(std::uint64_t seq); + // Check the burn probe; aborts when over the ceiling. True when aborted. + bool abort_if_breached(const BurnProbe& probe); + + State state() const; + const std::string& name() const; + int blast_pct() const; + + private: + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + std::string name_; + int blast_pct_; + std::int64_t duration_ms_; + double max_burn_; + mutable std::mutex mutex_; + State state_ = State::kIdle; + std::int64_t start_ms_ = 0; +}; + +} // namespace molo::chaos diff --git a/src/operability/chaos/hook.h b/src/operability/chaos/hook.h new file mode 100644 index 0000000..a7763ec --- /dev/null +++ b/src/operability/chaos/hook.h @@ -0,0 +1,57 @@ +// Single chaos hook: a named fault point with a deterministic action. +// Hooks are value objects; the Injector owns the registry (max 64) and +// decides when each hook fires. Header only. +#pragma once + +#include +#include + +namespace molo::chaos { + +class Hook { + public: + enum class Action : int { + kNone = 0, + kDelay = 1, // sleep delay_ms before continuing + kFail = 2, // return an injected failure + kAbort = 3 // abort the operation outright + }; + + Hook() = default; + Hook(std::string name, Action action, int fire_pct = 100, std::int64_t delay_ms = 0) + : name_(std::move(name)), action_(action), fire_pct_(fire_pct), delay_ms_(delay_ms) { + if (fire_pct_ < 0) fire_pct_ = 0; + if (fire_pct_ > 100) fire_pct_ = 100; + if (delay_ms_ < 0) delay_ms_ = 0; + } + + const std::string& name() const { return name_; } + Action action() const { return action_; } + int fire_pct() const { return fire_pct_; } + std::int64_t delay_ms() const { return delay_ms_; } + + // Deterministic sampler: hashes the sequence number so tests can predict + // exactly which calls fire without any randomness. + bool fires_for(std::uint64_t seq) const { + if (action_ == Action::kNone || fire_pct_ <= 0) return false; + if (fire_pct_ >= 100) return true; + return (mix(seq) % 100) < static_cast(fire_pct_); + } + + private: + static std::uint64_t mix(std::uint64_t x) { + x ^= x >> 33; + x *= 0xff51afd7ed558ccdULL; + x ^= x >> 33; + x *= 0xc4ceb9fe1a85ec53ULL; + x ^= x >> 33; + return x; + } + + std::string name_; + Action action_ = Action::kNone; + int fire_pct_ = 100; + std::int64_t delay_ms_ = 0; +}; + +} // namespace molo::chaos diff --git a/src/operability/chaos/injector.cpp b/src/operability/chaos/injector.cpp new file mode 100644 index 0000000..61d896c --- /dev/null +++ b/src/operability/chaos/injector.cpp @@ -0,0 +1,86 @@ +#include "operability/chaos/injector.h" + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::chaos { + +Injector::Injector(::molo::common::Clock& clock, ::molo::common::Observer& observer) + : clock_(clock), observer_(observer) { + static_assert(64 >= 1, "hook table must be non-empty"); + if (::molo::operability::Caps::kChaosMaxHooks < 64) { + // Static table sized to the cap; see header. + } +} + +bool Injector::add_hook(Hook hook) { + if (hook.name().empty()) return false; + std::lock_guard lock(mutex_); + for (std::size_t i = 0; i < count_; ++i) { + if (hooks_[i].name() == hook.name()) return false; + } + if (count_ >= ::molo::operability::Caps::kChaosMaxHooks || count_ >= hooks_.size()) { + return false; + } + hooks_[count_++] = std::move(hook); + observer_.counter("chaos_hooks_total", 1); + return true; +} + +bool Injector::remove_hook(const std::string& name) { + std::lock_guard lock(mutex_); + for (std::size_t i = 0; i < count_; ++i) { + if (hooks_[i].name() == name) { + for (std::size_t j = i + 1; j < count_; ++j) hooks_[j - 1] = hooks_[j]; + --count_; + observer_.counter("chaos_hooks_removed_total", 1); + return true; + } + } + return false; +} + +void Injector::clear() { + std::lock_guard lock(mutex_); + count_ = 0; +} + +std::size_t Injector::size() const { + std::lock_guard lock(mutex_); + return count_; +} + +Injector::Decision Injector::evaluate(std::uint64_t seq) { + std::lock_guard lock(mutex_); + Decision worst = Decision::kProceed; + for (std::size_t i = 0; i < count_; ++i) { + const Hook& h = hooks_[i]; + if (!h.fires_for(seq + i)) continue; + switch (h.action()) { + case Hook::Action::kNone: + break; + case Hook::Action::kDelay: + clock_.sleep_ms(h.delay_ms()); + observer_.counter("chaos_delayed_total", 1); + if (worst == Decision::kProceed) worst = Decision::kDelayed; + break; + case Hook::Action::kFail: + observer_.event("chaos.fired", h.name()); + observer_.counter("chaos_failed_total", 1); + if (worst == Decision::kProceed || worst == Decision::kDelayed) { + worst = Decision::kFailed; + } + break; + case Hook::Action::kAbort: + observer_.event("chaos.fired", h.name()); + observer_.counter("chaos_aborted_total", 1); + worst = Decision::kAborted; + break; + } + if (worst == Decision::kAborted) break; + } + return worst; +} + +} // namespace molo::chaos diff --git a/src/operability/chaos/injector.h b/src/operability/chaos/injector.h new file mode 100644 index 0000000..ab332a4 --- /dev/null +++ b/src/operability/chaos/injector.h @@ -0,0 +1,46 @@ +// Fault injector: owns up to 64 hooks and evaluates them on the request +// path. evaluate() is the single choke point: production calls it from +// middleware before dispatch, tests call it directly with a sequence id. +#pragma once + +#include +#include +#include +#include +#include + +#include "operability/chaos/hook.h" + +namespace molo::common { +class Clock; +class Observer; +} // namespace molo::common + +namespace molo::chaos { + +class Injector { + public: + enum class Decision : int { kProceed = 0, kDelayed = 1, kFailed = 2, kAborted = 3 }; + + Injector(::molo::common::Clock& clock, ::molo::common::Observer& observer); + + Injector(const Injector&) = delete; + Injector& operator=(const Injector&) = delete; + + bool add_hook(Hook hook); + bool remove_hook(const std::string& name); + void clear(); + std::size_t size() const; + + // Evaluate all hooks for one operation. seq drives deterministic sampling. + Decision evaluate(std::uint64_t seq); + + private: + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + mutable std::mutex mutex_; + std::array hooks_; + std::size_t count_ = 0; +}; + +} // namespace molo::chaos diff --git a/src/operability/coord/dist_lock.cpp b/src/operability/coord/dist_lock.cpp new file mode 100644 index 0000000..9dec03b --- /dev/null +++ b/src/operability/coord/dist_lock.cpp @@ -0,0 +1,63 @@ +#include "operability/coord/dist_lock.h" + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/kv_store.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::coord { + +namespace { + +constexpr std::int64_t kPollMs = 25; + +} // namespace + +DistLock::DistLock(::molo::common::KvStore& kv, ::molo::common::Clock& clock, + ::molo::common::Observer& observer, std::string key, std::string owner) + : kv_(kv), clock_(clock), observer_(observer), fence_(kv, key), key_(key), owner_(owner) {} + +std::uint64_t DistLock::try_lock(std::int64_t ttl_ms, std::int64_t wait_ms) { + { + std::lock_guard lock(mutex_); + if (locked_) return 0; + } + if (key_.empty() || owner_.empty() || ttl_ms <= 0) return 0; + if (wait_ms < 0) wait_ms = 0; + if (ttl_ms > ::molo::operability::Caps::kMaxLeaseTtlMs) { + ttl_ms = ::molo::operability::Caps::kMaxLeaseTtlMs; + } + const std::int64_t deadline = clock_.steady_ms() + wait_ms; + for (;;) { + if (kv_.acquire_ms(key_, owner_, ttl_ms)) { + const std::uint64_t token = fence_.next(ttl_ms); + if (token == 0) { + kv_.erase(key_); + return 0; + } + std::lock_guard lock(mutex_); + locked_ = true; + observer_.event("lock.acquired", key_); + observer_.counter("lock_acquired_total", 1); + return token; + } + if (clock_.steady_ms() >= deadline) return 0; + clock_.sleep_ms(kPollMs); + } +} + +void DistLock::unlock() { + std::lock_guard lock(mutex_); + if (!locked_) return; + locked_ = false; + std::string cur; + if (kv_.get(key_, cur) && cur == owner_) kv_.erase(key_); + observer_.event("lock.released", key_); +} + +bool DistLock::locked() const { + std::lock_guard lock(mutex_); + return locked_; +} + +} // namespace molo::coord diff --git a/src/operability/coord/dist_lock.h b/src/operability/coord/dist_lock.h new file mode 100644 index 0000000..3d8034e --- /dev/null +++ b/src/operability/coord/dist_lock.h @@ -0,0 +1,44 @@ +// Distributed mutex built on Lease. try_lock polls until the lease is held +// or the wait budget expires; on success it returns a fencing token from +// the key scoped fence so guarded writes can reject stale holders. +#pragma once + +#include +#include +#include + +#include "operability/coord/fence.h" + +namespace molo::common { +class Clock; +class KvStore; +class Observer; +} // namespace molo::common + +namespace molo::coord { + +class DistLock { + public: + DistLock(::molo::common::KvStore& kv, ::molo::common::Clock& clock, + ::molo::common::Observer& observer, std::string key, std::string owner); + + DistLock(const DistLock&) = delete; + DistLock& operator=(const DistLock&) = delete; + + // Returns the fencing token on success, 0 on timeout or error. + std::uint64_t try_lock(std::int64_t ttl_ms, std::int64_t wait_ms); + void unlock(); + bool locked() const; + + private: + ::molo::common::KvStore& kv_; + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + Fence fence_; + std::string key_; + std::string owner_; + mutable std::mutex mutex_; + bool locked_ = false; +}; + +} // namespace molo::coord diff --git a/src/operability/coord/election.cpp b/src/operability/coord/election.cpp new file mode 100644 index 0000000..2663431 --- /dev/null +++ b/src/operability/coord/election.cpp @@ -0,0 +1,74 @@ +#include "operability/coord/election.h" + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/kv_store.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::coord { + +LeaderElection::LeaderElection(::molo::common::KvStore& kv, ::molo::common::Clock& clock, + ::molo::common::Observer& observer, std::string key, + std::string node_id, std::int64_t ttl_ms, std::int64_t renew_ms) + : kv_(kv), + clock_(clock), + observer_(observer), + key_(std::move(key)), + node_id_(std::move(node_id)), + ttl_ms_(ttl_ms), + renew_ms_(renew_ms) { + if (ttl_ms_ <= 0) ttl_ms_ = ::molo::operability::Caps::kLeaderTtlMs; + if (ttl_ms_ > ::molo::operability::Caps::kMaxLeaseTtlMs) { + ttl_ms_ = ::molo::operability::Caps::kMaxLeaseTtlMs; + } + if (renew_ms_ <= 0 || renew_ms_ >= ttl_ms_) renew_ms_ = ttl_ms_ / 3; + if (renew_ms_ <= 0) renew_ms_ = 1; +} + +bool LeaderElection::tick() { + std::lock_guard lock(mutex_); + if (key_.empty() || node_id_.empty()) return false; + const std::int64_t now = clock_.steady_ms(); + if (leader_) { + if (now - last_action_ms_ < renew_ms_) return true; + if (kv_.renew_ms(key_, node_id_, ttl_ms_)) { + last_action_ms_ = now; + observer_.counter("election_renewed_total", 1); + return true; + } + leader_ = false; + observer_.event("election.lost", node_id_); + return false; + } + if (kv_.acquire_ms(key_, node_id_, ttl_ms_)) { + leader_ = true; + last_action_ms_ = now; + observer_.event("election.won", node_id_); + observer_.counter("election_won_total", 1); + return true; + } + return false; +} + +bool LeaderElection::is_leader() const { + std::lock_guard lock(mutex_); + return leader_; +} + +std::string LeaderElection::leader() const { + std::lock_guard lock(mutex_); + std::string cur; + if (kv_.get(key_, cur)) return cur; + return {}; +} + +void LeaderElection::resign() { + std::lock_guard lock(mutex_); + if (!leader_) return; + leader_ = false; + std::string cur; + if (kv_.get(key_, cur) && cur == node_id_) kv_.erase(key_); + observer_.event("election.resigned", node_id_); +} + +} // namespace molo::coord diff --git a/src/operability/coord/election.h b/src/operability/coord/election.h new file mode 100644 index 0000000..9038641 --- /dev/null +++ b/src/operability/coord/election.h @@ -0,0 +1,47 @@ +// Leader election on top of the KV store lease. The caller drives tick() +// from the event loop (or a timer); tick acquires the leader key when free +// and renews it while leader. Transitions are reported via the observer. +// The task scheduler migrates its internal lease onto this class. +#pragma once + +#include +#include +#include + +namespace molo::common { +class Clock; +class KvStore; +class Observer; +} // namespace molo::common + +namespace molo::coord { + +class LeaderElection { + public: + LeaderElection(::molo::common::KvStore& kv, ::molo::common::Clock& clock, + ::molo::common::Observer& observer, std::string key, std::string node_id, + std::int64_t ttl_ms = 10000, std::int64_t renew_ms = 3000); + + LeaderElection(const LeaderElection&) = delete; + LeaderElection& operator=(const LeaderElection&) = delete; + + // One election step. Returns true when this node is leader afterwards. + bool tick(); + bool is_leader() const; + std::string leader() const; + void resign(); + + private: + ::molo::common::KvStore& kv_; + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + std::string key_; + std::string node_id_; + std::int64_t ttl_ms_; + std::int64_t renew_ms_; + mutable std::mutex mutex_; + bool leader_ = false; + std::int64_t last_action_ms_ = 0; +}; + +} // namespace molo::coord diff --git a/src/operability/coord/fence.h b/src/operability/coord/fence.h new file mode 100644 index 0000000..bd20539 --- /dev/null +++ b/src/operability/coord/fence.h @@ -0,0 +1,77 @@ +// Fencing token guard. A fenced resource only accepts work carrying the +// newest token, so a stale lock holder cannot corrupt state after its lease +// expired. Tokens are monotonic per Fence; the distributed value lives in +// the KV store under the lock key and is advanced by CAS. Header only. +#pragma once + +#include +#include +#include +#include + +#include "supervisor/common/kv_store.hpp" + +namespace molo::coord { + +class Fence { + public: + explicit Fence(::molo::common::KvStore& kv, std::string key) + : kv_(kv), key_(std::move(key)), local_(0) {} + + Fence(const Fence&) = delete; + Fence& operator=(const Fence&) = delete; + + // Advance the fence and return the new token. Uses CAS so concurrent + // advancers serialize; each caller gets a distinct increasing token. + std::uint64_t next(std::int64_t ttl_ms = 60000) { + for (int attempt = 0; attempt < 16; ++attempt) { + std::string cur; + const bool have = kv_.get(fence_key(), cur); + const std::uint64_t cur_token = have ? parse(cur) : 0; + std::uint64_t want = cur_token + 1; + const std::uint64_t local = local_.load(); + if (want <= local) want = local + 1; + const std::string next_val = std::to_string(want); + const bool ok = have ? kv_.cas_ms(fence_key(), cur, next_val, ttl_ms) + : kv_.acquire_ms(fence_key(), next_val, ttl_ms); + if (ok) { + std::uint64_t expect = local; + while (expect < want && !local_.compare_exchange_weak(expect, want)) { + } + return want; + } + } + return 0; // contention: caller treats 0 as failure + } + + // Returns true when the token is still the newest known token. + bool check(std::uint64_t token) { + std::string cur; + if (!kv_.get(fence_key(), cur)) return token >= local_.load(); + const std::uint64_t remote = parse(cur); + std::uint64_t expect = local_.load(); + while (expect < remote && !local_.compare_exchange_weak(expect, remote)) { + } + return token >= remote; + } + + const std::string& key() const { return key_; } + + private: + static std::uint64_t parse(const std::string& s) { + std::uint64_t v = 0; + for (char c : s) { + if (c < '0' || c > '9') break; + v = v * 10 + static_cast(c - '0'); + } + return v; + } + + std::string fence_key() const { return key_ + "/fence"; } + + ::molo::common::KvStore& kv_; + std::string key_; + std::atomic local_; +}; + +} // namespace molo::coord diff --git a/src/operability/coord/lease.cpp b/src/operability/coord/lease.cpp new file mode 100644 index 0000000..988948b --- /dev/null +++ b/src/operability/coord/lease.cpp @@ -0,0 +1,74 @@ +#include "operability/coord/lease.h" + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/kv_store.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::coord { + +std::atomic Lease::live_{0}; + +Lease::Lease(::molo::common::KvStore& kv, ::molo::common::Clock& clock, + ::molo::common::Observer& observer, std::string key, std::string owner) + : kv_(kv), clock_(clock), observer_(observer), key_(std::move(key)), owner_(std::move(owner)) { + live_.fetch_add(1); +} + +Lease::~Lease() { + release(); + live_.fetch_sub(1); +} + +std::size_t Lease::live_count() { return live_.load(); } + +bool Lease::acquire(std::int64_t ttl_ms) { + std::lock_guard lock(mutex_); + if (live_.load() > ::molo::operability::Caps::kMaxLeases) return false; + if (key_.empty() || owner_.empty()) return false; + if (ttl_ms <= 0) return false; + if (ttl_ms > ::molo::operability::Caps::kMaxLeaseTtlMs) { + ttl_ms = ::molo::operability::Caps::kMaxLeaseTtlMs; + } + if (!kv_.acquire_ms(key_, owner_, ttl_ms)) return false; + holds_ = true; + acquired_ms_ = clock_.steady_ms(); + observer_.event("lease.acquired", key_); + observer_.counter("lease_acquired_total", 1); + return true; +} + +bool Lease::renew(std::int64_t ttl_ms) { + std::lock_guard lock(mutex_); + if (!holds_) return false; + if (ttl_ms <= 0) return false; + if (ttl_ms > ::molo::operability::Caps::kMaxLeaseTtlMs) { + ttl_ms = ::molo::operability::Caps::kMaxLeaseTtlMs; + } + if (!kv_.renew_ms(key_, owner_, ttl_ms)) { + holds_ = false; + observer_.event("lease.lost", key_); + return false; + } + observer_.counter("lease_renewed_total", 1); + return true; +} + +void Lease::release() { + std::lock_guard lock(mutex_); + if (!holds_) return; + holds_ = false; + std::string cur; + if (kv_.get(key_, cur) && cur == owner_) kv_.erase(key_); + observer_.event("lease.released", key_); +} + +bool Lease::holds() const { + std::lock_guard lock(mutex_); + return holds_; +} + +const std::string& Lease::key() const { return key_; } +const std::string& Lease::owner() const { return owner_; } + +} // namespace molo::coord diff --git a/src/operability/coord/lease.h b/src/operability/coord/lease.h new file mode 100644 index 0000000..d0d8acf --- /dev/null +++ b/src/operability/coord/lease.h @@ -0,0 +1,50 @@ +// Single distributed lease backed by the KV store. At most kMaxLeases live +// Lease objects may exist per process; acquire maps to KvStore::acquire and +// renew maps to KvStore::renew with the owner id as the expected value. +#pragma once + +#include +#include +#include +#include + +namespace molo::common { +class Clock; +class KvStore; +class Observer; +} // namespace molo::common + +namespace molo::coord { + +class Lease { + public: + Lease(::molo::common::KvStore& kv, ::molo::common::Clock& clock, + ::molo::common::Observer& observer, std::string key, std::string owner); + ~Lease(); + + Lease(const Lease&) = delete; + Lease& operator=(const Lease&) = delete; + + static std::size_t live_count(); + + bool acquire(std::int64_t ttl_ms); + bool renew(std::int64_t ttl_ms); + void release(); + bool holds() const; + const std::string& key() const; + const std::string& owner() const; + + private: + ::molo::common::KvStore& kv_; + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + std::string key_; + std::string owner_; + mutable std::mutex mutex_; + bool holds_ = false; + std::int64_t acquired_ms_ = 0; + + static std::atomic live_; +}; + +} // namespace molo::coord diff --git a/src/operability/debug/endpoint.cpp b/src/operability/debug/endpoint.cpp new file mode 100644 index 0000000..442824b --- /dev/null +++ b/src/operability/debug/endpoint.cpp @@ -0,0 +1,63 @@ +#include "operability/debug/endpoint.h" + +#include "operability/caps.h" + +namespace molo::debug { + +Endpoint::Endpoint(SnapshotFn snapshot, AuthCheck auth) + : snapshot_(std::move(snapshot)), auth_(std::move(auth)) {} + +bool Endpoint::enabled() { +#ifdef MOLO_ENABLE_DEBUG + return true; +#else + return false; +#endif +} + +Endpoint::Response Endpoint::handle(const std::string& path, const std::string& token) { + std::lock_guard lock(mutex_); + Response r; + if (!enabled()) { + r.status = 404; + r.body = "not found"; + return r; + } + if (path.size() > ::molo::operability::Caps::kDebugMaxPathBytes) { + r.status = 414; + r.body = "path too long"; + return r; + } + if (auth_) { + bool ok = false; + try { + ok = auth_(token); + } catch (...) { + ok = false; + } + if (!ok) { + r.status = 403; + r.body = "forbidden"; + return r; + } + } + if (path == "/debug/snapshot") { + r.status = 200; + r.content_type = "application/json"; + r.body = snapshot_ ? snapshot_() : "{}"; + } else if (path == "/debug/ping") { + r.status = 200; + r.content_type = "text/plain"; + r.body = "pong"; + } else { + r.status = 404; + r.body = "not found"; + return r; + } + if (r.body.size() > ::molo::operability::Caps::kDebugMaxResponseBytes) { + r.body.resize(::molo::operability::Caps::kDebugMaxResponseBytes); + } + return r; +} + +} // namespace molo::debug diff --git a/src/operability/debug/endpoint.h b/src/operability/debug/endpoint.h new file mode 100644 index 0000000..1f5b8ab --- /dev/null +++ b/src/operability/debug/endpoint.h @@ -0,0 +1,39 @@ +// Admin only debug endpoints served under /debug/*. Production mounts this +// on the admin listener only (public listener returns 404), behind the +// MOLO_ENABLE_DEBUG compile flag plus the injected auth check. Responses +// are capped at 1 MB. +#pragma once + +#include +#include +#include + +namespace molo::debug { + +class Endpoint { + public: + using SnapshotFn = std::function; + using AuthCheck = std::function; + + struct Response { + int status = 404; + std::string content_type = "text/plain"; + std::string body; + }; + + Endpoint(SnapshotFn snapshot = SnapshotFn{}, AuthCheck auth = AuthCheck{}); + + Endpoint(const Endpoint&) = delete; + Endpoint& operator=(const Endpoint&) = delete; + + static bool enabled(); + + Response handle(const std::string& path, const std::string& token); + + private: + SnapshotFn snapshot_; + AuthCheck auth_; + mutable std::mutex mutex_; +}; + +} // namespace molo::debug diff --git a/src/operability/debug/snapshot.cpp b/src/operability/debug/snapshot.cpp new file mode 100644 index 0000000..1df1310 --- /dev/null +++ b/src/operability/debug/snapshot.cpp @@ -0,0 +1,89 @@ +#include "operability/debug/snapshot.h" + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" + +namespace molo::debug { + +namespace { + +constexpr std::size_t kMaxSections = 64; + +std::string escape_json(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + switch (c) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + out.push_back(c); + break; + } + } + return out; +} + +} // namespace + +Snapshot::Snapshot(::molo::common::Clock& clock, std::string version) + : clock_(clock), version_(std::move(version)), boot_ms_(clock.wall_ms()) {} + +bool Snapshot::add_section(const std::string& name, const std::string& json) { + if (name.empty()) return false; + std::lock_guard lock(mutex_); + for (const auto& s : sections_) { + if (s.first == name) return false; + } + if (sections_.size() >= kMaxSections) return false; + sections_.emplace_back(name, json); + return true; +} + +bool Snapshot::remove_section(const std::string& name) { + std::lock_guard lock(mutex_); + for (auto it = sections_.begin(); it != sections_.end(); ++it) { + if (it->first == name) { + sections_.erase(it); + return true; + } + } + return false; +} + +std::string Snapshot::capture() { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.wall_ms(); + std::string out = "{"; + out += "\"version\":\"" + escape_json(version_) + "\""; + out += ",\"wall_ms\":" + std::to_string(now); + out += ",\"uptime_ms\":" + std::to_string(now - boot_ms_); + for (const auto& s : sections_) { + out += ",\"" + escape_json(s.first) + "\":" + (s.second.empty() ? "null" : s.second); + } + out += "}"; + if (out.size() > ::molo::operability::Caps::kDebugMaxResponseBytes) { + out.resize(::molo::operability::Caps::kDebugMaxResponseBytes); + } + return out; +} + +std::size_t Snapshot::section_count() const { + std::lock_guard lock(mutex_); + return sections_.size(); +} + +} // namespace molo::debug diff --git a/src/operability/debug/snapshot.h b/src/operability/debug/snapshot.h new file mode 100644 index 0000000..9f9f78b --- /dev/null +++ b/src/operability/debug/snapshot.h @@ -0,0 +1,40 @@ +// Point in time debug snapshot: version, clocks and registered sections +// rendered as one JSON document capped at 1 MB. Sections let each subsystem +// contribute without the snapshot knowing their types. +#pragma once + +#include +#include +#include +#include + +namespace molo::common { +class Clock; +} // namespace molo::common + +namespace molo::debug { + +class Snapshot { + public: + explicit Snapshot(::molo::common::Clock& clock, std::string version = "0.0.0"); + + Snapshot(const Snapshot&) = delete; + Snapshot& operator=(const Snapshot&) = delete; + + // Register a pre-rendered JSON fragment under name. Returns false when the + // section table is full or the name already exists. + bool add_section(const std::string& name, const std::string& json); + bool remove_section(const std::string& name); + + std::string capture(); + std::size_t section_count() const; + + private: + ::molo::common::Clock& clock_; + std::string version_; + std::int64_t boot_ms_; + mutable std::mutex mutex_; + std::vector> sections_; +}; + +} // namespace molo::debug diff --git a/src/operability/deploy/blue_green.cpp b/src/operability/deploy/blue_green.cpp new file mode 100644 index 0000000..89ff8e1 --- /dev/null +++ b/src/operability/deploy/blue_green.cpp @@ -0,0 +1,205 @@ +#include "operability/deploy/blue_green.h" + +#include + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::deploy { + +namespace { + +constexpr std::int64_t kPollMs = 50; + +} // namespace + +BlueGreen::BlueGreen(Config cfg, ::molo::common::Clock& clock, ::molo::common::Observer& observer, + ReadyProbe probe, SwitchHook on_switch, RetireHook on_retire) + : cfg_(cfg), + clock_(clock), + observer_(observer), + probe_(std::move(probe)), + on_switch_(std::move(on_switch)), + on_retire_(std::move(on_retire)) { + if (cfg_.green_instances > ::molo::operability::Caps::kMaxDeployInstances) { + cfg_.green_instances = ::molo::operability::Caps::kMaxDeployInstances; + } + if (cfg_.ready_timeout_ms > ::molo::operability::Caps::kDeployReadyTimeoutMs) { + cfg_.ready_timeout_ms = ::molo::operability::Caps::kDeployReadyTimeoutMs; + } + if (cfg_.ready_timeout_ms < 0) cfg_.ready_timeout_ms = 0; + if (cfg_.rollback_window_ms > ::molo::operability::Caps::kBlueGreenRollbackWindowMs) { + cfg_.rollback_window_ms = ::molo::operability::Caps::kBlueGreenRollbackWindowMs; + } + if (cfg_.rollback_window_ms < 0) cfg_.rollback_window_ms = 0; +} + +bool BlueGreen::start() { + std::lock_guard lock(mutex_); + if (state_ != State::kIdle) return false; + if (cfg_.green_instances == 0 || !probe_) return false; + state_ = State::kRunning; + stage_ = Stage::kDeployGreen; + serving_green_ = false; + observer_.event("deploy.start", "blue_green green=" + std::to_string(cfg_.green_instances)); + return true; +} + +bool BlueGreen::step() { + Stage stage; + { + std::lock_guard lock(mutex_); + if (state_ != State::kRunning) return state_ == State::kSucceeded; + stage = stage_; + } + switch (stage) { + case Stage::kDeployGreen: { + observer_.event("deploy.stage", "deploy_green"); + std::lock_guard lock(mutex_); + stage_ = Stage::kVerifyGreen; + return true; + } + case Stage::kVerifyGreen: { + if (!wait_green_ready()) { + std::lock_guard lock(mutex_); + state_ = State::kFailed; + observer_.event("deploy.failed", "green never ready"); + return false; + } + std::lock_guard lock(mutex_); + stage_ = Stage::kSwitch; + return true; + } + case Stage::kSwitch: { + bool ok = true; + if (on_switch_) { + try { + ok = on_switch_(); + } catch (...) { + ok = false; + } + } + std::lock_guard lock(mutex_); + if (!ok) { + state_ = State::kFailed; + observer_.event("deploy.failed", "switch refused"); + return false; + } + serving_green_ = true; + window_end_ms_ = clock_.steady_ms() + cfg_.rollback_window_ms; + stage_ = Stage::kRollbackWindow; + observer_.event("deploy.stage", "serving green"); + return true; + } + case Stage::kRollbackWindow: { + if (clock_.steady_ms() < window_end_ms_) return true; // stay here until window ends + std::lock_guard lock(mutex_); + stage_ = Stage::kRetireBlue; + return true; + } + case Stage::kRetireBlue: { + if (on_retire_) { + try { + on_retire_(true); + } catch (...) { + } + } + std::lock_guard lock(mutex_); + stage_ = Stage::kDone; + state_ = State::kSucceeded; + observer_.event("deploy.succeeded", "blue retired"); + return true; + } + case Stage::kDone: + return true; + } + return false; +} + +void BlueGreen::abort() { + std::lock_guard lock(mutex_); + if (state_ == State::kRunning || state_ == State::kPaused) { + state_ = State::kAborted; + observer_.event("deploy.aborted", "serving_green=" + std::to_string(serving_green_)); + } +} + +BlueGreen::State BlueGreen::state() const { + std::lock_guard lock(mutex_); + return state_; +} + +const char* BlueGreen::state_name() const { return name_of(state()); } + +bool BlueGreen::rollback() { + std::lock_guard lock(mutex_); + if (!serving_green_) return false; + if (state_ != State::kRunning && state_ != State::kFailed) return false; + if (state_ == State::kRunning && stage_ != Stage::kRollbackWindow) return false; + serving_green_ = false; + stage_ = Stage::kDone; + state_ = State::kRolledBack; + if (on_retire_) { + try { + on_retire_(false); + } catch (...) { + } + } + observer_.event("deploy.rollback", "back to blue"); + return true; +} + +BlueGreen::Stage BlueGreen::stage() const { + std::lock_guard lock(mutex_); + return stage_; +} + +bool BlueGreen::serving_green() const { + std::lock_guard lock(mutex_); + return serving_green_; +} + +const char* BlueGreen::name_of(State s) { + switch (s) { + case State::kIdle: + return "idle"; + case State::kRunning: + return "running"; + case State::kPaused: + return "paused"; + case State::kSucceeded: + return "succeeded"; + case State::kFailed: + return "failed"; + case State::kAborted: + return "aborted"; + case State::kRolledBack: + return "rolled_back"; + } + return "unknown"; +} + +bool BlueGreen::wait_green_ready() { + const std::int64_t deadline = clock_.steady_ms() + cfg_.ready_timeout_ms; + for (;;) { + bool all_ready = true; + for (std::size_t i = 0; i < cfg_.green_instances; ++i) { + bool ready = false; + try { + ready = probe_(i); + } catch (...) { + ready = false; + } + if (!ready) { + all_ready = false; + break; + } + } + if (all_ready) return true; + if (clock_.steady_ms() >= deadline) return false; + clock_.sleep_ms(kPollMs); + } +} + +} // namespace molo::deploy diff --git a/src/operability/deploy/blue_green.h b/src/operability/deploy/blue_green.h new file mode 100644 index 0000000..b47ef14 --- /dev/null +++ b/src/operability/deploy/blue_green.h @@ -0,0 +1,74 @@ +// Blue/green deployment: build the green fleet next to blue, verify it, +// switch traffic, then retire blue after a rollback window. Every stage is +// explicit so an operator or a frozen clock can drive it step by step. +#pragma once + +#include +#include +#include +#include + +#include "operability/deploy/strategy.h" + +namespace molo::common { +class Clock; +class Observer; +} // namespace molo::common + +namespace molo::deploy { + +class BlueGreen : public Strategy { + public: + enum class Stage : int { + kDeployGreen = 0, + kVerifyGreen = 1, + kSwitch = 2, + kRollbackWindow = 3, + kRetireBlue = 4, + kDone = 5 + }; + + struct Config { + std::size_t green_instances = 0; + std::int64_t ready_timeout_ms = 60000; + std::int64_t rollback_window_ms = 600000; + }; + + using ReadyProbe = std::function; + using SwitchHook = std::function; + using RetireHook = std::function; + + BlueGreen(Config cfg, ::molo::common::Clock& clock, ::molo::common::Observer& observer, + ReadyProbe probe, SwitchHook on_switch = SwitchHook{}, + RetireHook on_retire = RetireHook{}); + + bool start() override; + bool step() override; + void abort() override; + State state() const override; + const char* state_name() const override; + + // Roll back to blue. Valid during the rollback window and after failure. + bool rollback(); + + Stage stage() const; + bool serving_green() const; + + private: + static const char* name_of(State s); + bool wait_green_ready(); + + Config cfg_; + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + ReadyProbe probe_; + SwitchHook on_switch_; + RetireHook on_retire_; + mutable std::mutex mutex_; + State state_ = State::kIdle; + Stage stage_ = Stage::kDeployGreen; + bool serving_green_ = false; + std::int64_t window_end_ms_ = 0; +}; + +} // namespace molo::deploy diff --git a/src/operability/deploy/lifecycle.cpp b/src/operability/deploy/lifecycle.cpp new file mode 100644 index 0000000..0a4d7c7 --- /dev/null +++ b/src/operability/deploy/lifecycle.cpp @@ -0,0 +1,137 @@ +#include "operability/deploy/lifecycle.h" + +#include + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::deploy { + +Lifecycle::Lifecycle(::molo::common::Clock& clock, ::molo::common::Observer& observer) + : clock_(clock), observer_(observer) {} + +void Lifecycle::on_stop_listeners(Hook hook) { stop_listeners_ = std::move(hook); } +void Lifecycle::on_connections_drained(DrainedProbe probe) { + connections_drained_ = std::move(probe); +} +void Lifecycle::on_background_drained(DrainedProbe probe) { + background_drained_ = std::move(probe); +} +void Lifecycle::on_drop_background(DropHook hook) { drop_background_ = std::move(hook); } +void Lifecycle::on_serving(Hook hook) { on_serving_ = std::move(hook); } +void Lifecycle::on_complete(Hook hook) { on_complete_ = std::move(hook); } + +void Lifecycle::mark_serving() { + { + std::lock_guard lock(mutex_); + phase_ = Phase::kServing; + } + observer_.event("lifecycle.serving", ""); + if (on_serving_) { + try { + on_serving_(); + } catch (...) { + } + } +} + +bool Lifecycle::run_drain(std::int64_t deadline_ms, std::int64_t poll_ms) { + if (deadline_ms > ::molo::operability::Caps::kDrainDeadlineMs) { + deadline_ms = ::molo::operability::Caps::kDrainDeadlineMs; + } + if (deadline_ms < 0) deadline_ms = 0; + if (poll_ms <= 0) poll_ms = ::molo::operability::Caps::kDrainPollMs; + + const std::int64_t start = clock_.steady_ms(); + const std::int64_t deadline = start + deadline_ms; + bool dropped = false; + + { + std::lock_guard lock(mutex_); + phase_ = Phase::kStopListeners; + } + observer_.event("lifecycle.phase", "stop_listeners"); + if (stop_listeners_) { + try { + stop_listeners_(); + } catch (...) { + } + } + + { + std::lock_guard lock(mutex_); + phase_ = Phase::kDrainConnections; + } + const bool conn_ok = drain_until(connections_drained_, deadline, poll_ms, "connections"); + + { + std::lock_guard lock(mutex_); + phase_ = Phase::kDrainBackground; + } + const bool bg_ok = drain_until(background_drained_, deadline, poll_ms, "background"); + if (!bg_ok) { + dropped = true; + observer_.event("lifecycle.dropped", "background work exceeded drain deadline"); + if (drop_background_) { + try { + drop_background_("drain deadline exceeded"); + } catch (...) { + } + } + } + + { + std::lock_guard lock(mutex_); + phase_ = Phase::kFinalize; + } + observer_.event("lifecycle.phase", "finalize"); + if (on_complete_) { + try { + on_complete_(); + } catch (...) { + } + } + + { + std::lock_guard lock(mutex_); + phase_ = Phase::kDone; + clean_ = conn_ok && bg_ok && !dropped; + } + observer_.event("lifecycle.done", clean_ ? "clean" : "forced"); + return clean_; +} + +Lifecycle::Phase Lifecycle::phase() const { + std::lock_guard lock(mutex_); + return phase_; +} + +bool Lifecycle::clean() const { + std::lock_guard lock(mutex_); + return clean_; +} + +bool Lifecycle::drain_until(const DrainedProbe& probe, std::int64_t deadline_ms, + std::int64_t poll_ms, const char* what) { + if (!probe) return true; + for (;;) { + bool drained = false; + try { + drained = probe(); + } catch (...) { + drained = false; + } + if (drained) { + observer_.event("lifecycle.drained", what); + return true; + } + if (clock_.steady_ms() >= deadline_ms) { + observer_.event("lifecycle.timeout", what); + return false; + } + clock_.sleep_ms(poll_ms); + } +} + +} // namespace molo::deploy diff --git a/src/operability/deploy/lifecycle.h b/src/operability/deploy/lifecycle.h new file mode 100644 index 0000000..8f0b4f9 --- /dev/null +++ b/src/operability/deploy/lifecycle.h @@ -0,0 +1,71 @@ +// Process lifecycle: boot completion plus a four phase graceful drain. +// Phase 1 stops listeners, phase 2 drains connections, phase 3 drains +// background work (timed out work is handed to the drop hook, which wires +// to the dead letter queue), phase 4 finalizes. Production wires the hooks +// to Listener::stop, Connection::drain, ShutdownCoordinator and StartupGate. +#pragma once + +#include +#include +#include +#include + +namespace molo::common { +class Clock; +class Observer; +} // namespace molo::common + +namespace molo::deploy { + +class Lifecycle { + public: + using Hook = std::function; + using DrainedProbe = std::function; + using DropHook = std::function; + + enum class Phase : int { + kBoot = 0, + kServing = 1, + kStopListeners = 2, + kDrainConnections = 3, + kDrainBackground = 4, + kFinalize = 5, + kDone = 6 + }; + + Lifecycle(::molo::common::Clock& clock, ::molo::common::Observer& observer); + + void on_stop_listeners(Hook hook); + void on_connections_drained(DrainedProbe probe); + void on_background_drained(DrainedProbe probe); + void on_drop_background(DropHook hook); + void on_serving(Hook hook); + void on_complete(Hook hook); + + // Called once the server accepts traffic. Wires to StartupGate. + void mark_serving(); + + // Runs all four drain phases. Returns true when fully drained. + bool run_drain(std::int64_t deadline_ms, std::int64_t poll_ms = 50); + + Phase phase() const; + bool clean() const; + + private: + bool drain_until(const DrainedProbe& probe, std::int64_t deadline_ms, std::int64_t poll_ms, + const char* what); + + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + Hook stop_listeners_; + DrainedProbe connections_drained_; + DrainedProbe background_drained_; + DropHook drop_background_; + Hook on_serving_; + Hook on_complete_; + mutable std::mutex mutex_; + Phase phase_ = Phase::kBoot; + bool clean_ = false; +}; + +} // namespace molo::deploy diff --git a/src/operability/deploy/rolling.cpp b/src/operability/deploy/rolling.cpp new file mode 100644 index 0000000..48269df --- /dev/null +++ b/src/operability/deploy/rolling.cpp @@ -0,0 +1,152 @@ +#include "operability/deploy/rolling.h" + +#include + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::deploy { + +namespace { + +constexpr std::int64_t kPollMs = 50; + +} // namespace + +Rolling::Rolling(Config cfg, ::molo::common::Clock& clock, ::molo::common::Observer& observer, + ReadyProbe probe, UpgradeHook hook) + : cfg_(cfg), clock_(clock), observer_(observer), probe_(std::move(probe)), hook_(std::move(hook)) { + if (cfg_.total_instances > ::molo::operability::Caps::kMaxDeployInstances) { + cfg_.total_instances = ::molo::operability::Caps::kMaxDeployInstances; + } + if (cfg_.batch_size == 0) cfg_.batch_size = 1; + if (cfg_.batch_size > ::molo::operability::Caps::kMaxDeployBatch) { + cfg_.batch_size = ::molo::operability::Caps::kMaxDeployBatch; + } + if (cfg_.ready_timeout_ms > ::molo::operability::Caps::kDeployReadyTimeoutMs) { + cfg_.ready_timeout_ms = ::molo::operability::Caps::kDeployReadyTimeoutMs; + } + if (cfg_.ready_timeout_ms < 0) cfg_.ready_timeout_ms = 0; + if (cfg_.pause_between_batches_ms < 0) cfg_.pause_between_batches_ms = 0; +} + +bool Rolling::start() { + std::lock_guard lock(mutex_); + if (state_ != State::kIdle) return false; + if (cfg_.total_instances == 0 || !probe_) return false; + state_ = State::kRunning; + cursor_ = 0; + upgraded_ = 0; + failed_ = 0; + observer_.event("deploy.start", "rolling total=" + std::to_string(cfg_.total_instances)); + return true; +} + +bool Rolling::step() { + std::size_t begin = 0; + std::size_t end = 0; + { + std::lock_guard lock(mutex_); + if (state_ != State::kRunning) return state_ == State::kSucceeded; + begin = cursor_; + end = begin + cfg_.batch_size; + if (end > cfg_.total_instances) end = cfg_.total_instances; + } + for (std::size_t i = begin; i < end; ++i) { + if (hook_) hook_(i); + if (wait_ready(i)) { + std::lock_guard lock(mutex_); + ++upgraded_; + observer_.counter("deploy_upgraded_total", 1); + } else { + std::lock_guard lock(mutex_); + ++failed_; + observer_.counter("deploy_failed_total", 1); + observer_.event("deploy.failed", "index=" + std::to_string(i)); + if (failed_ > cfg_.max_unavailable) { + state_ = State::kFailed; + return false; + } + } + } + { + std::lock_guard lock(mutex_); + cursor_ = end; + observer_.event("deploy.batch", "cursor=" + std::to_string(cursor_)); + if (cursor_ >= cfg_.total_instances) { + state_ = failed_ == 0 ? State::kSucceeded : State::kFailed; + if (state_ == State::kSucceeded) observer_.event("deploy.succeeded", ""); + return state_ == State::kSucceeded; + } + } + if (cfg_.pause_between_batches_ms > 0) clock_.sleep_ms(cfg_.pause_between_batches_ms); + return true; +} + +void Rolling::abort() { + std::lock_guard lock(mutex_); + if (state_ == State::kRunning || state_ == State::kPaused) { + state_ = State::kAborted; + observer_.event("deploy.aborted", "cursor=" + std::to_string(cursor_)); + } +} + +Rolling::State Rolling::state() const { + std::lock_guard lock(mutex_); + return state_; +} + +const char* Rolling::state_name() const { return name_of(state()); } + +std::size_t Rolling::upgraded() const { + std::lock_guard lock(mutex_); + return upgraded_; +} + +std::size_t Rolling::failed() const { + std::lock_guard lock(mutex_); + return failed_; +} + +std::size_t Rolling::pending() const { + std::lock_guard lock(mutex_); + return cfg_.total_instances > cursor_ ? cfg_.total_instances - cursor_ : 0; +} + +const char* Rolling::name_of(State s) { + switch (s) { + case State::kIdle: + return "idle"; + case State::kRunning: + return "running"; + case State::kPaused: + return "paused"; + case State::kSucceeded: + return "succeeded"; + case State::kFailed: + return "failed"; + case State::kAborted: + return "aborted"; + case State::kRolledBack: + return "rolled_back"; + } + return "unknown"; +} + +bool Rolling::wait_ready(std::size_t index) { + const std::int64_t deadline = clock_.steady_ms() + cfg_.ready_timeout_ms; + for (;;) { + bool ready = false; + try { + ready = probe_(index); + } catch (...) { + ready = false; + } + if (ready) return true; + if (clock_.steady_ms() >= deadline) return false; + clock_.sleep_ms(kPollMs); + } +} + +} // namespace molo::deploy diff --git a/src/operability/deploy/rolling.h b/src/operability/deploy/rolling.h new file mode 100644 index 0000000..326a948 --- /dev/null +++ b/src/operability/deploy/rolling.h @@ -0,0 +1,62 @@ +// Rolling deployment: upgrade instances in batches, waiting for each batch +// to report ready before moving on. Readiness comes from an injected probe +// so production can poll readyz/startupz while tests use a stub. +#pragma once + +#include +#include +#include +#include + +#include "operability/deploy/strategy.h" + +namespace molo::common { +class Clock; +class Observer; +} // namespace molo::common + +namespace molo::deploy { + +class Rolling : public Strategy { + public: + struct Config { + std::size_t total_instances = 0; + std::size_t batch_size = 1; + std::int64_t pause_between_batches_ms = 0; + std::int64_t ready_timeout_ms = 60000; + std::size_t max_unavailable = 0; + }; + + using ReadyProbe = std::function; + using UpgradeHook = std::function; + + Rolling(Config cfg, ::molo::common::Clock& clock, ::molo::common::Observer& observer, + ReadyProbe probe, UpgradeHook hook = UpgradeHook{}); + + bool start() override; + bool step() override; + void abort() override; + State state() const override; + const char* state_name() const override; + + std::size_t upgraded() const; + std::size_t failed() const; + std::size_t pending() const; + + private: + static const char* name_of(State s); + bool wait_ready(std::size_t index); + + Config cfg_; + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + ReadyProbe probe_; + UpgradeHook hook_; + mutable std::mutex mutex_; + State state_ = State::kIdle; + std::size_t cursor_ = 0; + std::size_t upgraded_ = 0; + std::size_t failed_ = 0; +}; + +} // namespace molo::deploy diff --git a/src/operability/deploy/strategy.h b/src/operability/deploy/strategy.h new file mode 100644 index 0000000..a02ca3e --- /dev/null +++ b/src/operability/deploy/strategy.h @@ -0,0 +1,29 @@ +// Common interface for deployment strategies (rolling, blue/green). +// Strategies are driven step by step so the caller keeps the event loop +// responsive; every blocking wait goes through the injected clock. +#pragma once + +namespace molo::deploy { + +class Strategy { + public: + enum class State : int { + kIdle = 0, + kRunning = 1, + kPaused = 2, + kSucceeded = 3, + kFailed = 4, + kAborted = 5, + kRolledBack = 6 + }; + + virtual ~Strategy() = default; + + virtual bool start() = 0; + virtual bool step() = 0; + virtual void abort() = 0; + virtual State state() const = 0; + virtual const char* state_name() const = 0; +}; + +} // namespace molo::deploy diff --git a/src/operability/resilience/breaker.cpp b/src/operability/resilience/breaker.cpp new file mode 100644 index 0000000..3173eb4 --- /dev/null +++ b/src/operability/resilience/breaker.cpp @@ -0,0 +1,114 @@ +#include "operability/resilience/breaker.h" + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::resilience { + +Breaker::Breaker(::molo::common::Clock& clock, ::molo::common::Observer& observer, std::string name) + : clock_(clock), observer_(observer), name_(std::move(name)) {} + +bool Breaker::allow() { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + if (state_ == State::kClosed) return true; + if (state_ == State::kOpen) { + if (now - opened_ms_ < ::molo::operability::Caps::kBreakerOpenHoldMs) return false; + state_ = State::kHalfOpen; + half_open_inflight_ = 0; + half_open_ok_ = 0; + observer_.event("breaker.half_open", name_); + } + if (half_open_inflight_ >= ::molo::operability::Caps::kBreakerHalfOpenProbes) return false; + ++half_open_inflight_; + return true; +} + +void Breaker::on_success() { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + consecutive_failures_ = 0; + if (state_ == State::kClosed) { + samples_.push_back(now); + prune_locked(now); + return; + } + if (state_ == State::kHalfOpen) { + if (half_open_inflight_ > 0) --half_open_inflight_; + ++half_open_ok_; + if (half_open_ok_ >= ::molo::operability::Caps::kBreakerHalfOpenProbes) { + state_ = State::kClosed; + failures_.clear(); + samples_.clear(); + observer_.event("breaker.closed", name_); + } + } +} + +void Breaker::on_failure() { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + ++consecutive_failures_; + observer_.counter("breaker_failures_total", 1); + if (state_ == State::kHalfOpen) { + if (half_open_inflight_ > 0) --half_open_inflight_; + trip_locked(now); + return; + } + if (state_ == State::kOpen) return; + failures_.push_back(now); + samples_.push_back(now); + prune_locked(now); + if (consecutive_failures_ >= + static_cast(::molo::operability::Caps::kBreakerConsecutiveFailures)) { + trip_locked(now); + return; + } + if (samples_.size() >= + static_cast(::molo::operability::Caps::kBreakerWindowMinSamples) && + failures_.size() * 100 >= + samples_.size() * + static_cast(::molo::operability::Caps::kBreakerWindowFailurePct)) { + trip_locked(now); + } +} + +Breaker::State Breaker::state() const { + std::lock_guard lock(mutex_); + return state_; +} + +const char* Breaker::state_name() const { + switch (state()) { + case State::kClosed: + return "closed"; + case State::kOpen: + return "open"; + case State::kHalfOpen: + return "half_open"; + } + return "unknown"; +} + +const std::string& Breaker::name() const { return name_; } + +void Breaker::trip_locked(std::int64_t now) { + state_ = State::kOpen; + opened_ms_ = now; + half_open_inflight_ = 0; + half_open_ok_ = 0; + observer_.event("breaker.open", name_); + observer_.counter("breaker_opened_total", 1); +} + +void Breaker::prune_locked(std::int64_t now) { + const std::int64_t cutoff = now - ::molo::operability::Caps::kBreakerWindowMs; + while (!failures_.empty() && failures_.front() < cutoff) failures_.pop_front(); + while (!samples_.empty() && samples_.front() < cutoff) samples_.pop_front(); + constexpr std::size_t kCap = 4096; + while (failures_.size() > kCap) failures_.pop_front(); + while (samples_.size() > kCap) samples_.pop_front(); +} + +} // namespace molo::resilience diff --git a/src/operability/resilience/breaker.h b/src/operability/resilience/breaker.h new file mode 100644 index 0000000..d75a58a --- /dev/null +++ b/src/operability/resilience/breaker.h @@ -0,0 +1,55 @@ +// Circuit breaker, innermost resilience stage: never retry into an open +// breaker. Opens after 20 consecutive failures or a 50 percent failure rate +// over a 10 second window (minimum 10 samples). Half open admits 3 probes; +// all succeeding closes the breaker, any failing reopens it. +#pragma once + +#include +#include +#include +#include +#include + +namespace molo::common { +class Clock; +class Observer; +} // namespace molo::common + +namespace molo::resilience { + +class Breaker { + public: + enum class State : int { kClosed = 0, kOpen = 1, kHalfOpen = 2 }; + + Breaker(::molo::common::Clock& clock, ::molo::common::Observer& observer, std::string name); + + Breaker(const Breaker&) = delete; + Breaker& operator=(const Breaker&) = delete; + + // Consult before sending. False means fail fast without calling downstream. + bool allow(); + void on_success(); + void on_failure(); + + State state() const; + const char* state_name() const; + const std::string& name() const; + + private: + void trip_locked(std::int64_t now); + void prune_locked(std::int64_t now); + + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + std::string name_; + mutable std::mutex mutex_; + State state_ = State::kClosed; + std::size_t consecutive_failures_ = 0; + std::deque failures_; + std::deque samples_; + std::int64_t opened_ms_ = 0; + int half_open_inflight_ = 0; + int half_open_ok_ = 0; +}; + +} // namespace molo::resilience diff --git a/src/operability/resilience/bulkhead.cpp b/src/operability/resilience/bulkhead.cpp new file mode 100644 index 0000000..de604fe --- /dev/null +++ b/src/operability/resilience/bulkhead.cpp @@ -0,0 +1,75 @@ +#include "operability/resilience/bulkhead.h" + +#include + +#include "operability/caps.h" +#include "supervisor/common/observer.hpp" + +namespace molo::resilience { + +Bulkhead::Bulkhead(::molo::common::Observer& observer, std::size_t partitions, + std::size_t max_concurrent, std::size_t max_queue) + : observer_(observer), + partitions_(partitions), + max_concurrent_(max_concurrent), + max_queue_(max_queue) { + if (partitions_ == 0) partitions_ = 1; + if (partitions_ > ::molo::operability::Caps::kBulkheadMaxPartitions) { + partitions_ = ::molo::operability::Caps::kBulkheadMaxPartitions; + } + if (max_concurrent_ == 0) max_concurrent_ = 1; + if (max_concurrent_ > ::molo::operability::Caps::kBulkheadMaxConcurrent) { + max_concurrent_ = ::molo::operability::Caps::kBulkheadMaxConcurrent; + } + if (max_queue_ > ::molo::operability::Caps::kBulkheadMaxQueue) { + max_queue_ = ::molo::operability::Caps::kBulkheadMaxQueue; + } +} + +Bulkhead::Admission Bulkhead::try_acquire(const std::string& partition) { + std::lock_guard lock(mutex_); + Slot& slot = slots_[slot_of(partition)]; + if (slot.inflight < max_concurrent_) { + ++slot.inflight; + observer_.counter("bulkhead_admitted_total", 1); + return Admission::kAdmitted; + } + if (slot.queued < max_queue_) { + ++slot.queued; + observer_.counter("bulkhead_queued_total", 1); + return Admission::kQueued; + } + observer_.counter("bulkhead_rejected_total", 1); + return Admission::kRejected; +} + +void Bulkhead::release(const std::string& partition) { + std::lock_guard lock(mutex_); + Slot& slot = slots_[slot_of(partition)]; + if (slot.queued > 0) { + // A queued request takes the freed slot; inflight count is unchanged. + --slot.queued; + observer_.counter("bulkhead_promoted_total", 1); + return; + } + if (slot.inflight > 0) --slot.inflight; +} + +std::size_t Bulkhead::inflight(const std::string& partition) const { + std::lock_guard lock(mutex_); + return slots_[slot_of(partition)].inflight; +} + +std::size_t Bulkhead::queued(const std::string& partition) const { + std::lock_guard lock(mutex_); + return slots_[slot_of(partition)].queued; +} + +std::size_t Bulkhead::partition_count() const { return partitions_; } + +std::size_t Bulkhead::slot_of(const std::string& partition) const { + static const std::hash kHash; + return kHash(partition) % partitions_; +} + +} // namespace molo::resilience diff --git a/src/operability/resilience/bulkhead.h b/src/operability/resilience/bulkhead.h new file mode 100644 index 0000000..8763d42 --- /dev/null +++ b/src/operability/resilience/bulkhead.h @@ -0,0 +1,49 @@ +// Partitioned bulkhead: at most 32 partitions, each with 64 concurrent +// slots and a 128 deep queue. Excess work is rejected immediately so the +// caller can shed with 503 plus Retry-After instead of queueing forever. +#pragma once + +#include +#include +#include +#include +#include + +namespace molo::common { +class Observer; +} // namespace molo::common + +namespace molo::resilience { + +class Bulkhead { + public: + enum class Admission : int { kAdmitted = 0, kQueued = 1, kRejected = 2 }; + + Bulkhead(::molo::common::Observer& observer, std::size_t partitions = 8, + std::size_t max_concurrent = 64, std::size_t max_queue = 128); + + Bulkhead(const Bulkhead&) = delete; + Bulkhead& operator=(const Bulkhead&) = delete; + + Admission try_acquire(const std::string& partition); + void release(const std::string& partition); + std::size_t inflight(const std::string& partition) const; + std::size_t queued(const std::string& partition) const; + std::size_t partition_count() const; + + private: + struct Slot { + std::size_t inflight = 0; + std::size_t queued = 0; + }; + std::size_t slot_of(const std::string& partition) const; + + ::molo::common::Observer& observer_; + std::size_t partitions_; + std::size_t max_concurrent_; + std::size_t max_queue_; + mutable std::mutex mutex_; + std::array slots_; +}; + +} // namespace molo::resilience diff --git a/src/operability/resilience/deadline.cpp b/src/operability/resilience/deadline.cpp new file mode 100644 index 0000000..2d6f9e5 --- /dev/null +++ b/src/operability/resilience/deadline.cpp @@ -0,0 +1,36 @@ +#include "operability/resilience/deadline.h" + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" + +namespace molo::resilience { + +Deadline::Deadline(::molo::common::Clock& clock) : clock_(clock) {} + +std::optional Deadline::parse_header(const std::string& value) { + if (value.empty() || value.size() > 20) return std::nullopt; + std::int64_t v = 0; + for (char c : value) { + if (c < '0' || c > '9') return std::nullopt; + const std::int64_t next = v * 10 + static_cast(c - '0'); + if (next < v) return std::nullopt; // overflow + v = next; + } + return v; +} + +std::string Deadline::format_header(std::int64_t absolute_ms) { + if (absolute_ms < 0) absolute_ms = 0; + return std::to_string(absolute_ms); +} + +std::int64_t Deadline::remaining_ms(std::int64_t absolute_ms) const { + const std::int64_t now = clock_.wall_ms(); + const std::int64_t remaining = absolute_ms - now; + const std::int64_t budget = remaining - ::molo::operability::Caps::kDeadlineFloorMarginMs; + return budget > 0 ? budget : 0; +} + +bool Deadline::expired(std::int64_t absolute_ms) const { return remaining_ms(absolute_ms) == 0; } + +} // namespace molo::resilience diff --git a/src/operability/resilience/deadline.h b/src/operability/resilience/deadline.h new file mode 100644 index 0000000..d61fe4e --- /dev/null +++ b/src/operability/resilience/deadline.h @@ -0,0 +1,41 @@ +// Deadline propagation for the X-Deadline-Ms header. Inbound absolute +// deadlines are converted to a remaining budget minus a 50 ms floor margin +// before forwarding, so downstream work never starts with a budget that is +// already expired. Production wires this into clients before send_inner. +#pragma once + +#include +#include +#include + +namespace molo::common { +class Clock; +} // namespace molo::common + +namespace molo::resilience { + +class Deadline { + public: + explicit Deadline(::molo::common::Clock& clock); + + Deadline(const Deadline&) = delete; + Deadline& operator=(const Deadline&) = delete; + + // Parse an absolute epoch millis header value. Empty on malformed input. + static std::optional parse_header(const std::string& value); + + // Format an absolute epoch millis header value. + static std::string format_header(std::int64_t absolute_ms); + + // Remaining budget for outbound use: absolute minus now minus the 50 ms + // floor margin. Returns 0 when already expired or within the margin. + std::int64_t remaining_ms(std::int64_t absolute_ms) const; + + // True when the absolute deadline leaves no usable budget. + bool expired(std::int64_t absolute_ms) const; + + private: + ::molo::common::Clock& clock_; +}; + +} // namespace molo::resilience diff --git a/src/operability/resilience/shedder.cpp b/src/operability/resilience/shedder.cpp new file mode 100644 index 0000000..f0ab4a6 --- /dev/null +++ b/src/operability/resilience/shedder.cpp @@ -0,0 +1,44 @@ +#include "operability/resilience/shedder.h" + +#include "operability/caps.h" +#include "supervisor/common/observer.hpp" + +namespace molo::resilience { + +Shedder::Shedder(::molo::common::Observer& observer, OverloadProbe probe, + std::int64_t retry_after_s) + : observer_(observer), probe_(std::move(probe)), retry_after_s_(retry_after_s) { + if (retry_after_s_ < 0) retry_after_s_ = 0; + if (retry_after_s_ > ::molo::operability::Caps::kShedRetryAfterMaxS) { + retry_after_s_ = ::molo::operability::Caps::kShedRetryAfterMaxS; + } +} + +bool Shedder::should_shed() const { + std::lock_guard lock(mutex_); + if (!probe_) return false; + bool shed = false; + try { + shed = probe_(); + } catch (...) { + shed = false; + } + if (shed) observer_.counter("shed_total", 1); + return shed; +} + +Shedder::Rejection Shedder::reject() const { + std::lock_guard lock(mutex_); + Rejection r; + r.status = 503; + r.retry_after = std::to_string(retry_after_s_); + observer_.counter("shed_rejected_total", 1); + return r; +} + +std::int64_t Shedder::retry_after_s() const { + std::lock_guard lock(mutex_); + return retry_after_s_; +} + +} // namespace molo::resilience diff --git a/src/operability/resilience/shedder.h b/src/operability/resilience/shedder.h new file mode 100644 index 0000000..aca5c31 --- /dev/null +++ b/src/operability/resilience/shedder.h @@ -0,0 +1,44 @@ +// Load shedder: when the overload probe reports pressure, requests are +// rejected with 503 plus a Retry-After hint instead of queueing. The probe +// is injected so production can use queue depth or latency while tests use +// a flag. Sits outside the bulkhead: shed first, then bulkhead, then send. +#pragma once + +#include +#include +#include +#include + +namespace molo::common { +class Observer; +} // namespace molo::common + +namespace molo::resilience { + +class Shedder { + public: + using OverloadProbe = std::function; + + struct Rejection { + int status = 503; + std::string retry_after; // seconds, ASCII digits + }; + + Shedder(::molo::common::Observer& observer, OverloadProbe probe, + std::int64_t retry_after_s = 5); + + Shedder(const Shedder&) = delete; + Shedder& operator=(const Shedder&) = delete; + + bool should_shed() const; + Rejection reject() const; + std::int64_t retry_after_s() const; + + private: + ::molo::common::Observer& observer_; + OverloadProbe probe_; + std::int64_t retry_after_s_; + mutable std::mutex mutex_; +}; + +} // namespace molo::resilience diff --git a/src/operability/resilience/slo.cpp b/src/operability/resilience/slo.cpp new file mode 100644 index 0000000..473a2fe --- /dev/null +++ b/src/operability/resilience/slo.cpp @@ -0,0 +1,182 @@ +#include "operability/resilience/slo.h" + +#include + +#include "operability/caps.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::resilience { + +namespace { + +std::string trunc(const std::string& s) { + if (s.size() > 128) return s.substr(0, 128); + return s; +} + +std::string escape_label(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + if (c == '\\' || c == '"') out.push_back('\\'); + out.push_back(c == '\n' ? ' ' : c); + } + return out; +} + +} // namespace + +Slo::Slo(::molo::common::Clock& clock, ::molo::common::Observer& observer) + : Slo(clock, observer, Config{}) {} + +Slo::Slo(::molo::common::Clock& clock, ::molo::common::Observer& observer, Config cfg) + : clock_(clock), observer_(observer), cfg_(cfg) { + if (cfg_.target_ratio < 0.0) cfg_.target_ratio = 0.0; + if (cfg_.target_ratio > 1.0) cfg_.target_ratio = 1.0; + if (cfg_.window_ms <= 0) cfg_.window_ms = 3600000; +} + +void Slo::record(const std::string& route, bool ok, const std::string& reason) { + const std::string r = trunc(route); + std::lock_guard lock(mutex_); + auto it = routes_.find(r); + if (it == routes_.end()) { + if (routes_.size() >= ::molo::operability::Caps::kSloMaxRoutes) return; + it = routes_.emplace(r, RouteData{}).first; + } + RouteData& data = it->second; + Sample s; + s.at_ms = clock_.steady_ms(); + s.ok = ok; + data.samples.push_back(s); + while (data.samples.size() > ::molo::operability::Caps::kSloMaxSamplesPerRoute) { + data.samples.pop_front(); + } + if (!ok) { + bad_[{r, trunc(reason)}] += 1; + observer_.counter("slo_bad_events_total", 1); + } else { + observer_.counter("slo_good_events_total", 1); + } +} + +double Slo::budget_remaining(const std::string& route) { + const std::string r = trunc(route); + std::lock_guard lock(mutex_); + auto it = routes_.find(r); + if (it == routes_.end()) return 1.0; + prune_locked(it->second, clock_.steady_ms() - cfg_.window_ms); + const auto& samples = it->second.samples; + if (samples.empty()) return 1.0; + std::size_t errors = 0; + for (const auto& s : samples) { + if (!s.ok) ++errors; + } + const double budgeted = 1.0 - cfg_.target_ratio; + const auto allowed = + static_cast(static_cast(samples.size()) * budgeted + 0.999); + if (allowed == 0) return errors == 0 ? 1.0 : 0.0; + if (errors >= allowed) return 0.0; + return 1.0 - static_cast(errors) / static_cast(allowed); +} + +double Slo::burn_rate(const std::string& route, std::int64_t window_ms) { + const std::string r = trunc(route); + if (window_ms <= 0) return 0.0; + std::lock_guard lock(mutex_); + auto it = routes_.find(r); + if (it == routes_.end()) return 0.0; + const std::int64_t cutoff = clock_.steady_ms() - window_ms; + std::size_t total = 0; + std::size_t errors = 0; + for (const auto& s : it->second.samples) { + if (s.at_ms < cutoff) continue; + ++total; + if (!s.ok) ++errors; + } + if (total == 0) return 0.0; + const double budgeted = 1.0 - cfg_.target_ratio; + if (budgeted <= 0.0) return errors == 0 ? 0.0 : 1000000.0; + return (static_cast(errors) / static_cast(total)) / budgeted; +} + +std::int64_t Slo::bad_events(const std::string& route, const std::string& reason) { + std::lock_guard lock(mutex_); + auto it = bad_.find({trunc(route), trunc(reason)}); + return it == bad_.end() ? 0 : it->second; +} + +std::string Slo::render_prometheus() { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + std::string out; + out += "# HELP molo_slo_budget_remaining_ratio Fraction of error budget left.\n"; + out += "# TYPE molo_slo_budget_remaining_ratio gauge\n"; + out += "# HELP molo_slo_burn_rate Error ratio over budgeted ratio.\n"; + out += "# TYPE molo_slo_burn_rate gauge\n"; + out += "# HELP molo_slo_bad_events_total Bad events by route and reason.\n"; + out += "# TYPE molo_slo_bad_events_total counter\n"; + constexpr std::int64_t kWindows[2] = {300000, 3600000}; + const char* kNames[2] = {"5m", "1h"}; + char buf[64]; + for (auto& kv : routes_) { + prune_locked(kv.second, now - cfg_.window_ms); + const auto& samples = kv.second.samples; + std::size_t errors = 0; + for (const auto& s : samples) { + if (!s.ok) ++errors; + } + double remaining = 1.0; + const double budgeted = 1.0 - cfg_.target_ratio; + const auto allowed = + static_cast(static_cast(samples.size()) * budgeted + 0.999); + if (!samples.empty()) { + if (allowed == 0) { + remaining = errors == 0 ? 1.0 : 0.0; + } else if (errors >= allowed) { + remaining = 0.0; + } else { + remaining = 1.0 - static_cast(errors) / static_cast(allowed); + } + } + std::snprintf(buf, sizeof(buf), "%.6f", remaining); + out += "molo_slo_budget_remaining_ratio{route=\"" + escape_label(kv.first) + "\"} "; + out += buf; + out += "\n"; + for (int w = 0; w < 2; ++w) { + std::size_t total = 0; + std::size_t werr = 0; + const std::int64_t cutoff = now - kWindows[w]; + for (const auto& s : samples) { + if (s.at_ms < cutoff) continue; + ++total; + if (!s.ok) ++werr; + } + double burn = 0.0; + if (total > 0) { + if (budgeted <= 0.0) { + burn = werr == 0 ? 0.0 : 1000000.0; + } else { + burn = (static_cast(werr) / static_cast(total)) / budgeted; + } + } + std::snprintf(buf, sizeof(buf), "%.6f", burn); + out += "molo_slo_burn_rate{route=\"" + escape_label(kv.first) + "\",window=\"" + kNames[w] + + "\"} "; + out += buf; + out += "\n"; + } + } + for (const auto& kv : bad_) { + out += "molo_slo_bad_events_total{route=\"" + escape_label(kv.first.first) + "\",reason=\"" + + escape_label(kv.first.second) + "\"} " + std::to_string(kv.second) + "\n"; + } + return out; +} + +void Slo::prune_locked(RouteData& data, std::int64_t cutoff) { + while (!data.samples.empty() && data.samples.front().at_ms < cutoff) data.samples.pop_front(); +} + +} // namespace molo::resilience diff --git a/src/operability/resilience/slo.h b/src/operability/resilience/slo.h new file mode 100644 index 0000000..b63f23d --- /dev/null +++ b/src/operability/resilience/slo.h @@ -0,0 +1,63 @@ +// Error budget tracker. Records good and bad events per route template and +// exposes the SLO series: molo_slo_budget_remaining_ratio, +// molo_slo_burn_rate{window} and molo_slo_bad_events_total{route,reason}. +// Production scrapes render_prometheus next to the registry exporter. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace molo::common { +class Clock; +class Observer; +} // namespace molo::common + +namespace molo::resilience { + +class Slo { + public: + struct Config { + double target_ratio = 0.999; + std::int64_t window_ms = 3600000; + }; + + Slo(::molo::common::Clock& clock, ::molo::common::Observer& observer); + Slo(::molo::common::Clock& clock, ::molo::common::Observer& observer, Config cfg); + + Slo(const Slo&) = delete; + Slo& operator=(const Slo&) = delete; + + void record(const std::string& route, bool ok, const std::string& reason = ""); + + // Fraction of the error budget left in the window, 0 to 1. + double budget_remaining(const std::string& route); + // Observed error ratio divided by budgeted ratio over window_ms. + double burn_rate(const std::string& route, std::int64_t window_ms); + std::int64_t bad_events(const std::string& route, const std::string& reason); + + std::string render_prometheus(); + + private: + struct Sample { + std::int64_t at_ms = 0; + bool ok = true; + }; + struct RouteData { + std::deque samples; + }; + + void prune_locked(RouteData& data, std::int64_t cutoff); + + ::molo::common::Clock& clock_; + ::molo::common::Observer& observer_; + Config cfg_; + mutable std::mutex mutex_; + std::map routes_; + std::map, std::int64_t> bad_; +}; + +} // namespace molo::resilience diff --git a/src/realtime/SPEC.md b/src/realtime/SPEC.md new file mode 100644 index 0000000..cd4ff68 --- /dev/null +++ b/src/realtime/SPEC.md @@ -0,0 +1,135 @@ +# Realtime component specification (06-realtime.md) + +Wire-visible behavior of `src/realtime/`. This spec is written before the +code it describes; gaps are listed next to the code that will close them. + +## 1. Integration seams + +Parts 01-runtime, 02-observability, 03-security and 05-request-routing do +not exist in this isolated worktree yet. All cross-part contact goes +through small seam interfaces declared under `shared/`: + +- `shared/http_seam.h`: minimal `HttpRequest` / `HttpResponse` value + types. They mirror the `molo::http_msg` pair (method, path, query, + headers, body) with zero behavior. Real integration swaps these for + the 05 types without changing protocol logic. +- `shared/transport.h`: `ITransport` byte sink plus `MemoryTransport` + (test double) and `FdTransport` (writes to an OS file descriptor). + The 01-runtime connection will implement `ITransport`. +- `shared/connection_context.h`: auth identity, session id, + correlation id, route name. Populated by middleware before upgrade + (rule 1). Standalone use fills it by hand. +- `shared/realtime_log.h`: fixed log levels and line format. Wires to + 02-observability later; today it writes to `stderr`. +- `shared/realtime_metrics.h`: counters with static names. Wires to + 02-observability later; today it keeps process-local atomics. +- `shared/caps.h`: every static ceiling in one place. + +No file outside `shared/` includes anything outside `src/realtime/` +except the C++ standard library and OS headers. + +## 2. WebSocket (RFC 6455) + +- Handshake: `GET` with `Upgrade: websocket` (case-insensitive value, + token match), `Connection` containing token `Upgrade`, and a + `Sec-WebSocket-Key` that base64-decodes to exactly 16 bytes. + Optional `Sec-WebSocket-Protocol`: server selects at most one from + its allow-list, echoes it, else fails with 400. Success answers + `101` with `Upgrade: websocket`, `Connection: Upgrade`, + `Sec-WebSocket-Accept: base64(SHA-1(key + GUID))`. +- Framing: opcodes 0x0, 0x1, 0x2, 0x8, 0x9, 0xA. Control frames have + FIN set and payload at most 125 bytes. Server rejects unmasked + client frames (1002) and never masks its own frames. Text must be + valid UTF-8 (1007 on violation). Fragmented messages reassemble up + to `kMaxMessageBytes`; close body is a 2-byte code plus UTF-8 + reason, max 125 bytes total. +- Liveness: every ping is answered with a pong carrying the identical + payload. `HeartbeatManager` sends pings every `kPingInterval` and + fails peers silent for `kPongTimeout` with code 1001. +- Close codes emitted: 1000, 1001, 1002, 1003, 1007, 1008, 1009, 1011. + +## 3. SSE (WHATWG HTML 9.2) + +- Response headers: `Content-Type: text/event-stream`, + `Cache-Control: no-store`, `Connection: keep-alive`. Headers flush + immediately on stream start. +- Encoding: lines `field: value` with fields `event`, `data`, `id`, + `retry`, `:` comments; blank line dispatches. Multiline data emits + one `data:` line per line. `retry` is clamped to 1 s..300 s. +- Replay: `EventStore` keeps the last 1000 events or 5 minutes per + stream, whichever is smaller. `Last-Event-ID` replays missed events + in order. HTTP 204 tells the client to stop reconnecting. +- Client states: CONNECTING 0, OPEN 1, CLOSED 2. Fails on non-200 or + wrong content type, honors 301/307 redirect and 204 stop, sends the + stored last event id on reconnect. Comment keep-alive every 15 s. + +## 4. Engine.IO v4 + +- Path `/engine.io/`, query `EIO=4`, `transport=polling|websocket`, + `sid` after handshake. Polling GET long-holds up to 25 s. POST + bodies use length-prefixed packet payloads. +- Packet types: open 0, close 1, ping 2, pong 3, message 4, + upgrade 5, noop 6. Open JSON keys exactly `sid, upgrades, + pingInterval, pingTimeout, maxPayload`. Upgrade probe + `ping/probe` then `pong/probe`, then `upgrade`. +- At most one active polling GET and one WebSocket per session; + unknown sid answers 400. + +## 5. Socket.IO v5 + +- Packet types: CONNECT 0, DISCONNECT 1, EVENT 2, ACK 3, + CONNECT_ERROR 4, BINARY_EVENT 5, BINARY_ACK 6. Text encoding + `[-][,][]`, namespace omitted + when `/`. Each packet wraps as one Engine.IO message (prefix `4`). +- Namespaces start with `/`, no commas. Rooms are server-side sets; + broadcast excludes the sender by default with an explicit include + flag. EVENT payload is a JSON array whose first element is the + event name. Ack ids are per-connection monotonic integers with a + 30 s default timeout. Binary attachments use + `{"_placeholder":true,"num":N}` markers, max 16 attachments and + 8 MiB total per packet. +- Fan-out is O(subscribers) with per-recipient queue caps; broadcast + to 0 subscribers is a counted no-op, not an error. + +## 6. Streaming + +- Chunked bodies flush at least every 64 KiB or 100 ms during + activity. Passthrough uses a 256 KiB ring buffer with producer + pause. File passthrough supports single ranges and conditional + requests (If-None-Match precedence, 304/206/416). +- `BackpressureValve` reports pause/resume from high/low watermarks. + Compression (owned by 01-runtime) must never buffer a full stream. + +## 7. WebDAV (RFC 4918) + +- Methods: GET, PUT, DELETE, PROPFIND (allprop, propname, prop), + PROPPATCH (set/remove with 207 per-property status), MKCOL, + COPY/MOVE (`Overwrite`, `Destination`, `Depth`), LOCK (exclusive + and shared, `Timeout`, `Depth: 0`), UNLOCK (matching `Lock-Token`). +- `DAV: 1` header on OPTIONS and responses. XML bodies use the + `DAV:` namespace. `Depth: infinity` is rewritten to max depth 32. + Max XML body 1 MiB. Locks have finite timeouts, auto-expire, and + use unguessable random tokens. Quota exceeded answers 507. +- Client streams GET into a caller body (no full buffering), + conditional GET with ETag cache (max 128 entries, 64 MiB), quota + accounting hooks per transfer. + +## 8. Caps (defaults, see `shared/caps.h`) + +Max message 1 MiB, max frame payload 256 KiB, max frames per message +64, send queue 256 messages, connections per IP 128, per route 100k, +idle timeout 60 s, ping interval 25 s, pong timeout 20 s, close +handshake timeout 5 s. + +## 9. Gaps + +- G1: coroutine (`co_await`) handler loops need `molo::runtime::Task` + from 01-runtime. Handlers are written as explicit state machines + with the same transitions; adapting them is mechanical. +- G2: real TCP/TLS I/O comes from 01-runtime connections and the + 03-security TLS handle. `FdTransport` covers loopback tests only. +- G3: cross-instance broadcast waits on the 04-data pubsub bridge; + v1 delivery is at-most-once across instances. +- G4: persistent SSE replay waits on a 04-data `EventStore`; v1 is + in-memory with the documented interface. +- G5: WebDAV ACL (RFC 3744) and DeltaV are out of v1. diff --git a/src/realtime/engineio/engine_session.cpp b/src/realtime/engineio/engine_session.cpp new file mode 100644 index 0000000..3fb1302 --- /dev/null +++ b/src/realtime/engineio/engine_session.cpp @@ -0,0 +1,59 @@ +// Engine.IO session. See engine_session.h. + +#include "realtime/engineio/engine_session.h" + +#include "realtime/shared/mini_json.h" +#include "realtime/shared/random_token.h" + +namespace molo::realtime::engineio { + +EngineSession::EngineSession(shared::ConnectionContext context, + EngineConfig config) + : sid_(shared::random_id(20)), + config_(config), + context_(std::move(context)) {} + +EnginePacket EngineSession::open_packet() const { + shared::Json upgrades = shared::Json::make_array(); + upgrades.push(shared::Json::str("websocket")); + shared::Json body = shared::Json::make_object(); + body.set("sid", shared::Json::str(sid_)); + body.set("upgrades", std::move(upgrades)); + body.set("pingInterval", + shared::Json::integer(config_.ping_interval.count())); + body.set("pingTimeout", + shared::Json::integer(config_.ping_timeout.count())); + body.set("maxPayload", + shared::Json::integer(static_cast(config_.max_payload))); + EnginePacket p; + p.type = EnginePacketType::kOpen; + p.data = body.dump(); + return p; +} + +bool EngineSession::enqueue(EnginePacket packet) { + if (closed_ || queue_.size() >= config_.queue_depth) { + return false; + } + queue_.push_back(std::move(packet)); + return true; +} + +bool EngineSession::dequeue(EnginePacket& out) { + if (queue_.empty()) { + return false; + } + out = std::move(queue_.front()); + queue_.pop_front(); + return true; +} + +void EngineSession::upgrade_to_websocket() { + transport_ = EngineTransport::kWebSocket; +} + +void EngineSession::note_activity(std::chrono::steady_clock::time_point now) { + last_activity_ = now; +} + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/engine_session.h b/src/realtime/engineio/engine_session.h new file mode 100644 index 0000000..3b77b65 --- /dev/null +++ b/src/realtime/engineio/engine_session.h @@ -0,0 +1,79 @@ +#pragma once + +// One Engine.IO session: id, transport state, outbound packet queue, +// and heartbeat bookkeeping. At most one active polling GET and one +// WebSocket per session; the transports enforce that, the session +// records it. Message payloads flow up to Socket.IO. + +#include +#include +#include +#include + +#include "realtime/engineio/packet_codec.h" +#include "realtime/shared/caps.h" +#include "realtime/shared/connection_context.h" + +namespace molo::realtime::engineio { + +enum class EngineTransport : std::uint8_t { + kPolling, + kWebSocket, +}; + +struct EngineConfig { + std::chrono::milliseconds ping_interval = shared::Caps::kPingInterval; + std::chrono::milliseconds ping_timeout = shared::Caps::kPongTimeout; + std::size_t max_payload = 1024u * 1024u; + std::size_t queue_depth = 256; +}; + +class EngineSession { + public: + explicit EngineSession(shared::ConnectionContext context, + EngineConfig config = EngineConfig{}); + + const std::string& sid() const { return sid_; } + EngineTransport transport() const { return transport_; } + const EngineConfig& config() const { return config_; } + const shared::ConnectionContext& context() const { return context_; } + + // Open packet JSON: keys exactly sid, upgrades, pingInterval, + // pingTimeout, maxPayload. + EnginePacket open_packet() const; + + // Queues an outbound packet. Returns false when the queue is full + // (caller counts the shed). + bool enqueue(EnginePacket packet); + bool dequeue(EnginePacket& out); + bool has_pending() const { return !queue_.empty(); } + std::size_t queue_size() const { return queue_.size(); } + + // Polling GET bookkeeping: only one may be outstanding. + bool polling_in_flight() const { return polling_in_flight_; } + void set_polling_in_flight(bool v) { polling_in_flight_ = v; } + + void upgrade_to_websocket(); + bool upgraded() const { return transport_ == EngineTransport::kWebSocket; } + + void note_activity(std::chrono::steady_clock::time_point now); + std::chrono::steady_clock::time_point last_activity() const { + return last_activity_; + } + + void close() { closed_ = true; } + bool closed() const { return closed_; } + + private: + std::string sid_; + EngineTransport transport_ = EngineTransport::kPolling; + EngineConfig config_; + shared::ConnectionContext context_; + std::deque queue_; + bool polling_in_flight_ = false; + bool closed_ = false; + std::chrono::steady_clock::time_point last_activity_ = + std::chrono::steady_clock::now(); +}; + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/heartbeat_scheduler.cpp b/src/realtime/engineio/heartbeat_scheduler.cpp new file mode 100644 index 0000000..42a2593 --- /dev/null +++ b/src/realtime/engineio/heartbeat_scheduler.cpp @@ -0,0 +1,43 @@ +// Engine.IO heartbeat scheduler. See heartbeat_scheduler.h. + +#include "realtime/engineio/heartbeat_scheduler.h" + +#include "realtime/shared/realtime_metrics.h" + +namespace molo::realtime::engineio { + +HeartbeatScheduler::HeartbeatScheduler(EngineSession* session) + : session_(session) {} + +void HeartbeatScheduler::start(std::chrono::steady_clock::time_point now) { + started_ = true; + next_ping_ = now + session_->config().ping_interval; +} + +void HeartbeatScheduler::tick(std::chrono::steady_clock::time_point now) { + if (!started_ || timed_out_ || session_->closed()) { + return; + } + if (now - session_->last_activity() >= session_->config().ping_timeout) { + timed_out_ = true; + session_->close(); + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kHeartbeatTimeouts); + if (timeout_fn_) { + timeout_fn_(); + } + return; + } + if (now >= next_ping_) { + next_ping_ = now + session_->config().ping_interval; + EnginePacket ping; + ping.type = EnginePacketType::kPing; + if (send_fn_) { + send_fn_(ping); + } else { + session_->enqueue(ping); + } + } +} + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/heartbeat_scheduler.h b/src/realtime/engineio/heartbeat_scheduler.h new file mode 100644 index 0000000..a16a3fe --- /dev/null +++ b/src/realtime/engineio/heartbeat_scheduler.h @@ -0,0 +1,37 @@ +#pragma once + +// Engine.IO heartbeat scheduler. The server sends ping packets on +// pingInterval; the client must answer (any packet counts as +// activity) within pingTimeout or the session closes. Driven by the +// runtime timer wheel via tick(). + +#include +#include + +#include "realtime/engineio/engine_session.h" +#include "realtime/engineio/packet_codec.h" + +namespace molo::realtime::engineio { + +class HeartbeatScheduler { + public: + using SendFn = std::function; + using TimeoutFn = std::function; + + explicit HeartbeatScheduler(EngineSession* session); + + void on_send(SendFn fn) { send_fn_ = std::move(fn); } + void on_timeout(TimeoutFn fn) { timeout_fn_ = std::move(fn); } + void start(std::chrono::steady_clock::time_point now); + void tick(std::chrono::steady_clock::time_point now); + + private: + EngineSession* session_; + SendFn send_fn_; + TimeoutFn timeout_fn_; + std::chrono::steady_clock::time_point next_ping_; + bool started_ = false; + bool timed_out_ = false; +}; + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/packet_codec.cpp b/src/realtime/engineio/packet_codec.cpp new file mode 100644 index 0000000..be02ba7 --- /dev/null +++ b/src/realtime/engineio/packet_codec.cpp @@ -0,0 +1,74 @@ +// Engine.IO v4 packet codec. See packet_codec.h. + +#include "realtime/engineio/packet_codec.h" + +namespace molo::realtime::engineio { +namespace { +constexpr std::size_t kMaxPollPackets = 64; +constexpr std::size_t kMaxPollBytes = 1024u * 1024u; +} // namespace + +std::string PacketCodec::encode(const EnginePacket& packet) { + std::string out; + out.push_back( + static_cast('0' + static_cast(packet.type))); + out += packet.data; + return out; +} + +std::optional PacketCodec::decode(const std::string& wire) { + if (wire.empty() || wire[0] < '0' || wire[0] > '6') { + return std::nullopt; + } + EnginePacket p; + p.type = static_cast(wire[0] - '0'); + p.data = wire.substr(1); + return p; +} + +std::string PacketCodec::encode_polling( + const std::vector& packets) { + std::string out; + for (const EnginePacket& p : packets) { + std::string wire = encode(p); + out += std::to_string(wire.size()) + ":" + wire; + } + return out; +} + +std::optional> PacketCodec::decode_polling( + const std::string& payload) { + if (payload.size() > kMaxPollBytes) { + return std::nullopt; + } + std::vector out; + std::size_t i = 0; + while (i < payload.size()) { + std::size_t colon = payload.find(':', i); + if (colon == std::string::npos) { + return std::nullopt; + } + std::size_t len = 0; + try { + len = std::stoul(payload.substr(i, colon - i)); + } catch (...) { + return std::nullopt; + } + i = colon + 1; + if (i + len > payload.size()) { + return std::nullopt; + } + auto packet = decode(payload.substr(i, len)); + if (!packet.has_value()) { + return std::nullopt; + } + out.push_back(std::move(*packet)); + if (out.size() > kMaxPollPackets) { + return std::nullopt; + } + i += len; + } + return out; +} + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/packet_codec.h b/src/realtime/engineio/packet_codec.h new file mode 100644 index 0000000..1a9092d --- /dev/null +++ b/src/realtime/engineio/packet_codec.h @@ -0,0 +1,48 @@ +#pragma once + +// Engine.IO v4 packet codec. Types: open 0, close 1, ping 2, pong 3, +// message 4, upgrade 5, noop 6. Wire form is one digit plus payload +// ("4hello"). Polling payloads frame packets as length-prefixed +// segments ("5:4hello"). Binary is out of v1 scope on the Engine.IO +// layer (Socket.IO binary rides as base64 text or WS binary). + +#include +#include +#include +#include + +namespace molo::realtime::engineio { + +enum class EnginePacketType : std::uint8_t { + kOpen = 0, + kClose = 1, + kPing = 2, + kPong = 3, + kMessage = 4, + kUpgrade = 5, + kNoop = 6, +}; + +struct EnginePacket { + EnginePacketType type = EnginePacketType::kNoop; + std::string data; +}; + +class PacketCodec { + public: + // Encodes one packet to wire form (""). + static std::string encode(const EnginePacket& packet); + + // Decodes one wire packet. Returns nullopt on bad type digit. + static std::optional decode(const std::string& wire); + + // Encodes packets as one polling payload ("len:packet..."). + static std::string encode_polling(const std::vector& packets); + + // Decodes a polling payload. Returns nullopt on framing errors. + // At most 64 packets and 1 MiB total per payload. + static std::optional> decode_polling( + const std::string& payload); +}; + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/polling_transport.cpp b/src/realtime/engineio/polling_transport.cpp new file mode 100644 index 0000000..6c5e747 --- /dev/null +++ b/src/realtime/engineio/polling_transport.cpp @@ -0,0 +1,71 @@ +// Long-polling transport. See polling_transport.h. + +#include "realtime/engineio/polling_transport.h" + +namespace molo::realtime::engineio { + +shared::HttpResponse PollingTransport::answer_get(EngineSession& session, + bool& hold) { + hold = false; + if (session.polling_in_flight()) { + shared::HttpResponse res; + res.status = 400; + res.body = "duplicate polling GET"; + return res; + } + if (!session.has_pending()) { + session.set_polling_in_flight(true); + hold = true; + return shared::HttpResponse{}; // Placeholder; held open. + } + return flush_held(session); +} + +shared::HttpResponse PollingTransport::flush_held(EngineSession& session) { + session.set_polling_in_flight(false); + std::vector packets; + EnginePacket p; + while (session.dequeue(p)) { + packets.push_back(std::move(p)); + } + if (packets.empty()) { + EnginePacket noop; + noop.type = EnginePacketType::kNoop; + packets.push_back(noop); + } + shared::HttpResponse res; + res.status = 200; + shared::set_header(res, "Content-Type", "text/plain; charset=UTF-8"); + res.body = PacketCodec::encode_polling(packets); + return res; +} + +PollingTransport::PostResult PollingTransport::accept_post( + EngineSession& session, const shared::HttpRequest& req) { + PostResult r; + if (req.body.size() > session.config().max_payload) { + r.response.status = 400; + r.response.body = "payload over maxPayload"; + return r; + } + auto packets = PacketCodec::decode_polling(req.body); + if (!packets.has_value()) { + r.response.status = 400; + r.response.body = "bad polling payload"; + return r; + } + r.ok = true; + r.packets = std::move(*packets); + r.response.status = 200; + r.response.body = "ok"; + return r; +} + +shared::HttpResponse PollingTransport::unknown_sid() { + shared::HttpResponse res; + res.status = 400; + res.body = "unknown sid"; + return res; +} + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/polling_transport.h b/src/realtime/engineio/polling_transport.h new file mode 100644 index 0000000..ba53645 --- /dev/null +++ b/src/realtime/engineio/polling_transport.h @@ -0,0 +1,43 @@ +#pragma once + +// Long-polling transport: answers GET with pending packets as one +// polling payload (long-holds up to 25 s when empty) and decodes +// POST bodies into inbound packets. One outstanding GET per session +// (400 on duplicates); unknown sid answers 400. The hold itself is +// driven by the runtime; this class only renders the answers. + +#include +#include + +#include "realtime/engineio/engine_session.h" +#include "realtime/engineio/packet_codec.h" +#include "realtime/shared/caps.h" +#include "realtime/shared/http_seam.h" + +namespace molo::realtime::engineio { + +class PollingTransport { + public: + // Renders a GET answer: 200 with polling payload, draining the + // session queue. Sets hold=true when the queue was empty (caller + // long-holds). 400 when a GET is already in flight. + shared::HttpResponse answer_get(EngineSession& session, bool& hold); + + // Called when the hold expires or packets arrive: renders the + // pending payload and clears the in-flight flag. + shared::HttpResponse flush_held(EngineSession& session); + + // Decodes a POST body into inbound packets. 400 status on framing + // errors or oversized bodies. + struct PostResult { + bool ok = false; + std::vector packets; + shared::HttpResponse response; + }; + PostResult accept_post(EngineSession& session, + const shared::HttpRequest& req); + + static shared::HttpResponse unknown_sid(); +}; + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/upgrade_manager.cpp b/src/realtime/engineio/upgrade_manager.cpp new file mode 100644 index 0000000..ac1a47e --- /dev/null +++ b/src/realtime/engineio/upgrade_manager.cpp @@ -0,0 +1,26 @@ +// Polling to WebSocket upgrade. See upgrade_manager.h. + +#include "realtime/engineio/upgrade_manager.h" + +namespace molo::realtime::engineio { + +std::optional UpgradeManager::on_packet( + EngineSession& session, const EnginePacket& packet) { + if (packet.type == EnginePacketType::kPing && packet.data == "probe") { + probe_seen_ = true; + EnginePacket pong; + pong.type = EnginePacketType::kPong; + pong.data = "probe"; + return PacketCodec::encode(pong); + } + if (packet.type == EnginePacketType::kUpgrade) { + if (probe_seen_) { + upgraded_ = true; + session.upgrade_to_websocket(); + } + return std::string{""}; // Consumed, no answer. + } + return std::nullopt; // Flows to Socket.IO. +} + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/upgrade_manager.h b/src/realtime/engineio/upgrade_manager.h new file mode 100644 index 0000000..f4b09cd --- /dev/null +++ b/src/realtime/engineio/upgrade_manager.h @@ -0,0 +1,43 @@ +#pragma once + +// Upgrade state machine: polling to WebSocket. The client sends +// "2probe" on the new WebSocket; the server answers "3probe"; the +// client sends "5" (upgrade); the server switches the session +// transport and drains queued packets over the socket. Probe packets +// never reach Socket.IO. + +#include +#include + +#include "realtime/engineio/engine_session.h" +#include "realtime/engineio/packet_codec.h" + +namespace molo::realtime::engineio { + +class UpgradeManager { + public: + enum class State : std::uint8_t { + kIdle, + kProbeSeen, + kUpgraded, + }; + State state() const { + return upgraded_ ? State::kUpgraded + : (probe_seen_ ? State::kProbeSeen : State::kIdle); + } + + // Handles one inbound packet on the upgrading socket. Returns the + // wire answer when the packet is part of the probe ("3probe"), + // nullopt when the packet must flow to Socket.IO instead. + // upgraded flips true on "5". + std::optional on_packet(EngineSession& session, + const EnginePacket& packet); + + bool upgraded() const { return upgraded_; } + + private: + bool probe_seen_ = false; + bool upgraded_ = false; +}; + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/websocket_transport.cpp b/src/realtime/engineio/websocket_transport.cpp new file mode 100644 index 0000000..0811439 --- /dev/null +++ b/src/realtime/engineio/websocket_transport.cpp @@ -0,0 +1,42 @@ +// Engine.IO WebSocket transport. See websocket_transport.h. + +#include "realtime/engineio/websocket_transport.h" + +#include "realtime/engineio/polling_transport.h" + +namespace molo::realtime::engineio { + +WebSocketTransport::Admit WebSocketTransport::admit( + const shared::HttpRequest& req, bool sid_known) { + Admit a; + auto query = shared::parse_query(req.query); + auto eio = query.find("EIO"); + auto transport = query.find("transport"); + if (eio == query.end() || eio->second != "4") { + a.failure.status = 400; + a.failure.body = "EIO=4 required"; + return a; + } + if (transport == query.end() || transport->second != "websocket") { + a.failure.status = 400; + a.failure.body = "transport=websocket required"; + return a; + } + if (!sid_known) { + a.failure = PollingTransport::unknown_sid(); + return a; + } + a.ok = true; + return a; +} + +std::string WebSocketTransport::to_message(const EnginePacket& packet) { + return PacketCodec::encode(packet); +} + +std::optional WebSocketTransport::from_message( + const std::string& text) { + return PacketCodec::decode(text); +} + +} // namespace molo::realtime::engineio diff --git a/src/realtime/engineio/websocket_transport.h b/src/realtime/engineio/websocket_transport.h new file mode 100644 index 0000000..ca2228b --- /dev/null +++ b/src/realtime/engineio/websocket_transport.h @@ -0,0 +1,35 @@ +#pragma once + +// WebSocket transport for Engine.IO: exchanges raw Engine.IO packets +// ("") as WebSocket text messages over an upgraded +// connection. Validates transport=websocket with a known sid and +// routes decoded packets to the session. + +#include +#include + +#include "realtime/engineio/engine_session.h" +#include "realtime/engineio/packet_codec.h" +#include "realtime/shared/http_seam.h" + +namespace molo::realtime::engineio { + +class WebSocketTransport { + public: + // Validates the upgrade query (EIO=4, transport=websocket, known + // sid). Returns the failure response when invalid. + struct Admit { + bool ok = false; + shared::HttpResponse failure; + }; + static Admit admit(const shared::HttpRequest& req, bool sid_known); + + // Encodes a packet as one WebSocket text message body. + static std::string to_message(const EnginePacket& packet); + + // Decodes one WebSocket text message into a packet. Returns + // nullopt on framing errors (caller fails the connection). + static std::optional from_message(const std::string& text); +}; + +} // namespace molo::realtime::engineio diff --git a/src/realtime/realtime.mk b/src/realtime/realtime.mk new file mode 100644 index 0000000..c9fd7e2 --- /dev/null +++ b/src/realtime/realtime.mk @@ -0,0 +1,90 @@ +# Part makefile for 06-realtime. Included by the root Makefile. +# Owns: src/realtime/**, tests/src/realtime/**, build outputs only. + +REALTIME_SRCS := \ + src/realtime/shared/http_seam.cpp \ + src/realtime/shared/transport.cpp \ + src/realtime/shared/sha1.cpp \ + src/realtime/shared/base64.cpp \ + src/realtime/shared/utf8.cpp \ + src/realtime/shared/random_token.cpp \ + src/realtime/shared/send_queue.cpp \ + src/realtime/shared/reconnect_policy.cpp \ + src/realtime/shared/realtime_log.cpp \ + src/realtime/shared/realtime_metrics.cpp \ + src/realtime/shared/base_realtime_client.cpp \ + src/realtime/shared/backpressure_valve.cpp \ + src/realtime/shared/mini_json.cpp \ + src/realtime/websocket/server/handshake_validator.cpp \ + src/realtime/websocket/server/frame_parser.cpp \ + src/realtime/websocket/server/frame_serializer.cpp \ + src/realtime/websocket/server/websocket_response.cpp \ + src/realtime/websocket/server/websocket_handler.cpp \ + src/realtime/websocket/server/websocket_view.cpp \ + src/realtime/websocket/server/heartbeat_manager.cpp \ + src/realtime/websocket/client/client_handshake.cpp \ + src/realtime/websocket/client/websocket_client.cpp \ + src/realtime/sse/event_encoder.cpp \ + src/realtime/sse/event_store.cpp \ + src/realtime/sse/sse_response.cpp \ + src/realtime/sse/sse_client.cpp \ + src/realtime/streaming/flush_policy.cpp \ + src/realtime/streaming/chunked_body.cpp \ + src/realtime/streaming/passthrough_body.cpp \ + src/realtime/engineio/packet_codec.cpp \ + src/realtime/engineio/engine_session.cpp \ + src/realtime/engineio/polling_transport.cpp \ + src/realtime/engineio/websocket_transport.cpp \ + src/realtime/engineio/upgrade_manager.cpp \ + src/realtime/engineio/heartbeat_scheduler.cpp \ + src/realtime/socketio/packet_codec.cpp \ + src/realtime/socketio/namespace.cpp \ + src/realtime/socketio/room_manager.cpp \ + src/realtime/socketio/event_dispatcher.cpp \ + src/realtime/socketio/ack_tracker.cpp \ + src/realtime/socketio/socketio_server.cpp \ + src/realtime/socketio/client/socketio_client.cpp \ + src/realtime/webdav/server/resource_tree.cpp \ + src/realtime/webdav/server/multistatus_builder.cpp \ + src/realtime/webdav/server/dav_xml.cpp \ + src/realtime/webdav/server/propfind_handler.cpp \ + src/realtime/webdav/server/proppatch_handler.cpp \ + src/realtime/webdav/server/mkcol_handler.cpp \ + src/realtime/webdav/server/copy_move_handler.cpp \ + src/realtime/webdav/server/lock_manager.cpp \ + src/realtime/webdav/server/quota_guard.cpp \ + src/realtime/webdav/server/dav_view.cpp \ + src/realtime/webdav/client/webdav_client.cpp + +REALTIME_OBJS := $(patsubst src/%.cpp,build/obj/%.o,$(REALTIME_SRCS)) +REALTIME_LIB := build/lib/librealtime.a + +REALTIME_TESTS := \ + build/test/realtime/ws/handshake_test \ + build/test/realtime/ws/frames_test \ + build/test/realtime/ws/echo_test \ + build/test/realtime/ws/heartbeat_test \ + build/test/realtime/sse/feed_test \ + build/test/realtime/sse/client_test \ + build/test/realtime/stream/passthrough_test \ + build/test/realtime/engineio/handshake_test \ + build/test/realtime/socketio/chat_test \ + build/test/realtime/dav/files_test \ + build/test/realtime/shared/foundation_test \ + build/test/realtime/shared/robustness_test + +$(REALTIME_LIB): $(REALTIME_OBJS) + mkdir -p $(dir $@) + $(AR) rcs $@ $^ + +build/obj/realtime/%.o: src/realtime/%.cpp + mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) -Isrc -c $< -o $@ + +build/test/realtime/%_test: tests/src/realtime/%_test.cpp $(REALTIME_LIB) + mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) -Isrc -Isrc/realtime -Itests/src/realtime $< $(REALTIME_LIB) -o $@ + +build/test/realtime/%: tests/src/realtime/%.cpp $(REALTIME_LIB) + mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) -Isrc -Isrc/realtime -Itests/src/realtime $< $(REALTIME_LIB) -o $@ diff --git a/src/realtime/shared/backpressure_valve.cpp b/src/realtime/shared/backpressure_valve.cpp new file mode 100644 index 0000000..83d0e79 --- /dev/null +++ b/src/realtime/shared/backpressure_valve.cpp @@ -0,0 +1,32 @@ +// Backpressure valve. See backpressure_valve.h. + +#include "realtime/shared/backpressure_valve.h" + +namespace molo::realtime::shared { + +BackpressureValve::BackpressureValve(std::size_t low_watermark, + std::size_t high_watermark) + : low_(low_watermark), + high_(high_watermark < low_watermark ? low_watermark : high_watermark) { +} + +void BackpressureValve::note_buffered(std::size_t n) { + level_ += n; + if (level_ >= high_) { + paused_ = true; + } +} + +void BackpressureValve::note_flushed(std::size_t n) { + level_ = n >= level_ ? 0 : level_ - n; + if (level_ <= low_) { + paused_ = false; + } +} + +void BackpressureValve::reset() { + level_ = 0; + paused_ = false; +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/backpressure_valve.h b/src/realtime/shared/backpressure_valve.h new file mode 100644 index 0000000..6f7521e --- /dev/null +++ b/src/realtime/shared/backpressure_valve.h @@ -0,0 +1,33 @@ +#pragma once + +// Backpressure valve with high/low watermarks. Producers call +// note_buffered(bytes) as queued data grows and note_flushed(bytes) +// as it drains. paused() tells the producer to stop reading; the +// producer resumes when the level drops back under the low mark. +// Used by streaming passthrough, SSE, and WebDAV GET. + +#include + +namespace molo::realtime::shared { + +class BackpressureValve { + public: + BackpressureValve(std::size_t low_watermark, std::size_t high_watermark); + + void note_buffered(std::size_t n); + void note_flushed(std::size_t n); + void reset(); + + bool paused() const { return paused_; } + std::size_t level() const { return level_; } + std::size_t high_watermark() const { return high_; } + std::size_t low_watermark() const { return low_; } + + private: + std::size_t low_; + std::size_t high_; + std::size_t level_ = 0; + bool paused_ = false; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/base64.cpp b/src/realtime/shared/base64.cpp new file mode 100644 index 0000000..5d54cba --- /dev/null +++ b/src/realtime/shared/base64.cpp @@ -0,0 +1,110 @@ +// Base64 per RFC 4648. See base64.h. + +#include "realtime/shared/base64.h" + +namespace molo::realtime::shared { + +std::string base64_encode(const uint8_t* data, std::size_t len) { + static const char kAlphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + out.reserve(((len + 2) / 3) * 4); + std::size_t i = 0; + while (i + 3 <= len) { + uint32_t n = (static_cast(data[i]) << 16) | + (static_cast(data[i + 1]) << 8) | + static_cast(data[i + 2]); + out.push_back(kAlphabet[(n >> 18) & 63u]); + out.push_back(kAlphabet[(n >> 12) & 63u]); + out.push_back(kAlphabet[(n >> 6) & 63u]); + out.push_back(kAlphabet[n & 63u]); + i += 3; + } + std::size_t rem = len - i; + if (rem == 1) { + uint32_t n = static_cast(data[i]) << 16; + out.push_back(kAlphabet[(n >> 18) & 63u]); + out.push_back(kAlphabet[(n >> 12) & 63u]); + out.push_back('='); + out.push_back('='); + } else if (rem == 2) { + uint32_t n = (static_cast(data[i]) << 16) | + (static_cast(data[i + 1]) << 8); + out.push_back(kAlphabet[(n >> 18) & 63u]); + out.push_back(kAlphabet[(n >> 12) & 63u]); + out.push_back(kAlphabet[(n >> 6) & 63u]); + out.push_back('='); + } + return out; +} + +std::string base64_encode(const std::string& data) { + return base64_encode(reinterpret_cast(data.data()), + data.size()); +} + +std::string base64_encode(const std::vector& data) { + return base64_encode(data.data(), data.size()); +} + +std::optional> base64_decode(const std::string& text) { + static int table[256]; + static bool ready = false; + if (!ready) { + for (int i = 0; i < 256; ++i) { + table[i] = -1; + } + for (int i = 0; i < 26; ++i) { + table['A' + i] = i; + table['a' + i] = 26 + i; + } + for (int i = 0; i < 10; ++i) { + table['0' + i] = 52 + i; + } + table['+'] = 62; + table['/'] = 63; + ready = true; + } + if (text.empty()) { + return std::vector{}; + } + if (text.size() % 4 != 0) { + return std::nullopt; + } + std::vector out; + out.reserve((text.size() / 4) * 3); + for (std::size_t i = 0; i < text.size(); i += 4) { + int v[4]; + int pad = 0; + for (int j = 0; j < 4; ++j) { + unsigned char c = static_cast(text[i + j]); + if (text[i + j] == '=') { + // Padding is only legal in the final quantum. + if (i + 4 != text.size() || (j != 2 && j != 3)) { + return std::nullopt; + } + v[j] = 0; + ++pad; + } else { + if (pad > 0 || table[c] < 0) { + return std::nullopt; + } + v[j] = table[c]; + } + } + uint32_t n = (static_cast(v[0]) << 18) | + (static_cast(v[1]) << 12) | + (static_cast(v[2]) << 6) | + static_cast(v[3]); + out.push_back(static_cast((n >> 16) & 0xFFu)); + if (pad < 2) { + out.push_back(static_cast((n >> 8) & 0xFFu)); + } + if (pad < 1) { + out.push_back(static_cast(n & 0xFFu)); + } + } + return out; +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/base64.h b/src/realtime/shared/base64.h new file mode 100644 index 0000000..885f96e --- /dev/null +++ b/src/realtime/shared/base64.h @@ -0,0 +1,21 @@ +#pragma once + +// Base64 per RFC 4648 section 4 (standard alphabet with padding). +// Used for Sec-WebSocket-Key/Accept and basic auth material. + +#include +#include +#include +#include + +namespace molo::realtime::shared { + +std::string base64_encode(const uint8_t* data, std::size_t len); +std::string base64_encode(const std::string& data); +std::string base64_encode(const std::vector& data); + +// Returns nullopt on invalid input (bad alphabet, bad padding, +// embedded whitespace). Output is capped at 4/3 of input length. +std::optional> base64_decode(const std::string& text); + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/base_realtime_client.cpp b/src/realtime/shared/base_realtime_client.cpp new file mode 100644 index 0000000..bf44a51 --- /dev/null +++ b/src/realtime/shared/base_realtime_client.cpp @@ -0,0 +1,63 @@ +// Base realtime client. See base_realtime_client.h. + +#include "realtime/shared/base_realtime_client.h" + +#include "realtime/shared/base64.h" +#include "realtime/shared/random_token.h" +#include "realtime/shared/realtime_log.h" + +namespace molo::realtime::shared { + +std::vector> ClientAuth::headers() const { + std::vector> out; + if (!bearer_token.empty()) { + out.emplace_back("Authorization", "Bearer " + bearer_token); + } else if (!basic_user.empty()) { + std::string raw = basic_user + ":" + basic_password; + out.emplace_back("Authorization", "Basic " + base64_encode(raw)); + } + if (!cookie_header.empty()) { + out.emplace_back("Cookie", cookie_header); + } + return out; +} + +BaseRealtimeClient::BaseRealtimeClient(std::string component) + : component_(std::move(component)) { + context_.correlation_id = random_id(12); +} + +std::chrono::milliseconds BaseRealtimeClient::note_failure() { + ++failures_; + return reconnect_.next(); +} + +void BaseRealtimeClient::note_connected() { + failures_ = 0; + reconnect_.reset(); + set_state(ClientState::kOpen); +} + +void BaseRealtimeClient::set_state(ClientState s) { + if (state_ == s) { + return; + } + state_ = s; + if (state_cb_) { + state_cb_(s); + } +} + +void BaseRealtimeClient::log_info(const std::string& msg) const { + RealtimeLog::info(component_, context_.correlation_id, msg); +} + +void BaseRealtimeClient::log_warn(const std::string& msg) const { + RealtimeLog::warn(component_, context_.correlation_id, msg); +} + +void BaseRealtimeClient::log_error(const std::string& msg) const { + RealtimeLog::error(component_, context_.correlation_id, msg); +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/base_realtime_client.h b/src/realtime/shared/base_realtime_client.h new file mode 100644 index 0000000..b871d4e --- /dev/null +++ b/src/realtime/shared/base_realtime_client.h @@ -0,0 +1,82 @@ +#pragma once + +// Base class of every realtime client (WebSocket, SSE, Socket.IO, +// WebDAV). Owns auth material, reconnect/backoff, and logging in one +// place so client behavior stays identical across protocols (rule 8). + +#include +#include +#include +#include +#include +#include + +#include "realtime/shared/connection_context.h" +#include "realtime/shared/reconnect_policy.h" + +namespace molo::realtime::shared { + +enum class ClientState : std::uint8_t { + kConnecting, + kOpen, + kClosed, +}; + +struct ClientAuth { + // Bearer token, basic credentials, or cookie header value. Empty + // means anonymous. Exactly one of these should be set. + std::string bearer_token; + std::string basic_user; + std::string basic_password; + std::string cookie_header; + + bool anonymous() const { + return bearer_token.empty() && basic_user.empty() && + cookie_header.empty(); + } + // Renders Authorization / Cookie header pairs for a handshake. + std::vector> headers() const; +}; + +class BaseRealtimeClient { + public: + explicit BaseRealtimeClient(std::string component); + virtual ~BaseRealtimeClient() = default; + + BaseRealtimeClient(const BaseRealtimeClient&) = delete; + BaseRealtimeClient& operator=(const BaseRealtimeClient&) = delete; + + ClientState state() const { return state_; } + void set_auth(ClientAuth auth) { auth_ = std::move(auth); } + const ClientAuth& auth() const { return auth_; } + ConnectionContext& context() { return context_; } + const ConnectionContext& context() const { return context_; } + ReconnectPolicy& reconnect_policy() { return reconnect_; } + + // Records a failed attempt and returns the delay before the next + // one. Records success with note_connected(). + std::chrono::milliseconds note_failure(); + void note_connected(); + std::uint32_t consecutive_failures() const { return failures_; } + + using StateCallback = std::function; + void on_state(StateCallback cb) { state_cb_ = std::move(cb); } + + protected: + void set_state(ClientState s); + void log_info(const std::string& msg) const; + void log_warn(const std::string& msg) const; + void log_error(const std::string& msg) const; + + ClientAuth auth_; + ConnectionContext context_; + + private: + std::string component_; + ClientState state_ = ClientState::kClosed; + ReconnectPolicy reconnect_; + std::uint32_t failures_ = 0; + StateCallback state_cb_; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/caps.h b/src/realtime/shared/caps.h new file mode 100644 index 0000000..90d5698 --- /dev/null +++ b/src/realtime/shared/caps.h @@ -0,0 +1,57 @@ +#pragma once + +// Static ceilings for every realtime knob. Plain constants, no +// autotuning. Every value is overridable through the config structs +// that take it as a default, never grown at runtime. + +#include +#include +#include + +namespace molo::realtime::shared { + +using namespace std::chrono_literals; + +struct Caps { + // WebSocket framing and messages. + static constexpr std::size_t kMaxMessageBytes = 1024u * 1024u; + static constexpr std::size_t kMaxFramePayloadBytes = 256u * 1024u; + static constexpr std::size_t kMaxFramesPerMessage = 64u; + static constexpr std::size_t kMaxControlPayloadBytes = 125u; + // Send queues and connections. + static constexpr std::size_t kSendQueueDepth = 256u; + static constexpr std::size_t kMaxConnectionsPerIp = 128u; + static constexpr std::size_t kMaxConnectionsPerRoute = 100000u; + static constexpr std::size_t kMaxRoutes = 4096u; + // Timeouts and heartbeats. + static constexpr std::chrono::seconds kIdleTimeout = 60s; + static constexpr std::chrono::seconds kPingInterval = 25s; + static constexpr std::chrono::seconds kPongTimeout = 20s; + static constexpr std::chrono::seconds kCloseHandshakeTimeout = 5s; + static constexpr std::chrono::seconds kEnginePollHold = 25s; + static constexpr std::chrono::seconds kAckTimeout = 30s; + // SSE. + static constexpr std::size_t kSseReplayMaxEvents = 1000u; + static constexpr std::chrono::minutes kSseReplayMaxAge = 5min; + static constexpr std::chrono::seconds kSseKeepAlive = 15s; + static constexpr std::chrono::milliseconds kSseRetryMin = 1000ms; + static constexpr std::chrono::milliseconds kSseRetryMax = 300000ms; + // Socket.IO binary. + static constexpr std::size_t kMaxAttachments = 16u; + static constexpr std::size_t kMaxAttachmentBytes = 8u * 1024u * 1024u; + // Streaming. + static constexpr std::size_t kFlushBytes = 64u * 1024u; + static constexpr std::chrono::milliseconds kFlushInterval = 100ms; + static constexpr std::size_t kPassthroughRingBytes = 256u * 1024u; + // WebDAV. + static constexpr std::size_t kMaxXmlBodyBytes = 1024u * 1024u; + static constexpr std::size_t kMaxDavDepth = 32u; + static constexpr std::size_t kDavCacheEntries = 128u; + static constexpr std::size_t kDavCacheBytes = 64u * 1024u * 1024u; + // Reconnect backoff. + static constexpr std::chrono::milliseconds kBackoffBase = 250ms; + static constexpr std::chrono::seconds kBackoffMax = 30s; + static constexpr std::uint32_t kBackoffMaxAttempts = 25u; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/connection_context.h b/src/realtime/shared/connection_context.h new file mode 100644 index 0000000..2d8632d --- /dev/null +++ b/src/realtime/shared/connection_context.h @@ -0,0 +1,37 @@ +#pragma once + +// Auth/session/logging handle shared by every realtime connection. +// Populated by the middleware chain before upgrade (rule 1); protocol +// engines only read it. Every connection type derives from or holds +// one of these. + +#include +#include +#include + +namespace molo::realtime::shared { + +struct ConnectionContext { + // Correlation id flowing through logs, metrics and traces. + std::string correlation_id; + // Route the connection upgraded on, e.g. "/ws/chat". + std::string route; + // Authenticated identity, empty when anonymous. + std::string user_id; + std::string auth_scheme; + // Session id when a cookie session resolved before upgrade. + std::string session_id; + // Remote peer address for per-IP caps and logging. + std::string remote_ip; + // Negotiated subprotocol, empty when none. + std::string subprotocol; + // Wall-clock time the connection opened. + std::chrono::steady_clock::time_point opened_at = + std::chrono::steady_clock::now(); + // Monotonic connection number within the process. + std::uint64_t connection_no = 0; + + bool authenticated() const { return !user_id.empty(); } +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/http_seam.cpp b/src/realtime/shared/http_seam.cpp new file mode 100644 index 0000000..a89911b --- /dev/null +++ b/src/realtime/shared/http_seam.cpp @@ -0,0 +1,119 @@ +// Minimal HTTP seam helpers. See http_seam.h. + +#include "realtime/shared/http_seam.h" + +#include + +namespace molo::realtime::shared { +namespace { + +std::string lower(std::string s) { + for (char& c : s) { + c = static_cast(std::tolower(static_cast(c))); + } + return s; +} + +template +std::vector values_of(const Msg& msg, const std::string& name) { + std::vector out; + const std::string want = lower(name); + for (const auto& [k, v] : msg.headers) { + if (lower(k) == want) { + out.push_back(v); + } + } + return out; +} + +std::string trim(const std::string& s) { + std::size_t b = 0; + while (b < s.size() && + std::isspace(static_cast(s[b])) != 0) { + ++b; + } + std::size_t e = s.size(); + while (e > b && std::isspace(static_cast(s[e - 1])) != 0) { + --e; + } + return s.substr(b, e - b); +} + +} // namespace + +std::vector header_values(const HttpRequest& req, + const std::string& name) { + return values_of(req, name); +} + +std::vector header_values(const HttpResponse& res, + const std::string& name) { + return values_of(res, name); +} + +std::string header_first(const HttpRequest& req, const std::string& name) { + auto all = values_of(req, name); + return all.empty() ? std::string{} : all.front(); +} + +std::string header_first(const HttpResponse& res, const std::string& name) { + auto all = values_of(res, name); + return all.empty() ? std::string{} : all.front(); +} + +void set_header(HttpResponse& res, const std::string& name, + const std::string& value) { + const std::string want = lower(name); + for (auto it = res.headers.begin(); it != res.headers.end();) { + if (lower(it->first) == want) { + it = res.headers.erase(it); + } else { + ++it; + } + } + res.headers.emplace(name, value); +} + +std::map parse_query(const std::string& query) { + std::map out; + std::size_t i = 0; + while (i <= query.size()) { + std::size_t amp = query.find('&', i); + if (amp == std::string::npos) { + amp = query.size(); + } + std::string part = query.substr(i, amp - i); + if (!part.empty()) { + std::size_t eq = part.find('='); + std::string key = part.substr(0, eq); + std::string val = eq == std::string::npos ? "" : part.substr(eq + 1); + for (char& c : key) { + if (c == '+') c = ' '; + } + for (char& c : val) { + if (c == '+') c = ' '; + } + out[key] = val; + } + i = amp + 1; + } + return out; +} + +bool header_has_token(const std::string& value, const std::string& token) { + const std::string want = lower(trim(token)); + std::size_t i = 0; + while (i <= value.size()) { + std::size_t comma = value.find(',', i); + if (comma == std::string::npos) { + comma = value.size(); + } + if (lower(trim(value.substr(i, comma - i))) == want) { + return true; + } + i = comma + 1; + } + return false; +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/http_seam.h b/src/realtime/shared/http_seam.h new file mode 100644 index 0000000..ff04102 --- /dev/null +++ b/src/realtime/shared/http_seam.h @@ -0,0 +1,50 @@ +#pragma once + +// Minimal HTTP request/response value types. This seam mirrors the +// molo::http_msg pair owned by 05-request-routing.md so realtime +// protocol logic compiles and tests standalone. Real integration +// swaps these types for the 05 ones; no protocol file may add +// behavior here. + +#include +#include +#include +#include + +namespace molo::realtime::shared { + +struct HttpRequest { + std::string method = "GET"; + std::string path = "/"; + std::string query; + std::multimap headers; + std::string body; + std::string remote_ip; +}; + +struct HttpResponse { + std::uint16_t status = 200; + std::multimap headers; + std::string body; + bool stream = false; +}; + +// Case-insensitive header lookup. Returns all values in order. +std::vector header_values(const HttpRequest& req, + const std::string& name); +std::vector header_values(const HttpResponse& res, + const std::string& name); +std::string header_first(const HttpRequest& req, const std::string& name); +std::string header_first(const HttpResponse& res, const std::string& name); +void set_header(HttpResponse& res, const std::string& name, + const std::string& value); + +// Parses a query string ("a=1&b=2") into key/value pairs. No decoding +// beyond '+' to space is done here; full decoding lives in 05. +std::map parse_query(const std::string& query); + +// True when a comma-separated header value contains the token +// (case-insensitive, surrounding whitespace ignored). +bool header_has_token(const std::string& value, const std::string& token); + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/mini_json.cpp b/src/realtime/shared/mini_json.cpp new file mode 100644 index 0000000..55aaf95 --- /dev/null +++ b/src/realtime/shared/mini_json.cpp @@ -0,0 +1,409 @@ +// Minimal JSON codec. See mini_json.h. + +#include "realtime/shared/mini_json.h" + +#include + +namespace molo::realtime::shared { +namespace { + +constexpr std::size_t kMaxDepth = 32; +constexpr std::size_t kMaxInput = 1024u * 1024u; + +void escape_into(const std::string& s, std::string& out) { + out.push_back('"'); + for (unsigned char c : s) { + switch (c) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + if (c < 0x20u) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out.push_back(static_cast(c)); + } + } + } + out.push_back('"'); +} + +class Parser { + public: + explicit Parser(const std::string& text) : s_(text) {} + + std::optional run() { + auto v = value(0); + if (!v.has_value()) { + return std::nullopt; + } + skip_ws(); + if (pos_ != s_.size()) { + return std::nullopt; + } + return v; + } + + private: + void skip_ws() { + while (pos_ < s_.size() && + std::isspace(static_cast(s_[pos_])) != 0) { + ++pos_; + } + } + + std::optional value(std::size_t depth) { + if (depth > kMaxDepth || pos_ >= s_.size()) { + return std::nullopt; + } + skip_ws(); + if (pos_ >= s_.size()) { + return std::nullopt; + } + char c = s_[pos_]; + if (c == '{') { + return object(depth); + } + if (c == '[') { + return array(depth); + } + if (c == '"') { + auto s = string(); + if (!s.has_value()) { + return std::nullopt; + } + return Json::str(*s); + } + if (c == 't') { + return literal("true", Json::boolean(true)); + } + if (c == 'f') { + return literal("false", Json::boolean(false)); + } + if (c == 'n') { + return literal("null", Json::null()); + } + return number(); + } + + std::optional literal(const char* word, Json v) { + for (const char* p = word; *p != '\0'; ++p) { + if (pos_ >= s_.size() || s_[pos_] != *p) { + return std::nullopt; + } + ++pos_; + } + return v; + } + + std::optional number() { + std::size_t start = pos_; + if (pos_ < s_.size() && s_[pos_] == '-') { + ++pos_; + } + bool any = false; + while (pos_ < s_.size() && + std::isdigit(static_cast(s_[pos_])) != 0) { + ++pos_; + any = true; + } + if (!any) { + return std::nullopt; + } + // Fractions and exponents collapse to the integer part: realtime + // wires only use integers, and accepting the literal keeps codec + // interop while staying exact for our values. + if (pos_ < s_.size() && s_[pos_] == '.') { + ++pos_; + while (pos_ < s_.size() && + std::isdigit(static_cast(s_[pos_])) != 0) { + ++pos_; + } + } + if (pos_ < s_.size() && (s_[pos_] == 'e' || s_[pos_] == 'E')) { + ++pos_; + if (pos_ < s_.size() && (s_[pos_] == '+' || s_[pos_] == '-')) { + ++pos_; + } + while (pos_ < s_.size() && + std::isdigit(static_cast(s_[pos_])) != 0) { + ++pos_; + } + } + try { + long long v = std::stoll(s_.substr(start, pos_ - start)); + return Json::integer(v); + } catch (...) { + return std::nullopt; + } + } + + std::optional string() { + ++pos_; // Opening quote. + std::string out; + while (pos_ < s_.size()) { + char c = s_[pos_++]; + if (c == '"') { + return out; + } + if (c != '\\') { + if (static_cast(c) < 0x20u) { + return std::nullopt; + } + out.push_back(c); + continue; + } + if (pos_ >= s_.size()) { + return std::nullopt; + } + char e = s_[pos_++]; + switch (e) { + case '"': + case '\\': + case '/': + out.push_back(e); + break; + case 'n': + out.push_back('\n'); + break; + case 'r': + out.push_back('\r'); + break; + case 't': + out.push_back('\t'); + break; + case 'b': + out.push_back('\b'); + break; + case 'f': + out.push_back('\f'); + break; + case 'u': { + if (pos_ + 4 > s_.size()) { + return std::nullopt; + } + unsigned cp = 0; + for (int i = 0; i < 4; ++i) { + char h = s_[pos_++]; + cp <<= 4; + if (h >= '0' && h <= '9') { + cp |= static_cast(h - '0'); + } else if (h >= 'a' && h <= 'f') { + cp |= static_cast(h - 'a' + 10); + } else if (h >= 'A' && h <= 'F') { + cp |= static_cast(h - 'A' + 10); + } else { + return std::nullopt; + } + } + // Encode as UTF-8, replacing surrogates with U+FFFD. + if ((cp >= 0xD800u && cp <= 0xDFFFu) || cp > 0x10FFFFu) { + cp = 0xFFFDu; + } + if (cp < 0x80u) { + out.push_back(static_cast(cp)); + } else if (cp < 0x800u) { + out.push_back(static_cast(0xC0u | (cp >> 6))); + out.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } else { + out.push_back(static_cast(0xE0u | (cp >> 12))); + out.push_back(static_cast(0x80u | ((cp >> 6) & 0x3Fu))); + out.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + break; + } + default: + return std::nullopt; + } + } + return std::nullopt; + } + + std::optional array(std::size_t depth) { + ++pos_; // '['. + Json arr = Json::make_array(); + skip_ws(); + if (pos_ < s_.size() && s_[pos_] == ']') { + ++pos_; + return arr; + } + while (true) { + auto v = value(depth + 1); + if (!v.has_value()) { + return std::nullopt; + } + arr.push(std::move(*v)); + skip_ws(); + if (pos_ >= s_.size()) { + return std::nullopt; + } + if (s_[pos_] == ']') { + ++pos_; + return arr; + } + if (s_[pos_] != ',') { + return std::nullopt; + } + ++pos_; + } + } + + std::optional object(std::size_t depth) { + ++pos_; // '{'. + Json obj = Json::make_object(); + skip_ws(); + if (pos_ < s_.size() && s_[pos_] == '}') { + ++pos_; + return obj; + } + while (true) { + skip_ws(); + if (pos_ >= s_.size() || s_[pos_] != '"') { + return std::nullopt; + } + auto key = string(); + if (!key.has_value()) { + return std::nullopt; + } + skip_ws(); + if (pos_ >= s_.size() || s_[pos_] != ':') { + return std::nullopt; + } + ++pos_; + auto v = value(depth + 1); + if (!v.has_value()) { + return std::nullopt; + } + obj.set(*key, std::move(*v)); + skip_ws(); + if (pos_ >= s_.size()) { + return std::nullopt; + } + if (s_[pos_] == '}') { + ++pos_; + return obj; + } + if (s_[pos_] != ',') { + return std::nullopt; + } + ++pos_; + } + } + + const std::string& s_; + std::size_t pos_ = 0; +}; + +} // namespace + +Json Json::null() { + return Json{}; +} +Json Json::boolean(bool v) { + Json j; + j.type_ = Type::kBool; + j.boolean_ = v; + return j; +} +Json Json::integer(long long v) { + Json j; + j.type_ = Type::kInt; + j.integer_ = v; + return j; +} +Json Json::str(const std::string& v) { + Json j; + j.type_ = Type::kString; + j.string_ = v; + return j; +} +Json Json::make_array() { + Json j; + j.type_ = Type::kArray; + return j; +} +Json Json::make_object() { + Json j; + j.type_ = Type::kObject; + return j; +} + +void Json::push(Json v) { + array_.push_back(std::move(v)); +} +void Json::set(const std::string& key, Json v) { + object_[key] = std::move(v); +} +const Json* Json::find(const std::string& key) const { + auto it = object_.find(key); + return it == object_.end() ? nullptr : &it->second; +} + +std::string Json::dump() const { + switch (type_) { + case Type::kNull: + return "null"; + case Type::kBool: + return boolean_ ? "true" : "false"; + case Type::kInt: + return std::to_string(integer_); + case Type::kString: { + std::string out; + escape_into(string_, out); + return out; + } + case Type::kArray: { + std::string out = "["; + bool first = true; + for (const auto& v : array_) { + if (!first) { + out.push_back(','); + } + first = false; + out += v.dump(); + } + out.push_back(']'); + return out; + } + case Type::kObject: { + std::string out = "{"; + bool first = true; + for (const auto& [k, v] : object_) { + if (!first) { + out.push_back(','); + } + first = false; + escape_into(k, out); + out.push_back(':'); + out += v.dump(); + } + out.push_back('}'); + return out; + } + } + return "null"; +} + +std::optional Json::parse(const std::string& text) { + if (text.size() > kMaxInput) { + return std::nullopt; + } + Parser p(text); + return p.run(); +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/mini_json.h b/src/realtime/shared/mini_json.h new file mode 100644 index 0000000..decd70f --- /dev/null +++ b/src/realtime/shared/mini_json.h @@ -0,0 +1,61 @@ +#pragma once + +// Minimal JSON value, reader and writer for the subset used on +// realtime wires: null, bool, number (integer), string, array, +// object. Depth and size capped. Not a general JSON library. + +#include +#include +#include +#include +#include +#include + +namespace molo::realtime::shared { + +class Json { + public: + enum class Type : std::uint8_t { + kNull, + kBool, + kInt, + kString, + kArray, + kObject, + }; + + Json() = default; + static Json null(); + static Json boolean(bool v); + static Json integer(long long v); + static Json str(const std::string& v); + static Json make_array(); + static Json make_object(); + + Type type() const { return type_; } + bool boolean() const { return boolean_; } + long long integer() const { return integer_; } + const std::string& string() const { return string_; } + const std::vector& array() const { return array_; } + const std::map& object() const { return object_; } + + void push(Json v); + void set(const std::string& key, Json v); + const Json* find(const std::string& key) const; + + std::string dump() const; + + // Parses one JSON value, rejecting trailing garbage. Depth capped + // at 32, input capped at 1 MiB. Returns nullopt on any error. + static std::optional parse(const std::string& text); + + private: + Type type_ = Type::kNull; + bool boolean_ = false; + long long integer_ = 0; + std::string string_; + std::vector array_; + std::map object_; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/random_token.cpp b/src/realtime/shared/random_token.cpp new file mode 100644 index 0000000..bcdef2b --- /dev/null +++ b/src/realtime/shared/random_token.cpp @@ -0,0 +1,63 @@ +// OS randomness. See random_token.h. + +#include "realtime/shared/random_token.h" + +#include +#include +#include + +namespace molo::realtime::shared { + +bool fill_random(uint8_t* data, std::size_t len) { + std::size_t off = 0; + while (off < len) { + ssize_t n = ::getrandom(data + off, len - off, 0); + if (n > 0) { + off += static_cast(n); + continue; + } + // Fallback: read /dev/urandom directly. + int fd = ::open("/dev/urandom", O_RDONLY | O_CLOEXEC); + if (fd < 0) { + return false; + } + while (off < len) { + ssize_t m = ::read(fd, data + off, len - off); + if (m <= 0) { + ::close(fd); + return false; + } + off += static_cast(m); + } + ::close(fd); + } + return true; +} + +std::string random_hex(std::size_t nbytes) { + static const char kHex[] = "0123456789abcdef"; + std::vector buf(nbytes, 0); + fill_random(buf.data(), buf.size()); + std::string out; + out.reserve(nbytes * 2); + for (uint8_t b : buf) { + out.push_back(kHex[b >> 4]); + out.push_back(kHex[b & 0xFu]); + } + return out; +} + +std::string random_id(std::size_t nbytes) { + static const char kAlphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + std::vector buf(nbytes, 0); + fill_random(buf.data(), buf.size()); + std::string out; + out.reserve(nbytes); + for (uint8_t b : buf) { + out.push_back(kAlphabet[b % 62u]); + } + return out; +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/random_token.h b/src/realtime/shared/random_token.h new file mode 100644 index 0000000..ff86a73 --- /dev/null +++ b/src/realtime/shared/random_token.h @@ -0,0 +1,23 @@ +#pragma once + +// Unguessable random tokens from the OS CSPRNG (getrandom with a +// /dev/urandom fallback). Used for WebDAV lock tokens, Engine.IO +// session ids, and client nonces. + +#include +#include +#include + +namespace molo::realtime::shared { + +// Fills len bytes from the OS CSPRNG. Returns false only when no +// OS randomness source is available. +bool fill_random(uint8_t* data, std::size_t len); + +// Lowercase hex token of exactly 2*nbytes characters. +std::string random_hex(std::size_t nbytes); + +// Opaque id for sessions: base62-ish from random bytes. +std::string random_id(std::size_t nbytes = 16); + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/realtime_log.cpp b/src/realtime/shared/realtime_log.cpp new file mode 100644 index 0000000..8d7e8f9 --- /dev/null +++ b/src/realtime/shared/realtime_log.cpp @@ -0,0 +1,69 @@ +// Fixed-format realtime logging. See realtime_log.h. + +#include "realtime/shared/realtime_log.h" + +#include +#include + +namespace molo::realtime::shared { +namespace { +std::atomic g_level{static_cast(LogLevel::kInfo)}; + +const char* name_of(LogLevel level) { + switch (level) { + case LogLevel::kDebug: + return "DEBUG"; + case LogLevel::kInfo: + return "INFO"; + case LogLevel::kWarn: + return "WARN"; + case LogLevel::kError: + return "ERROR"; + } + return "INFO"; +} +} // namespace + +void RealtimeLog::set_level(LogLevel level) { + g_level.store(static_cast(level)); +} + +LogLevel RealtimeLog::level() { + return static_cast(g_level.load()); +} + +void RealtimeLog::log(LogLevel level, const std::string& component, + const std::string& correlation_id, + const std::string& message) { + if (static_cast(level) < g_level.load()) { + return; + } + std::cerr << name_of(level) << " correlation=" << correlation_id << " " + << component << ": " << message << "\n"; +} + +void RealtimeLog::debug(const std::string& component, + const std::string& correlation_id, + const std::string& message) { + log(LogLevel::kDebug, component, correlation_id, message); +} + +void RealtimeLog::info(const std::string& component, + const std::string& correlation_id, + const std::string& message) { + log(LogLevel::kInfo, component, correlation_id, message); +} + +void RealtimeLog::warn(const std::string& component, + const std::string& correlation_id, + const std::string& message) { + log(LogLevel::kWarn, component, correlation_id, message); +} + +void RealtimeLog::error(const std::string& component, + const std::string& correlation_id, + const std::string& message) { + log(LogLevel::kError, component, correlation_id, message); +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/realtime_log.h b/src/realtime/shared/realtime_log.h new file mode 100644 index 0000000..5bdba79 --- /dev/null +++ b/src/realtime/shared/realtime_log.h @@ -0,0 +1,41 @@ +#pragma once + +// Fixed log levels and line format for realtime components. Wires to +// 02-observability later; today it renders lines to stderr. Format: +// " correlation= conn= : ". + +#include + +namespace molo::realtime::shared { + +enum class LogLevel : int { + kDebug = 0, + kInfo = 1, + kWarn = 2, + kError = 3, +}; + +class RealtimeLog { + public: + static void set_level(LogLevel level); + static LogLevel level(); + + static void log(LogLevel level, const std::string& component, + const std::string& correlation_id, + const std::string& message); + + static void debug(const std::string& component, + const std::string& correlation_id, + const std::string& message); + static void info(const std::string& component, + const std::string& correlation_id, + const std::string& message); + static void warn(const std::string& component, + const std::string& correlation_id, + const std::string& message); + static void error(const std::string& component, + const std::string& correlation_id, + const std::string& message); +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/realtime_metrics.cpp b/src/realtime/shared/realtime_metrics.cpp new file mode 100644 index 0000000..ff82279 --- /dev/null +++ b/src/realtime/shared/realtime_metrics.cpp @@ -0,0 +1,76 @@ +// Process-local realtime counters. See realtime_metrics.h. + +#include "realtime/shared/realtime_metrics.h" + +namespace molo::realtime::shared { + +const char* metric_name(RealtimeMetric m) { + switch (m) { + case RealtimeMetric::kConnectionsOpened: + return "molo_realtime_connections_opened"; + case RealtimeMetric::kConnectionsClosed: + return "molo_realtime_connections_closed"; + case RealtimeMetric::kMessagesReceived: + return "molo_realtime_messages_received"; + case RealtimeMetric::kMessagesSent: + return "molo_realtime_messages_sent"; + case RealtimeMetric::kBytesReceived: + return "molo_realtime_bytes_received"; + case RealtimeMetric::kBytesSent: + return "molo_realtime_bytes_sent"; + case RealtimeMetric::kShedMessages: + return "molo_realtime_shed_messages"; + case RealtimeMetric::kHeartbeatTimeouts: + return "molo_realtime_heartbeat_timeouts"; + case RealtimeMetric::kHandshakeFailures: + return "molo_realtime_handshake_failures"; + case RealtimeMetric::kBroadcastFanout: + return "molo_realtime_broadcast_fanout"; + case RealtimeMetric::kBroadcastNoop: + return "molo_realtime_broadcast_noop"; + case RealtimeMetric::kAcksTimedOut: + return "molo_realtime_acks_timed_out"; + case RealtimeMetric::kDavQuotaExceeded: + return "molo_realtime_dav_quota_exceeded"; + case RealtimeMetric::kCount_: + return "molo_realtime_unknown"; + } + return "molo_realtime_unknown"; +} + +RealtimeMetrics::RealtimeMetrics() { + reset_all(); +} + +RealtimeMetrics& RealtimeMetrics::instance() { + static RealtimeMetrics inst; + return inst; +} + +void RealtimeMetrics::inc(RealtimeMetric m, std::uint64_t n) { + values_[static_cast(m)].fetch_add(n, + std::memory_order_relaxed); +} + +std::uint64_t RealtimeMetrics::get(RealtimeMetric m) const { + return values_[static_cast(m)].load(std::memory_order_relaxed); +} + +void RealtimeMetrics::reset_all() { + for (std::size_t i = 0; + i < static_cast(RealtimeMetric::kCount_); ++i) { + values_[i].store(0, std::memory_order_relaxed); + } +} + +std::vector RealtimeMetrics::snapshot() const { + std::vector out; + for (std::size_t i = 0; + i < static_cast(RealtimeMetric::kCount_); ++i) { + out.push_back({metric_name(static_cast(i)), + values_[i].load(std::memory_order_relaxed)}); + } + return out; +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/realtime_metrics.h b/src/realtime/shared/realtime_metrics.h new file mode 100644 index 0000000..d2f023d --- /dev/null +++ b/src/realtime/shared/realtime_metrics.h @@ -0,0 +1,53 @@ +#pragma once + +// Process-local counters with static metric names. Wires to +// 02-observability (Prometheus/OpenTelemetry export) later; the names +// are stable so dashboards can be built against them now. + +#include +#include +#include +#include + +namespace molo::realtime::shared { + +enum class RealtimeMetric : std::uint32_t { + kConnectionsOpened, + kConnectionsClosed, + kMessagesReceived, + kMessagesSent, + kBytesReceived, + kBytesSent, + kShedMessages, + kHeartbeatTimeouts, + kHandshakeFailures, + kBroadcastFanout, + kBroadcastNoop, + kAcksTimedOut, + kDavQuotaExceeded, + kCount_, +}; + +const char* metric_name(RealtimeMetric m); + +class RealtimeMetrics { + public: + static RealtimeMetrics& instance(); + + void inc(RealtimeMetric m, std::uint64_t n = 1); + std::uint64_t get(RealtimeMetric m) const; + void reset_all(); + + struct Sample { + std::string name; + std::uint64_t value; + }; + std::vector snapshot() const; + + private: + RealtimeMetrics(); + std::atomic values_[static_cast( + RealtimeMetric::kCount_)]; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/reconnect_policy.cpp b/src/realtime/shared/reconnect_policy.cpp new file mode 100644 index 0000000..18fbb6e --- /dev/null +++ b/src/realtime/shared/reconnect_policy.cpp @@ -0,0 +1,51 @@ +// Jittered exponential backoff. See reconnect_policy.h. + +#include "realtime/shared/reconnect_policy.h" + +#include "realtime/shared/random_token.h" + +namespace molo::realtime::shared { + +ReconnectPolicy::ReconnectPolicy(std::chrono::milliseconds base, + std::chrono::milliseconds max, + std::uint32_t max_attempts) + : base_(base), max_(max), max_attempts_(max_attempts) { + uint64_t seed = 0; + fill_random(reinterpret_cast(&seed), sizeof(seed)); + rng_.seed(seed); +} + +std::chrono::milliseconds ReconnectPolicy::bound_for( + std::uint32_t attempt) const { + std::chrono::milliseconds bound = base_; + for (std::uint32_t i = 0; i < attempt && bound < max_; ++i) { + if (bound > max_ / 2) { + bound = max_; + } else { + bound *= 2; + } + if (bound > max_) { + bound = max_; + } + } + return bound; +} + +std::chrono::milliseconds ReconnectPolicy::delay_for(std::uint32_t attempt) { + std::chrono::milliseconds bound = bound_for(attempt); + if (bound.count() <= 1) { + return bound; + } + std::uniform_int_distribution dist(0, bound.count()); + return std::chrono::milliseconds(dist(rng_)); +} + +std::chrono::milliseconds ReconnectPolicy::next() { + std::chrono::milliseconds d = delay_for(attempt_); + if (attempt_ < max_attempts_) { + ++attempt_; + } + return d; +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/reconnect_policy.h b/src/realtime/shared/reconnect_policy.h new file mode 100644 index 0000000..a0401b7 --- /dev/null +++ b/src/realtime/shared/reconnect_policy.h @@ -0,0 +1,43 @@ +#pragma once + +// Jittered exponential backoff shared by all realtime clients. +// delay(attempt) = min(max, base * 2^attempt) spread by full jitter +// in [0, delay]. Attempt counts saturate at the static cap; reset() +// restarts the sequence after a successful connection. + +#include +#include +#include + +#include "realtime/shared/caps.h" + +namespace molo::realtime::shared { + +class ReconnectPolicy { + public: + explicit ReconnectPolicy( + std::chrono::milliseconds base = Caps::kBackoffBase, + std::chrono::milliseconds max = Caps::kBackoffMax, + std::uint32_t max_attempts = Caps::kBackoffMaxAttempts); + + // Delay before the given 0-based attempt. Past max_attempts the + // delay stays at max (callers decide when to give up). + std::chrono::milliseconds delay_for(std::uint32_t attempt); + + // Pure exponential bound without jitter, for tests. + std::chrono::milliseconds bound_for(std::uint32_t attempt) const; + + std::uint32_t max_attempts() const { return max_attempts_; } + void reset() { attempt_ = 0; } + // Next delay in the running sequence, advancing the attempt count. + std::chrono::milliseconds next(); + + private: + std::chrono::milliseconds base_; + std::chrono::milliseconds max_; + std::uint32_t max_attempts_; + std::uint32_t attempt_ = 0; + std::mt19937_64 rng_; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/send_queue.cpp b/src/realtime/shared/send_queue.cpp new file mode 100644 index 0000000..060c39b --- /dev/null +++ b/src/realtime/shared/send_queue.cpp @@ -0,0 +1,49 @@ +// Bounded send queue. See send_queue.h. + +#include "realtime/shared/send_queue.h" + +namespace molo::realtime::shared { + +SendQueue::SendQueue(std::size_t capacity, ShedPolicy policy) + : capacity_(capacity == 0 ? 1 : capacity), policy_(policy) {} + +bool SendQueue::push(SendItem item) { + if (closed_) { + return false; + } + if (items_.size() < capacity_) { + items_.push_back(std::move(item)); + return true; + } + switch (policy_) { + case ShedPolicy::kDropOldest: { + const SendItem& oldest = items_.front(); + shed_bytes_ += oldest.bytes.size(); + ++shed_count_; + items_.pop_front(); + items_.push_back(std::move(item)); + return true; + } + case ShedPolicy::kDropNewest: + shed_bytes_ += item.bytes.size(); + ++shed_count_; + return false; + case ShedPolicy::kClose: + shed_bytes_ += item.bytes.size(); + ++shed_count_; + must_close_ = true; + return false; + } + return false; +} + +bool SendQueue::pop(SendItem& out) { + if (items_.empty()) { + return false; + } + out = std::move(items_.front()); + items_.pop_front(); + return true; +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/send_queue.h b/src/realtime/shared/send_queue.h new file mode 100644 index 0000000..0950477 --- /dev/null +++ b/src/realtime/shared/send_queue.h @@ -0,0 +1,68 @@ +#pragma once + +// Bounded per-connection send queue (rule 9). When full, the route +// policy decides: drop oldest, drop newest, or close with a code. +// Every shed event is counted for metrics. Unbounded queues are +// forbidden: capacity is fixed at construction. + +#include +#include +#include +#include +#include + +#include "realtime/shared/caps.h" + +namespace molo::realtime::shared { + +enum class ShedPolicy : std::uint8_t { + kDropOldest, + kDropNewest, + kClose, +}; + +struct SendItem { + std::vector bytes; + bool is_text = false; +}; + +class SendQueue { + public: + explicit SendQueue( + std::size_t capacity = Caps::kSendQueueDepth, + ShedPolicy policy = ShedPolicy::kDropOldest); + + // Enqueues bytes. Returns true when the item is queued; false when + // the policy shed it (drop newest / close) or the queue is closed. + // With kDropOldest the oldest item is shed and counted, then the + // new item is queued and true is returned. + bool push(SendItem item); + + // Pops the oldest item. Returns false when empty. + bool pop(SendItem& out); + + bool empty() const { return items_.empty(); } + std::size_t size() const { return items_.size(); } + std::size_t capacity() const { return capacity_; } + std::size_t shed_count() const { return shed_count_; } + std::size_t shed_bytes() const { return shed_bytes_; } + ShedPolicy policy() const { return policy_; } + + // Marks the queue closed: push fails, pop drains what is left. + void close() { closed_ = true; } + bool closed() const { return closed_; } + // True when the close policy fired and the connection must close. + bool must_close() const { return must_close_; } + void clear_must_close() { must_close_ = false; } + + private: + std::deque items_; + std::size_t capacity_; + ShedPolicy policy_; + std::size_t shed_count_ = 0; + std::size_t shed_bytes_ = 0; + bool closed_ = false; + bool must_close_ = false; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/sha1.cpp b/src/realtime/shared/sha1.cpp new file mode 100644 index 0000000..cb4bcc8 --- /dev/null +++ b/src/realtime/shared/sha1.cpp @@ -0,0 +1,128 @@ +// SHA-1 per FIPS 180-4. See sha1.h. + +#include "realtime/shared/sha1.h" + +#include + +namespace molo::realtime::shared { +namespace { + +uint32_t rotl(uint32_t v, unsigned n) { + return (v << n) | (v >> (32 - n)); +} + +uint32_t load_be32(const uint8_t* p) { + return (static_cast(p[0]) << 24) | + (static_cast(p[1]) << 16) | + (static_cast(p[2]) << 8) | static_cast(p[3]); +} + +void store_be32(uint8_t* p, uint32_t v) { + p[0] = static_cast(v >> 24); + p[1] = static_cast(v >> 16); + p[2] = static_cast(v >> 8); + p[3] = static_cast(v); +} + +} // namespace + +Sha1::Sha1() { + h_[0] = 0x67452301u; + h_[1] = 0xEFCDAB89u; + h_[2] = 0x98BADCFEu; + h_[3] = 0x10325476u; + h_[4] = 0xC3D2E1F0u; +} + +void Sha1::update(const uint8_t* data, std::size_t len) { + total_len_ += len; + while (len > 0) { + std::size_t take = 64 - buf_len_; + if (take > len) { + take = len; + } + std::memcpy(buf_ + buf_len_, data, take); + buf_len_ += take; + data += take; + len -= take; + if (buf_len_ == 64) { + process_block(buf_); + buf_len_ = 0; + } + } +} + +void Sha1::update(const std::string& data) { + update(reinterpret_cast(data.data()), data.size()); +} + +Sha1::Digest Sha1::final() { + uint64_t bit_len = total_len_ * 8u; + uint8_t pad = 0x80; + update(&pad, 1); + pad = 0x00; + while (buf_len_ != 56) { + update(&pad, 1); + } + uint8_t len_bytes[8]; + for (int i = 7; i >= 0; --i) { + len_bytes[i] = static_cast(bit_len & 0xFFu); + bit_len >>= 8; + } + // Append length without re-entering padding: buffer has room now. + std::memcpy(buf_ + 56, len_bytes, 8); + total_len_ += 8; + process_block(buf_); + buf_len_ = 0; + Digest out; + for (int i = 0; i < 5; ++i) { + store_be32(out.data() + i * 4, h_[i]); + } + return out; +} + +Sha1::Digest Sha1::hash(const std::string& data) { + Sha1 s; + s.update(data); + return s.final(); +} + +void Sha1::process_block(const uint8_t block[64]) { + uint32_t w[80]; + for (int i = 0; i < 16; ++i) { + w[i] = load_be32(block + i * 4); + } + for (int i = 16; i < 80; ++i) { + w[i] = rotl(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1); + } + uint32_t a = h_[0], b = h_[1], c = h_[2], d = h_[3], e = h_[4]; + for (int i = 0; i < 80; ++i) { + uint32_t f, k; + if (i < 20) { + f = (b & c) | ((~b) & d); + k = 0x5A827999u; + } else if (i < 40) { + f = b ^ c ^ d; + k = 0x6ED9EBA1u; + } else if (i < 60) { + f = (b & c) | (b & d) | (c & d); + k = 0x8F1BBCDCu; + } else { + f = b ^ c ^ d; + k = 0xCA62C1D6u; + } + uint32_t tmp = rotl(a, 5) + f + e + k + w[i]; + e = d; + d = c; + c = rotl(b, 30); + b = a; + a = tmp; + } + h_[0] += a; + h_[1] += b; + h_[2] += c; + h_[3] += d; + h_[4] += e; +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/sha1.h b/src/realtime/shared/sha1.h new file mode 100644 index 0000000..cfff042 --- /dev/null +++ b/src/realtime/shared/sha1.h @@ -0,0 +1,34 @@ +#pragma once + +// SHA-1 per FIPS 180-4, implemented in-house. Sole use is the +// WebSocket handshake accept computation (RFC 6455 section 1.3). +// Not a general password or integrity primitive. + +#include +#include +#include + +namespace molo::realtime::shared { + +class Sha1 { + public: + static constexpr std::size_t kDigestBytes = 20; + using Digest = std::array; + + Sha1(); + void update(const uint8_t* data, std::size_t len); + void update(const std::string& data); + Digest final(); + + static Digest hash(const std::string& data); + + private: + void process_block(const uint8_t block[64]); + + uint32_t h_[5]; + uint8_t buf_[64]; + std::size_t buf_len_ = 0; + uint64_t total_len_ = 0; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/transport.cpp b/src/realtime/shared/transport.cpp new file mode 100644 index 0000000..6c1f020 --- /dev/null +++ b/src/realtime/shared/transport.cpp @@ -0,0 +1,48 @@ +// Byte transport seam. See transport.h. + +#include "realtime/shared/transport.h" + +#include +#include + +namespace molo::realtime::shared { + +bool MemoryTransport::send(const uint8_t* data, std::size_t len) { + if (!open_ || blocked_) { + return false; + } + if (cap_bytes_ > 0 && sent_.size() + len > cap_bytes_) { + return false; + } + sent_.insert(sent_.end(), data, data + len); + return true; +} + +std::string MemoryTransport::sent_text() const { + return std::string(sent_.begin(), sent_.end()); +} + +bool FdTransport::send(const uint8_t* data, std::size_t len) { + if (!open_) { + return false; + } + std::size_t off = 0; + while (off < len) { + ssize_t n = ::write(fd_, data + off, len - off); + if (n <= 0) { + open_ = false; + return false; + } + off += static_cast(n); + } + return true; +} + +void FdTransport::shutdown_write() { + if (open_) { + ::shutdown(fd_, SHUT_WR); + open_ = false; + } +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/transport.h b/src/realtime/shared/transport.h new file mode 100644 index 0000000..c4cfdf8 --- /dev/null +++ b/src/realtime/shared/transport.h @@ -0,0 +1,69 @@ +#pragma once + +// Byte transport seam. Protocol engines write frames through +// ITransport and never touch sockets directly. The 01-runtime +// connection implements this interface in production; tests use +// MemoryTransport or FdTransport. + +#include +#include +#include +#include + +namespace molo::realtime::shared { + +class ITransport { + public: + virtual ~ITransport() = default; + // Queues bytes for the peer. Returns false when the peer is gone + // or the transport-level buffer refuses more data. + virtual bool send(const uint8_t* data, std::size_t len) = 0; + virtual bool send(const std::vector& data) { + return send(data.data(), data.size()); + } + virtual bool send(const std::string& data) { + return send(reinterpret_cast(data.data()), data.size()); + } + virtual void shutdown_write() = 0; + virtual bool open() const = 0; +}; + +// In-memory transport double. Records everything sent; optionally +// blocks after a byte cap to simulate a slow consumer. +class MemoryTransport : public ITransport { + public: + explicit MemoryTransport(std::size_t cap_bytes = 0) : cap_bytes_(cap_bytes) {} + + using ITransport::send; + bool send(const uint8_t* data, std::size_t len) override; + void shutdown_write() override { open_ = false; } + bool open() const override { return open_; } + + const std::vector& sent() const { return sent_; } + std::string sent_text() const; + void clear() { sent_.clear(); } + void set_blocked(bool blocked) { blocked_ = blocked; } + + private: + std::vector sent_; + std::size_t cap_bytes_; + bool open_ = true; + bool blocked_ = false; +}; + +// File-descriptor transport for loopback tests. Writes go to the fd +// with a single write(2) call per send; short writes fail the send. +class FdTransport : public ITransport { + public: + explicit FdTransport(int fd) : fd_(fd) {} + using ITransport::send; + bool send(const uint8_t* data, std::size_t len) override; + void shutdown_write() override; + bool open() const override { return open_; } + + private: + int fd_; + bool open_ = true; +}; + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/utf8.cpp b/src/realtime/shared/utf8.cpp new file mode 100644 index 0000000..7f21743 --- /dev/null +++ b/src/realtime/shared/utf8.cpp @@ -0,0 +1,58 @@ +// Strict UTF-8 validation. See utf8.h. + +#include "realtime/shared/utf8.h" + +namespace molo::realtime::shared { + +bool is_valid_utf8(const uint8_t* data, std::size_t len) { + std::size_t i = 0; + while (i < len) { + uint8_t c = data[i]; + std::size_t want = 0; + uint32_t min = 0; + if (c < 0x80u) { + ++i; + continue; + } else if ((c & 0xE0u) == 0xC0u) { + want = 1; + min = 0x80u; + if (c < 0xC2u) { + return false; // Overlong. + } + } else if ((c & 0xF0u) == 0xE0u) { + want = 2; + min = 0x800u; + } else if ((c & 0xF8u) == 0xF0u) { + want = 3; + min = 0x10000u; + if (c > 0xF4u) { + return false; // Above U+10FFFF. + } + } else { + return false; + } + if (i + want >= len) { + return false; // Truncated. + } + uint32_t cp = c & ((1u << (6 - want)) - 1u); + for (std::size_t k = 1; k <= want; ++k) { + uint8_t d = data[i + k]; + if ((d & 0xC0u) != 0x80u) { + return false; + } + cp = (cp << 6) | (d & 0x3Fu); + } + if (cp < min || (cp >= 0xD800u && cp <= 0xDFFFu) || cp > 0x10FFFFu) { + return false; + } + i += want + 1; + } + return true; +} + +bool is_valid_utf8(const std::string& text) { + return is_valid_utf8(reinterpret_cast(text.data()), + text.size()); +} + +} // namespace molo::realtime::shared diff --git a/src/realtime/shared/utf8.h b/src/realtime/shared/utf8.h new file mode 100644 index 0000000..940b48e --- /dev/null +++ b/src/realtime/shared/utf8.h @@ -0,0 +1,15 @@ +#pragma once + +// Strict UTF-8 validation per RFC 3629: rejects overlong forms, +// surrogates, values above U+10FFFF, and truncated sequences. +// Used for WebSocket text frames and close reasons. + +#include +#include + +namespace molo::realtime::shared { + +bool is_valid_utf8(const uint8_t* data, std::size_t len); +bool is_valid_utf8(const std::string& text); + +} // namespace molo::realtime::shared diff --git a/src/realtime/socketio/ack_tracker.cpp b/src/realtime/socketio/ack_tracker.cpp new file mode 100644 index 0000000..c1034d6 --- /dev/null +++ b/src/realtime/socketio/ack_tracker.cpp @@ -0,0 +1,40 @@ +// Ack tracking. See ack_tracker.h. + +#include "realtime/socketio/ack_tracker.h" + +#include "realtime/shared/realtime_metrics.h" + +namespace molo::realtime::socketio { + +AckTracker::AckTracker(std::chrono::milliseconds timeout) : timeout_(timeout) {} + +long long AckTracker::expect(std::chrono::steady_clock::time_point now) { + long long id = next_id_++; + pending_[id] = now; + return id; +} + +bool AckTracker::resolve(long long ack_id) { + return pending_.erase(ack_id) > 0; +} + +std::size_t AckTracker::tick(std::chrono::steady_clock::time_point now) { + std::size_t expired = 0; + for (auto it = pending_.begin(); it != pending_.end();) { + if (now - it->second < timeout_) { + ++it; + continue; + } + long long id = it->first; + it = pending_.erase(it); + ++expired; + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kAcksTimedOut); + if (timeout_cb_) { + timeout_cb_(id); + } + } + return expired; +} + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/ack_tracker.h b/src/realtime/socketio/ack_tracker.h new file mode 100644 index 0000000..6145819 --- /dev/null +++ b/src/realtime/socketio/ack_tracker.h @@ -0,0 +1,45 @@ +#pragma once + +// Tracks outbound events that expect an acknowledgement. Ack ids are +// per-connection monotonic integers; entries expire after the ack +// timeout (30 s default) and count a metric. The runtime drives +// tick(); the connection calls resolve() on inbound ACK packets. + +#include +#include +#include +#include + +#include "realtime/shared/caps.h" +#include "realtime/shared/mini_json.h" + +namespace molo::realtime::socketio { + +class AckTracker { + public: + using TimeoutCallback = std::function; + + explicit AckTracker(std::chrono::milliseconds timeout = + shared::Caps::kAckTimeout); + + // Registers a pending ack, returning its fresh id. + long long expect(std::chrono::steady_clock::time_point now); + + // Resolves an ack id. Returns true when it was pending. + bool resolve(long long ack_id); + + // Expires overdue entries, firing the timeout callback per id. + // Returns the expired count. + std::size_t tick(std::chrono::steady_clock::time_point now); + + void on_timeout(TimeoutCallback cb) { timeout_cb_ = std::move(cb); } + std::size_t pending() const { return pending_.size(); } + + private: + std::chrono::milliseconds timeout_; + TimeoutCallback timeout_cb_; + long long next_id_ = 0; + std::map pending_; +}; + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/client/socketio_client.cpp b/src/realtime/socketio/client/socketio_client.cpp new file mode 100644 index 0000000..c8a5975 --- /dev/null +++ b/src/realtime/socketio/client/socketio_client.cpp @@ -0,0 +1,194 @@ +// Matching Socket.IO client. See socketio_client.h. + +#include "realtime/socketio/client/socketio_client.h" + +namespace molo::realtime::socketio::client { + +SocketIoClient::SocketIoClient(bool polling_only) + : shared::BaseRealtimeClient("socketio-client"), + polling_only_(polling_only) {} + +void SocketIoClient::on_event(const std::string& nsp, const std::string& event, + EventHandler handler) { + dispatcher_.on(nsp, event, std::move(handler)); +} + +void SocketIoClient::on_packet(const SocketPacket& packet) { + if (packet.type == SocketPacketType::kBinaryEvent || + packet.type == SocketPacketType::kBinaryAck) { + if (!packet.attachments.empty() || true) { + pending_binary_ = packet; + pending_frames_.clear(); + has_pending_binary_ = true; + if (packet.attachments.empty()) { + // Zero attachments: handle immediately. + has_pending_binary_ = false; + SocketPacket plain = packet; + plain.type = packet.type == SocketPacketType::kBinaryEvent + ? SocketPacketType::kEvent + : SocketPacketType::kAck; + on_packet(plain); + } + } + return; + } + switch (packet.type) { + case SocketPacketType::kConnect: + connected_.insert(packet.nsp); + if (connect_cb_) { + connect_cb_(packet.nsp); + } + break; + case SocketPacketType::kDisconnect: + connected_.erase(packet.nsp); + break; + case SocketPacketType::kConnectError: { + std::string msg; + const shared::Json* m = packet.data.find("message"); + if (m != nullptr && m->type() == shared::Json::Type::kString) { + msg = m->string(); + } + if (error_cb_) { + error_cb_(packet.nsp, msg); + } + break; + } + case SocketPacketType::kEvent: { + long long ack_id = packet.ack_id; + std::string nsp = packet.nsp; + AckFn ack; + if (ack_id >= 0) { + ack = [this, nsp, ack_id](shared::Json args) { + SocketPacket p; + p.type = SocketPacketType::kAck; + p.nsp = nsp; + p.ack_id = ack_id; + p.data = std::move(args); + send_packet(std::move(p)); + }; + } + dispatcher_.dispatch("", packet.nsp, packet.data, std::move(ack)); + break; + } + case SocketPacketType::kAck: { + auto it = pending_acks_.find(packet.ack_id); + if (it != pending_acks_.end()) { + AckFn fn = std::move(it->second); + pending_acks_.erase(it); + if (fn) { + fn(packet.data); + } + } + break; + } + default: + break; + } +} + +void SocketIoClient::on_binary(const std::vector& frame) { + if (!has_pending_binary_) { + return; + } + pending_frames_.push_back(frame); + if (pending_frames_.size() < pending_binary_.attachments.size()) { + return; + } + SocketPacket packet = pending_binary_; + has_pending_binary_ = false; + auto frames = std::move(pending_frames_); + pending_frames_.clear(); + if (!PacketCodec::attach(packet, frames)) { + return; + } + packet.type = packet.type == SocketPacketType::kBinaryEvent + ? SocketPacketType::kEvent + : SocketPacketType::kAck; + on_packet(packet); +} + +void SocketIoClient::connect_nsp(const std::string& nsp, shared::Json auth) { + SocketPacket p; + p.type = SocketPacketType::kConnect; + p.nsp = nsp; + p.data = std::move(auth); + send_packet(std::move(p)); +} + +void SocketIoClient::disconnect_nsp(const std::string& nsp) { + SocketPacket p; + p.type = SocketPacketType::kDisconnect; + p.nsp = nsp; + send_packet(std::move(p)); + connected_.erase(nsp); +} + +void SocketIoClient::emit(const std::string& nsp, const std::string& event, + shared::Json args, AckFn ack) { + shared::Json data = shared::Json::make_array(); + data.push(shared::Json::str(event)); + if (args.type() == shared::Json::Type::kArray) { + for (const auto& a : args.array()) { + data.push(a); + } + } else { + data.push(std::move(args)); + } + SocketPacket p; + p.type = SocketPacketType::kEvent; + p.nsp = nsp; + if (ack) { + p.ack_id = next_ack_++; + pending_acks_[p.ack_id] = std::move(ack); + } + p.data = std::move(data); + send_packet(std::move(p)); +} + +void SocketIoClient::emit_binary( + const std::string& nsp, const std::string& event, shared::Json args, + const std::vector>& attachments, AckFn ack) { + shared::Json data = shared::Json::make_array(); + data.push(shared::Json::str(event)); + for (std::size_t i = 0; i < attachments.size(); ++i) { + shared::Json ph = shared::Json::make_object(); + ph.set("_placeholder", shared::Json::boolean(true)); + ph.set("num", shared::Json::integer(static_cast(i))); + data.push(std::move(ph)); + } + if (args.type() == shared::Json::Type::kArray) { + for (const auto& a : args.array()) { + data.push(a); + } + } else if (args.type() != shared::Json::Type::kNull) { + data.push(std::move(args)); + } + SocketPacket p; + p.type = SocketPacketType::kBinaryEvent; + p.nsp = nsp; + if (ack) { + p.ack_id = next_ack_++; + pending_acks_[p.ack_id] = std::move(ack); + } + p.data = std::move(data); + p.attachments = attachments; + send_packet(std::move(p)); +} + +bool SocketIoClient::connected_nsp(const std::string& nsp) const { + return connected_.count(nsp) > 0; +} + +void SocketIoClient::send_packet(SocketPacket packet) { + if (!emit_fn_) { + return; + } + auto text = PacketCodec::encode_text(packet); + if (!text.has_value()) { + log_warn("encode failed"); + return; + } + emit_fn_(*text, packet.attachments); +} + +} // namespace molo::realtime::socketio::client diff --git a/src/realtime/socketio/client/socketio_client.h b/src/realtime/socketio/client/socketio_client.h new file mode 100644 index 0000000..9f45e24 --- /dev/null +++ b/src/realtime/socketio/client/socketio_client.h @@ -0,0 +1,80 @@ +#pragma once + +// Matching Socket.IO client (rule 8): identical namespace and event +// API to the server side, ack support, binary events, and a +// long-polling fallback mode that disables WebSocket framing. The +// caller owns the Engine.IO transport and feeds decoded Socket.IO +// text packets plus binary frames; the client renders outbound wire +// strings through its emit callback. + +#include +#include +#include +#include +#include + +#include "realtime/shared/base_realtime_client.h" +#include "realtime/shared/mini_json.h" +#include "realtime/socketio/event_dispatcher.h" +#include "realtime/socketio/packet_codec.h" + +namespace molo::realtime::socketio::client { + +class SocketIoClient : public shared::BaseRealtimeClient { + public: + // Renders one outbound text packet plus binary frames. + using EmitFn = std::function>& binary)>; + using ConnectCallback = std::function; + using ErrorCallback = + std::function; + + // polling_only=true forces long-polling mode (no WebSocket). + explicit SocketIoClient(bool polling_only = false); + + bool polling_only() const { return polling_only_; } + + void on_emit(EmitFn fn) { emit_fn_ = std::move(fn); } + void on_event(const std::string& nsp, const std::string& event, + EventHandler handler); + void on_connect(ConnectCallback cb) { connect_cb_ = std::move(cb); } + void on_connect_error(ErrorCallback cb) { error_cb_ = std::move(cb); } + + // Feeds one decoded text packet from the transport. + void on_packet(const SocketPacket& packet); + void on_binary(const std::vector& frame); + + // Joins a namespace (sends CONNECT with optional auth object). + void connect_nsp(const std::string& nsp, + shared::Json auth = shared::Json::make_object()); + void disconnect_nsp(const std::string& nsp); + + // Emits an event. ack callback answers when the server acks. + void emit(const std::string& nsp, const std::string& event, + shared::Json args = shared::Json::make_array(), + AckFn ack = AckFn{}); + // Emits a binary event with attachments. + void emit_binary(const std::string& nsp, const std::string& event, + shared::Json args, + const std::vector>& attachments, + AckFn ack = AckFn{}); + + bool connected_nsp(const std::string& nsp) const; + + private: + void send_packet(SocketPacket packet); + + bool polling_only_; + EmitFn emit_fn_; + EventDispatcher dispatcher_; + ConnectCallback connect_cb_; + ErrorCallback error_cb_; + std::set connected_; + std::map pending_acks_; + long long next_ack_ = 0; + SocketPacket pending_binary_; + std::vector> pending_frames_; + bool has_pending_binary_ = false; +}; + +} // namespace molo::realtime::socketio::client diff --git a/src/realtime/socketio/event_dispatcher.cpp b/src/realtime/socketio/event_dispatcher.cpp new file mode 100644 index 0000000..c5ca905 --- /dev/null +++ b/src/realtime/socketio/event_dispatcher.cpp @@ -0,0 +1,36 @@ +// Event dispatch. See event_dispatcher.h. + +#include "realtime/socketio/event_dispatcher.h" + +namespace molo::realtime::socketio { + +void EventDispatcher::on(const std::string& nsp, const std::string& event, + EventHandler handler) { + handlers_[nsp][event] = std::move(handler); +} + +bool EventDispatcher::dispatch(const std::string& sid, const std::string& nsp, + const shared::Json& data, AckFn ack) const { + if (data.type() != shared::Json::Type::kArray || data.array().empty() || + data.array()[0].type() != shared::Json::Type::kString) { + return false; + } + auto ns = handlers_.find(nsp); + if (ns == handlers_.end()) { + return false; + } + auto h = ns->second.find(data.array()[0].string()); + if (h == ns->second.end()) { + return false; + } + h->second(sid, data, std::move(ack)); + return true; +} + +bool EventDispatcher::has(const std::string& nsp, + const std::string& event) const { + auto ns = handlers_.find(nsp); + return ns != handlers_.end() && ns->second.count(event) > 0; +} + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/event_dispatcher.h b/src/realtime/socketio/event_dispatcher.h new file mode 100644 index 0000000..dc445c4 --- /dev/null +++ b/src/realtime/socketio/event_dispatcher.h @@ -0,0 +1,38 @@ +#pragma once + +// Event dispatch: maps (namespace, event name) to handlers. EVENT +// payloads are JSON arrays whose first element is the event name. +// Handlers receive the sender sid, the full argument array, and an +// ack callback that answers when the sender passed an ack id. + +#include +#include +#include +#include + +#include "realtime/shared/mini_json.h" + +namespace molo::realtime::socketio { + +using AckFn = std::function; +using EventHandler = std::function; + +class EventDispatcher { + public: + // Registers a handler for one event in one namespace. + void on(const std::string& nsp, const std::string& event, + EventHandler handler); + + // Dispatches an EVENT packet's data array. Returns false when no + // handler matches or the payload shape is wrong. + bool dispatch(const std::string& sid, const std::string& nsp, + const shared::Json& data, AckFn ack) const; + + bool has(const std::string& nsp, const std::string& event) const; + + private: + std::map> handlers_; +}; + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/namespace.cpp b/src/realtime/socketio/namespace.cpp new file mode 100644 index 0000000..d93c6be --- /dev/null +++ b/src/realtime/socketio/namespace.cpp @@ -0,0 +1,30 @@ +// Socket.IO namespace. See namespace.h. + +#include "realtime/socketio/namespace.h" + +namespace molo::realtime::socketio { + +Namespace::Namespace(std::string name, NamespaceAccept accept) + : name_(std::move(name)), accept_(std::move(accept)) {} + +std::string Namespace::check_connect(const std::string& sid, + const shared::Json& auth) const { + if (!accept_) { + return {}; + } + return accept_(sid, auth); +} + +void Namespace::add(const std::string& sid) { + sids_.insert(sid); +} + +void Namespace::remove(const std::string& sid) { + sids_.erase(sid); +} + +bool Namespace::contains(const std::string& sid) const { + return sids_.count(sid) > 0; +} + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/namespace.h b/src/realtime/socketio/namespace.h new file mode 100644 index 0000000..ca3135e --- /dev/null +++ b/src/realtime/socketio/namespace.h @@ -0,0 +1,42 @@ +#pragma once + +// One Socket.IO namespace: middleware acceptance plus the set of +// connected session ids. CONNECT carries auth data; the accept +// callback refuses with a CONNECT_ERROR reason or admits. Namespaces +// start with '/', no commas. + +#include +#include +#include + +#include "realtime/shared/mini_json.h" + +namespace molo::realtime::socketio { + +// Return empty to admit, or a reason string to refuse. +using NamespaceAccept = std::function; + +class Namespace { + public: + explicit Namespace(std::string name, + NamespaceAccept accept = NamespaceAccept{}); + + const std::string& name() const { return name_; } + + // Runs acceptance for a CONNECT. Empty result admits. + std::string check_connect(const std::string& sid, + const shared::Json& auth) const; + + void add(const std::string& sid); + void remove(const std::string& sid); + bool contains(const std::string& sid) const; + std::size_t size() const { return sids_.size(); } + + private: + std::string name_; + NamespaceAccept accept_; + std::set sids_; +}; + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/packet_codec.cpp b/src/realtime/socketio/packet_codec.cpp new file mode 100644 index 0000000..1c92bfe --- /dev/null +++ b/src/realtime/socketio/packet_codec.cpp @@ -0,0 +1,181 @@ +// Socket.IO v5 packet codec. See packet_codec.h. + +#include "realtime/socketio/packet_codec.h" + +namespace molo::realtime::socketio { +namespace { + +// Counts {"_placeholder":true,"num":N} markers in document order. +bool collect_placeholders(const shared::Json& v, std::vector& nums) { + using Type = shared::Json::Type; + if (v.type() == Type::kObject) { + const shared::Json* ph = v.find("_placeholder"); + const shared::Json* num = v.find("num"); + if (ph != nullptr && num != nullptr && ph->type() == Type::kBool && + ph->boolean() && num->type() == Type::kInt && + v.object().size() == 2) { + if (num->integer() < 0) { + return false; + } + nums.push_back(static_cast(num->integer())); + return true; + } + for (const auto& [k, child] : v.object()) { + if (!collect_placeholders(child, nums)) { + return false; + } + } + return true; + } + if (v.type() == Type::kArray) { + for (const auto& child : v.array()) { + if (!collect_placeholders(child, nums)) { + return false; + } + } + } + return true; +} + +bool is_binary(SocketPacketType t) { + return t == SocketPacketType::kBinaryEvent || + t == SocketPacketType::kBinaryAck; +} + +} // namespace + +bool PacketCodec::valid_nsp(const std::string& nsp) { + return nsp.size() >= 1 && nsp[0] == '/' && + nsp.find(',') == std::string::npos; +} + +std::optional PacketCodec::encode_text( + const SocketPacket& packet) { + if (!valid_nsp(packet.nsp)) { + return std::nullopt; + } + std::vector nums; + if (is_binary(packet.type)) { + if (!collect_placeholders(packet.data, nums)) { + return std::nullopt; + } + if (nums.size() != packet.attachments.size() || + nums.size() > shared::Caps::kMaxAttachments) { + return std::nullopt; + } + std::size_t total = 0; + for (const auto& b : packet.attachments) { + total += b.size(); + } + if (total > shared::Caps::kMaxAttachmentBytes) { + return std::nullopt; + } + } + std::string out; + out.push_back( + static_cast('0' + static_cast(packet.type))); + if (is_binary(packet.type)) { + out += std::to_string(nums.size()) + "-"; + } + if (packet.nsp != "/") { + out += packet.nsp + ","; + } + if (packet.ack_id >= 0) { + out += std::to_string(packet.ack_id); + } + if (packet.type == SocketPacketType::kDisconnect && + packet.data.type() == shared::Json::Type::kArray && + packet.data.array().empty()) { + return out; // DISCONNECT usually carries no payload. + } + out += packet.data.dump(); + return out; +} + +std::optional PacketCodec::decode_text(const std::string& text) { + if (text.empty() || text[0] < '0' || text[0] > '6') { + return std::nullopt; + } + SocketPacket p; + p.type = static_cast(text[0] - '0'); + std::size_t i = 1; + std::size_t attachments = 0; + if (is_binary(p.type)) { + std::size_t dash = text.find('-', i); + if (dash == std::string::npos) { + return std::nullopt; + } + try { + long long n = std::stoll(text.substr(i, dash - i)); + if (n < 0 || static_cast(n) > shared::Caps::kMaxAttachments) { + return std::nullopt; + } + attachments = static_cast(n); + } catch (...) { + return std::nullopt; + } + i = dash + 1; + } + if (i < text.size() && text[i] == '/') { + std::size_t comma = text.find(',', i); + if (comma == std::string::npos) { + return std::nullopt; + } + p.nsp = text.substr(i, comma - i); + if (!valid_nsp(p.nsp)) { + return std::nullopt; + } + i = comma + 1; + } + std::size_t digits = 0; + while (i + digits < text.size() && + text[i + digits] >= '0' && text[i + digits] <= '9') { + ++digits; + } + if (digits > 0) { + try { + p.ack_id = std::stoll(text.substr(i, digits)); + } catch (...) { + return std::nullopt; + } + i += digits; + } + if (i < text.size()) { + auto data = shared::Json::parse(text.substr(i)); + if (!data.has_value()) { + return std::nullopt; + } + p.data = std::move(*data); + } else { + p.data = shared::Json::make_array(); + } + if (is_binary(p.type)) { + std::vector nums; + if (!collect_placeholders(p.data, nums) || nums.size() != attachments) { + return std::nullopt; + } + p.attachments.resize(attachments); + } + return p; +} + +bool PacketCodec::attach(SocketPacket& packet, + const std::vector>& buffers) { + if (!is_binary(packet.type)) { + return buffers.empty(); + } + if (buffers.size() != packet.attachments.size()) { + return false; + } + std::size_t total = 0; + for (const auto& b : buffers) { + total += b.size(); + } + if (total > shared::Caps::kMaxAttachmentBytes) { + return false; + } + packet.attachments = buffers; + return true; +} + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/packet_codec.h b/src/realtime/socketio/packet_codec.h new file mode 100644 index 0000000..53d4e49 --- /dev/null +++ b/src/realtime/socketio/packet_codec.h @@ -0,0 +1,60 @@ +#pragma once + +// Socket.IO v5 packet codec. Types: CONNECT 0, DISCONNECT 1, +// EVENT 2, ACK 3, CONNECT_ERROR 4, BINARY_EVENT 5, BINARY_ACK 6. +// Text encoding: [-][,][], +// namespace omitted when "/". Binary payloads travel as follow-up +// message packets; the codec extracts placeholder objects +// {"_placeholder":true,"num":N} and reattaches buffers in order. + +#include +#include +#include +#include + +#include "realtime/shared/caps.h" +#include "realtime/shared/mini_json.h" + +namespace molo::realtime::socketio { + +enum class SocketPacketType : std::uint8_t { + kConnect = 0, + kDisconnect = 1, + kEvent = 2, + kAck = 3, + kConnectError = 4, + kBinaryEvent = 5, + kBinaryAck = 6, +}; + +struct SocketPacket { + SocketPacketType type = SocketPacketType::kEvent; + std::string nsp = "/"; + // ack_id >= 0 means present. + long long ack_id = -1; + shared::Json data = shared::Json::make_array(); + // Binary attachments in placeholder order. + std::vector> attachments; +}; + +class PacketCodec { + public: + // Encodes the text packet (placeholders inline for binary types). + // Returns nullopt when the namespace is invalid or attachment + // counts exceed caps. + static std::optional encode_text(const SocketPacket& packet); + + // Decodes a text packet, expecting attachment_count binary frames + // to follow for binary types. Returns nullopt on any error. + static std::optional decode_text(const std::string& text); + + // Replaces placeholders in packet.data with the buffers, in order. + // Returns false on count mismatch or malformed placeholders. + // Buffers beyond the cap are rejected. + static bool attach(SocketPacket& packet, + const std::vector>& buffers); + + static bool valid_nsp(const std::string& nsp); +}; + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/room_manager.cpp b/src/realtime/socketio/room_manager.cpp new file mode 100644 index 0000000..8354c26 --- /dev/null +++ b/src/realtime/socketio/room_manager.cpp @@ -0,0 +1,128 @@ +// Server-side rooms. See room_manager.h. + +#include "realtime/socketio/room_manager.h" + +#include "realtime/shared/realtime_metrics.h" +#include "realtime/socketio/packet_codec.h" + +namespace molo::realtime::socketio { + +bool RoomManager::valid_room(const std::string& room) { + return !room.empty() && room.size() <= 256 && + room.find('\0') == std::string::npos; +} + +bool RoomManager::join(const std::string& nsp, const std::string& room, + const std::string& sid) { + if (!PacketCodec::valid_nsp(nsp) || !valid_room(room) || sid.empty()) { + return false; + } + rooms_[nsp][room].insert(sid); + return true; +} + +bool RoomManager::leave(const std::string& nsp, const std::string& room, + const std::string& sid) { + auto ns = rooms_.find(nsp); + if (ns == rooms_.end()) { + return false; + } + auto r = ns->second.find(room); + if (r == ns->second.end()) { + return false; + } + bool erased = r->second.erase(sid) > 0; + if (r->second.empty()) { + ns->second.erase(r); + } + return erased; +} + +void RoomManager::leave_all(const std::string& nsp, const std::string& sid) { + auto ns = rooms_.find(nsp); + if (ns == rooms_.end()) { + return; + } + for (auto it = ns->second.begin(); it != ns->second.end();) { + it->second.erase(sid); + if (it->second.empty()) { + it = ns->second.erase(it); + } else { + ++it; + } + } +} + +std::vector RoomManager::members( + const std::string& nsp, const std::string& room) const { + auto ns = rooms_.find(nsp); + if (ns == rooms_.end()) { + return {}; + } + auto r = ns->second.find(room); + if (r == ns->second.end()) { + return {}; + } + return std::vector(r->second.begin(), r->second.end()); +} + +std::vector RoomManager::rooms_of( + const std::string& nsp, const std::string& sid) const { + std::vector out; + auto ns = rooms_.find(nsp); + if (ns == rooms_.end()) { + return out; + } + for (const auto& [room, sids] : ns->second) { + if (sids.count(sid) > 0) { + out.push_back(room); + } + } + return out; +} + +std::size_t RoomManager::room_size(const std::string& nsp, + const std::string& room) const { + auto ns = rooms_.find(nsp); + if (ns == rooms_.end()) { + return 0; + } + auto r = ns->second.find(room); + return r == ns->second.end() ? 0 : r->second.size(); +} + +std::size_t RoomManager::broadcast( + const std::string& nsp, const std::string& room, + const std::string& exclude_sid, + const std::function& deliver) { + auto ns = rooms_.find(nsp); + if (ns == rooms_.end()) { + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kBroadcastNoop); + return 0; + } + auto r = ns->second.find(room); + if (r == ns->second.end()) { + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kBroadcastNoop); + return 0; + } + std::size_t n = 0; + for (const std::string& sid : r->second) { + if (!exclude_sid.empty() && sid == exclude_sid) { + continue; + } + deliver(sid); + ++n; + } + if (n == 0) { + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kBroadcastNoop); + } else { + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kBroadcastFanout, n); + } + return n; +} + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/room_manager.h b/src/realtime/socketio/room_manager.h new file mode 100644 index 0000000..e75b334 --- /dev/null +++ b/src/realtime/socketio/room_manager.h @@ -0,0 +1,46 @@ +#pragma once + +// Server-side room sets: join, leave, membership, broadcast fan-out +// in O(subscribers). Broadcast to 0 subscribers is a counted no-op, +// never an error. Per-recipient delivery goes through caller +// callbacks so queue caps stay with the connection. + +#include +#include +#include +#include +#include +#include + +namespace molo::realtime::socketio { + +class RoomManager { + public: + // Joins sid to room. Returns false on bad names. + bool join(const std::string& nsp, const std::string& room, + const std::string& sid); + bool leave(const std::string& nsp, const std::string& room, + const std::string& sid); + // Removes sid from every room (on disconnect). + void leave_all(const std::string& nsp, const std::string& sid); + + std::vector members(const std::string& nsp, + const std::string& room) const; + std::vector rooms_of(const std::string& nsp, + const std::string& sid) const; + std::size_t room_size(const std::string& nsp, + const std::string& room) const; + + // Calls deliver(sid) for every member except exclude_sid (empty + // means include all). Returns the recipient count. + std::size_t broadcast(const std::string& nsp, const std::string& room, + const std::string& exclude_sid, + const std::function& deliver); + + private: + static bool valid_room(const std::string& room); + // nsp -> room -> members. + std::map>> rooms_; +}; + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/socketio_server.cpp b/src/realtime/socketio/socketio_server.cpp new file mode 100644 index 0000000..d135c17 --- /dev/null +++ b/src/realtime/socketio/socketio_server.cpp @@ -0,0 +1,184 @@ +// Socket.IO server. See socketio_server.h. + +#include "realtime/socketio/socketio_server.h" + +#include "realtime/shared/realtime_log.h" +#include "realtime/shared/realtime_metrics.h" + +namespace molo::realtime::socketio { + +SocketIoServer::SocketIoServer() { + namespaces_.emplace("/", Namespace("/")); +} + +Namespace& SocketIoServer::get_or_create_namespace(const std::string& nsp) { + auto it = namespaces_.find(nsp); + if (it != namespaces_.end()) { + return it->second; + } + return namespaces_.emplace(nsp, Namespace(nsp)).first->second; +} + +bool SocketIoServer::has_namespace(const std::string& nsp) const { + return namespaces_.count(nsp) > 0; +} + +void SocketIoServer::on_packet(const std::string& sid, + const SocketPacket& packet) { + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kMessagesReceived); + if (packet.type == SocketPacketType::kBinaryEvent || + packet.type == SocketPacketType::kBinaryAck) { + if (!packet.attachments.empty()) { + pending_binary_[sid] = packet; + pending_frames_[sid].clear(); + } + return; + } + switch (packet.type) { + case SocketPacketType::kConnect: + handle_connect(sid, packet); + break; + case SocketPacketType::kDisconnect: + disconnect_sid(sid, packet.nsp); + break; + case SocketPacketType::kEvent: + handle_event(sid, packet); + break; + case SocketPacketType::kAck: + if (packet.ack_id >= 0) { + acks_.resolve(packet.ack_id); + } + break; + default: + break; // CONNECT_ERROR inbound is a client bug; ignore. + } +} + +void SocketIoServer::on_binary(const std::string& sid, + const std::vector& frame) { + auto it = pending_binary_.find(sid); + if (it == pending_binary_.end()) { + return; + } + pending_frames_[sid].push_back(frame); + if (pending_frames_[sid].size() < it->second.attachments.size()) { + return; + } + SocketPacket packet = it->second; + pending_binary_.erase(it); + auto frames = std::move(pending_frames_[sid]); + pending_frames_.erase(sid); + if (!PacketCodec::attach(packet, frames)) { + return; + } + if (packet.type == SocketPacketType::kBinaryEvent) { + packet.type = SocketPacketType::kEvent; + handle_event(sid, packet); + } else if (packet.ack_id >= 0) { + acks_.resolve(packet.ack_id); + } +} + +bool SocketIoServer::emit_to(const std::string& sid, const std::string& nsp, + const std::string& event, shared::Json args, + long long ack_id) { + shared::Json data = shared::Json::make_array(); + data.push(shared::Json::str(event)); + if (args.type() == shared::Json::Type::kArray) { + for (const auto& a : args.array()) { + data.push(a); + } + } else { + data.push(std::move(args)); + } + SocketPacket p; + p.type = SocketPacketType::kEvent; + p.nsp = nsp; + p.ack_id = ack_id; + p.data = std::move(data); + send_packet(sid, p); + return true; +} + +std::size_t SocketIoServer::emit_to_room( + const std::string& nsp, const std::string& room, const std::string& event, + shared::Json args, const std::string& sender_sid, bool include_sender) { + std::string exclude = include_sender ? "" : sender_sid; + return rooms_.broadcast(nsp, room, exclude, [&](const std::string& sid) { + emit_to(sid, nsp, event, args); + }); +} + +void SocketIoServer::disconnect_sid(const std::string& sid, + const std::string& nsp) { + rooms_.leave_all(nsp, sid); + auto it = namespaces_.find(nsp); + if (it != namespaces_.end()) { + it->second.remove(sid); + } + SocketPacket p; + p.type = SocketPacketType::kDisconnect; + p.nsp = nsp; + send_packet(sid, p); +} + +void SocketIoServer::send_packet(const std::string& sid, + const SocketPacket& packet) { + if (!sink_) { + return; + } + auto text = PacketCodec::encode_text(packet); + if (!text.has_value()) { + shared::RealtimeLog::warn("socketio", sid, "encode failed"); + return; + } + sink_(sid, *text, packet.attachments); + shared::RealtimeMetrics::instance().inc(shared::RealtimeMetric::kMessagesSent); +} + +void SocketIoServer::handle_connect(const std::string& sid, + const SocketPacket& packet) { + Namespace& ns = get_or_create_namespace(packet.nsp); + std::string refusal = ns.check_connect(sid, packet.data); + if (!refusal.empty()) { + SocketPacket err; + err.type = SocketPacketType::kConnectError; + err.nsp = packet.nsp; + err.data = shared::Json::make_object(); + err.data.set("message", shared::Json::str(refusal)); + send_packet(sid, err); + return; + } + ns.add(sid); + SocketPacket ok; + ok.type = SocketPacketType::kConnect; + ok.nsp = packet.nsp; + ok.data = shared::Json::make_object(); + ok.data.set("sid", shared::Json::str(sid)); + send_packet(sid, ok); +} + +void SocketIoServer::handle_event(const std::string& sid, + const SocketPacket& packet) { + auto ns = namespaces_.find(packet.nsp); + if (ns == namespaces_.end() || !ns->second.contains(sid)) { + return; + } + long long ack_id = packet.ack_id; + std::string nsp = packet.nsp; + AckFn ack; + if (ack_id >= 0) { + ack = [this, sid, nsp, ack_id](shared::Json args) { + SocketPacket p; + p.type = SocketPacketType::kAck; + p.nsp = nsp; + p.ack_id = ack_id; + p.data = std::move(args); + send_packet(sid, p); + }; + } + dispatcher_.dispatch(sid, packet.nsp, packet.data, std::move(ack)); +} + +} // namespace molo::realtime::socketio diff --git a/src/realtime/socketio/socketio_server.h b/src/realtime/socketio/socketio_server.h new file mode 100644 index 0000000..496c02a --- /dev/null +++ b/src/realtime/socketio/socketio_server.h @@ -0,0 +1,78 @@ +#pragma once + +// Socket.IO server: owns namespaces, rooms, dispatch, and per-sid +// outbound packet sinks. Transports (polling or WebSocket) feed +// decoded Socket.IO text packets plus binary frames via on_packet(); +// the server answers CONNECT/DISCONNECT/EVENT/ACK per the v5 spec +// and emits replies through the registered sink. Each packet the +// server emits is one Engine.IO message away from the wire (the +// caller adds the "4" prefix). + +#include +#include +#include +#include +#include + +#include "realtime/shared/connection_context.h" +#include "realtime/socketio/ack_tracker.h" +#include "realtime/socketio/event_dispatcher.h" +#include "realtime/socketio/namespace.h" +#include "realtime/socketio/packet_codec.h" +#include "realtime/socketio/room_manager.h" + +namespace molo::realtime::socketio { + +class SocketIoServer { + public: + // Sends one text packet plus optional binary frames to a sid. + using Sink = + std::function>& binary)>; + + SocketIoServer(); + + Namespace& get_or_create_namespace(const std::string& nsp); + bool has_namespace(const std::string& nsp) const; + EventDispatcher& dispatcher() { return dispatcher_; } + RoomManager& rooms() { return rooms_; } + + void set_sink(Sink sink) { sink_ = std::move(sink); } + + // Handles one decoded text packet from a sid. Binary frames for + // binary packets arrive via on_binary() afterwards. + void on_packet(const std::string& sid, const SocketPacket& packet); + // Handles one binary frame for a pending binary packet. + void on_binary(const std::string& sid, const std::vector& frame); + + // Emits an event to one sid. ack_id >= 0 requests an ack. + bool emit_to(const std::string& sid, const std::string& nsp, + const std::string& event, shared::Json args, + long long ack_id = -1); + // Emits to a room. Excludes the sender by default; include_sender + // overrides. Returns the recipient count. + std::size_t emit_to_room(const std::string& nsp, const std::string& room, + const std::string& event, shared::Json args, + const std::string& sender_sid = "", + bool include_sender = false); + + void disconnect_sid(const std::string& sid, const std::string& nsp = "/"); + + AckTracker& acks() { return acks_; } + + private: + void send_packet(const std::string& sid, const SocketPacket& packet); + void handle_connect(const std::string& sid, const SocketPacket& packet); + void handle_event(const std::string& sid, const SocketPacket& packet); + + std::map namespaces_; + EventDispatcher dispatcher_; + RoomManager rooms_; + AckTracker acks_; + Sink sink_; + // Pending binary packets awaiting frames: sid -> packet. + std::map pending_binary_; + std::map>> pending_frames_; +}; + +} // namespace molo::realtime::socketio diff --git a/src/realtime/sse/event_encoder.cpp b/src/realtime/sse/event_encoder.cpp new file mode 100644 index 0000000..27f3496 --- /dev/null +++ b/src/realtime/sse/event_encoder.cpp @@ -0,0 +1,70 @@ +// SSE wire encoder. See event_encoder.h. + +#include "realtime/sse/event_encoder.h" + +namespace molo::realtime::sse { + +std::string EventEncoder::encode(const SseEvent& event) { + std::string out; + if (!event.event.empty()) { + out += "event: " + event.event + "\n"; + } + std::size_t start = 0; + // Normalize CRLF/CR to LF and emit one data line per line. + std::string flat; + flat.reserve(event.data.size()); + for (std::size_t i = 0; i < event.data.size(); ++i) { + char c = event.data[i]; + if (c == '\r') { + if (i + 1 < event.data.size() && event.data[i + 1] == '\n') { + ++i; + } + flat.push_back('\n'); + } else { + flat.push_back(c); + } + } + while (true) { + std::size_t nl = flat.find('\n', start); + std::string line = nl == std::string::npos + ? flat.substr(start) + : flat.substr(start, nl - start); + out += "data: " + line + "\n"; + if (nl == std::string::npos) { + break; + } + start = nl + 1; + } + if (!event.id.empty()) { + out += "id: " + event.id + "\n"; + } + if (event.retry.has_value()) { + out += "retry: " + std::to_string(clamp_retry(*event.retry).count()) + + "\n"; + } + out += "\n"; + return out; +} + +std::string EventEncoder::comment(const std::string& text) { + std::string clean; + for (char c : text) { + if (c != '\r' && c != '\n') { + clean.push_back(c); + } + } + return ": " + clean + "\n\n"; +} + +std::chrono::milliseconds EventEncoder::clamp_retry( + std::chrono::milliseconds d) { + if (d < shared::Caps::kSseRetryMin) { + return shared::Caps::kSseRetryMin; + } + if (d > shared::Caps::kSseRetryMax) { + return shared::Caps::kSseRetryMax; + } + return d; +} + +} // namespace molo::realtime::sse diff --git a/src/realtime/sse/event_encoder.h b/src/realtime/sse/event_encoder.h new file mode 100644 index 0000000..e04fabf --- /dev/null +++ b/src/realtime/sse/event_encoder.h @@ -0,0 +1,33 @@ +#pragma once + +// SSE wire encoder (WHATWG HTML 9.2). Renders events as `field: +// value` lines plus a blank dispatch line. Multiline data emits one +// `data:` line per line. Byte-exact: LF line endings, one space +// after the colon, trailing blank line. + +#include +#include +#include +#include + +#include "realtime/shared/caps.h" + +namespace molo::realtime::sse { + +struct SseEvent { + std::string event; // Empty means the default "message" type. + std::string data; + std::string id; // Empty means no id line. + std::optional retry; +}; + +class EventEncoder { + public: + static std::string encode(const SseEvent& event); + // Keep-alive comment line, e.g. ": ping\n\n". + static std::string comment(const std::string& text); + // Clamps retry delay to 1 s..300 s per spec. + static std::chrono::milliseconds clamp_retry(std::chrono::milliseconds d); +}; + +} // namespace molo::realtime::sse diff --git a/src/realtime/sse/event_store.cpp b/src/realtime/sse/event_store.cpp new file mode 100644 index 0000000..8ab9027 --- /dev/null +++ b/src/realtime/sse/event_store.cpp @@ -0,0 +1,58 @@ +// In-memory SSE replay buffer. See event_store.h. + +#include "realtime/sse/event_store.h" + +namespace molo::realtime::sse { + +EventStore::EventStore(std::size_t max_events, std::chrono::minutes max_age) + : max_events_(max_events == 0 ? 1 : max_events), max_age_(max_age) {} + +SseEvent EventStore::append(SseEvent event, + std::chrono::steady_clock::time_point now) { + if (event.id.empty()) { + event.id = std::to_string(next_id_++); + } + entries_.push_back(Entry{event, now}); + evict(now); + return event; +} + +std::vector EventStore::replay_after( + const std::string& last_id, std::chrono::steady_clock::time_point now) { + evict(now); + std::vector out; + bool collect = last_id.empty(); + bool known = last_id.empty(); + for (const Entry& e : entries_) { + if (!collect && e.event.id == last_id) { + collect = true; + known = true; + continue; + } + if (collect) { + out.push_back(e.event); + } + } + if (!known) { + out.clear(); + for (const Entry& e : entries_) { + out.push_back(e.event); + } + } + return out; +} + +void EventStore::clear() { + entries_.clear(); +} + +void EventStore::evict(std::chrono::steady_clock::time_point now) { + while (entries_.size() > max_events_) { + entries_.pop_front(); + } + while (!entries_.empty() && now - entries_.front().at > max_age_) { + entries_.pop_front(); + } +} + +} // namespace molo::realtime::sse diff --git a/src/realtime/sse/event_store.h b/src/realtime/sse/event_store.h new file mode 100644 index 0000000..99031f8 --- /dev/null +++ b/src/realtime/sse/event_store.h @@ -0,0 +1,56 @@ +#pragma once + +// In-memory SSE replay buffer with the documented EventStore +// interface (SPEC.md gap G4: a persistent 04-data store implements +// the same interface later). Holds the last 1000 events or 5 minutes +// per stream, whichever is smaller. Ids are caller-assigned +// monotonic strings; replay_after(id) returns newer events in order. + +#include +#include +#include +#include +#include +#include + +#include "realtime/shared/caps.h" +#include "realtime/sse/event_encoder.h" + +namespace molo::realtime::sse { + +class EventStore { + public: + explicit EventStore( + std::size_t max_events = shared::Caps::kSseReplayMaxEvents, + std::chrono::minutes max_age = shared::Caps::kSseReplayMaxAge); + + // Appends an event, assigning an id from the monotonic counter + // when empty. Returns the stored event with its final id. + SseEvent append(SseEvent event, + std::chrono::steady_clock::time_point now = + std::chrono::steady_clock::now()); + + // Events newer than last_id (exclusive), oldest first. Unknown or + // empty last_id replays everything within the window. + std::vector replay_after( + const std::string& last_id, + std::chrono::steady_clock::time_point now = + std::chrono::steady_clock::now()); + + std::size_t size() const { return entries_.size(); } + void clear(); + + private: + struct Entry { + SseEvent event; + std::chrono::steady_clock::time_point at; + }; + void evict(std::chrono::steady_clock::time_point now); + + std::deque entries_; + std::size_t max_events_; + std::chrono::minutes max_age_; + std::uint64_t next_id_ = 1; +}; + +} // namespace molo::realtime::sse diff --git a/src/realtime/sse/sse_client.cpp b/src/realtime/sse/sse_client.cpp new file mode 100644 index 0000000..a8b9f57 --- /dev/null +++ b/src/realtime/sse/sse_client.cpp @@ -0,0 +1,133 @@ +// EventSource-compatible SSE client. See sse_client.h. + +#include "realtime/sse/sse_client.h" + +namespace molo::realtime::sse { + +SseClient::SseClient() : shared::BaseRealtimeClient("sse-client") {} + +bool SseClient::on_head(const shared::HttpResponse& head) { + redirect_url_.clear(); + failure_reason_.clear(); + stopped_ = false; + if (head.status == 204) { + stopped_ = true; + set_state(shared::ClientState::kClosed); + log_info("server asked to stop (204)"); + return false; + } + if (head.status == 301 || head.status == 307) { + redirect_url_ = shared::header_first(head, "Location"); + set_state(shared::ClientState::kConnecting); + log_info("redirect to " + redirect_url_); + return false; + } + if (head.status != 200) { + failure_reason_ = "expected 200, got " + std::to_string(head.status); + set_state(shared::ClientState::kConnecting); + return false; + } + std::string ctype = shared::header_first(head, "Content-Type"); + if (ctype.find("text/event-stream") == std::string::npos) { + failure_reason_ = "wrong content type: " + ctype; + set_state(shared::ClientState::kConnecting); + return false; + } + note_connected(); + log_info("open"); + if (open_cb_) { + open_cb_(); + } + return true; +} + +void SseClient::on_bytes(const uint8_t* data, std::size_t len) { + buffer_.append(reinterpret_cast(data), len); + dispatch_buffer(); +} + +void SseClient::on_bytes(const std::string& data) { + on_bytes(reinterpret_cast(data.data()), data.size()); +} + +void SseClient::close() { + set_state(shared::ClientState::kClosed); +} + +void SseClient::dispatch_buffer() { + // Split on LF, tolerating CRLF. A blank line dispatches. + std::size_t pos = 0; + while (true) { + std::size_t nl = buffer_.find('\n', pos); + if (nl == std::string::npos) { + break; + } + std::string line = buffer_.substr(pos, nl - pos); + pos = nl + 1; + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + if (line.empty()) { + if (pending_has_data_) { + SseEvent e; + e.event = pending_event_; + e.data = pending_data_; + e.id = pending_id_; + if (!pending_id_.empty()) { + last_event_id_ = pending_id_; + } + pending_event_.clear(); + pending_data_.clear(); + pending_has_data_ = false; + pending_id_.clear(); + if (event_cb_) { + event_cb_(e); + } + } else { + pending_event_.clear(); + pending_id_.clear(); + } + continue; + } + if (line[0] == ':') { + continue; // Comment / keep-alive. + } + std::size_t colon = line.find(':'); + std::string field = colon == std::string::npos ? line + : line.substr(0, colon); + std::string value = colon == std::string::npos + ? "" + : line.substr(colon + 1); + if (!value.empty() && value[0] == ' ') { + value.erase(0, 1); + } + if (field == "event") { + pending_event_ = value; + } else if (field == "data") { + if (pending_has_data_) { + pending_data_ += "\n"; + } + pending_data_ += value; + pending_has_data_ = true; + } else if (field == "id") { + pending_id_ = value; + } else if (field == "retry") { + try { + long long ms = std::stoll(value); + if (ms >= 0) { + retry_delay_ = EventEncoder::clamp_retry( + std::chrono::milliseconds(ms)); + } + } catch (...) { + // Ignore malformed retry values. + } + } + } + buffer_.erase(0, pos); + // Cap buffered partial lines at 1 MiB to bound memory. + if (buffer_.size() > 1024u * 1024u) { + buffer_.erase(0, buffer_.size() - 1024u * 1024u); + } +} + +} // namespace molo::realtime::sse diff --git a/src/realtime/sse/sse_client.h b/src/realtime/sse/sse_client.h new file mode 100644 index 0000000..9aafd10 --- /dev/null +++ b/src/realtime/sse/sse_client.h @@ -0,0 +1,67 @@ +#pragma once + +// EventSource-compatible SSE client (rule 8). States CONNECTING 0, +// OPEN 1, CLOSED 2. Parses the event stream incrementally, tracks +// Last-Event-ID, honors retry:, redirect (301/307) and 204 stop. +// Reconnect timing comes from BaseRealtimeClient; the caller owns +// the socket and feeds bytes plus status lines. + +#include +#include +#include +#include +#include + +#include "realtime/shared/base_realtime_client.h" +#include "realtime/shared/http_seam.h" +#include "realtime/sse/event_encoder.h" + +namespace molo::realtime::sse { + +class SseClient : public shared::BaseRealtimeClient { + public: + using EventCallback = std::function; + using OpenCallback = std::function; + + SseClient(); + + // Inspects the response head of a (re)connect attempt. Returns + // true when the stream opens. Sets redirect_url on 301/307, + // stopped on 204, failure_reason otherwise. + bool on_head(const shared::HttpResponse& head); + + std::string redirect_url() const { return redirect_url_; } + bool stopped() const { return stopped_; } + std::string failure_reason() const { return failure_reason_; } + + // Value for the Last-Event-ID header on reconnect, empty when none. + std::string last_event_id() const { return last_event_id_; } + std::chrono::milliseconds retry_delay() const { return retry_delay_; } + + // Feeds stream bytes, dispatching complete events. + void on_bytes(const uint8_t* data, std::size_t len); + void on_bytes(const std::string& data); + + void on_event(EventCallback cb) { event_cb_ = std::move(cb); } + void on_open(OpenCallback cb) { open_cb_ = std::move(cb); } + + void close(); + + private: + void dispatch_buffer(); + + std::string buffer_; + std::string last_event_id_; + std::string pending_event_; + std::string pending_data_; + bool pending_has_data_ = false; + std::string pending_id_; + std::chrono::milliseconds retry_delay_{3000}; + std::string redirect_url_; + std::string failure_reason_; + bool stopped_ = false; + EventCallback event_cb_; + OpenCallback open_cb_; +}; + +} // namespace molo::realtime::sse diff --git a/src/realtime/sse/sse_response.cpp b/src/realtime/sse/sse_response.cpp new file mode 100644 index 0000000..49c7a6d --- /dev/null +++ b/src/realtime/sse/sse_response.cpp @@ -0,0 +1,53 @@ +// Server-side SSE stream. See sse_response.h. + +#include "realtime/sse/sse_response.h" + +namespace molo::realtime::sse { + +SseResponse::SseResponse(EventStore* store) + : store_(store != nullptr ? store : &owned_) {} + +shared::HttpResponse SseResponse::start(const std::string& last_event_id) { + (void)last_event_id; + shared::HttpResponse res; + res.status = 200; + res.stream = true; + shared::set_header(res, "Content-Type", "text/event-stream"); + shared::set_header(res, "Cache-Control", "no-store"); + shared::set_header(res, "Connection", "keep-alive"); + return res; +} + +std::string SseResponse::replay_bytes(const std::string& last_event_id) { + std::string out; + if (last_event_id.empty()) { + return out; + } + for (const SseEvent& e : store_->replay_after(last_event_id)) { + out += EventEncoder::encode(e); + ++sent_count_; + } + return out; +} + +std::string SseResponse::send(SseEvent event) { + if (!retry_sent_ && initial_retry_.count() > 0) { + retry_sent_ = true; + event.retry = initial_retry_; + } + SseEvent stored = store_->append(std::move(event)); + ++sent_count_; + return EventEncoder::encode(stored); +} + +std::string SseResponse::keep_alive() { + return EventEncoder::comment("ping"); +} + +shared::HttpResponse SseResponse::stop() { + shared::HttpResponse res; + res.status = 204; + return res; +} + +} // namespace molo::realtime::sse diff --git a/src/realtime/sse/sse_response.h b/src/realtime/sse/sse_response.h new file mode 100644 index 0000000..0a664f8 --- /dev/null +++ b/src/realtime/sse/sse_response.h @@ -0,0 +1,50 @@ +#pragma once + +// Server-side SSE stream. Renders the response headers (rule 6), +// replays missed events on Last-Event-ID, assigns monotonic ids, +// sends keep-alive comments, and honors 204 stop. The runtime flushes +// headers immediately at start() and pumps send() output. + +#include +#include +#include + +#include "realtime/shared/connection_context.h" +#include "realtime/shared/http_seam.h" +#include "realtime/sse/event_encoder.h" +#include "realtime/sse/event_store.h" + +namespace molo::realtime::sse { + +class SseResponse { + public: + explicit SseResponse(EventStore* store = nullptr); + + // Builds the response head. When last_event_id is set the caller + // should emit replay_bytes() right after the headers. + shared::HttpResponse start(const std::string& last_event_id = ""); + + // Encoded replay of missed events for a reconnecting client. + std::string replay_bytes(const std::string& last_event_id); + + // Encodes one event, storing it for replay. Returns wire bytes. + std::string send(SseEvent event); + + // Keep-alive comment bytes. + static std::string keep_alive(); + + // 204 response telling the client to stop reconnecting. + static shared::HttpResponse stop(); + + void set_retry(std::chrono::milliseconds d) { initial_retry_ = d; } + std::uint64_t sent_count() const { return sent_count_; } + + private: + EventStore* store_; + EventStore owned_; + std::chrono::milliseconds initial_retry_{0}; + bool retry_sent_ = false; + std::uint64_t sent_count_ = 0; +}; + +} // namespace molo::realtime::sse diff --git a/src/realtime/streaming/chunked_body.cpp b/src/realtime/streaming/chunked_body.cpp new file mode 100644 index 0000000..cf7074f --- /dev/null +++ b/src/realtime/streaming/chunked_body.cpp @@ -0,0 +1,58 @@ +// Chunked body writer. See chunked_body.h. + +#include "realtime/streaming/chunked_body.h" + +#include + +namespace molo::realtime::streaming { + +ChunkedBody::ChunkedBody(FlushPolicy policy) : policy_(policy) {} + +std::string ChunkedBody::write(const uint8_t* data, std::size_t len) { + if (finished_ || len == 0) { + return {}; + } + buffer_.append(reinterpret_cast(data), len); + total_bytes_ += len; + policy_.note_written(len); + if (policy_.should_flush(std::chrono::steady_clock::now())) { + return flush(); + } + return {}; +} + +std::string ChunkedBody::write(const std::string& data) { + return write(reinterpret_cast(data.data()), data.size()); +} + +std::string ChunkedBody::flush() { + if (buffer_.empty() || finished_) { + return {}; + } + std::string out = frame_chunk( + reinterpret_cast(buffer_.data()), buffer_.size()); + buffer_.clear(); + policy_.note_flushed(std::chrono::steady_clock::now()); + return out; +} + +std::string ChunkedBody::finish() { + if (finished_) { + return {}; + } + finished_ = true; + std::string out = flush(); + out += "0\r\n\r\n"; + return out; +} + +std::string ChunkedBody::frame_chunk(const uint8_t* data, std::size_t len) { + char head[32]; + std::snprintf(head, sizeof(head), "%zx\r\n", len); + std::string out = head; + out.append(reinterpret_cast(data), len); + out += "\r\n"; + return out; +} + +} // namespace molo::realtime::streaming diff --git a/src/realtime/streaming/chunked_body.h b/src/realtime/streaming/chunked_body.h new file mode 100644 index 0000000..da6e71a --- /dev/null +++ b/src/realtime/streaming/chunked_body.h @@ -0,0 +1,43 @@ +#pragma once + +// Chunked transfer-encoded body writer. The caller writes payload +// bytes; the body renders chunk framing (hex length, CRLF, data, +// CRLF) and a terminating zero chunk. Explicit flush points come +// from FlushPolicy; finish() is idempotent. + +#include +#include +#include + +#include "realtime/streaming/flush_policy.h" + +namespace molo::realtime::streaming { + +class ChunkedBody { + public: + explicit ChunkedBody(FlushPolicy policy = FlushPolicy{}); + + // Appends payload, returning framed wire bytes (possibly empty + // when the policy holds data back). + std::string write(const uint8_t* data, std::size_t len); + std::string write(const std::string& data); + + // Forces out everything buffered, returning framed bytes. + std::string flush(); + // Terminator chunk. Idempotent. + std::string finish(); + + std::size_t buffered() const { return buffer_.size(); } + std::size_t total_bytes() const { return total_bytes_; } + bool finished() const { return finished_; } + + private: + std::string frame_chunk(const uint8_t* data, std::size_t len); + + FlushPolicy policy_; + std::string buffer_; + std::size_t total_bytes_ = 0; + bool finished_ = false; +}; + +} // namespace molo::realtime::streaming diff --git a/src/realtime/streaming/flush_policy.cpp b/src/realtime/streaming/flush_policy.cpp new file mode 100644 index 0000000..6fa22f4 --- /dev/null +++ b/src/realtime/streaming/flush_policy.cpp @@ -0,0 +1,35 @@ +// Flush policy. See flush_policy.h. + +#include "realtime/streaming/flush_policy.h" + +namespace molo::realtime::streaming { + +FlushPolicy::FlushPolicy(std::size_t max_bytes, + std::chrono::milliseconds max_interval) + : max_bytes_(max_bytes), max_interval_(max_interval) {} + +void FlushPolicy::note_written(std::size_t n) { + pending_ += n; +} + +bool FlushPolicy::should_flush( + std::chrono::steady_clock::time_point now) const { + if (pending_ == 0) { + return false; + } + if (pending_ >= max_bytes_) { + return true; + } + if (!ever_flushed_) { + return true; + } + return now - last_flush_ >= max_interval_; +} + +void FlushPolicy::note_flushed(std::chrono::steady_clock::time_point now) { + pending_ = 0; + last_flush_ = now; + ever_flushed_ = true; +} + +} // namespace molo::realtime::streaming diff --git a/src/realtime/streaming/flush_policy.h b/src/realtime/streaming/flush_policy.h new file mode 100644 index 0000000..311bbd2 --- /dev/null +++ b/src/realtime/streaming/flush_policy.h @@ -0,0 +1,32 @@ +#pragma once + +// Flush policy for streaming bodies: flush at least every 64 KiB or +// 100 ms during activity. Tracks bytes since the last flush and the +// last flush time; should_flush(now) reports when either fires. + +#include +#include + +#include "realtime/shared/caps.h" + +namespace molo::realtime::streaming { + +class FlushPolicy { + public: + explicit FlushPolicy( + std::size_t max_bytes = shared::Caps::kFlushBytes, + std::chrono::milliseconds max_interval = shared::Caps::kFlushInterval); + + void note_written(std::size_t n); + bool should_flush(std::chrono::steady_clock::time_point now) const; + void note_flushed(std::chrono::steady_clock::time_point now); + + private: + std::size_t max_bytes_; + std::chrono::milliseconds max_interval_; + std::size_t pending_ = 0; + std::chrono::steady_clock::time_point last_flush_; + bool ever_flushed_ = false; +}; + +} // namespace molo::realtime::streaming diff --git a/src/realtime/streaming/passthrough_body.cpp b/src/realtime/streaming/passthrough_body.cpp new file mode 100644 index 0000000..7ec83d9 --- /dev/null +++ b/src/realtime/streaming/passthrough_body.cpp @@ -0,0 +1,83 @@ +// Bounded passthrough pump. See passthrough_body.h. + +#include "realtime/streaming/passthrough_body.h" + +namespace molo::realtime::streaming { + +ByteRange parse_range(const std::string& header, std::size_t length) { + ByteRange r; + r.satisfiable = false; + const std::string prefix = "bytes="; + if (header.compare(0, prefix.size(), prefix) != 0) { + return r; + } + std::string spec = header.substr(prefix.size()); + if (spec.find(',') != std::string::npos) { + return r; // Multiple ranges: 416 path. + } + std::size_t dash = spec.find('-'); + if (dash == std::string::npos || length == 0) { + return r; + } + try { + if (dash == 0) { + std::size_t n = std::stoul(spec.substr(1)); + if (n == 0) { + return r; + } + r.first = n >= length ? 0 : length - n; + r.last = length - 1; + } else { + r.first = std::stoul(spec.substr(0, dash)); + std::string tail = spec.substr(dash + 1); + r.last = tail.empty() ? length - 1 : std::stoul(tail); + if (r.first >= length) { + return r; + } + if (r.last >= length) { + r.last = length - 1; + } + if (r.last < r.first) { + return r; + } + } + } catch (...) { + r.satisfiable = false; + return r; + } + r.satisfiable = true; + return r; +} + +PassthroughBody::PassthroughBody(std::size_t ring_bytes) + : ring_bytes_(ring_bytes == 0 ? 4096 : ring_bytes), + valve_(ring_bytes_ / 4, ring_bytes_) {} + +std::size_t PassthroughBody::pump(Producer producer, Consumer consumer, + std::size_t max_bytes) { + std::vector ring(ring_bytes_); + moved_ = 0; + while (!aborted_) { + if (max_bytes > 0 && moved_ >= max_bytes) { + break; + } + std::size_t want = ring_bytes_; + if (max_bytes > 0 && moved_ + want > max_bytes) { + want = max_bytes - moved_; + } + std::size_t n = producer(ring.data(), want); + if (n == 0) { + break; // Producer done. + } + valve_.note_buffered(n); + if (!consumer(ring.data(), n)) { + aborted_ = true; // Client disconnect. + break; + } + valve_.note_flushed(n); + moved_ += n; + } + return moved_; +} + +} // namespace molo::realtime::streaming diff --git a/src/realtime/streaming/passthrough_body.h b/src/realtime/streaming/passthrough_body.h new file mode 100644 index 0000000..926da7a --- /dev/null +++ b/src/realtime/streaming/passthrough_body.h @@ -0,0 +1,58 @@ +#pragma once + +// Bounded passthrough pump for proxy/file/WebDAV GET bodies. Bytes +// flow producer -> 256 KiB ring -> consumer without full buffering. +// The valve pauses the producer at the high mark and resumes at the +// low mark. Range and conditional state for file passthrough rides +// alongside; HTTP semantics stay in the caller. + +#include +#include +#include +#include +#include +#include + +#include "realtime/shared/backpressure_valve.h" +#include "realtime/shared/caps.h" + +namespace molo::realtime::streaming { + +struct ByteRange { + std::size_t first = 0; + std::size_t last = 0; // Inclusive. last < first means unsatisfiable. + bool satisfiable = true; +}; + +// Parses a single "bytes=F-L" / "bytes=F-" / "bytes=-N" range +// against a body length. Multiple ranges are rejected (416 path). +ByteRange parse_range(const std::string& header, std::size_t length); + +class PassthroughBody { + public: + using Producer = std::function; + using Consumer = std::function; + + explicit PassthroughBody( + std::size_t ring_bytes = shared::Caps::kPassthroughRingBytes); + + // Pumps until the producer returns 0 (done), the consumer returns + // false (disconnect), or max_bytes is reached. Returns bytes moved. + // The valve pauses the producer while the ring is full: the loop + // simply does not call the producer again until the consumer + // drains below the low mark. + std::size_t pump(Producer producer, Consumer consumer, + std::size_t max_bytes = 0); + + void abort() { aborted_ = true; } + bool aborted() const { return aborted_; } + std::size_t moved() const { return moved_; } + + private: + std::size_t ring_bytes_; + shared::BackpressureValve valve_; + bool aborted_ = false; + std::size_t moved_ = 0; +}; + +} // namespace molo::realtime::streaming diff --git a/src/realtime/webdav/client/webdav_client.cpp b/src/realtime/webdav/client/webdav_client.cpp new file mode 100644 index 0000000..35e4c4c --- /dev/null +++ b/src/realtime/webdav/client/webdav_client.cpp @@ -0,0 +1,165 @@ +// Matching WebDAV client. See webdav_client.h. + +#include "realtime/webdav/client/webdav_client.h" + +#include "realtime/webdav/server/dav_xml.h" + +namespace molo::realtime::webdav::client { + +WebDavClient::WebDavClient(std::size_t cache_entries, std::size_t cache_bytes) + : shared::BaseRealtimeClient("webdav-client"), + max_entries_(cache_entries), + max_bytes_(cache_bytes) {} + +void WebDavClient::add_auth(shared::HttpRequest& req) const { + for (const auto& [k, v] : auth().headers()) { + req.headers.emplace(k, v); + } +} + +shared::HttpRequest WebDavClient::propfind(const std::string& path, + const std::string& depth, + const std::string& body) const { + shared::HttpRequest req; + req.method = "PROPFIND"; + req.path = path; + req.headers.emplace("Depth", depth); + req.body = body; + if (!body.empty()) { + req.headers.emplace("Content-Type", "application/xml"); + } + add_auth(req); + return req; +} + +shared::HttpRequest WebDavClient::get(const std::string& path, + bool conditional) const { + shared::HttpRequest req; + req.method = "GET"; + req.path = path; + if (conditional) { + auto it = etags_.find(path); + if (it != etags_.end()) { + req.headers.emplace("If-None-Match", it->second); + } + } + add_auth(req); + return req; +} + +shared::HttpRequest WebDavClient::put(const std::string& path, + const std::string& body) const { + shared::HttpRequest req; + req.method = "PUT"; + req.path = path; + req.body = body; + add_auth(req); + return req; +} + +shared::HttpRequest WebDavClient::remove(const std::string& path) const { + shared::HttpRequest req; + req.method = "DELETE"; + req.path = path; + add_auth(req); + return req; +} + +shared::HttpRequest WebDavClient::mkcol(const std::string& path) const { + shared::HttpRequest req; + req.method = "MKCOL"; + req.path = path; + add_auth(req); + return req; +} + +std::vector WebDavClient::parse_propfind(const std::string& body) { + std::vector out; + auto xml = server::parse_dav_xml(body, 4u * 1024u * 1024u); + if (!xml.has_value() || xml->local != "multistatus") { + return out; + } + for (const server::DavXmlNode* resp : + server::find_children(*xml, "response")) { + DavEntry e; + for (const server::DavXmlNode* href : + server::find_children(*resp, "href")) { + e.href = href->text; + } + for (const server::DavXmlNode* ps : + server::find_children(*resp, "propstat")) { + for (const server::DavXmlNode* prop : + server::find_children(*ps, "prop")) { + for (const auto& child : prop->children) { + if (child.local == "resourcetype") { + e.is_collection = + !server::find_children(child, "collection").empty(); + } else if (child.local == "getcontentlength") { + try { + e.content_length = std::stoul(child.text); + } catch (...) { + } + } else if (child.local == "getetag") { + e.etag = child.text; + } + } + } + } + out.push_back(e); + } + return out; +} + +WebDavClient::GetResult WebDavClient::on_get_response( + const shared::HttpRequest& req, const shared::HttpResponse& res, + BodySink sink) { + GetResult r; + if (res.status == 304) { + r.ok = true; + r.cached = true; + return r; + } + if (res.status != 200 && res.status != 206) { + return r; + } + constexpr std::size_t kSlice = 64u * 1024u; + const uint8_t* data = + reinterpret_cast(res.body.data()); + std::size_t off = 0; + while (off < res.body.size()) { + std::size_t n = res.body.size() - off; + if (n > kSlice) { + n = kSlice; + } + if (quota_hook_ && !quota_hook_("down", n)) { + return r; + } + if (sink && !sink(data + off, n)) { + return r; + } + off += n; + r.bytes += n; + } + std::string etag = shared::header_first(res, "ETag"); + if (!etag.empty()) { + if (etags_.size() >= max_entries_) { + etags_.erase(etags_.begin()); + } + etags_[req.path] = etag; + cache_bytes_used_ += etag.size() + req.path.size(); + if (cache_bytes_used_ > max_bytes_) { + clear_cache(); + etags_[req.path] = etag; + cache_bytes_used_ = etag.size() + req.path.size(); + } + } + r.ok = true; + return r; +} + +void WebDavClient::clear_cache() { + etags_.clear(); + cache_bytes_used_ = 0; +} + +} // namespace molo::realtime::webdav::client diff --git a/src/realtime/webdav/client/webdav_client.h b/src/realtime/webdav/client/webdav_client.h new file mode 100644 index 0000000..20572c5 --- /dev/null +++ b/src/realtime/webdav/client/webdav_client.h @@ -0,0 +1,83 @@ +#pragma once + +// Matching WebDAV client (rule 8): PROPFIND listings, streaming GET +// directly into a caller body (no full buffering), conditional GET +// with an ETag cache (max 128 entries, 64 MiB), PUT/DELETE/MKCOL, +// and quota accounting hooks per transfer. The caller owns HTTP: the +// client renders requests and consumes responses. + +#include +#include +#include +#include +#include + +#include "realtime/shared/base_realtime_client.h" +#include "realtime/shared/caps.h" +#include "realtime/shared/http_seam.h" + +namespace molo::realtime::webdav::client { + +struct DavEntry { + std::string href; + bool is_collection = false; + std::size_t content_length = 0; + std::string etag; +}; + +class WebDavClient : public shared::BaseRealtimeClient { + public: + using BodySink = + std::function; + // Quota hook: direction "up"/"down", bytes moved. Return false to + // abort the transfer. + using QuotaHook = + std::function; + + explicit WebDavClient( + std::size_t cache_entries = shared::Caps::kDavCacheEntries, + std::size_t cache_bytes = shared::Caps::kDavCacheBytes); + + // Renders requests. Auth headers are added automatically. + shared::HttpRequest propfind(const std::string& path, + const std::string& depth = "1", + const std::string& body = "") const; + shared::HttpRequest get(const std::string& path, + bool conditional = true) const; + shared::HttpRequest put(const std::string& path, + const std::string& body) const; + shared::HttpRequest remove(const std::string& path) const; + shared::HttpRequest mkcol(const std::string& path) const; + + // Parses a 207 PROPFIND body into entries. + static std::vector parse_propfind(const std::string& body); + + // Consumes a GET response: streams res.body in 64 KiB slices to + // the sink (quota hook per slice), updates the ETag cache. + // Returns false when the sink or quota aborts. 304 answers replay + // nothing and return cached=true. + struct GetResult { + bool ok = false; + bool cached = false; + std::size_t bytes = 0; + }; + GetResult on_get_response(const shared::HttpRequest& req, + const shared::HttpResponse& res, BodySink sink); + + void set_quota_hook(QuotaHook hook) { quota_hook_ = std::move(hook); } + void clear_cache(); + + std::size_t cache_entries() const { return etags_.size(); } + std::size_t cache_bytes() const { return cache_bytes_used_; } + + private: + void add_auth(shared::HttpRequest& req) const; + + std::map etags_; // path -> etag. + std::size_t cache_bytes_used_ = 0; + std::size_t max_entries_; + std::size_t max_bytes_; + QuotaHook quota_hook_; +}; + +} // namespace molo::realtime::webdav::client diff --git a/src/realtime/webdav/server/copy_move_handler.cpp b/src/realtime/webdav/server/copy_move_handler.cpp new file mode 100644 index 0000000..514e6a8 --- /dev/null +++ b/src/realtime/webdav/server/copy_move_handler.cpp @@ -0,0 +1,89 @@ +// COPY and MOVE handler. See copy_move_handler.h. + +#include "realtime/webdav/server/copy_move_handler.h" + +namespace molo::realtime::webdav::server { + +shared::HttpResponse CopyMoveHandler::handle_copy( + const shared::HttpRequest& req, ResourceTree& tree, + const LockManager* locks) { + return run(false, req, tree, locks); +} + +shared::HttpResponse CopyMoveHandler::handle_move( + const shared::HttpRequest& req, ResourceTree& tree, + const LockManager* locks) { + return run(true, req, tree, locks); +} + +shared::HttpResponse CopyMoveHandler::run(bool is_move, + const shared::HttpRequest& req, + ResourceTree& tree, + const LockManager* locks) { + std::string dst_raw = shared::header_first(req, "Destination"); + if (dst_raw.empty()) { + shared::HttpResponse res; + res.status = 400; + res.body = "missing Destination"; + return res; + } + // Accept absolute paths and absolute URIs (strip scheme/host). + std::string dst_path = dst_raw; + std::size_t scheme = dst_raw.find("://"); + if (scheme != std::string::npos) { + std::size_t slash = dst_raw.find('/', scheme + 3); + dst_path = slash == std::string::npos ? "/" : dst_raw.substr(slash); + } + auto src = tree.normalize(req.path); + auto dst = tree.normalize(dst_path); + if (!src.has_value() || !dst.has_value()) { + shared::HttpResponse res; + res.status = 400; + res.body = "bad path"; + return res; + } + if (!tree.exists(*src)) { + shared::HttpResponse res; + res.status = 404; + res.body = "not found"; + return res; + } + std::string overwrite = shared::header_first(req, "Overwrite"); + bool ow = overwrite.empty() || overwrite == "T"; + if (!ow && tree.exists(*dst)) { + shared::HttpResponse res; + res.status = 412; + res.body = "destination exists"; + return res; + } + if (locks != nullptr) { + std::string if_header = shared::header_first(req, "If"); + if (locks->is_locked(*src) && !locks->satisfies(*src, if_header)) { + shared::HttpResponse res; + res.status = 423; + res.body = "source locked"; + return res; + } + if (locks->is_locked(*dst) && !locks->satisfies(*dst, if_header)) { + shared::HttpResponse res; + res.status = 423; + res.body = "destination locked"; + return res; + } + } + bool created = false; + bool ok = is_move ? tree.move(*src, *dst, ow, created) + : tree.copy(*src, *dst, ow, created); + if (!ok) { + shared::HttpResponse res; + res.status = 409; + res.body = "copy/move failed"; + return res; + } + shared::HttpResponse res; + res.status = created ? 201 : 204; + shared::set_header(res, "DAV", "1"); + return res; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/copy_move_handler.h b/src/realtime/webdav/server/copy_move_handler.h new file mode 100644 index 0000000..db859ff --- /dev/null +++ b/src/realtime/webdav/server/copy_move_handler.h @@ -0,0 +1,28 @@ +#pragma once + +// COPY and MOVE handler. Honors Destination (same-server absolute +// path), Overwrite (default true), and Depth for collections. New +// destinations answer 201, overwritten ones 204. Locked sources or +// destinations without the token answer 423. + +#include "realtime/shared/http_seam.h" +#include "realtime/webdav/server/lock_manager.h" +#include "realtime/webdav/server/resource_tree.h" + +namespace molo::realtime::webdav::server { + +class CopyMoveHandler { + public: + shared::HttpResponse handle_copy(const shared::HttpRequest& req, + ResourceTree& tree, + const LockManager* locks = nullptr); + shared::HttpResponse handle_move(const shared::HttpRequest& req, + ResourceTree& tree, + const LockManager* locks = nullptr); + + private: + shared::HttpResponse run(bool is_move, const shared::HttpRequest& req, + ResourceTree& tree, const LockManager* locks); +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/dav_view.cpp b/src/realtime/webdav/server/dav_view.cpp new file mode 100644 index 0000000..ebeefde --- /dev/null +++ b/src/realtime/webdav/server/dav_view.cpp @@ -0,0 +1,257 @@ +// Router-mountable WebDAV endpoint. See dav_view.h. + +#include "realtime/webdav/server/dav_view.h" + +#include "realtime/shared/realtime_metrics.h" +#include "realtime/streaming/passthrough_body.h" +#include "realtime/webdav/server/dav_xml.h" + +namespace molo::realtime::webdav::server { + +DavView::DavView(std::string root, DavConfig config) + : tree_(std::move(root)), + quota_(config.quota_bytes, config.quota_entries) {} + +shared::HttpResponse DavView::dispatch(const shared::HttpRequest& req) { + shared::HttpResponse res; + if (req.method == "OPTIONS") { + res = handle_options(req); + } else if (req.method == "GET" || req.method == "HEAD") { + res = handle_get(req); + } else if (req.method == "PUT") { + res = handle_put(req); + } else if (req.method == "DELETE") { + res = handle_delete(req); + } else if (req.method == "PROPFIND") { + res = propfind_.handle(req, tree_); + } else if (req.method == "PROPPATCH") { + res = proppatch_.handle(req, tree_); + } else if (req.method == "MKCOL") { + res = mkcol_.handle(req, tree_); + } else if (req.method == "COPY") { + res = copy_move_.handle_copy(req, tree_, &locks_); + } else if (req.method == "MOVE") { + res = copy_move_.handle_move(req, tree_, &locks_); + } else if (req.method == "LOCK") { + res = handle_lock(req); + } else if (req.method == "UNLOCK") { + res = handle_unlock(req); + } else { + res.status = 405; + res.body = "method not allowed"; + } + if (shared::header_first(res, "DAV").empty()) { + shared::set_header(res, "DAV", "1"); + } + return res; +} + +shared::HttpResponse DavView::handle_options(const shared::HttpRequest& req) { + (void)req; + shared::HttpResponse res; + res.status = 200; + shared::set_header(res, "DAV", "1"); + shared::set_header(res, "Allow", + "OPTIONS, GET, HEAD, PUT, DELETE, PROPFIND, PROPPATCH, " + "MKCOL, COPY, MOVE, LOCK, UNLOCK"); + return res; +} + +shared::HttpResponse DavView::handle_get(const shared::HttpRequest& req) { + auto norm = tree_.normalize(req.path); + const DavResource* r = norm.has_value() ? tree_.get(*norm) : nullptr; + if (r == nullptr || r->is_collection) { + shared::HttpResponse res; + res.status = 404; + res.body = "not found"; + return res; + } + std::string etag = r->etag(); + std::string inm = shared::header_first(req, "If-None-Match"); + if (!inm.empty() && (inm == "*" || inm == etag)) { + shared::HttpResponse res; + res.status = 304; + shared::set_header(res, "ETag", etag); + return res; + } + std::string body(r->body.begin(), r->body.end()); + std::string range_header = shared::header_first(req, "Range"); + if (!range_header.empty()) { + streaming::ByteRange range = + streaming::parse_range(range_header, body.size()); + if (!range.satisfiable) { + shared::HttpResponse res; + res.status = 416; + shared::set_header(res, "Content-Range", + "bytes */" + std::to_string(body.size())); + return res; + } + shared::HttpResponse res; + res.status = 206; + shared::set_header(res, "ETag", etag); + shared::set_header(res, "Accept-Ranges", "bytes"); + shared::set_header(res, "Content-Range", + "bytes " + std::to_string(range.first) + "-" + + std::to_string(range.last) + "/" + + std::to_string(body.size())); + res.body = body.substr(range.first, range.last - range.first + 1); + if (req.method == "HEAD") { + res.body.clear(); + } + return res; + } + shared::HttpResponse res; + res.status = 200; + shared::set_header(res, "ETag", etag); + shared::set_header(res, "Accept-Ranges", "bytes"); + res.body = req.method == "HEAD" ? "" : body; + return res; +} + +shared::HttpResponse DavView::handle_put(const shared::HttpRequest& req) { + auto norm = tree_.normalize(req.path); + if (!norm.has_value()) { + shared::HttpResponse res; + res.status = 400; + res.body = "bad path"; + return res; + } + const DavResource* cur = tree_.get(*norm); + if (cur != nullptr && cur->is_collection) { + shared::HttpResponse res; + res.status = 405; + res.body = "is a collection"; + return res; + } + std::size_t old = cur != nullptr ? cur->body.size() : 0; + if (!quota_.check_write(tree_.total_bytes(), old, req.body.size()) || + (cur == nullptr && + !quota_.check_entries(tree_.entry_count(), 1))) { + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kDavQuotaExceeded); + shared::HttpResponse res; + res.status = 507; + res.body = "quota exceeded"; + return res; + } + bool existed = cur != nullptr; + std::vector bytes(req.body.begin(), req.body.end()); + if (!tree_.put(*norm, std::move(bytes))) { + shared::HttpResponse res; + res.status = 409; + res.body = "parent missing"; + return res; + } + shared::HttpResponse res; + res.status = existed ? 204 : 201; + return res; +} + +shared::HttpResponse DavView::handle_delete(const shared::HttpRequest& req) { + auto norm = tree_.normalize(req.path); + if (!norm.has_value() || !tree_.exists(*norm)) { + shared::HttpResponse res; + res.status = 404; + res.body = "not found"; + return res; + } + if (!tree_.remove(*norm)) { + shared::HttpResponse res; + res.status = 409; + res.body = "collection not empty"; + return res; + } + shared::HttpResponse res; + res.status = 204; + return res; +} + +shared::HttpResponse DavView::handle_lock(const shared::HttpRequest& req) { + auto norm = tree_.normalize(req.path); + if (!norm.has_value()) { + shared::HttpResponse res; + res.status = 400; + res.body = "bad path"; + return res; + } + std::string depth = shared::header_first(req, "Depth"); + if (!depth.empty() && depth != "0") { + shared::HttpResponse res; + res.status = 400; + res.body = "Depth must be 0 for LOCK"; + return res; + } + // Empty body with If header: refresh. + if (req.body.empty()) { + std::string if_header = shared::header_first(req, "If"); + std::string token = LockManager::parse_lock_token(if_header); + std::chrono::seconds timeout = + locks_.parse_timeout(shared::header_first(req, "Timeout")); + if (token.empty() || !locks_.refresh(token, timeout)) { + shared::HttpResponse res; + res.status = 412; + res.body = "no such lock"; + return res; + } + shared::HttpResponse res; + res.status = 200; + return res; + } + auto xml = parse_dav_xml(req.body, 65536); + if (!xml.has_value() || xml->local != "lockinfo") { + shared::HttpResponse res; + res.status = 400; + res.body = "bad LOCK body"; + return res; + } + LockScope scope = LockScope::kExclusive; + std::string owner; + for (const DavXmlNode* ls : find_children(*xml, "lockscope")) { + if (!find_children(*ls, "shared").empty()) { + scope = LockScope::kShared; + } + } + for (const DavXmlNode* o : find_children(*xml, "owner")) { + owner = o->text; + if (!o->children.empty()) { + owner = o->children[0].text; + } + } + std::chrono::seconds timeout = + locks_.parse_timeout(shared::header_first(req, "Timeout")); + auto created = locks_.lock(*norm, scope, owner, timeout); + if (!created.has_value()) { + shared::HttpResponse res; + res.status = 423; + res.body = "locked"; + return res; + } + shared::HttpResponse res; + res.status = 200; + shared::set_header(res, "Content-Type", "application/xml; charset=utf-8"); + shared::set_header(res, "Lock-Token", "<" + created->token + ">"); + res.body = + "\n" + "\n" + "\n\n" + created->token + + "\n\n"; + return res; +} + +shared::HttpResponse DavView::handle_unlock(const shared::HttpRequest& req) { + (void)req; + std::string token = shared::header_first(req, "Lock-Token"); + if (token.empty() || !locks_.unlock(token)) { + shared::HttpResponse res; + res.status = 409; + res.body = "no such lock"; + return res; + } + shared::HttpResponse res; + res.status = 204; + return res; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/dav_view.h b/src/realtime/webdav/server/dav_view.h new file mode 100644 index 0000000..21f58e0 --- /dev/null +++ b/src/realtime/webdav/server/dav_view.h @@ -0,0 +1,52 @@ +#pragma once + +// Router-mountable WebDAV endpoint. Dispatches OPTIONS, GET, PUT, +// DELETE, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK to the +// method handlers over one resource tree, one lock manager, and one +// quota guard. GET supports conditional requests (If-None-Match +// precedence, 304) and single ranges (206/416). Every response +// carries the DAV: 1 header. + +#include "realtime/shared/http_seam.h" +#include "realtime/webdav/server/copy_move_handler.h" +#include "realtime/webdav/server/lock_manager.h" +#include "realtime/webdav/server/mkcol_handler.h" +#include "realtime/webdav/server/propfind_handler.h" +#include "realtime/webdav/server/proppatch_handler.h" +#include "realtime/webdav/server/quota_guard.h" +#include "realtime/webdav/server/resource_tree.h" + +namespace molo::realtime::webdav::server { + +struct DavConfig { + std::size_t quota_bytes = 1024u * 1024u * 1024u; + std::size_t quota_entries = 100000u; +}; + +class DavView { + public: + explicit DavView(std::string root = "/dav", DavConfig config = DavConfig{}); + + shared::HttpResponse dispatch(const shared::HttpRequest& req); + + ResourceTree& tree() { return tree_; } + LockManager& locks() { return locks_; } + + private: + shared::HttpResponse handle_options(const shared::HttpRequest& req); + shared::HttpResponse handle_get(const shared::HttpRequest& req); + shared::HttpResponse handle_put(const shared::HttpRequest& req); + shared::HttpResponse handle_delete(const shared::HttpRequest& req); + shared::HttpResponse handle_lock(const shared::HttpRequest& req); + shared::HttpResponse handle_unlock(const shared::HttpRequest& req); + + ResourceTree tree_; + LockManager locks_; + QuotaGuard quota_; + PropfindHandler propfind_; + ProppatchHandler proppatch_; + MkcolHandler mkcol_; + CopyMoveHandler copy_move_; +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/dav_xml.cpp b/src/realtime/webdav/server/dav_xml.cpp new file mode 100644 index 0000000..985df4a --- /dev/null +++ b/src/realtime/webdav/server/dav_xml.cpp @@ -0,0 +1,179 @@ +// Minimal DAV XML scanner. See dav_xml.h. + +#include "realtime/webdav/server/dav_xml.h" + +namespace molo::realtime::webdav::server { +namespace { + +constexpr std::size_t kMaxDepth = 16; +constexpr std::size_t kMaxNodes = 4096; + +std::string decode_entities(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (std::size_t i = 0; i < s.size();) { + if (s[i] == '&') { + std::size_t semi = s.find(';', i); + std::string ent = semi == std::string::npos + ? "" + : s.substr(i, semi - i + 1); + if (ent == "&") { + out.push_back('&'); + } else if (ent == "<") { + out.push_back('<'); + } else if (ent == ">") { + out.push_back('>'); + } else if (ent == """) { + out.push_back('"'); + } else if (ent == "'") { + out.push_back('\''); + } else { + return ""; // Unknown entity: fail the parse. + } + i = semi + 1; + } else { + out.push_back(s[i]); + ++i; + } + } + return out; +} + +std::string strip_prefix(const std::string& tag) { + std::size_t colon = tag.find(':'); + return colon == std::string::npos ? tag : tag.substr(colon + 1); +} + +class Scanner { + public: + explicit Scanner(const std::string& s) : s_(s) {} + + std::optional run() { + skip_prolog(); + auto root = element(0); + if (!root.has_value()) { + return std::nullopt; + } + skip_ws(); + if (pos_ != s_.size()) { + return std::nullopt; + } + return root; + } + + private: + void skip_ws() { + while (pos_ < s_.size() && (s_[pos_] == ' ' || s_[pos_] == '\t' || + s_[pos_] == '\r' || s_[pos_] == '\n')) { + ++pos_; + } + } + + void skip_prolog() { + skip_ws(); + if (s_.compare(pos_, 5, "", pos_); + pos_ = end == std::string::npos ? s_.size() : end + 2; + } + skip_ws(); + } + + std::optional element(std::size_t depth) { + if (depth > kMaxDepth || ++nodes_ > kMaxNodes || pos_ >= s_.size() || + s_[pos_] != '<') { + return std::nullopt; + } + ++pos_; + if (pos_ < s_.size() && (s_[pos_] == '!' || s_[pos_] == '?')) { + return std::nullopt; // No comments/PIs inside bodies. + } + std::size_t end = s_.find('>', pos_); + if (end == std::string::npos) { + return std::nullopt; + } + std::string head = s_.substr(pos_, end - pos_); + pos_ = end + 1; + bool self_close = false; + if (!head.empty() && head.back() == '/') { + self_close = true; + head.pop_back(); + } + std::size_t sp = head.find_first_of(" \t\r\n"); + std::string tag = head.substr(0, sp); + if (tag.empty()) { + return std::nullopt; + } + DavXmlNode node; + node.local = strip_prefix(tag); + if (self_close) { + return node; + } + std::string text; + while (true) { + if (pos_ >= s_.size()) { + return std::nullopt; + } + if (s_[pos_] == '<') { + if (s_.compare(pos_, 2, "', pos_); + if (close_end == std::string::npos) { + return std::nullopt; + } + std::string close_tag = s_.substr(pos_ + 2, close_end - pos_ - 2); + if (strip_prefix(close_tag) != node.local) { + return std::nullopt; + } + pos_ = close_end + 1; + break; + } + // Trim trailing whitespace-only text before a child. + bool only_ws = true; + for (char c : text) { + if (c != ' ' && c != '\t' && c != '\r' && c != '\n') { + only_ws = false; + } + } + if (only_ws) { + text.clear(); + } + auto child = element(depth + 1); + if (!child.has_value()) { + return std::nullopt; + } + node.children.push_back(std::move(*child)); + } else { + text.push_back(s_[pos_++]); + } + } + node.text = decode_entities(text); + return node; + } + + const std::string& s_; + std::size_t pos_ = 0; + std::size_t nodes_ = 0; +}; + +} // namespace + +std::optional parse_dav_xml(const std::string& body, + std::size_t max_bytes) { + if (body.size() > max_bytes || body.empty()) { + return std::nullopt; + } + Scanner s(body); + return s.run(); +} + +std::vector find_children(const DavXmlNode& node, + const std::string& local) { + std::vector out; + for (const auto& c : node.children) { + if (c.local == local) { + out.push_back(&c); + } + } + return out; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/dav_xml.h b/src/realtime/webdav/server/dav_xml.h new file mode 100644 index 0000000..9398935 --- /dev/null +++ b/src/realtime/webdav/server/dav_xml.h @@ -0,0 +1,31 @@ +#pragma once + +// Minimal XML scanner for DAV request bodies. Handles the shapes +// PROPFIND and PROPPATCH clients send: one root element with known +// DAV: children (allprop, propname, prop, set, remove) and arbitrary +// dead-property tags with text values. Not a general parser: no DTD, +// no entities beyond the five predefined ones, depth capped. + +#include +#include +#include +#include + +namespace molo::realtime::webdav::server { + +struct DavXmlNode { + std::string local; // Local tag name, prefix stripped. + std::string text; // Decoded direct text. + std::vector children; +}; + +// Parses one document. Returns nullopt on malformed input or when +// the body exceeds max_bytes. +std::optional parse_dav_xml(const std::string& body, + std::size_t max_bytes); + +// Direct children with the given local name. +std::vector find_children(const DavXmlNode& node, + const std::string& local); + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/lock_manager.cpp b/src/realtime/webdav/server/lock_manager.cpp new file mode 100644 index 0000000..b7f2891 --- /dev/null +++ b/src/realtime/webdav/server/lock_manager.cpp @@ -0,0 +1,163 @@ +// Advisory lock manager. See lock_manager.h. + +#include "realtime/webdav/server/lock_manager.h" + +#include "realtime/shared/random_token.h" + +namespace molo::realtime::webdav::server { + +LockManager::LockManager(std::chrono::seconds default_timeout, + std::chrono::seconds max_timeout) + : default_timeout_(default_timeout), max_timeout_(max_timeout) {} + +std::optional LockManager::lock(const std::string& path, + LockScope scope, + const std::string& owner, + std::chrono::seconds timeout) { + expire(); + for (const auto& [token, l] : locks_) { + if (l.path == path) { + if (l.scope == LockScope::kExclusive || scope == LockScope::kExclusive) { + return std::nullopt; // Conflict. + } + } + } + if (timeout <= std::chrono::seconds(0)) { + timeout = default_timeout_; + } + if (timeout > max_timeout_) { + timeout = max_timeout_; + } + DavLock l; + l.token = "opaquelocktoken:" + shared::random_hex(16); + l.path = path; + l.scope = scope; + l.owner = owner; + l.expires_at = std::chrono::steady_clock::now() + timeout; + locks_[l.token] = l; + return l; +} + +bool LockManager::refresh(const std::string& token, std::chrono::seconds timeout) { + expire(); + auto it = locks_.find(token); + if (it == locks_.end()) { + // Try with angle brackets stripped variants. + std::string bare = token; + if (!bare.empty() && bare.front() == '<') { + bare.erase(0, 1); + } + if (!bare.empty() && bare.back() == '>') { + bare.pop_back(); + } + it = locks_.find(bare); + if (it == locks_.end()) { + return false; + } + } + if (timeout <= std::chrono::seconds(0)) { + timeout = default_timeout_; + } + if (timeout > max_timeout_) { + timeout = max_timeout_; + } + it->second.expires_at = std::chrono::steady_clock::now() + timeout; + return true; +} + +bool LockManager::unlock(const std::string& token) { + expire(); + std::string bare = parse_lock_token(token); + if (bare.empty()) { + bare = token; + } + return locks_.erase(bare) > 0; +} + +bool LockManager::is_locked(const std::string& path) const { + expire(); + for (const auto& [token, l] : locks_) { + if (l.path == path) { + return true; + } + } + return false; +} + +std::vector LockManager::locks_for(const std::string& path) const { + expire(); + std::vector out; + for (const auto& [token, l] : locks_) { + if (l.path == path) { + out.push_back(l); + } + } + return out; +} + +bool LockManager::satisfies(const std::string& path, + const std::string& if_header) const { + for (const auto& [token, l] : locks_) { + if (l.path == path && l.expires_at > std::chrono::steady_clock::now() && + if_header.find(token) != std::string::npos) { + return true; + } + } + return false; +} + +std::chrono::seconds LockManager::parse_timeout( + const std::string& header) const { + if (header.empty()) { + return default_timeout_; + } + std::size_t pos = header.find("Second-"); + if (pos != std::string::npos) { + try { + long long s = std::stoll(header.substr(pos + 7)); + if (s > 0) { + std::chrono::seconds t(s); + return t > max_timeout_ ? max_timeout_ : t; + } + } catch (...) { + } + } + if (header.find("Infinite") != std::string::npos) { + return max_timeout_; + } + return default_timeout_; +} + +std::string LockManager::parse_lock_token(const std::string& header) { + std::size_t start = header.find("opaquelocktoken:"); + if (start == std::string::npos) { + // Bare token guess: trimmed header without brackets. + std::string t = header; + while (!t.empty() && (t.front() == ' ' || t.front() == '<')) { + t.erase(0, 1); + } + while (!t.empty() && (t.back() == ' ' || t.back() == '>')) { + t.pop_back(); + } + return t; + } + std::size_t end = start; + while (end < header.size() && header[end] != '>' && + header[end] != ' ' && header[end] != ')') { + ++end; + } + return header.substr(start, end - start); +} + +void LockManager::expire() const { + auto now = std::chrono::steady_clock::now(); + for (auto it = locks_.begin(); it != locks_.end();) { + if (it->second.expires_at <= now) { + it = locks_.erase(it); + } else { + ++it; + } + } +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/lock_manager.h b/src/realtime/webdav/server/lock_manager.h new file mode 100644 index 0000000..14add20 --- /dev/null +++ b/src/realtime/webdav/server/lock_manager.h @@ -0,0 +1,68 @@ +#pragma once + +// Advisory lock manager (rule 11). Locks are exclusive or shared +// with Depth 0, carry unguessable random tokens, have finite +// timeouts, and auto-expire on access. LOCK without a body refreshes +// a lock named by the If header; UNLOCK takes the Lock-Token header. + +#include +#include +#include +#include +#include +#include + +namespace molo::realtime::webdav::server { + +enum class LockScope : std::uint8_t { kExclusive, kShared }; + +struct DavLock { + std::string token; // Opaque lock token URI. + std::string path; + LockScope scope = LockScope::kExclusive; + std::string owner; + std::chrono::steady_clock::time_point expires_at; +}; + +class LockManager { + public: + explicit LockManager(std::chrono::seconds default_timeout = std::chrono::seconds(3600), + std::chrono::seconds max_timeout = std::chrono::seconds(86400)); + + // Creates a lock. Returns nullopt when an exclusive lock (or any + // lock against an exclusive request) already covers the path. + std::optional lock(const std::string& path, LockScope scope, + const std::string& owner, + std::chrono::seconds timeout); + + // Refreshes a lock named by token. False when unknown or expired. + bool refresh(const std::string& token, std::chrono::seconds timeout); + + // Releases the lock by token. False when unknown. + bool unlock(const std::string& token); + + bool is_locked(const std::string& path) const; + std::vector locks_for(const std::string& path) const; + + // True when the If header (or bare token) presents a live token + // covering the path. + bool satisfies(const std::string& path, const std::string& if_header) const; + + // Parses a Timeout header ("Second-60", "Infinite,..."). Infinite + // is rewritten to max_timeout_. + std::chrono::seconds parse_timeout(const std::string& header) const; + + // Extracts the first or bare token. + static std::string parse_lock_token(const std::string& header); + + std::size_t size() const { return locks_.size(); } + + private: + void expire() const; + + mutable std::map locks_; // token -> lock. + std::chrono::seconds default_timeout_; + std::chrono::seconds max_timeout_; +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/mkcol_handler.cpp b/src/realtime/webdav/server/mkcol_handler.cpp new file mode 100644 index 0000000..33574cd --- /dev/null +++ b/src/realtime/webdav/server/mkcol_handler.cpp @@ -0,0 +1,40 @@ +// MKCOL handler. See mkcol_handler.h. + +#include "realtime/webdav/server/mkcol_handler.h" + +namespace molo::realtime::webdav::server { + +shared::HttpResponse MkcolHandler::handle(const shared::HttpRequest& req, + ResourceTree& tree) { + auto norm = tree.normalize(req.path); + if (!norm.has_value()) { + shared::HttpResponse res; + res.status = 400; + res.body = "bad path"; + return res; + } + if (!req.body.empty()) { + shared::HttpResponse res; + res.status = 415; + res.body = "MKCOL with body is not supported"; + return res; + } + if (tree.exists(*norm)) { + shared::HttpResponse res; + res.status = 405; + res.body = "exists"; + return res; + } + if (!tree.mkcol(*norm)) { + shared::HttpResponse res; + res.status = 409; + res.body = "parent missing"; + return res; + } + shared::HttpResponse res; + res.status = 201; + shared::set_header(res, "DAV", "1"); + return res; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/mkcol_handler.h b/src/realtime/webdav/server/mkcol_handler.h new file mode 100644 index 0000000..0a568cd --- /dev/null +++ b/src/realtime/webdav/server/mkcol_handler.h @@ -0,0 +1,18 @@ +#pragma once + +// MKCOL handler: creates a collection. Missing parent answers 409, +// existing path answers 405, success answers 201. A non-empty body +// is rejected (415 in this v1, extended MKCOL is out of scope). + +#include "realtime/shared/http_seam.h" +#include "realtime/webdav/server/resource_tree.h" + +namespace molo::realtime::webdav::server { + +class MkcolHandler { + public: + shared::HttpResponse handle(const shared::HttpRequest& req, + ResourceTree& tree); +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/multistatus_builder.cpp b/src/realtime/webdav/server/multistatus_builder.cpp new file mode 100644 index 0000000..2cbcbb3 --- /dev/null +++ b/src/realtime/webdav/server/multistatus_builder.cpp @@ -0,0 +1,104 @@ +// 207 Multistatus builder. See multistatus_builder.h. + +#include "realtime/webdav/server/multistatus_builder.h" + +namespace molo::realtime::webdav::server { + +void MultistatusBuilder::add(DavResponseEntry entry) { + entries_.push_back(std::move(entry)); +} + +std::string MultistatusBuilder::escape(const std::string& text) { + std::string out; + out.reserve(text.size()); + for (char c : text) { + switch (c) { + case '&': + out += "&"; + break; + case '<': + out += "<"; + break; + case '>': + out += ">"; + break; + case '"': + out += """; + break; + default: + out.push_back(c); + } + } + return out; +} + +std::string MultistatusBuilder::tag_for(const std::string& clark) { + // Clark "{ns}local": DAV: namespace uses D:, others use Z:. + if (clark.compare(0, 5, "{DAV:") == 0) { + std::size_t end = clark.find('}'); + if (end != std::string::npos) { + return "D:" + clark.substr(end + 1); + } + } + std::size_t end = clark.find('}'); + std::string local = end == std::string::npos ? clark : clark.substr(end + 1); + return "Z:" + local; +} + +std::string MultistatusBuilder::build() const { + std::string out = + "\n" + "\n"; + bool needs_z = false; + for (const auto& e : entries_) { + for (const auto& [name, value] : e.propstat.ok) { + if (tag_for(name).compare(0, 2, "Z:") == 0) { + needs_z = true; + } + } + } + (void)needs_z; + for (const auto& e : entries_) { + out += " \n " + escape(e.href) + "\n"; + if (!e.status.empty()) { + out += " " + escape(e.status) + "\n"; + } + if (!e.propstat.ok.empty()) { + out += " \n \n"; + for (const auto& [name, value] : e.propstat.ok) { + std::string tag = tag_for(name); + if (value.empty()) { + out += " <" + tag + "/>\n"; + } else { + out += " <" + tag + ">" + escape(value) + "\n"; + } + } + out += " \n HTTP/1.1 200 OK\n"; + out += " \n"; + } + if (!e.propstat.missing.empty()) { + out += " \n \n"; + for (const auto& name : e.propstat.missing) { + out += " <" + tag_for(name) + "/>\n"; + } + out += " \n HTTP/1.1 404 Not Found\n"; + out += " \n"; + } + if (!e.propstat.failed.empty()) { + out += " \n \n"; + for (const auto& name : e.propstat.failed) { + out += " <" + tag_for(name) + "/>\n"; + } + out += + " \n HTTP/1.1 422 Unprocessable " + "Entity\n"; + out += " \n"; + } + out += " \n"; + } + out += "\n"; + return out; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/multistatus_builder.h b/src/realtime/webdav/server/multistatus_builder.h new file mode 100644 index 0000000..43f7c56 --- /dev/null +++ b/src/realtime/webdav/server/multistatus_builder.h @@ -0,0 +1,43 @@ +#pragma once + +// Builds 207 Multistatus XML bodies in the DAV: namespace. One +// response element per resource, each carrying propstat groups with +// per-property status lines. All text is XML-escaped. + +#include +#include + +namespace molo::realtime::webdav::server { + +struct DavPropstat { + // (clark name, value XML or empty) for found properties. + std::vector> ok; + // Clark names for missing properties (404 group). + std::vector missing; + // (clark name) for properties that failed to update (422 group). + std::vector failed; +}; + +struct DavResponseEntry { + std::string href; + DavPropstat propstat; + // Optional whole-resource status instead of propstat (rare). + std::string status; +}; + +class MultistatusBuilder { + public: + void add(DavResponseEntry entry); + std::string build() const; + std::size_t size() const { return entries_.size(); } + + static std::string escape(const std::string& text); + // Renders a Clark name "{ns}local" as a namespaced tag using the + // D: prefix for DAV: and dead properties verbatim. + static std::string tag_for(const std::string& clark); + + private: + std::vector entries_; +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/propfind_handler.cpp b/src/realtime/webdav/server/propfind_handler.cpp new file mode 100644 index 0000000..1dea135 --- /dev/null +++ b/src/realtime/webdav/server/propfind_handler.cpp @@ -0,0 +1,159 @@ +// PROPFIND handler. See propfind_handler.h. + +#include "realtime/webdav/server/propfind_handler.h" + +#include "realtime/webdav/server/dav_xml.h" +#include "realtime/webdav/server/multistatus_builder.h" + +namespace molo::realtime::webdav::server { +namespace { + +constexpr char kDavNs[] = "{DAV:}"; + +void add_live_props(const DavResource& r, DavPropstat& ps) { + if (r.is_collection) { + ps.ok.emplace_back(std::string(kDavNs) + "resourcetype", + ""); + } else { + ps.ok.emplace_back(std::string(kDavNs) + "resourcetype", ""); + ps.ok.emplace_back(std::string(kDavNs) + "getcontentlength", + std::to_string(r.body.size())); + } + ps.ok.emplace_back(std::string(kDavNs) + "getetag", r.etag()); + ps.ok.emplace_back(std::string(kDavNs) + "displayname", ""); +} + +} // namespace + +PropfindHandler::PropfindHandler(std::size_t max_depth, std::size_t max_xml) + : max_depth_(max_depth), max_xml_(max_xml) {} + +PropfindHandler::Depth PropfindHandler::parse_depth( + const std::string& header) const { + Depth d; + if (header.empty() || header == "infinity") { + d.ok = true; + d.value = header.empty() ? max_depth_ : max_depth_; + return d; + } + if (header == "0") { + d.ok = true; + d.value = 0; + return d; + } + if (header == "1") { + d.ok = true; + d.value = 1; + return d; + } + return d; +} + +shared::HttpResponse PropfindHandler::handle(const shared::HttpRequest& req, + ResourceTree& tree) { + Depth depth = parse_depth(shared::header_first(req, "Depth")); + if (!depth.ok) { + shared::HttpResponse res; + res.status = 400; + res.body = "bad Depth header"; + return res; + } + auto norm = tree.normalize(req.path); + if (!norm.has_value() || !tree.exists(*norm)) { + shared::HttpResponse res; + res.status = 404; + res.body = "not found"; + return res; + } + // Parse the request body: allprop, propname, or prop selection. + enum class Mode { kAllprop, kPropname, kProp }; + Mode mode = Mode::kAllprop; + std::vector wanted; + if (!req.body.empty()) { + auto xml = parse_dav_xml(req.body, max_xml_); + if (!xml.has_value() || xml->local != "propfind") { + shared::HttpResponse res; + res.status = 400; + res.body = "bad PROPFIND body"; + return res; + } + if (!find_children(*xml, "allprop").empty()) { + mode = Mode::kAllprop; + } else if (!find_children(*xml, "propname").empty()) { + mode = Mode::kPropname; + } else { + auto props = find_children(*xml, "prop"); + if (props.empty()) { + shared::HttpResponse res; + res.status = 400; + res.body = "PROPFIND needs allprop, propname, or prop"; + return res; + } + mode = Mode::kProp; + for (const DavXmlNode* prop : props) { + for (const auto& child : prop->children) { + wanted.push_back(std::string(kDavNs) + child.local); + } + } + } + } + + MultistatusBuilder builder; + for (const std::string& path : tree.list(*norm, depth.value)) { + const DavResource* r = tree.get(path); + if (r == nullptr) { + continue; + } + DavResponseEntry entry; + entry.href = path; + if (mode == Mode::kPropname) { + entry.propstat.ok.emplace_back(std::string(kDavNs) + "resourcetype", ""); + entry.propstat.ok.emplace_back(std::string(kDavNs) + "getetag", ""); + entry.propstat.ok.emplace_back(std::string(kDavNs) + "displayname", ""); + if (!r->is_collection) { + entry.propstat.ok.emplace_back(std::string(kDavNs) + + "getcontentlength", + ""); + } + for (const auto& [name, value] : r->props) { + (void)value; + entry.propstat.ok.emplace_back(name, ""); + } + } else if (mode == Mode::kAllprop) { + add_live_props(*r, entry.propstat); + for (const auto& [name, value] : r->props) { + entry.propstat.ok.emplace_back(name, value); + } + } else { + for (const std::string& name : wanted) { + if (name == std::string(kDavNs) + "resourcetype") { + entry.propstat.ok.emplace_back( + name, r->is_collection ? "" : ""); + } else if (name == std::string(kDavNs) + "getcontentlength" && + !r->is_collection) { + entry.propstat.ok.emplace_back(name, std::to_string(r->body.size())); + } else if (name == std::string(kDavNs) + "getetag") { + entry.propstat.ok.emplace_back(name, r->etag()); + } else if (name == std::string(kDavNs) + "displayname") { + entry.propstat.ok.emplace_back(name, ""); + } else { + auto it = r->props.find(name); + if (it != r->props.end()) { + entry.propstat.ok.emplace_back(name, it->second); + } else { + entry.propstat.missing.push_back(name); + } + } + } + } + builder.add(std::move(entry)); + } + shared::HttpResponse res; + res.status = 207; + shared::set_header(res, "Content-Type", "application/xml; charset=utf-8"); + shared::set_header(res, "DAV", "1"); + res.body = builder.build(); + return res; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/propfind_handler.h b/src/realtime/webdav/server/propfind_handler.h new file mode 100644 index 0000000..7e3c9ea --- /dev/null +++ b/src/realtime/webdav/server/propfind_handler.h @@ -0,0 +1,38 @@ +#pragma once + +// PROPFIND handler: allprop, propname, and prop selections over the +// resource tree. Honors Depth 0/1/infinity (infinity rewritten to +// the max depth cap, rule 10). Answers 207 with a DAV: multistatus +// body, 404 for a missing root path. An empty body means allprop. + +#include + +#include "realtime/shared/caps.h" +#include "realtime/shared/http_seam.h" +#include "realtime/webdav/server/resource_tree.h" + +namespace molo::realtime::webdav::server { + +class PropfindHandler { + public: + explicit PropfindHandler( + std::size_t max_depth = shared::Caps::kMaxDavDepth, + std::size_t max_xml = shared::Caps::kMaxXmlBodyBytes); + + shared::HttpResponse handle(const shared::HttpRequest& req, + ResourceTree& tree); + + // Parses the Depth header: "0", "1", "infinity". Anything else is + // a 400. Infinity is rewritten to max_depth_. + struct Depth { + bool ok = false; + std::size_t value = 0; + }; + Depth parse_depth(const std::string& header) const; + + private: + std::size_t max_depth_; + std::size_t max_xml_; +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/proppatch_handler.cpp b/src/realtime/webdav/server/proppatch_handler.cpp new file mode 100644 index 0000000..5e04ab1 --- /dev/null +++ b/src/realtime/webdav/server/proppatch_handler.cpp @@ -0,0 +1,70 @@ +// PROPPATCH handler. See proppatch_handler.h. + +#include "realtime/webdav/server/proppatch_handler.h" + +#include "realtime/webdav/server/dav_xml.h" +#include "realtime/webdav/server/multistatus_builder.h" + +namespace molo::realtime::webdav::server { + +ProppatchHandler::ProppatchHandler(std::size_t max_xml) : max_xml_(max_xml) {} + +shared::HttpResponse ProppatchHandler::handle(const shared::HttpRequest& req, + ResourceTree& tree) { + auto norm = tree.normalize(req.path); + DavResource* r = norm.has_value() ? tree.get(*norm) : nullptr; + if (r == nullptr) { + shared::HttpResponse res; + res.status = 404; + res.body = "not found"; + return res; + } + auto xml = parse_dav_xml(req.body, max_xml_); + if (!xml.has_value() || xml->local != "propertyupdate") { + shared::HttpResponse res; + res.status = 400; + res.body = "bad PROPPATCH body"; + return res; + } + DavResponseEntry entry; + entry.href = *norm; + auto apply_prop = [&](const DavXmlNode& prop, bool is_set) { + for (const auto& child : prop.children) { + std::string name = "{custom:}" + child.local; + bool live = child.local == "resourcetype" || + child.local == "getcontentlength" || + child.local == "getetag" || + child.local == "displayname"; + if (live) { + entry.propstat.failed.push_back(std::string("{DAV:}") + child.local); + continue; + } + if (is_set) { + r->props[name] = child.text; + } else { + r->props.erase(name); + } + entry.propstat.ok.emplace_back(name, ""); + } + }; + for (const DavXmlNode* set : find_children(*xml, "set")) { + for (const DavXmlNode* prop : find_children(*set, "prop")) { + apply_prop(*prop, true); + } + } + for (const DavXmlNode* rem : find_children(*xml, "remove")) { + for (const DavXmlNode* prop : find_children(*rem, "prop")) { + apply_prop(*prop, false); + } + } + MultistatusBuilder builder; + builder.add(std::move(entry)); + shared::HttpResponse res; + res.status = 207; + shared::set_header(res, "Content-Type", "application/xml; charset=utf-8"); + shared::set_header(res, "DAV", "1"); + res.body = builder.build(); + return res; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/proppatch_handler.h b/src/realtime/webdav/server/proppatch_handler.h new file mode 100644 index 0000000..62089e0 --- /dev/null +++ b/src/realtime/webdav/server/proppatch_handler.h @@ -0,0 +1,27 @@ +#pragma once + +// PROPPATCH handler: set/remove dead properties with 207 +// per-property status. Live DAV: properties cannot be patched (422 +// per property). Missing resource answers 404. + +#include + +#include "realtime/shared/caps.h" +#include "realtime/shared/http_seam.h" +#include "realtime/webdav/server/resource_tree.h" + +namespace molo::realtime::webdav::server { + +class ProppatchHandler { + public: + explicit ProppatchHandler( + std::size_t max_xml = shared::Caps::kMaxXmlBodyBytes); + + shared::HttpResponse handle(const shared::HttpRequest& req, + ResourceTree& tree); + + private: + std::size_t max_xml_; +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/quota_guard.cpp b/src/realtime/webdav/server/quota_guard.cpp new file mode 100644 index 0000000..a292cb2 --- /dev/null +++ b/src/realtime/webdav/server/quota_guard.cpp @@ -0,0 +1,21 @@ +// Quota guard. See quota_guard.h. + +#include "realtime/webdav/server/quota_guard.h" + +namespace molo::realtime::webdav::server { + +QuotaGuard::QuotaGuard(std::size_t max_bytes, std::size_t max_entries) + : max_bytes_(max_bytes), max_entries_(max_entries) {} + +bool QuotaGuard::check_write(std::size_t used_bytes, std::size_t old_bytes, + std::size_t new_bytes) const { + std::size_t after = used_bytes >= old_bytes ? used_bytes - old_bytes : 0; + return after + new_bytes <= max_bytes_; +} + +bool QuotaGuard::check_entries(std::size_t used_entries, + std::size_t added) const { + return used_entries + added <= max_entries_; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/quota_guard.h b/src/realtime/webdav/server/quota_guard.h new file mode 100644 index 0000000..0db32b2 --- /dev/null +++ b/src/realtime/webdav/server/quota_guard.h @@ -0,0 +1,30 @@ +#pragma once + +// Quota guard: byte and entry ceilings for the resource tree. +// check_write() runs before PUT/COPY/MOVE and reports 507 when the +// write would exceed quota. Counts come from the tree itself so the +// guard never drifts. + +#include + +namespace molo::realtime::webdav::server { + +class QuotaGuard { + public: + explicit QuotaGuard(std::size_t max_bytes = 1024u * 1024u * 1024u, + std::size_t max_entries = 100000u); + + // True when a write of new_bytes (replacing old_bytes) fits. + bool check_write(std::size_t used_bytes, std::size_t old_bytes, + std::size_t new_bytes) const; + bool check_entries(std::size_t used_entries, std::size_t added) const; + + std::size_t max_bytes() const { return max_bytes_; } + std::size_t max_entries() const { return max_entries_; } + + private: + std::size_t max_bytes_; + std::size_t max_entries_; +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/resource_tree.cpp b/src/realtime/webdav/server/resource_tree.cpp new file mode 100644 index 0000000..dfc0128 --- /dev/null +++ b/src/realtime/webdav/server/resource_tree.cpp @@ -0,0 +1,237 @@ +// In-memory WebDAV resource tree. See resource_tree.h. + +#include "realtime/webdav/server/resource_tree.h" + +#include + +namespace molo::realtime::webdav::server { + +std::string DavResource::etag() const { + // Weak etag from length plus a byte hash. + uint64_t h = 1469598103934665603ULL; + for (uint8_t b : body) { + h ^= b; + h *= 1099511628211ULL; + } + char buf[64]; + std::snprintf(buf, sizeof(buf), "W/\"%zx-%llx\"", body.size(), + static_cast(h)); + return buf; +} + +ResourceTree::ResourceTree(std::string root) : root_(std::move(root)) { + if (root_.empty() || root_[0] != '/') { + root_ = "/dav"; + } + while (root_.size() > 1 && root_.back() == '/') { + root_.pop_back(); + } + DavResource col; + col.is_collection = true; + col.modified = clock_; + entries_[root_] = col; +} + +std::optional ResourceTree::normalize( + const std::string& path) const { + if (path.empty() || path[0] != '/') { + return std::nullopt; + } + std::vector segs; + std::size_t i = 0; + while (i < path.size()) { + while (i < path.size() && path[i] == '/') { + ++i; + } + std::size_t j = i; + while (j < path.size() && path[j] != '/') { + ++j; + } + if (j > i) { + std::string seg = path.substr(i, j - i); + if (seg == "..") { + if (segs.empty()) { + return std::nullopt; + } + segs.pop_back(); + } else if (seg != "." && seg.find('\0') == std::string::npos) { + segs.push_back(seg); + } + } + i = j; + } + std::string out = "/"; + for (std::size_t k = 0; k < segs.size(); ++k) { + if (k > 0) { + out.push_back('/'); + } + out += segs[k]; + } + if (out != root_ && out.compare(0, root_.size() + 1, root_ + "/") != 0) { + return std::nullopt; // Outside the jail. + } + return out; +} + +bool ResourceTree::exists(const std::string& path) const { + return entries_.count(path) > 0; +} + +DavResource* ResourceTree::get(const std::string& path) { + auto it = entries_.find(path); + return it == entries_.end() ? nullptr : &it->second; +} + +const DavResource* ResourceTree::get(const std::string& path) const { + auto it = entries_.find(path); + return it == entries_.end() ? nullptr : &it->second; +} + +bool ResourceTree::put(const std::string& path, std::vector body) { + auto it = entries_.find(path); + if (it != entries_.end()) { + if (it->second.is_collection) { + return false; + } + total_bytes_ -= it->second.body.size(); + it->second.body = std::move(body); + total_bytes_ += it->second.body.size(); + it->second.modified = clock_; + return true; + } + std::size_t slash = path.rfind('/'); + std::string parent = slash == 0 ? "/" : path.substr(0, slash); + auto p = entries_.find(parent); + if (p == entries_.end() || !p->second.is_collection) { + return false; + } + DavResource r; + r.body = std::move(body); + r.modified = clock_; + total_bytes_ += r.body.size(); + entries_[path] = std::move(r); + return true; +} + +bool ResourceTree::mkcol(const std::string& path) { + if (entries_.count(path) > 0) { + return false; + } + std::size_t slash = path.rfind('/'); + std::string parent = slash == 0 ? "/" : path.substr(0, slash); + auto p = entries_.find(parent); + if (p == entries_.end() || !p->second.is_collection) { + return false; + } + DavResource r; + r.is_collection = true; + r.modified = clock_; + entries_[path] = std::move(r); + return true; +} + +bool ResourceTree::remove(const std::string& path) { + if (path == root_) { + return false; + } + auto it = entries_.find(path); + if (it == entries_.end()) { + return false; + } + if (it->second.is_collection) { + std::string prefix = path + "/"; + for (const auto& [k, v] : entries_) { + if (k.compare(0, prefix.size(), prefix) == 0) { + return false; // Non-empty collection. + } + } + } else { + total_bytes_ -= it->second.body.size(); + } + entries_.erase(it); + return true; +} + +bool ResourceTree::copy(const std::string& src, const std::string& dst, + bool overwrite, bool& created) { + created = false; + auto s = entries_.find(src); + if (s == entries_.end() || src == dst) { + return false; + } + if (entries_.count(dst) > 0 && !overwrite) { + return false; + } + std::string prefix = src + "/"; + for (const auto& [k, v] : entries_) { + if (k == src || k.compare(0, prefix.size(), prefix) == 0) { + std::string rel = k.substr(src.size()); + std::string target = dst + rel; + if (entries_.count(target) == 0) { + created = true; + } else if (!entries_[target].is_collection) { + total_bytes_ -= entries_[target].body.size(); + } + entries_[target] = v; + if (!v.is_collection) { + total_bytes_ += v.body.size(); + } + } + } + return true; +} + +bool ResourceTree::move(const std::string& src, const std::string& dst, + bool overwrite, bool& created) { + if (!copy(src, dst, overwrite, created)) { + return false; + } + std::string prefix = src + "/"; + for (auto it = entries_.begin(); it != entries_.end();) { + if (it->first == src || it->first.compare(0, prefix.size(), prefix) == 0) { + if (!it->second.is_collection) { + total_bytes_ -= it->second.body.size(); + } + it = entries_.erase(it); + } else { + ++it; + } + } + return true; +} + +std::vector ResourceTree::list(const std::string& path, + std::size_t depth) const { + std::vector out; + if (entries_.count(path) == 0) { + return out; + } + out.push_back(path); + if (depth == 0) { + return out; + } + std::string prefix = path == "/" ? "/" : path + "/"; + std::size_t base = 0; + for (char c : prefix) { + if (c == '/') { + ++base; + } + } + for (const auto& [k, v] : entries_) { + if (k == path || k.compare(0, prefix.size(), prefix) != 0) { + continue; + } + std::size_t slashes = 0; + for (char c : k) { + if (c == '/') { + ++slashes; + } + } + if (slashes - base + 1 <= depth) { + out.push_back(k); + } + } + return out; +} + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/webdav/server/resource_tree.h b/src/realtime/webdav/server/resource_tree.h new file mode 100644 index 0000000..e14242b --- /dev/null +++ b/src/realtime/webdav/server/resource_tree.h @@ -0,0 +1,74 @@ +#pragma once + +// In-memory WebDAV resource tree rooted at one collection. Paths +// are absolute, normalized, and jailed to the root: ".." escapes +// fail. Collections hold children; resources hold bytes plus live +// and dead properties. Quota accounting hooks fire on every write. + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::realtime::webdav::server { + +struct DavResource { + bool is_collection = false; + std::vector body; + // Dead (user) properties by Clark name "{ns}local". + std::map props; + std::uint64_t modified = 0; // Seconds since epoch, test-settable. + std::string etag() const; +}; + +class ResourceTree { + public: + explicit ResourceTree(std::string root = "/dav"); + + const std::string& root() const { return root_; } + + // Normalizes an absolute request path. Returns nullopt on escape + // attempts or malformed input. + std::optional normalize(const std::string& path) const; + + bool exists(const std::string& path) const; + DavResource* get(const std::string& path); + const DavResource* get(const std::string& path) const; + + // Creates or replaces a resource, creating no parents (they must + // exist). Returns false when the parent is missing. + bool put(const std::string& path, std::vector body); + // Creates a collection. False when the parent is missing or the + // path already exists. + bool mkcol(const std::string& path); + // Deletes a resource or an empty collection. False otherwise. + bool remove(const std::string& path); + // Deep copy. False when the source is missing or overwrite is + // false and the destination exists. + bool copy(const std::string& src, const std::string& dst, bool overwrite, + bool& created); + // Move: copy then delete. Same failure modes as copy. + bool move(const std::string& src, const std::string& dst, bool overwrite, + bool& created); + + // Lists paths under path up to depth (0, 1, or capped N), self + // first, then children in name order. + std::vector list(const std::string& path, + std::size_t depth) const; + + std::size_t total_bytes() const { return total_bytes_; } + std::size_t entry_count() const { return entries_.size(); } + void set_clock(std::uint64_t now) { clock_ = now; } + + private: + // Normalized absolute path -> resource. Root always exists. + std::map entries_; + std::string root_; + std::size_t total_bytes_ = 0; + std::uint64_t clock_ = 1700000000u; +}; + +} // namespace molo::realtime::webdav::server diff --git a/src/realtime/websocket/client/client_handshake.cpp b/src/realtime/websocket/client/client_handshake.cpp new file mode 100644 index 0000000..0b8bf96 --- /dev/null +++ b/src/realtime/websocket/client/client_handshake.cpp @@ -0,0 +1,68 @@ +// Client handshake. See client_handshake.h. + +#include "realtime/websocket/client/client_handshake.h" + +#include "realtime/shared/base64.h" +#include "realtime/shared/random_token.h" +#include "realtime/websocket/server/handshake_validator.h" + +namespace molo::realtime::websocket::client { + +shared::HttpRequest ClientHandshake::build_request( + const std::string& host, const std::string& path, + const std::vector>& extra_headers, + std::string& out_key) { + uint8_t raw[16]; + shared::fill_random(raw, sizeof(raw)); + out_key = shared::base64_encode(raw, sizeof(raw)); + shared::HttpRequest req; + req.method = "GET"; + req.path = path; + req.headers.emplace("Host", host); + req.headers.emplace("Upgrade", "websocket"); + req.headers.emplace("Connection", "Upgrade"); + req.headers.emplace("Sec-WebSocket-Key", out_key); + req.headers.emplace("Sec-WebSocket-Version", "13"); + for (const auto& [k, v] : extra_headers) { + req.headers.emplace(k, v); + } + return req; +} + +ClientHandshake::Answer ClientHandshake::check_response( + const shared::HttpResponse& res, const std::string& key) { + Answer a; + if (res.status != 101) { + a.failure_reason = "expected 101, got " + std::to_string(res.status); + return a; + } + if (!shared::header_has_token(shared::header_first(res, "Upgrade"), + "websocket")) { + a.failure_reason = "missing Upgrade: websocket"; + return a; + } + if (!shared::header_has_token(shared::header_first(res, "Connection"), + "Upgrade")) { + a.failure_reason = "missing Connection: Upgrade"; + return a; + } + std::string want = server::HandshakeValidator::accept_for(key); + if (shared::header_first(res, "Sec-WebSocket-Accept") != want) { + a.failure_reason = "Sec-WebSocket-Accept mismatch"; + return a; + } + a.ok = true; + a.subprotocol = shared::header_first(res, "Sec-WebSocket-Protocol"); + return a; +} + +std::string ClientHandshake::render(const shared::HttpRequest& req) { + std::string out = req.method + " " + req.path + " HTTP/1.1\r\n"; + for (const auto& [k, v] : req.headers) { + out += k + ": " + v + "\r\n"; + } + out += "\r\n"; + return out; +} + +} // namespace molo::realtime::websocket::client diff --git a/src/realtime/websocket/client/client_handshake.h b/src/realtime/websocket/client/client_handshake.h new file mode 100644 index 0000000..697760b --- /dev/null +++ b/src/realtime/websocket/client/client_handshake.h @@ -0,0 +1,38 @@ +#pragma once + +// Client side of the opening handshake: builds the GET request with +// a fresh 16-byte key and validates the 101 answer, checking the +// accept hash against the sent key. Pure data in/out; socket I/O +// belongs to the caller. + +#include +#include + +#include "realtime/shared/base_realtime_client.h" +#include "realtime/shared/http_seam.h" + +namespace molo::realtime::websocket::client { + +class ClientHandshake { + public: + // Renders the handshake request for host/path. Extra headers + // carry auth material from ClientAuth::headers(). + static shared::HttpRequest build_request( + const std::string& host, const std::string& path, + const std::vector>& extra_headers, + std::string& out_key); + + struct Answer { + bool ok = false; + std::string subprotocol; + std::string failure_reason; + }; + // Validates a 101 response against the key sent. + static Answer check_response(const shared::HttpResponse& res, + const std::string& key); + + // Renders a request to wire bytes (request line plus headers). + static std::string render(const shared::HttpRequest& req); +}; + +} // namespace molo::realtime::websocket::client diff --git a/src/realtime/websocket/client/websocket_client.cpp b/src/realtime/websocket/client/websocket_client.cpp new file mode 100644 index 0000000..8c77cc9 --- /dev/null +++ b/src/realtime/websocket/client/websocket_client.cpp @@ -0,0 +1,153 @@ +// Matching WebSocket client. See websocket_client.h. + +#include "realtime/websocket/client/websocket_client.h" + +#include "realtime/shared/random_token.h" +#include "realtime/shared/realtime_metrics.h" +#include "realtime/shared/utf8.h" +#include "realtime/websocket/server/frame_serializer.h" + +namespace molo::realtime::websocket::client { + +WebSocketClient::WebSocketClient() + : shared::BaseRealtimeClient("websocket-client"), parser_(false) {} + +void WebSocketClient::bind(shared::ITransport* transport, + const std::string& host, const std::string& path) { + transport_ = transport; + context().route = path; + (void)host; + set_state(shared::ClientState::kConnecting); + log_info("connecting to " + host + path); +} + +void WebSocketClient::note_open(const std::string& subprotocol) { + subprotocol_ = subprotocol; + note_connected(); + log_info("open"); +} + +bool WebSocketClient::on_bytes(const uint8_t* data, std::size_t len) { + if (state() == shared::ClientState::kClosed) { + return false; + } + std::size_t off = 0; + while (off < len && state() != shared::ClientState::kClosed) { + std::size_t consumed = 0; + server::ParseStep step = parser_.feed(data + off, len - off, consumed); + off += consumed; + if (step.outcome == server::ParseOutcome::kNeedMore) { + break; + } + if (step.outcome == server::ParseOutcome::kFailed) { + log_warn("server frame failure: " + step.fail_reason); + finish_close(1002, step.fail_reason); + return false; + } + handle_frame(step.frame); + } + return state() != shared::ClientState::kClosed; +} + +bool WebSocketClient::send_text(const std::string& text) { + if (state() != shared::ClientState::kOpen || transport_ == nullptr) { + return false; + } + send_frame(true, server::Opcode::kText, + reinterpret_cast(text.data()), text.size()); + return true; +} + +bool WebSocketClient::send_binary(const uint8_t* data, std::size_t len) { + if (state() != shared::ClientState::kOpen || transport_ == nullptr) { + return false; + } + send_frame(true, server::Opcode::kBinary, data, len); + return true; +} + +void WebSocketClient::send_ping(const std::vector& payload) { + if (state() != shared::ClientState::kOpen || transport_ == nullptr) { + return; + } + send_frame(true, server::Opcode::kPing, payload.data(), payload.size()); +} + +void WebSocketClient::close(std::uint16_t code, const std::string& reason) { + if (state() == shared::ClientState::kClosed || transport_ == nullptr) { + return; + } + std::vector body; + body.push_back(static_cast((code >> 8) & 0xFFu)); + body.push_back(static_cast(code & 0xFFu)); + body.insert(body.end(), reason.begin(), reason.end()); + send_frame(true, server::Opcode::kClose, body.data(), body.size()); + finish_close(code, reason); +} + +void WebSocketClient::send_frame(bool fin, server::Opcode opcode, + const uint8_t* data, std::size_t len) { + uint8_t mask[4]; + shared::fill_random(mask, sizeof(mask)); + transport_->send( + server::FrameSerializer::masked_frame(fin, opcode, data, len, mask)); + shared::RealtimeMetrics::instance().inc(shared::RealtimeMetric::kBytesSent, + len); +} + +void WebSocketClient::handle_frame(const server::Frame& frame) { + using server::Opcode; + if (frame.opcode == Opcode::kPing) { + send_frame(true, Opcode::kPong, frame.payload.data(), + frame.payload.size()); + return; + } + if (frame.opcode == Opcode::kPong) { + return; + } + if (frame.opcode == Opcode::kClose) { + std::uint16_t code = 1005; + std::string reason; + if (frame.payload.size() >= 2) { + code = static_cast((frame.payload[0] << 8) | + frame.payload[1]); + reason.assign(frame.payload.begin() + 2, frame.payload.end()); + } + finish_close(code, reason); + return; + } + if (frame.opcode == Opcode::kText || frame.opcode == Opcode::kBinary) { + if (!frame.fin) { + in_message_ = true; + message_is_text_ = frame.opcode == Opcode::kText; + message_buf_ = frame.payload; + return; + } + if (message_cb_) { + message_cb_(frame.opcode == Opcode::kText, frame.payload); + } + return; + } + if (in_message_) { + message_buf_.insert(message_buf_.end(), frame.payload.begin(), + frame.payload.end()); + if (frame.fin) { + in_message_ = false; + if (message_cb_) { + message_cb_(message_is_text_, message_buf_); + } + message_buf_.clear(); + } + } +} + +void WebSocketClient::finish_close(std::uint16_t code, + const std::string& reason) { + set_state(shared::ClientState::kClosed); + log_info("closed code=" + std::to_string(code) + " " + reason); + if (close_cb_) { + close_cb_(code, reason); + } +} + +} // namespace molo::realtime::websocket::client diff --git a/src/realtime/websocket/client/websocket_client.h b/src/realtime/websocket/client/websocket_client.h new file mode 100644 index 0000000..a684ea8 --- /dev/null +++ b/src/realtime/websocket/client/websocket_client.h @@ -0,0 +1,67 @@ +#pragma once + +// Matching WebSocket client (rule 8). Speaks the same handshake and +// framing as the server, masks its frames, answers pings, tracks +// reconnect state through BaseRealtimeClient, and pumps bytes +// through a caller-provided transport so tests can use loopback +// sockets and production can use runtime connections. + +#include +#include +#include +#include + +#include "realtime/shared/base_realtime_client.h" +#include "realtime/shared/transport.h" +#include "realtime/websocket/server/frame_parser.h" +#include "realtime/websocket/server/opcode.h" + +namespace molo::realtime::websocket::client { + +class WebSocketClient : public shared::BaseRealtimeClient { + public: + using MessageCallback = + std::function& payload)>; + using CloseCallback = + std::function; + + WebSocketClient(); + + // Binds the byte transport (owned by the caller) and moves to + // connecting. The caller performs the HTTP handshake with + // ClientHandshake, then calls note_open() on the 101 answer. + void bind(shared::ITransport* transport, const std::string& host, + const std::string& path); + void note_open(const std::string& subprotocol = ""); + const std::string& handshake_key() const { return key_; } + const std::string& subprotocol() const { return subprotocol_; } + + // Feeds server bytes. Returns false when the connection died. + bool on_bytes(const uint8_t* data, std::size_t len); + + bool send_text(const std::string& text); + bool send_binary(const uint8_t* data, std::size_t len); + void send_ping(const std::vector& payload = {}); + void close(std::uint16_t code = 1000, const std::string& reason = ""); + + void on_message(MessageCallback cb) { message_cb_ = std::move(cb); } + void on_close(CloseCallback cb) { close_cb_ = std::move(cb); } + + private: + void send_frame(bool fin, server::Opcode opcode, const uint8_t* data, + std::size_t len); + void handle_frame(const server::Frame& frame); + void finish_close(std::uint16_t code, const std::string& reason); + + shared::ITransport* transport_ = nullptr; + server::FrameParser parser_; + std::string key_; + std::string subprotocol_; + std::vector message_buf_; + bool message_is_text_ = false; + bool in_message_ = false; + MessageCallback message_cb_; + CloseCallback close_cb_; +}; + +} // namespace molo::realtime::websocket::client diff --git a/src/realtime/websocket/server/frame_parser.cpp b/src/realtime/websocket/server/frame_parser.cpp new file mode 100644 index 0000000..1110a31 --- /dev/null +++ b/src/realtime/websocket/server/frame_parser.cpp @@ -0,0 +1,125 @@ +// Incremental frame parser. See frame_parser.h. + +#include "realtime/websocket/server/frame_parser.h" + +namespace molo::realtime::websocket::server { + +FrameParser::FrameParser(bool require_mask, std::size_t max_frame_payload) + : require_mask_(require_mask), max_frame_payload_(max_frame_payload) {} + +void FrameParser::reset() { + buf_.clear(); + failed_ = false; +} + +ParseStep FrameParser::feed(const uint8_t* data, std::size_t len, + std::size_t& consumed) { + consumed = 0; + ParseStep step; + if (failed_) { + step.outcome = ParseOutcome::kFailed; + step.fail_code = CloseCode::kProtocolError; + step.fail_reason = "parser already failed"; + return step; + } + buf_.insert(buf_.end(), data, data + len); + consumed = len; + if (buf_.size() < 2) { + return step; // kNeedMore. + } + return parse_header(); +} + +ParseStep FrameParser::parse_header() { + ParseStep step; + uint8_t b0 = buf_[0]; + uint8_t b1 = buf_[1]; + bool fin = (b0 & 0x80u) != 0; + bool rsv = (b0 & 0x70u) != 0; + auto opcode = static_cast(b0 & 0x0Fu); + bool masked = (b1 & 0x80u) != 0; + uint64_t plen = b1 & 0x7Fu; + + auto fail = [&](CloseCode code, const std::string& reason) { + failed_ = true; + step.outcome = ParseOutcome::kFailed; + step.fail_code = code; + step.fail_reason = reason; + return step; + }; + if (rsv) { + return fail(CloseCode::kProtocolError, "RSV bits set without extension"); + } + if (!is_control(opcode) && !is_data(opcode)) { + return fail(CloseCode::kProtocolError, "unknown opcode"); + } + if (is_control(opcode) && !fin) { + return fail(CloseCode::kProtocolError, "fragmented control frame"); + } + if (require_mask_ && !masked) { + return fail(CloseCode::kProtocolError, "client frame without mask"); + } + if (!require_mask_ && masked) { + return fail(CloseCode::kProtocolError, "server frame must not be masked"); + } + std::size_t off = 2; + if (plen == 126u) { + if (buf_.size() < 4) { + return step; + } + plen = (static_cast(buf_[2]) << 8) | buf_[3]; + off = 4; + if (plen < 126u) { + return fail(CloseCode::kProtocolError, "non-minimal length encoding"); + } + } else if (plen == 127u) { + if (buf_.size() < 10) { + return step; + } + plen = 0; + for (int i = 0; i < 8; ++i) { + plen = (plen << 8) | buf_[2 + i]; + } + off = 10; + if ((plen >> 63) != 0) { + return fail(CloseCode::kProtocolError, "length bit 63 set"); + } + if (plen < 65536u) { + return fail(CloseCode::kProtocolError, "non-minimal length encoding"); + } + } + if (is_control(opcode) && plen > shared::Caps::kMaxControlPayloadBytes) { + return fail(CloseCode::kProtocolError, "control payload over 125 bytes"); + } + if (plen > max_frame_payload_) { + return fail(CloseCode::kMessageTooBig, "frame payload over cap"); + } + uint8_t mask[4] = {0, 0, 0, 0}; + if (masked) { + if (buf_.size() < off + 4) { + return step; + } + for (int i = 0; i < 4; ++i) { + mask[i] = buf_[off + i]; + } + off += 4; + } + if (buf_.size() < off + plen) { + return step; + } + Frame frame; + frame.fin = fin; + frame.opcode = opcode; + frame.payload.assign(buf_.begin() + off, buf_.begin() + off + plen); + if (masked) { + for (std::size_t i = 0; i < frame.payload.size(); ++i) { + frame.payload[i] ^= mask[i % 4]; + } + } + buf_.erase(buf_.begin(), buf_.begin() + off + static_cast(plen)); + step.outcome = ParseOutcome::kFrame; + step.frame = std::move(frame); + return step; +} + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/frame_parser.h b/src/realtime/websocket/server/frame_parser.h new file mode 100644 index 0000000..6ef0d82 --- /dev/null +++ b/src/realtime/websocket/server/frame_parser.h @@ -0,0 +1,62 @@ +#pragma once + +// Incremental WebSocket frame parser (RFC 6455 section 5). Feed raw +// bytes; the parser emits complete frames or a protocol failure with +// the close code to answer. Client frames MUST be masked (rule 2): +// unmasked frames fail with 1002. Control frames MUST have FIN set +// and payload at most 125 bytes (rule 4). + +#include +#include +#include +#include + +#include "realtime/shared/caps.h" +#include "realtime/websocket/server/opcode.h" + +namespace molo::realtime::websocket::server { + +struct Frame { + bool fin = true; + Opcode opcode = Opcode::kText; + std::vector payload; // Already unmasked. +}; + +enum class ParseOutcome : std::uint8_t { + kNeedMore, + kFrame, + kFailed, +}; + +struct ParseStep { + ParseOutcome outcome = ParseOutcome::kNeedMore; + Frame frame; + CloseCode fail_code = CloseCode::kProtocolError; + std::string fail_reason; +}; + +class FrameParser { + public: + // require_mask=true for server-side parsing of client bytes, + // false for client-side parsing of server bytes. + explicit FrameParser( + bool require_mask = true, + std::size_t max_frame_payload = shared::Caps::kMaxFramePayloadBytes); + + // Feeds bytes, returning one step per call. Callers loop until + // kNeedMore or kFailed. Consumed count tells how many bytes of + // the input were used. + ParseStep feed(const uint8_t* data, std::size_t len, std::size_t& consumed); + bool failed() const { return failed_; } + void reset(); + + private: + ParseStep parse_header(); + + bool require_mask_; + std::size_t max_frame_payload_; + std::vector buf_; + bool failed_ = false; +}; + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/frame_serializer.cpp b/src/realtime/websocket/server/frame_serializer.cpp new file mode 100644 index 0000000..b721434 --- /dev/null +++ b/src/realtime/websocket/server/frame_serializer.cpp @@ -0,0 +1,92 @@ +// Frame serializer. See frame_serializer.h. + +#include "realtime/websocket/server/frame_serializer.h" + +namespace molo::realtime::websocket::server { +namespace { + +void append_length(std::vector& out, std::size_t len, bool masked) { + uint8_t flag = masked ? 0x80u : 0x00u; + if (len < 126u) { + out.push_back(static_cast(len) | flag); + } else if (len < 65536u) { + out.push_back(126u | flag); + out.push_back(static_cast((len >> 8) & 0xFFu)); + out.push_back(static_cast(len & 0xFFu)); + } else { + out.push_back(127u | flag); + for (int i = 7; i >= 0; --i) { + out.push_back(static_cast((len >> (i * 8)) & 0xFFu)); + } + } +} + +} // namespace + +std::vector FrameSerializer::frame(bool fin, Opcode opcode, + const uint8_t* payload, + std::size_t len) { + std::vector out; + out.reserve(10 + len); + out.push_back((fin ? 0x80u : 0x00u) | + static_cast(opcode)); + append_length(out, len, false); + out.insert(out.end(), payload, payload + len); + return out; +} + +std::vector FrameSerializer::frame( + bool fin, Opcode opcode, const std::vector& payload) { + return frame(fin, opcode, payload.data(), payload.size()); +} + +std::vector FrameSerializer::text(const std::string& text) { + return frame(true, Opcode::kText, + reinterpret_cast(text.data()), text.size()); +} + +std::vector FrameSerializer::binary( + const std::vector& payload) { + return frame(true, Opcode::kBinary, payload); +} + +std::vector FrameSerializer::masked_frame( + bool fin, Opcode opcode, const uint8_t* payload, std::size_t len, + const uint8_t mask[4]) { + std::vector out; + out.reserve(14 + len); + out.push_back((fin ? 0x80u : 0x00u) | + static_cast(opcode)); + append_length(out, len, true); + out.insert(out.end(), mask, mask + 4); + for (std::size_t i = 0; i < len; ++i) { + out.push_back(static_cast(payload[i] ^ mask[i % 4])); + } + return out; +} + +std::vector FrameSerializer::close_frame(CloseCode code, + const std::string& reason) { + std::vector body; + auto raw = static_cast(code); + body.push_back(static_cast((raw >> 8) & 0xFFu)); + body.push_back(static_cast(raw & 0xFFu)); + std::size_t room = 125 - body.size(); + std::size_t take = reason.size() < room ? reason.size() : room; + body.insert(body.end(), reason.begin(), reason.begin() + take); + return frame(true, Opcode::kClose, body); +} + +std::vector FrameSerializer::ping( + const std::vector& payload) { + std::size_t take = payload.size() < 125 ? payload.size() : 125; + return frame(true, Opcode::kPing, payload.data(), take); +} + +std::vector FrameSerializer::pong( + const std::vector& payload) { + std::size_t take = payload.size() < 125 ? payload.size() : 125; + return frame(true, Opcode::kPong, payload.data(), take); +} + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/frame_serializer.h b/src/realtime/websocket/server/frame_serializer.h new file mode 100644 index 0000000..e709b93 --- /dev/null +++ b/src/realtime/websocket/server/frame_serializer.h @@ -0,0 +1,40 @@ +#pragma once + +// WebSocket frame serializer (RFC 6455 section 5). Server-side +// frames are never masked (rule 2). Client-side masking lives in +// the websocket client, which uses with_mask(). + +#include +#include +#include + +#include "realtime/websocket/server/opcode.h" + +namespace molo::realtime::websocket::server { + +class FrameSerializer { + public: + // Serializes one frame without masking. + static std::vector frame(bool fin, Opcode opcode, + const uint8_t* payload, std::size_t len); + static std::vector frame(bool fin, Opcode opcode, + const std::vector& payload); + static std::vector text(const std::string& text); + static std::vector binary(const std::vector& payload); + + // Serializes one masked frame (client to server) with the given + // 4-byte mask key. + static std::vector masked_frame(bool fin, Opcode opcode, + const uint8_t* payload, + std::size_t len, + const uint8_t mask[4]); + + // Close frame body: 2-byte big-endian code plus UTF-8 reason, + // truncated to 125 bytes total. + static std::vector close_frame(CloseCode code, + const std::string& reason); + static std::vector ping(const std::vector& payload); + static std::vector pong(const std::vector& payload); +}; + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/handshake_validator.cpp b/src/realtime/websocket/server/handshake_validator.cpp new file mode 100644 index 0000000..559d7c9 --- /dev/null +++ b/src/realtime/websocket/server/handshake_validator.cpp @@ -0,0 +1,94 @@ +// Opening handshake validation. See handshake_validator.h. + +#include "realtime/websocket/server/handshake_validator.h" + +#include "realtime/shared/base64.h" +#include "realtime/shared/realtime_metrics.h" +#include "realtime/shared/sha1.h" + +namespace molo::realtime::websocket::server { +namespace { +constexpr char kGuid[] = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + +HandshakeResult fail(const std::string& reason) { + HandshakeResult r; + r.ok = false; + r.failure_reason = reason; + r.response.status = 400; + shared::set_header(r.response, "Content-Type", "text/plain"); + r.response.body = "Bad Request: " + reason; + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kHandshakeFailures); + return r; +} +} // namespace + +HandshakeValidator::HandshakeValidator(HandshakeConfig config) + : config_(std::move(config)) {} + +std::string HandshakeValidator::accept_for(const std::string& key) { + shared::Sha1::Digest d = shared::Sha1::hash(key + kGuid); + return shared::base64_encode(d.data(), d.size()); +} + +HandshakeResult HandshakeValidator::validate( + const shared::HttpRequest& req) const { + if (req.method != "GET") { + return fail("WebSocket handshake requires GET"); + } + if (!shared::header_has_token(shared::header_first(req, "Upgrade"), + "websocket")) { + return fail("missing Upgrade: websocket"); + } + if (!shared::header_has_token(shared::header_first(req, "Connection"), + "Upgrade")) { + return fail("missing Connection: Upgrade"); + } + std::string key = shared::header_first(req, "Sec-WebSocket-Key"); + if (key.empty()) { + return fail("missing Sec-WebSocket-Key"); + } + auto raw = shared::base64_decode(key); + if (!raw.has_value() || raw->size() != 16) { + return fail("Sec-WebSocket-Key must decode to 16 bytes"); + } + std::string version = shared::header_first(req, "Sec-WebSocket-Version"); + if (!version.empty() && version != "13") { + HandshakeResult r = fail("unsupported Sec-WebSocket-Version"); + shared::set_header(r.response, "Sec-WebSocket-Version", "13"); + return r; + } + std::string picked; + std::string asked = shared::header_first(req, "Sec-WebSocket-Protocol"); + if (!asked.empty()) { + bool found = false; + for (const std::string& offered : config_.subprotocols) { + if (shared::header_has_token(asked, offered)) { + picked = offered; + found = true; + break; + } + } + if (!found && (config_.require_subprotocol || + !config_.subprotocols.empty())) { + return fail("no supported Sec-WebSocket-Protocol"); + } + } else if (config_.require_subprotocol) { + return fail("Sec-WebSocket-Protocol required"); + } + + HandshakeResult r; + r.ok = true; + r.accept_key = accept_for(key); + r.subprotocol = picked; + r.response.status = 101; + shared::set_header(r.response, "Upgrade", "websocket"); + shared::set_header(r.response, "Connection", "Upgrade"); + shared::set_header(r.response, "Sec-WebSocket-Accept", r.accept_key); + if (!picked.empty()) { + shared::set_header(r.response, "Sec-WebSocket-Protocol", picked); + } + return r; +} + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/handshake_validator.h b/src/realtime/websocket/server/handshake_validator.h new file mode 100644 index 0000000..a1151ab --- /dev/null +++ b/src/realtime/websocket/server/handshake_validator.h @@ -0,0 +1,46 @@ +#pragma once + +// Opening handshake validation per RFC 6455 section 4. Requires +// Upgrade/Connection headers and a 16-byte Sec-WebSocket-Key, picks +// at most one subprotocol, and builds the 101 response with the +// accept hash. Pure function of request plus config: no I/O. + +#include +#include +#include + +#include "realtime/shared/http_seam.h" + +namespace molo::realtime::websocket::server { + +struct HandshakeConfig { + // Subprotocols the server offers, in preference order. Empty + // means the server speaks no subprotocol: a request asking for + // one is still accepted when require_subprotocol is false, with + // no protocol echoed. + std::vector subprotocols; + bool require_subprotocol = false; +}; + +struct HandshakeResult { + bool ok = false; + shared::HttpResponse response; + std::string accept_key; + std::string subprotocol; + std::string failure_reason; +}; + +class HandshakeValidator { + public: + explicit HandshakeValidator(HandshakeConfig config = HandshakeConfig{}); + + HandshakeResult validate(const shared::HttpRequest& req) const; + + // base64(SHA-1(key + GUID)) per RFC 6455 section 1.3. + static std::string accept_for(const std::string& key); + + private: + HandshakeConfig config_; +}; + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/heartbeat_manager.cpp b/src/realtime/websocket/server/heartbeat_manager.cpp new file mode 100644 index 0000000..c8ed37a --- /dev/null +++ b/src/realtime/websocket/server/heartbeat_manager.cpp @@ -0,0 +1,48 @@ +// Application heartbeat. See heartbeat_manager.h. + +#include "realtime/websocket/server/heartbeat_manager.h" + +#include "realtime/shared/realtime_metrics.h" + +namespace molo::realtime::websocket::server { + +HeartbeatManager::HeartbeatManager(std::chrono::milliseconds interval, + std::chrono::milliseconds timeout) + : interval_(interval), timeout_(timeout) {} + +void HeartbeatManager::start(std::chrono::steady_clock::time_point now) { + if (started_) { + return; + } + started_ = true; + last_activity_ = now; + next_ping_ = now + interval_; +} + +void HeartbeatManager::note_activity(std::chrono::steady_clock::time_point now) { + last_activity_ = now; +} + +bool HeartbeatManager::tick(std::chrono::steady_clock::time_point now) { + if (!started_ || timed_out_) { + return !timed_out_; + } + if (now - last_activity_ >= timeout_) { + timed_out_ = true; + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kHeartbeatTimeouts); + if (timeout_fn_) { + timeout_fn_(); + } + return false; + } + if (now >= next_ping_) { + next_ping_ = now + interval_; + if (ping_fn_) { + ping_fn_(); + } + } + return true; +} + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/heartbeat_manager.h b/src/realtime/websocket/server/heartbeat_manager.h new file mode 100644 index 0000000..e3df63d --- /dev/null +++ b/src/realtime/websocket/server/heartbeat_manager.h @@ -0,0 +1,49 @@ +#pragma once + +// Application heartbeat for one connection. Sends pings every +// interval; when no frame at all arrives within the pong timeout the +// peer is dead and the manager closes it with 1001. The runtime +// calls tick(now) from its timer wheel; any inbound frame should +// call note_activity(). + +#include +#include +#include + +#include "realtime/shared/caps.h" + +namespace molo::realtime::websocket::server { + +class HeartbeatManager { + public: + using PingFn = std::function; + using TimeoutFn = std::function; + + explicit HeartbeatManager( + std::chrono::milliseconds interval = shared::Caps::kPingInterval, + std::chrono::milliseconds timeout = shared::Caps::kPongTimeout); + + void on_ping(PingFn fn) { ping_fn_ = std::move(fn); } + void on_timeout(TimeoutFn fn) { timeout_fn_ = std::move(fn); } + + // Starts the schedule at now. Idempotent. + void start(std::chrono::steady_clock::time_point now); + // Any inbound frame resets the dead-peer deadline. + void note_activity(std::chrono::steady_clock::time_point now); + // Fires due pings; fires the timeout callback once when the peer + // is silent past the deadline. Returns false after timeout. + bool tick(std::chrono::steady_clock::time_point now); + bool timed_out() const { return timed_out_; } + + private: + std::chrono::milliseconds interval_; + std::chrono::milliseconds timeout_; + PingFn ping_fn_; + TimeoutFn timeout_fn_; + std::chrono::steady_clock::time_point last_activity_; + std::chrono::steady_clock::time_point next_ping_; + bool started_ = false; + bool timed_out_ = false; +}; + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/opcode.h b/src/realtime/websocket/server/opcode.h new file mode 100644 index 0000000..ca8d16a --- /dev/null +++ b/src/realtime/websocket/server/opcode.h @@ -0,0 +1,40 @@ +#pragma once + +// WebSocket opcodes and close codes per RFC 6455 sections 5.2 and +// 7.4. One enum header shared by parser, serializer, views and +// clients so codes stay consistent. + +#include + +namespace molo::realtime::websocket::server { + +enum class Opcode : std::uint8_t { + kContinuation = 0x0, + kText = 0x1, + kBinary = 0x2, + kClose = 0x8, + kPing = 0x9, + kPong = 0xA, +}; + +inline bool is_control(Opcode op) { + return static_cast(op) >= 0x8u; +} + +inline bool is_data(Opcode op) { + return op == Opcode::kText || op == Opcode::kBinary || + op == Opcode::kContinuation; +} + +enum class CloseCode : std::uint16_t { + kNormal = 1000, + kGoingAway = 1001, + kProtocolError = 1002, + kUnsupportedData = 1003, + kBadUtf8 = 1007, + kPolicyViolation = 1008, + kMessageTooBig = 1009, + kInternalError = 1011, +}; + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/websocket_handler.cpp b/src/realtime/websocket/server/websocket_handler.cpp new file mode 100644 index 0000000..decb7c4 --- /dev/null +++ b/src/realtime/websocket/server/websocket_handler.cpp @@ -0,0 +1,21 @@ +// WebSocket handler base. See websocket_handler.h. + +#include "realtime/websocket/server/websocket_handler.h" + +namespace molo::realtime::websocket::server { + +bool WebSocketHandler::send_text(const std::string& text) { + return connection_ != nullptr && connection_->send_text(text); +} + +bool WebSocketHandler::send_binary(const std::vector& payload) { + return connection_ != nullptr && connection_->send_binary(payload); +} + +void WebSocketHandler::close(CloseCode code, const std::string& reason) { + if (connection_ != nullptr) { + connection_->close(code, reason); + } +} + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/websocket_handler.h b/src/realtime/websocket/server/websocket_handler.h new file mode 100644 index 0000000..d074959 --- /dev/null +++ b/src/realtime/websocket/server/websocket_handler.h @@ -0,0 +1,46 @@ +#pragma once + +// Base class for WebSocket endpoint logic. Users subclass and +// override on_text / on_binary / on_open / on_close. The handler +// never touches sockets: it sends through the connection, which the +// runtime binds after the handshake. Routers mount WebSocketView, +// which owns one handler per connection. + +#include +#include +#include + +#include "realtime/websocket/server/websocket_response.h" + +namespace molo::realtime::websocket::server { + +class WebSocketHandler { + public: + virtual ~WebSocketHandler() = default; + + virtual void on_open() {} + virtual void on_text(const std::string& message) { + (void)message; + } + virtual void on_binary(const std::vector& payload) { + (void)payload; + } + virtual void on_close(std::uint16_t code, const std::string& reason) { + (void)code; + (void)reason; + } + + // Bound by the view before on_open. Null until then. + WebSocketConnection* connection() { return connection_; } + + protected: + bool send_text(const std::string& text); + bool send_binary(const std::vector& payload); + void close(CloseCode code, const std::string& reason); + + private: + friend class WebSocketView; + WebSocketConnection* connection_ = nullptr; +}; + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/websocket_response.cpp b/src/realtime/websocket/server/websocket_response.cpp new file mode 100644 index 0000000..d912775 --- /dev/null +++ b/src/realtime/websocket/server/websocket_response.cpp @@ -0,0 +1,234 @@ +// WebSocket connection and 101 response. See websocket_response.h. + +#include "realtime/websocket/server/websocket_response.h" + +#include "realtime/shared/realtime_log.h" +#include "realtime/shared/realtime_metrics.h" +#include "realtime/shared/utf8.h" +#include "realtime/websocket/server/frame_serializer.h" + +namespace molo::realtime::websocket::server { + +WebSocketConnection::WebSocketConnection(shared::ITransport* transport, + shared::ConnectionContext context, + WebSocketConfig config) + : transport_(transport), + context_(std::move(context)), + config_(config), + parser_(true), + send_queue_(config.send_queue_depth, config.shed_policy) { + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kConnectionsOpened); +} + +bool WebSocketConnection::on_bytes(const uint8_t* data, std::size_t len) { + shared::RealtimeMetrics::instance().inc(shared::RealtimeMetric::kBytesReceived, + len); + std::size_t off = 0; + while (off < len && state_ != State::kClosed) { + std::size_t consumed = 0; + ParseStep step = parser_.feed(data + off, len - off, consumed); + off += consumed; + if (step.outcome == ParseOutcome::kNeedMore) { + break; + } + if (step.outcome == ParseOutcome::kFailed) { + fail(step.fail_code, step.fail_reason); + return false; + } + handle_frame(step.frame); + } + return state_ != State::kClosed; +} + +bool WebSocketConnection::send_text(const std::string& text) { + shared::SendItem item; + item.is_text = true; + item.bytes.assign(text.begin(), text.end()); + return send_queue_.push(std::move(item)); +} + +bool WebSocketConnection::send_binary(const std::vector& payload) { + shared::SendItem item; + item.is_text = false; + item.bytes = payload; + return send_queue_.push(std::move(item)); +} + +bool WebSocketConnection::flush() { + shared::SendItem item; + while (send_queue_.pop(item)) { + std::vector wire = + item.is_text ? FrameSerializer::frame( + true, Opcode::kText, item.bytes.data(), + item.bytes.size()) + : FrameSerializer::frame( + true, Opcode::kBinary, item.bytes.data(), + item.bytes.size()); + if (!transport_->send(wire)) { + send_queue_.close(); + finish_close(static_cast(CloseCode::kGoingAway), + "transport failed"); + return false; + } + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kMessagesSent); + shared::RealtimeMetrics::instance().inc(shared::RealtimeMetric::kBytesSent, + wire.size()); + } + if (send_queue_.must_close()) { + send_queue_.clear_must_close(); + close(CloseCode::kPolicyViolation, "send queue full"); + return false; + } + return state_ != State::kClosed; +} + +void WebSocketConnection::close(CloseCode code, const std::string& reason) { + if (state_ != State::kOpen) { + return; + } + state_ = State::kClosing; + transport_->send(FrameSerializer::close_frame(code, reason)); + finish_close(static_cast(code), reason); +} + +void WebSocketConnection::handle_frame(const Frame& frame) { + switch (frame.opcode) { + case Opcode::kPing: { + transport_->send(FrameSerializer::pong(frame.payload)); + if (ping_cb_) { + ping_cb_(); + } + return; + } + case Opcode::kPong: + return; + case Opcode::kClose: { + std::uint16_t code = + static_cast(CloseCode::kNormal); + std::string reason; + if (frame.payload.size() == 1) { + fail(CloseCode::kProtocolError, "close body of 1 byte"); + return; + } + if (frame.payload.size() >= 2) { + code = static_cast((frame.payload[0] << 8) | + frame.payload[1]); + reason.assign(frame.payload.begin() + 2, frame.payload.end()); + if (!shared::is_valid_utf8(reason)) { + fail(CloseCode::kBadUtf8, "close reason is not UTF-8"); + return; + } + } + if (state_ == State::kOpen) { + state_ = State::kClosing; + transport_->send(FrameSerializer::close_frame( + static_cast(code), reason)); + } + finish_close(code, reason); + return; + } + default: + break; + } + // Data frames: reassembly with caps. + if (frame.opcode == Opcode::kText || frame.opcode == Opcode::kBinary) { + if (in_message_) { + fail(CloseCode::kProtocolError, "new message before FIN"); + return; + } + if (!frame.fin) { + in_message_ = true; + message_is_text_ = frame.opcode == Opcode::kText; + message_buf_ = frame.payload; + message_frames_ = 1; + if (message_buf_.size() > config_.max_message_bytes) { + fail(CloseCode::kMessageTooBig, "message over cap"); + } + return; + } + if (frame.opcode == Opcode::kText && + !shared::is_valid_utf8(frame.payload.data(), + frame.payload.size())) { + fail(CloseCode::kBadUtf8, "text frame is not UTF-8"); + return; + } + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kMessagesReceived); + if (message_cb_) { + message_cb_(frame.opcode == Opcode::kText, frame.payload); + } + return; + } + // Continuation. + if (!in_message_) { + fail(CloseCode::kProtocolError, "stray continuation"); + return; + } + ++message_frames_; + if (message_frames_ > config_.max_frames_per_message) { + fail(CloseCode::kProtocolError, "too many frames per message"); + return; + } + if (message_buf_.size() + frame.payload.size() > config_.max_message_bytes) { + fail(CloseCode::kMessageTooBig, "message over cap"); + return; + } + message_buf_.insert(message_buf_.end(), frame.payload.begin(), + frame.payload.end()); + if (frame.fin) { + in_message_ = false; + if (message_is_text_ && + !shared::is_valid_utf8(message_buf_.data(), message_buf_.size())) { + fail(CloseCode::kBadUtf8, "text message is not UTF-8"); + return; + } + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kMessagesReceived); + if (message_cb_) { + message_cb_(message_is_text_, message_buf_); + } + message_buf_.clear(); + message_buf_.shrink_to_fit(); + } +} + +void WebSocketConnection::fail(CloseCode code, const std::string& reason) { + shared::RealtimeLog::warn("websocket", context_.correlation_id, + "protocol failure: " + reason); + if (state_ == State::kOpen) { + state_ = State::kClosing; + transport_->send(FrameSerializer::close_frame(code, reason)); + } + finish_close(static_cast(code), reason); +} + +void WebSocketConnection::finish_close(std::uint16_t code, + const std::string& reason) { + if (state_ == State::kClosed) { + return; + } + state_ = State::kClosed; + send_queue_.close(); + shared::RealtimeMetrics::instance().inc( + shared::RealtimeMetric::kConnectionsClosed); + if (close_cb_) { + close_cb_(code, reason); + } +} + +shared::HttpResponse WebSocketResponse::switching_protocols( + const std::string& accept_key, const std::string& subprotocol) { + shared::HttpResponse res; + res.status = 101; + shared::set_header(res, "Upgrade", "websocket"); + shared::set_header(res, "Connection", "Upgrade"); + shared::set_header(res, "Sec-WebSocket-Accept", accept_key); + if (!subprotocol.empty()) { + shared::set_header(res, "Sec-WebSocket-Protocol", subprotocol); + } + return res; +} + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/websocket_response.h b/src/realtime/websocket/server/websocket_response.h new file mode 100644 index 0000000..df9f55b --- /dev/null +++ b/src/realtime/websocket/server/websocket_response.h @@ -0,0 +1,103 @@ +#pragma once + +// The 101 upgrade response plus the live connection it opens. The +// response carries handshake headers; open() binds a transport and a +// context and returns the connection that pumps bytes. Message +// reassembly, UTF-8 checks and control-frame handling live here so +// handlers only see complete messages. + +#include +#include +#include +#include +#include + +#include "realtime/shared/caps.h" +#include "realtime/shared/connection_context.h" +#include "realtime/shared/http_seam.h" +#include "realtime/shared/send_queue.h" +#include "realtime/shared/transport.h" +#include "realtime/websocket/server/frame_parser.h" +#include "realtime/websocket/server/opcode.h" + +namespace molo::realtime::websocket::server { + +struct WebSocketConfig { + std::size_t max_message_bytes = shared::Caps::kMaxMessageBytes; + std::size_t max_frames_per_message = shared::Caps::kMaxFramesPerMessage; + std::size_t send_queue_depth = shared::Caps::kSendQueueDepth; + shared::ShedPolicy shed_policy = shared::ShedPolicy::kDropOldest; +}; + +class WebSocketConnection { + public: + using MessageCallback = + std::function& payload)>; + using CloseCallback = + std::function; + + WebSocketConnection(shared::ITransport* transport, + shared::ConnectionContext context, + WebSocketConfig config = WebSocketConfig{}); + + WebSocketConnection(const WebSocketConnection&) = delete; + WebSocketConnection& operator=(const WebSocketConnection&) = delete; + + // Feeds peer bytes. Returns false when the connection failed and + // must be torn down (close frame already queued or sent). + bool on_bytes(const uint8_t* data, std::size_t len); + + // Queues a text or binary message for the peer. Returns false when + // shed by policy. + bool send_text(const std::string& text); + bool send_binary(const std::vector& payload); + + // Flushes the send queue to the transport. Returns false when the + // transport failed or the close policy fired. + bool flush(); + + // Starts the closing handshake with the given code. + void close(CloseCode code, const std::string& reason); + bool open() const { return state_ == State::kOpen; } + bool closing() const { return state_ == State::kClosing; } + + void on_message(MessageCallback cb) { message_cb_ = std::move(cb); } + void on_close(CloseCallback cb) { close_cb_ = std::move(cb); } + void on_ping(std::function cb) { ping_cb_ = std::move(cb); } + + const shared::ConnectionContext& context() const { return context_; } + shared::SendQueue& send_queue() { return send_queue_; } + + private: + enum class State : std::uint8_t { kOpen, kClosing, kClosed }; + + void handle_frame(const Frame& frame); + void fail(CloseCode code, const std::string& reason); + void finish_close(std::uint16_t code, const std::string& reason); + + shared::ITransport* transport_; + shared::ConnectionContext context_; + WebSocketConfig config_; + FrameParser parser_; + shared::SendQueue send_queue_; + State state_ = State::kOpen; + // Reassembly buffer for fragmented messages. + std::vector message_buf_; + bool message_is_text_ = false; + bool in_message_ = false; + std::size_t message_frames_ = 0; + + MessageCallback message_cb_; + CloseCallback close_cb_; + std::function ping_cb_; +}; + +class WebSocketResponse { + public: + // Builds the 101 handshake response headers. The accept key comes + // from HandshakeValidator; this class only renders the response. + static shared::HttpResponse switching_protocols( + const std::string& accept_key, const std::string& subprotocol); +}; + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/websocket_view.cpp b/src/realtime/websocket/server/websocket_view.cpp new file mode 100644 index 0000000..d749d86 --- /dev/null +++ b/src/realtime/websocket/server/websocket_view.cpp @@ -0,0 +1,51 @@ +// Router-mountable WebSocket view. See websocket_view.h. + +#include "realtime/websocket/server/websocket_view.h" + +#include "realtime/shared/realtime_log.h" + +namespace molo::realtime::websocket::server { + +WebSocketView::WebSocketView(HandlerFactory factory, HandshakeConfig handshake, + WebSocketConfig socket) + : factory_(std::move(factory)), + validator_(std::move(handshake)), + socket_config_(socket) {} + +WebSocketView::Admit WebSocketView::admit( + const shared::HttpRequest& req) const { + HandshakeResult r = validator_.validate(req); + Admit admit; + admit.ok = r.ok; + admit.response = r.response; + admit.subprotocol = r.subprotocol; + admit.failure_reason = r.failure_reason; + return admit; +} + +WebSocketConnection* WebSocketView::accept(shared::ITransport* transport, + shared::ConnectionContext context) { + auto handler = factory_(); + auto conn = std::make_unique(transport, context, + socket_config_); + WebSocketConnection* raw = conn.get(); + handler->connection_ = raw; + WebSocketHandler* h = handler.get(); + raw->on_message([h](bool is_text, const std::vector& payload) { + if (is_text) { + h->on_text(std::string(payload.begin(), payload.end())); + } else { + h->on_binary(payload); + } + }); + raw->on_close([h](std::uint16_t code, const std::string& reason) { + h->on_close(code, reason); + }); + connections_.push_back(Entry{std::move(handler), std::move(conn)}); + shared::RealtimeLog::info("websocket", context.correlation_id, + "open route=" + context.route); + h->on_open(); + return raw; +} + +} // namespace molo::realtime::websocket::server diff --git a/src/realtime/websocket/server/websocket_view.h b/src/realtime/websocket/server/websocket_view.h new file mode 100644 index 0000000..40244ff --- /dev/null +++ b/src/realtime/websocket/server/websocket_view.h @@ -0,0 +1,58 @@ +#pragma once + +// Router-mountable WebSocket endpoint. The view runs the handshake +// (after middleware, auth and session per rule 1), answers 101, and +// binds one handler plus one connection per upgrade. Factories keep +// handler construction in user code. + +#include +#include +#include + +#include "realtime/shared/connection_context.h" +#include "realtime/shared/http_seam.h" +#include "realtime/shared/transport.h" +#include "realtime/websocket/server/handshake_validator.h" +#include "realtime/websocket/server/websocket_handler.h" +#include "realtime/websocket/server/websocket_response.h" + +namespace molo::realtime::websocket::server { + +class WebSocketView { + public: + using HandlerFactory = std::function()>; + + explicit WebSocketView(HandlerFactory factory, + HandshakeConfig handshake = HandshakeConfig{}, + WebSocketConfig socket = WebSocketConfig{}); + + // Runs the handshake. On success returns ok=true with the 101 + // response; the caller then passes the live transport to + // accept(). On failure returns the 4xx response to send. + struct Admit { + bool ok = false; + shared::HttpResponse response; + std::string subprotocol; + std::string failure_reason; + }; + Admit admit(const shared::HttpRequest& req) const; + + // Binds transport plus context, wires a fresh handler, and fires + // on_open. The view owns both for the life of the connection. + WebSocketConnection* accept(shared::ITransport* transport, + shared::ConnectionContext context); + + std::size_t open_connections() const { return connections_.size(); } + + private: + HandlerFactory factory_; + HandshakeValidator validator_; + WebSocketConfig socket_config_; + struct Entry { + std::unique_ptr handler; + std::unique_ptr connection; + }; + std::vector connections_; +}; + +} // namespace molo::realtime::websocket::server diff --git a/src/runtime/base64.h b/src/runtime/base64.h new file mode 100644 index 0000000..bca1bdf --- /dev/null +++ b/src/runtime/base64.h @@ -0,0 +1,92 @@ +// Shared Base64/Base64url codec (RFC 4648). Used by cookie signing +// (v1..) and the H1-to-h2c upgrade (HTTP2-Settings). One home +// so no feature reimplements it. +#pragma once + +#include +#include +#include +#include +#include + +namespace molo::runtime { + +class Base64Url { + public: + static std::string Encode(std::string_view in); + static std::optional Decode(std::string_view in); + + private: + static int Digit(char c); +}; + +inline std::string Base64Url::Encode(std::string_view in) { + static const char kAlphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + std::string out; + out.reserve((in.size() * 4 + 2) / 3); + std::size_t i = 0; + while (i + 3 <= in.size()) { + uint32_t n = ((uint32_t)(unsigned char)in[i] << 16) | + ((uint32_t)(unsigned char)in[i + 1] << 8) | + (uint32_t)(unsigned char)in[i + 2]; + out.push_back(kAlphabet[(n >> 18) & 63]); + out.push_back(kAlphabet[(n >> 12) & 63]); + out.push_back(kAlphabet[(n >> 6) & 63]); + out.push_back(kAlphabet[n & 63]); + i += 3; + } + std::size_t rem = in.size() - i; + if (rem == 1) { + uint32_t n = (uint32_t)(unsigned char)in[i] << 16; + out.push_back(kAlphabet[(n >> 18) & 63]); + out.push_back(kAlphabet[(n >> 12) & 63]); + } else if (rem == 2) { + uint32_t n = ((uint32_t)(unsigned char)in[i] << 16) | + ((uint32_t)(unsigned char)in[i + 1] << 8); + out.push_back(kAlphabet[(n >> 18) & 63]); + out.push_back(kAlphabet[(n >> 12) & 63]); + out.push_back(kAlphabet[(n >> 6) & 63]); + } + return out; +} + +inline int Base64Url::Digit(char c) { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '-' || c == '+') return 62; + if (c == '_' || c == '/') return 63; + return -1; +} + +inline std::optional Base64Url::Decode(std::string_view in) { + // Tolerate padding; reject other garbage loudly. + std::string s(in); + while (!s.empty() && s.back() == '=') s.pop_back(); + if (s.size() % 4 == 1) return std::nullopt; + std::string out; + out.reserve(s.size() * 3 / 4); + for (std::size_t i = 0; i < s.size();) { + uint32_t n = 0; + std::size_t group = 0; + for (; group < 4 && i < s.size(); ++group, ++i) { + int d = Digit(s[i]); + if (d < 0) return std::nullopt; + n = (n << 6) | (uint32_t)d; + } + if (group == 4) { + out.push_back((char)((n >> 16) & 0xff)); + out.push_back((char)((n >> 8) & 0xff)); + out.push_back((char)(n & 0xff)); + } else if (group == 3) { + out.push_back((char)((n >> 10) & 0xff)); + out.push_back((char)((n >> 2) & 0xff)); + } else if (group == 2) { + out.push_back((char)((n >> 4) & 0xff)); + } + } + return out; +} + +} // namespace molo::runtime diff --git a/src/runtime/caps.h b/src/runtime/caps.h new file mode 100644 index 0000000..cba18e2 --- /dev/null +++ b/src/runtime/caps.h @@ -0,0 +1,80 @@ +// Spec: plans/01-runtime.md section 6. Static ceilings for every runtime knob. +// Every limit here is a compile-time default; runtime config may lower it and +// may raise it only up to the paired ceiling. No autotuning anywhere. +#pragma once + +#include +#include +#include + +namespace molo::runtime::caps { + +// 6.1 Event loop and threading. +inline constexpr std::size_t kLoopWorkersMax = 64; +inline constexpr std::size_t kThreadPoolThreadsDefault = 4; +inline constexpr std::size_t kThreadPoolThreadsCeiling = 64; +inline constexpr std::size_t kThreadPoolQueueCap = 1024; +inline constexpr int kTimerGranularityMs = 10; +inline constexpr std::size_t kTimerWheelSlots = 1024; +inline constexpr std::size_t kTimersPerLoopCap = 65536; +inline constexpr std::size_t kPollerFdsCeiling = 65536; +inline constexpr std::size_t kPollerBatchMax = 256; +inline constexpr std::size_t kChannelCapDefault = 4096; +inline constexpr std::size_t kChannelCapCeiling = 1048576; +inline constexpr std::size_t kDeferredQueueCap = 65536; + +// 6.2 Connection and request lifecycle. +inline constexpr std::chrono::milliseconds kHeaderTimeout{10000}; +inline constexpr std::chrono::milliseconds kBodyTimeout{30000}; +inline constexpr std::chrono::milliseconds kIdleKeepAliveTimeout{75000}; +inline constexpr std::chrono::milliseconds kHandlerBudget{30000}; +inline constexpr std::chrono::milliseconds kShutdownDrainTimeout{30000}; +inline constexpr std::chrono::milliseconds kShutdownForcedTimeout{5000}; +inline constexpr std::chrono::milliseconds kTlsHandshakeTimeout{5000}; +inline constexpr std::size_t kMaxConnectionsDefault = 16384; +inline constexpr std::size_t kMaxConnectionsCeiling = 65536; +inline constexpr std::size_t kMaxRequestsPerH1Connection = 1000; +inline constexpr std::size_t kH2MaxConcurrentStreamsDefault = 100; +inline constexpr std::size_t kH2MaxConcurrentStreamsCeiling = 1000; +inline constexpr std::size_t kHeaderBlockMax = 65536; +inline constexpr std::size_t kHeaderFieldMax = 100; +inline constexpr std::size_t kHeaderLineMax = 16384; +inline constexpr std::size_t kUriMax = 8192; +inline constexpr std::size_t kBodyMaxDefault = 10 * 1024 * 1024; +inline constexpr std::size_t kBodyMaxCeiling = 100 * 1024 * 1024; +inline constexpr std::size_t kReadChunkSize = 16384; +inline constexpr std::size_t kWriteHighWatermark = 256 * 1024; +inline constexpr std::size_t kWriteLowWatermark = 64 * 1024; +inline constexpr std::size_t kWriteBufferCeiling = 16 * 1024 * 1024; +// 6.7 Static: bounded streaming chunk for file-backed response bodies. +inline constexpr std::size_t kFileChunkSize = 64 * 1024; + +// 6.4 HTTP/2. +inline constexpr std::size_t kH2InitialWindow = 65535; +inline constexpr std::size_t kH2MaxFrameDefault = 16384; +inline constexpr std::size_t kH2MaxFrameCeiling = 16777215; +inline constexpr std::size_t kH2HeaderTableDefault = 4096; +inline constexpr std::size_t kH2HeaderTableCeiling = 16384; +inline constexpr std::size_t kH2HeaderListMax = 65536; +inline constexpr std::size_t kH2MaxFrameHeaderListCap = 100; + +// 6.5 Compression. +inline constexpr std::size_t kCompressThreshold = 1024; +inline constexpr std::size_t kDecompressMaxAbsolute = 10 * 1024 * 1024; +inline constexpr std::size_t kDecompressMaxRatio = 10; +inline constexpr std::size_t kCompressChunkSize = 65536; +inline constexpr int kGzipLevel = 6; +inline constexpr int kBrotliQualityDynamic = 4; +inline constexpr int kBrotliQualityStatic = 11; + +// 6.6 Cookies. +inline constexpr std::size_t kCookieCountMax = 50; +inline constexpr std::size_t kCookieTotalMax = 8192; +inline constexpr std::size_t kSetCookieMax = 4096; + +// 6.7 Static files. +inline constexpr std::size_t kStaticChunkSize = 65536; +inline constexpr std::size_t kStaticPathMax = 4096; +inline constexpr int kStaticSymlinkDepthMax = 8; + +} // namespace molo::runtime::caps diff --git a/src/runtime/channel.h b/src/runtime/channel.h new file mode 100644 index 0000000..439a797 --- /dev/null +++ b/src/runtime/channel.h @@ -0,0 +1,162 @@ +// Spec: plans/01-runtime.md section 3. Bounded inter-loop (and inter-thread) +// message queue. Only the owning loop drains it; anyone may push. +#pragma once + +#include +#include +#include +#include +#include + +#include "caps.h" +#include "event_loop.h" + +#if defined(__linux__) +#include +#include +#else +#include +#include +#endif + +namespace molo::runtime { + +template +class Channel { + public: + explicit Channel(std::size_t cap = caps::kChannelCapDefault) + : cap_(cap == 0 ? 1 : cap) { + if (cap_ > caps::kChannelCapCeiling) cap_ = caps::kChannelCapCeiling; +#if defined(__linux__) + notify_fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); +#else + if (pipe(pipe_fds_) == 0) { + int flags = fcntl(pipe_fds_[0], F_GETFL, 0); + fcntl(pipe_fds_[0], F_SETFL, flags | O_NONBLOCK); + flags = fcntl(pipe_fds_[1], F_GETFL, 0); + fcntl(pipe_fds_[1], F_SETFL, flags | O_NONBLOCK); + notify_fd_ = pipe_fds_[0]; + } +#endif + } + + ~Channel() { detach(); close_fds(); } + + Channel(const Channel&) = delete; + Channel& operator=(const Channel&) = delete; + + bool valid() const { return notify_fd_ >= 0; } + + // False when full or closed. + bool try_push(T value) { + bool need_wake = false; + { + std::lock_guard lock(mutex_); + if (closed_ || queue_.size() >= cap_) return false; + need_wake = queue_.empty(); + queue_.push_back(std::move(value)); + } + if (need_wake) Signal(); + return true; + } + + bool try_pop(T& out) { + std::lock_guard lock(mutex_); + if (queue_.empty()) return false; + out = std::move(queue_.front()); + queue_.pop_front(); + return true; + } + + void close() { + { + std::lock_guard lock(mutex_); + closed_ = true; + } + Signal(); + } + + bool closed() const { + std::lock_guard lock(mutex_); + return closed_; + } + + std::size_t size() const { + std::lock_guard lock(mutex_); + return queue_.size(); + } + + int notify_fd() const { return notify_fd_; } + + bool attach(EventLoop& loop, std::function on_readable) { + if (!valid() || !on_readable) return false; + attached_loop_ = &loop; + return loop.poller().add_fd(notify_fd_, IPoller::kRead, + [this, cb = std::move(on_readable)](int, uint32_t) { + DrainSignal(); + cb(); + }); + } + + void detach() { + if (attached_loop_) { + attached_loop_->poller().remove_fd(notify_fd_); + attached_loop_ = nullptr; + } + } + + private: + void Signal() { +#if defined(__linux__) + if (notify_fd_ < 0) return; + uint64_t v = 1; + ssize_t r = write(notify_fd_, &v, sizeof(v)); + (void)r; +#else + if (pipe_fds_[1] < 0) return; + char c = 1; + ssize_t r = write(pipe_fds_[1], &c, 1); + (void)r; +#endif + } + + void DrainSignal() { +#if defined(__linux__) + uint64_t v = 0; + while (read(notify_fd_, &v, sizeof(v)) == (ssize_t)sizeof(v)) { + if (v == 0) break; + } +#else + char buf[64]; + while (read(notify_fd_, buf, sizeof(buf)) > 0) { + } +#endif + } + + void close_fds() { +#if defined(__linux__) + if (notify_fd_ >= 0) { + detach(); + ::close(notify_fd_); + notify_fd_ = -1; + } +#else + if (pipe_fds_[0] >= 0) ::close(pipe_fds_[0]); + if (pipe_fds_[1] >= 0) ::close(pipe_fds_[1]); + pipe_fds_[0] = pipe_fds_[1] = -1; + notify_fd_ = -1; +#endif + } + + std::size_t cap_; + mutable std::mutex mutex_; + std::deque queue_; + bool closed_ = false; + int notify_fd_ = -1; +#if !defined(__linux__) + int pipe_fds_[2] = {-1, -1}; +#endif + EventLoop* attached_loop_ = nullptr; +}; + +} // namespace molo::runtime diff --git a/src/runtime/compress/brotli.cpp b/src/runtime/compress/brotli.cpp new file mode 100644 index 0000000..544e1cc --- /dev/null +++ b/src/runtime/compress/brotli.cpp @@ -0,0 +1,345 @@ +// Spec: plans/01-runtime.md 6.5, RFC 7932 sections 9.1, 9.2. In-house +// Brotli, framing subset (see header for the documented v1 gap). +#include "brotli.h" + +#include + +#include "runtime/caps.h" + +namespace molo::runtime::compress { + +// ----- Encoder ----- + +void BrotliEncoder::reset() { + header_done_ = false; + finished_ = false; + pending_in_.clear(); + stage_.clear(); + acc_ = 0; + have_ = 0; +} + +void BrotliEncoder::PutBits(uint32_t bits, int count) { + acc_ |= bits << have_; + have_ += count; + while (have_ >= 8) { + stage_.push_back((char)(acc_ & 0xff)); + acc_ >>= 8; + have_ -= 8; + } +} + +void BrotliEncoder::PadToByte() { + if (have_ > 0) { + stage_.push_back((char)(acc_ & 0xff)); + acc_ = 0; + have_ = 0; + } +} + +CodecResult BrotliEncoder::feed(const char* in, std::size_t n, char* out, + std::size_t cap, bool finish) { + CodecResult r; + if (finished_) { + if (n != 0) { + r.status = CodecStatus::Error; + return r; + } + std::size_t k = stage_.size() < cap ? stage_.size() : cap; + if (k > 0) { + memcpy(out, stage_.data(), k); + stage_.erase(0, k); + } + r.out_used = k; + r.status = stage_.empty() ? CodecStatus::Done : CodecStatus::NeedOutput; + return r; + } + if (n > 0 && in != nullptr) pending_in_.append(in, n); + r.in_used = n; + if (!header_done_) { + PutBits(0, 1); // WBITS 16: single 0 bit + header_done_ = true; + } + // Emit full 64 KiB non-final blocks eagerly for streaming; the tail goes + // out at finish. MNIBBLES 4 covers MLEN up to 65536. + while (pending_in_.size() >= 65536 || + (finish && !pending_in_.empty())) { + std::size_t chunk = pending_in_.size(); + if (chunk > 65536) chunk = 65536; + PutBits(0, 1); // ISLAST = 0 + PutBits(0, 2); // MNIBBLES = 4 ("00") + PutBits((uint32_t)(chunk - 1), 16); // MLEN - 1 + PutBits(1, 1); // ISUNCOMPRESSED = 1 + PadToByte(); // ignored bits must be zero: they are + stage_.append(pending_in_.data(), chunk); + pending_in_.erase(0, chunk); + } + if (finish) { + PutBits(1, 1); // ISLAST = 1 + PutBits(1, 1); // ISLASTEMPTY = 1 + PadToByte(); + finished_ = true; + } + std::size_t k = stage_.size() < cap ? stage_.size() : cap; + if (k > 0) { + memcpy(out, stage_.data(), k); + stage_.erase(0, k); + } + r.out_used = k; + if (finished_ && stage_.empty()) { + r.status = CodecStatus::Done; + } else if (!stage_.empty()) { + r.status = CodecStatus::NeedOutput; + } else { + r.status = CodecStatus::Ok; + } + return r; +} + +// ----- Decoder ----- + +void BrotliDecoder::reset() { + staging_.clear(); + bitpos_ = 0; + header_done_ = false; + done_ = false; + unsupported_ = false; + total_out_ = 0; + total_in_ = 0; + copy_left_ = 0; +} + +bool BrotliDecoder::Need(std::size_t k) { + return staging_.size() * 8 - bitpos_ >= k; +} + +uint32_t BrotliDecoder::Take(std::size_t k) { + uint32_t v = 0; + for (std::size_t i = 0; i < k; ++i) { + std::size_t p = bitpos_ + i; + unsigned char c = (unsigned char)staging_[p / 8]; + v |= (uint32_t)((c >> (p % 8)) & 1) << i; + } + bitpos_ += k; + return v; +} + +bool BrotliDecoder::AlignZero() { + while (bitpos_ % 8 != 0) { + if (!Need(1)) return false; + if (Take(1) != 0) return false; + } + return true; +} + +// Returns 1 block done, 0 need more input, -1 corrupt, -2 unsupported. +int BrotliDecoder::ParseBlock(char* out, std::size_t cap, + std::size_t& produced, bool& need_output) { + need_output = false; + if (copy_left_ > 0) { + // Resume a pending uncompressed copy (byte aligned by construction). + while (copy_left_ > 0) { + if (produced >= cap) { + need_output = true; + return 1; + } + if (total_out_ >= caps::kDecompressMaxAbsolute) return -1; + if (!Need(8)) return 0; + char b = (char)Take(8); + out[produced++] = b; + ++total_out_; + --copy_left_; + } + return 1; + } + if (!Need(1)) return 0; + bool islast = Take(1) != 0; + if (islast) { + if (!Need(1)) { + bitpos_ -= 1; + return 0; + } + bool isempty = Take(1) != 0; + if (!isempty) { + // Final non-empty block: ISUNCOMPRESSED is absent, so the block is + // entropy-coded. Documented v1 gap. + return -2; + } + // Stream ends here: rest of the byte must be zero padding. + while (bitpos_ % 8 != 0) { + if (!Need(1)) return 0; + if (Take(1) != 0) return -1; + } + done_ = true; + return 1; + } + if (!Need(2)) { + bitpos_ -= 1; + return 0; + } + uint32_t mn = Take(2); + if (mn == 3) { + // Empty block: reserved + metadata skip. + if (!Need(3)) { + bitpos_ -= 3; // ISLAST + MNIBBLES + return 0; + } + if (Take(1) != 0) return -1; // reserved + uint32_t mskip = Take(2); // MSKIPBYTES + std::size_t mlen = 0; + if (mskip > 0) { + if (!Need(mskip * 8)) { + bitpos_ -= 6; // rewind to block start + return 0; + } + mlen = (std::size_t)Take(mskip * 8) + 1; + } + // Align (zeros) then skip metadata bytes. + std::size_t snap = bitpos_; + while (bitpos_ % 8 != 0) { + if (!Need(1)) { + bitpos_ = snap - 6; + return 0; + } + if (Take(1) != 0) return -1; + } + if (staging_.size() - bitpos_ / 8 < mlen) { + bitpos_ = snap - 6; + return 0; + } + bitpos_ += mlen * 8; + return 1; + } + uint32_t nibbles = 4 + mn; // 4, 5, 6 + if (!Need(nibbles * 4)) { + bitpos_ -= 3; + return 0; + } + uint32_t mlen = Take(nibbles * 4) + 1; + if (nibbles > 4) { + // Last nibble must be non-zero. + uint32_t top = (mlen - 1) >> ((nibbles - 1) * 4); + if ((top & 0xf) == 0) return -1; + } + if (!Need(1)) { + bitpos_ -= 3 + nibbles * 4; + return 0; + } + bool uncompressed = Take(1) != 0; + if (!uncompressed) return -2; // entropy-coded: documented gap + std::size_t snap = bitpos_; + while (bitpos_ % 8 != 0) { + if (!Need(1)) { + bitpos_ = snap - (4 + nibbles * 4); + return 0; + } + if (Take(1) != 0) return -1; + } + copy_left_ = mlen; + while (copy_left_ > 0) { + if (produced >= cap) { + need_output = true; + return 1; + } + if (total_out_ >= caps::kDecompressMaxAbsolute) return -1; + if (!Need(8)) return 0; + char b = (char)Take(8); + out[produced++] = b; + ++total_out_; + --copy_left_; + } + return 1; +} + +CodecResult BrotliDecoder::feed(const char* in, std::size_t n, char* out, + std::size_t cap, bool finish) { + CodecResult r; + if (done_ || unsupported_) { + r.status = unsupported_ ? CodecStatus::Unsupported : CodecStatus::Error; + return r; + } + if (n > 0 && in != nullptr) { + if (staging_.size() + n > caps::kBodyMaxCeiling) { + r.status = CodecStatus::Error; + return r; + } + staging_.append(in, n); + total_in_ += n; + } + r.in_used = n; + if (!header_done_) { + // WBITS variable-length code, rewound on truncation. + std::size_t snap = bitpos_; + if (!Need(1)) { + r.status = finish ? CodecStatus::Error : CodecStatus::Ok; + return r; + } + if (Take(1) == 0) { + header_done_ = true; // WBITS 16 + } else { + if (!Need(3)) { + bitpos_ = snap; + r.status = finish ? CodecStatus::Error : CodecStatus::Ok; + return r; + } + uint32_t v = Take(3); + if (v == 0) { + if (!Need(3)) { + bitpos_ = snap; + r.status = finish ? CodecStatus::Error : CodecStatus::Ok; + return r; + } + uint32_t w = Take(3); + if (w == 1) { + r.status = CodecStatus::Error; // 0010001 is invalid + return r; + } + header_done_ = true; // 17 or 10..15 + } else { + header_done_ = true; // 18..24 + } + } + } + bool need_output = false; + for (;;) { + if (done_) break; + std::size_t produced = 0; + int st = ParseBlock(out + r.out_used, cap - r.out_used, produced, + need_output); + r.out_used += produced; + if (st == 1) { + if (done_) break; + if (need_output) { + r.status = CodecStatus::NeedOutput; + return r; + } + continue; // next block + } + if (st == -2) { + unsupported_ = true; + r.status = CodecStatus::Unsupported; + return r; + } + if (st == -1 || finish) { + r.status = CodecStatus::Error; + return r; + } + r.status = CodecStatus::Ok; + return r; + } + // Trailing bytes after the final block are invalid. + if (bitpos_ / 8 != staging_.size()) { + r.status = CodecStatus::Error; + return r; + } + if (total_out_ > caps::kDecompressMaxAbsolute || + (total_in_ > 0 && + total_out_ > total_in_ * caps::kDecompressMaxRatio)) { + r.status = CodecStatus::Error; + return r; + } + r.status = CodecStatus::Done; + return r; +} + +} // namespace molo::runtime::compress diff --git a/src/runtime/compress/brotli.h b/src/runtime/compress/brotli.h new file mode 100644 index 0000000..b261707 --- /dev/null +++ b/src/runtime/compress/brotli.h @@ -0,0 +1,65 @@ +// Spec: plans/01-runtime.md 6.5, RFC 7932. In-house Brotli, framing subset. +// +// The encoder emits window bits plus uncompressed meta-blocks (ISLASTEMPTY +// final block): always valid Brotli, streaming, bounded. The decoder parses +// the stream header and meta-block headers, decodes uncompressed blocks and +// skips metadata blocks; entropy-coded blocks return Unsupported, which the +// caller maps to 415/passthrough. This is a documented v1 gap: full +// entropy-code decoding is future work, tracked next to this file. +#pragma once + +#include +#include +#include + +#include "compressor.h" + +namespace molo::runtime::compress { + +class BrotliEncoder : public IEncoder { + public: + BrotliEncoder() = default; + CodecResult feed(const char* in, std::size_t n, char* out, std::size_t cap, + bool finish) override; + void reset() override; + const char* name() const override { return "br"; } + + private: + void PutBits(uint32_t bits, int count); + void PadToByte(); + bool header_done_ = false; + bool finished_ = false; + std::string pending_in_; + std::string stage_; + uint32_t acc_ = 0; + int have_ = 0; +}; + +class BrotliDecoder : public IDecoder { + public: + BrotliDecoder() = default; + CodecResult feed(const char* in, std::size_t n, char* out, std::size_t cap, + bool finish) override; + void reset() override; + const char* name() const override { return "br"; } + + private: + // Bit cursor over staging_. Returns false when truncated. + bool Need(std::size_t k); + uint32_t Take(std::size_t k); // caller ensures Need(k) + bool AlignZero(); // pad to byte, must be zeros + int ParseBlock(char* out, std::size_t cap, std::size_t& produced, + bool& need_output); + + std::string staging_; + std::size_t bitpos_ = 0; + bool header_done_ = false; + bool done_ = false; + bool unsupported_ = false; + std::size_t total_out_ = 0; + std::size_t total_in_ = 0; + // Persistent uncompressed-block copy state. + std::size_t copy_left_ = 0; +}; + +} // namespace molo::runtime::compress diff --git a/src/runtime/compress/compressor.h b/src/runtime/compress/compressor.h new file mode 100644 index 0000000..9e9f9b5 --- /dev/null +++ b/src/runtime/compress/compressor.h @@ -0,0 +1,46 @@ +// Spec: plans/01-runtime.md 6.5. Streaming codec interface. Both codecs are +// implemented in-house, work in bounded chunks, and never buffer a full +// passthrough body. +#pragma once + +#include + +namespace molo::runtime::compress { + +enum class CodecStatus { + Ok, // progress made; call again for more + NeedOutput, // output buffer full, input remains + Done, // stream complete + Error, // corrupt input, cap hit, or API misuse + Unsupported, // valid framing the v1 decoder does not implement +}; + +struct CodecResult { + CodecStatus status = CodecStatus::Ok; + std::size_t in_used = 0; + std::size_t out_used = 0; +}; + +class IEncoder { + public: + virtual ~IEncoder() = default; + // finish=true flushes the last block and the trailer. After Done only + // drain feeds (n=0) are allowed until reset(). + virtual CodecResult feed(const char* in, std::size_t n, char* out, + std::size_t cap, bool finish) = 0; + virtual void reset() = 0; + virtual const char* name() const = 0; +}; + +class IDecoder { + public: + virtual ~IDecoder() = default; + // finish=true means no more input will ever arrive: an incomplete stream + // is an Error. After Done, feed is an error until reset(). + virtual CodecResult feed(const char* in, std::size_t n, char* out, + std::size_t cap, bool finish) = 0; + virtual void reset() = 0; + virtual const char* name() const = 0; +}; + +} // namespace molo::runtime::compress diff --git a/src/runtime/compress/gzip.cpp b/src/runtime/compress/gzip.cpp new file mode 100644 index 0000000..cfe5dff --- /dev/null +++ b/src/runtime/compress/gzip.cpp @@ -0,0 +1,773 @@ +// Spec: plans/01-runtime.md 6.5, RFC 1952, RFC 1951 (DEFLATE). In-house +// gzip: stored-block encoder, full inflate decoder with bomb guard. +#include "gzip.h" + +#include + +#include "runtime/caps.h" + +namespace molo::runtime::compress { + +namespace { +constexpr std::array kCrcTable = [] { + std::array t{}; + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = i; + for (int k = 0; k < 8; ++k) { + c = (c & 1) ? (0xedb88320u ^ (c >> 1)) : (c >> 1); + } + t[i] = c; + } + return t; +}(); + +constexpr uint32_t kLenBase[29] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, + 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, + 67, 83, 99, 115, 131, 163, 195, 227, 258}; +constexpr int kLenExtra[29] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, + 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0}; +constexpr uint32_t kDistBase[30] = { + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, + 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, + 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577}; +constexpr int kDistExtra[30] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13}; +constexpr int kClOrder[19] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, + 11, 4, 12, 3, 13, 2, 14, 1, 15}; +} // namespace + +uint32_t Crc32Update(uint32_t crc, const char* data, std::size_t len) { + for (std::size_t i = 0; i < len; ++i) { + crc = kCrcTable[(crc ^ (unsigned char)data[i]) & 0xff] ^ (crc >> 8); + } + return crc; +} + +// ----- Encoder ----- + +GzipEncoder::GzipEncoder() { reset(); } + +void GzipEncoder::reset() { + header_done_ = false; + trailer_done_ = false; + finished_ = false; + final_emitted_ = false; + pending_in_.clear(); + hist_.clear(); + stage_.clear(); + crc_ = 0xffffffffu; + isize_ = 0; + acc_ = 0; + have_ = 0; + head_.assign(4096, -1); + prev_.clear(); +} + +void GzipEncoder::PutBit(int bit) { + if (bit) acc_ |= (1u << have_); + if (++have_ >= 8) { + stage_.push_back((char)(acc_ & 0xff)); + acc_ = 0; + have_ = 0; + } +} + +void GzipEncoder::PutBitsLsb(uint32_t bits, int count) { + for (int i = 0; i < count; ++i) PutBit((bits >> i) & 1); +} + +void GzipEncoder::PutCode(uint32_t code, int len) { + for (int i = len - 1; i >= 0; --i) PutBit((code >> i) & 1); +} + +void GzipEncoder::PutLitLen(int sym) { + if (sym <= 143) { + PutCode((uint32_t)(0x30 + sym), 8); + } else if (sym <= 255) { + PutCode((uint32_t)(0x190 + sym - 144), 9); + } else if (sym <= 279) { + PutCode((uint32_t)(sym - 256), 7); + } else { + PutCode((uint32_t)(0xc0 + sym - 280), 8); + } +} + +void GzipEncoder::PutDist(int sym) { PutCode((uint32_t)sym, 5); } + +void GzipEncoder::FlushBits() { + if (have_ > 0) { + stage_.push_back((char)(acc_ & 0xff)); + acc_ = 0; + have_ = 0; + } +} + +int GzipEncoder::Hash3(const char* p) { + return (((unsigned char)p[0] << 10) ^ ((unsigned char)p[1] << 5) ^ + (unsigned char)p[2]) & + 4095; +} + +void GzipEncoder::EncodeBlock(const char* data, std::size_t start, + std::size_t len, bool last) { + // Block header: BFINAL + BTYPE 01 (fixed Huffman), LSB-first. + PutBit(last ? 1 : 0); + PutBit(1); + PutBit(0); + std::size_t end = start + len; + prev_.assign(end, -1); + head_.assign(4096, -1); // chains are per-block over this view + // Seed chains with the history bytes. + for (std::size_t i = 0; i < start; ++i) { + if (i + 3 > end) break; + int h = Hash3(data + i); + prev_[i] = head_[h]; + head_[h] = (int)i; + } + std::size_t pos = start; + while (pos < end) { + std::size_t best_len = 0; + std::size_t best_dist = 0; + if (pos + 3 <= end) { + int h = Hash3(data + pos); + int cand = head_[h]; + int chain = 0; + std::size_t max_len = end - pos; + if (max_len > 258) max_len = 258; + while (cand >= 0 && chain < 64) { + std::size_t cpos = (std::size_t)cand; + if (pos > cpos && pos - cpos <= 32768) { + std::size_t l = 0; + while (l < max_len && data[cpos + l] == data[pos + l]) ++l; + if (l > best_len) { + best_len = l; + best_dist = pos - cpos; + if (l >= max_len) break; + } + } else if (pos <= cpos) { + break; + } + cand = prev_[cpos]; + ++chain; + } + prev_[pos] = head_[h]; + head_[h] = (int)pos; + } + if (best_len >= 3) { + int li = 28; + for (int i = 0; i < 29; ++i) { + uint32_t hi = (i == 28) ? 258 : kLenBase[i + 1] - 1; + if (best_len >= kLenBase[i] && best_len <= hi) { + li = i; + break; + } + } + PutLitLen(257 + li); + if (kLenExtra[li] > 0) { + PutBitsLsb((uint32_t)(best_len - kLenBase[li]), kLenExtra[li]); + } + int di = 29; + for (int i = 0; i < 30; ++i) { + uint32_t hi = (i == 29) ? 32768 : kDistBase[i + 1] - 1; + if (best_dist >= kDistBase[i] && best_dist <= hi) { + di = i; + break; + } + } + PutDist(di); + if (kDistExtra[di] > 0) { + PutBitsLsb((uint32_t)(best_dist - kDistBase[di]), kDistExtra[di]); + } + // Insert skipped positions into the chains. + for (std::size_t k = 1; k < best_len; ++k) { + if (pos + k + 3 > end) break; + int h = Hash3(data + pos + k); + prev_[pos + k] = head_[h]; + head_[h] = (int)(pos + k); + } + pos += best_len; + } else { + PutLitLen((unsigned char)data[pos]); + ++pos; + } + } + PutLitLen(256); // end of block +} + +CodecResult GzipEncoder::feed(const char* in, std::size_t n, char* out, + std::size_t cap, bool finish) { + CodecResult r; + if (finished_) { + // Drain-only feeds after finish are allowed; new input is not. + if (n != 0) { + r.status = CodecStatus::Error; + return r; + } + std::size_t k = stage_.size() < cap ? stage_.size() : cap; + if (k > 0) { + memcpy(out, stage_.data(), k); + stage_.erase(0, k); + } + r.out_used = k; + r.status = stage_.empty() ? CodecStatus::Done : CodecStatus::NeedOutput; + return r; + } + if (n > 0 && in != nullptr) pending_in_.append(in, n); + r.in_used = n; + if (!header_done_) { + char hdr[10] = {(char)0x1f, (char)0x8b, (char)0x08, (char)0x00, + (char)0x00, (char)0x00, (char)0x00, (char)0x00, + (char)0x00, (char)0x03}; + stage_.append(hdr, 10); + header_done_ = true; + } + // Encode 32 KiB fixed-Huffman blocks eagerly; the tail goes out at finish. + while (pending_in_.size() >= 32768 || + (finish && !pending_in_.empty())) { + std::size_t chunk = pending_in_.size(); + if (chunk > 32768) chunk = 32768; + bool last = finish && chunk == pending_in_.size(); + std::string view = hist_; + view.append(pending_in_.data(), chunk); + crc_ = Crc32Update(crc_, pending_in_.data(), chunk); + isize_ += (uint32_t)chunk; + EncodeBlock(view.data(), hist_.size(), chunk, last); + if (last) final_emitted_ = true; + hist_.append(pending_in_.data(), chunk); + if (hist_.size() > 32768) hist_.erase(0, hist_.size() - 32768); + pending_in_.erase(0, chunk); + } + if (finish && !trailer_done_ && pending_in_.empty()) { + if (!final_emitted_) { + // Empty input (or an exact block multiple): one final fixed block + // holding only end-of-block. + EncodeBlock(hist_.data(), hist_.size(), 0, true); + final_emitted_ = true; + } + FlushBits(); + char tr[8]; + uint32_t crc = crc_ ^ 0xffffffffu; + tr[0] = (char)(crc & 0xff); + tr[1] = (char)((crc >> 8) & 0xff); + tr[2] = (char)((crc >> 16) & 0xff); + tr[3] = (char)((crc >> 24) & 0xff); + tr[4] = (char)(isize_ & 0xff); + tr[5] = (char)((isize_ >> 8) & 0xff); + tr[6] = (char)((isize_ >> 16) & 0xff); + tr[7] = (char)((isize_ >> 24) & 0xff); + stage_.append(tr, 8); + trailer_done_ = true; + finished_ = true; + } else if (finish && !pending_in_.empty()) { + // Unreachable: the loop above drains pending when finish is set. + r.status = CodecStatus::Error; + return r; + } + std::size_t k = stage_.size() < cap ? stage_.size() : cap; + if (k > 0) { + memcpy(out, stage_.data(), k); + stage_.erase(0, k); + } + r.out_used = k; + if (finished_ && stage_.empty()) { + r.status = CodecStatus::Done; + } else if (!stage_.empty()) { + r.status = CodecStatus::NeedOutput; + } else { + r.status = CodecStatus::Ok; + } + return r; +} + +// ----- Decoder ----- + +void GzipDecoder::reset() { + staging_.clear(); + bytepos_ = 0; + bitpos_ = 0; + header_done_ = false; + stream_end_ = false; + done_ = false; + crc_ = 0xffffffffu; + isize_ = 0; + total_out_ = 0; + total_in_ = 0; + window_.clear(); + fixed_ready_ = false; + dynamic_ready_ = false; + in_block_ = false; + block_final_ = false; + block_type_ = 0; + stored_left_ = 0; + match_stage_ = MatchStage::None; + match_len_ = 0; + match_dist_ = 0; + len_extra_ = 0; + dist_extra_ = 0; +} + +bool GzipDecoder::BitReader::need(std::size_t k) { + while (have_ < k && bytepos_ < len_) { + acc_ |= (uint64_t)(unsigned char)data_[bytepos_++] << have_; + have_ += 8; + if (have_ >= 56) break; + } + return have_ >= k; +} + +void GzipDecoder::BitReader::rewind(std::size_t bp) { + bitpos_ = bp; + bytepos_ = bp / 8; + acc_ = 0; + have_ = 0; + std::size_t skip = bp % 8; + if (skip > 0 && bytepos_ < len_) { + acc_ = (uint64_t)((unsigned char)data_[bytepos_++] >> skip); + have_ = 8 - skip; + } +} + +uint32_t GzipDecoder::BitReader::take(std::size_t k) { + uint32_t v = 0; + if (k > 0) { + v = (uint32_t)(acc_ & ((k >= 64) ? ~0ull : ((1ull << k) - 1ull))); + acc_ >>= k; + have_ -= k; + bitpos_ += k; + } + return v; +} + +bool GzipDecoder::HuffmanTable::build(const std::vector& lengths) { + count.fill(0); + offset.fill(0); + for (int l : lengths) { + if (l < 0 || l > 15) return false; + if (l > 0) count[(std::size_t)l]++; + } + int offs = 0; + for (int len = 1; len <= 15; ++len) { + offset[(std::size_t)len] = offs; + offs += count[(std::size_t)len]; + } + // Over-subscribed check. + int left = 1; + for (int len = 1; len <= 15; ++len) { + left <<= 1; + left -= count[(std::size_t)len]; + if (left < 0) return false; + } + symbols.assign((std::size_t)offs, 0); + std::array next = offset; + for (std::size_t s = 0; s < lengths.size(); ++s) { + int l = lengths[s]; + if (l > 0) symbols[(std::size_t)next[(std::size_t)l]++] = (int)s; + } + return true; +} + +int GzipDecoder::HuffmanTable::decode(BitReader& br) const { + std::size_t snap = br.bitpos(); + int code = 0; + int first = 0; + int index = 0; + for (int len = 1; len <= 15; ++len) { + if (!br.need(1)) { + br.rewind(snap); + return -2; + } + code |= (int)br.take(1); + int cnt = count[(std::size_t)len]; + if (code - first < cnt) { + std::size_t pos = (std::size_t)(offset[(std::size_t)len] + code - first); + if (pos >= symbols.size()) return -1; + return symbols[pos]; + } + index += cnt; + first = (first + cnt) << 1; + code <<= 1; + (void)index; + } + return -1; +} + +bool GzipDecoder::BuildFixed() { + if (fixed_ready_) return true; + std::vector lit(288, 0); + for (int i = 0; i <= 143; ++i) lit[(std::size_t)i] = 8; + for (int i = 144; i <= 255; ++i) lit[(std::size_t)i] = 9; + for (int i = 256; i <= 279; ++i) lit[(std::size_t)i] = 7; + for (int i = 280; i <= 287; ++i) lit[(std::size_t)i] = 8; + std::vector dist(32, 5); + if (!litlen_.build(lit)) return false; + if (!dist_.build(dist)) return false; + fixed_ready_ = true; + dynamic_ready_ = false; + return true; +} + +bool GzipDecoder::BuildDynamic(BitReader& br) { + if (!br.need(14)) return false; + int hlit = (int)br.take(5) + 257; + int hdist = (int)br.take(5) + 1; + int hclen = (int)br.take(4) + 4; + if (hlit > 288 || hdist > 32) return false; + std::vector cl_lens(19, 0); + for (int i = 0; i < hclen; ++i) { + if (!br.need(3)) return false; + cl_lens[(std::size_t)kClOrder[i]] = (int)br.take(3); + } + HuffmanTable cl; + if (!cl.build(cl_lens)) return false; + std::vector lens((std::size_t)(hlit + hdist), 0); + std::size_t i = 0; + while (i < lens.size()) { + int sym = cl.decode(br); + if (sym < 0) return false; + if (sym <= 15) { + lens[i++] = sym; + } else if (sym == 16) { + if (i == 0) return false; + if (!br.need(2)) return false; + int rep = (int)br.take(2) + 3; + if (i + (std::size_t)rep > lens.size()) return false; + int prev = lens[i - 1]; + for (int k = 0; k < rep; ++k) lens[i++] = prev; + } else if (sym == 17) { + if (!br.need(3)) return false; + int rep = (int)br.take(3) + 3; + if (i + (std::size_t)rep > lens.size()) return false; + for (int k = 0; k < rep; ++k) lens[i++] = 0; + } else if (sym == 18) { + if (!br.need(7)) return false; + int rep = (int)br.take(7) + 11; + if (i + (std::size_t)rep > lens.size()) return false; + for (int k = 0; k < rep; ++k) lens[i++] = 0; + } else { + return false; + } + } + // End-of-block code must exist. + if (lens[256] == 0) return false; + std::vector lit(lens.begin(), lens.begin() + hlit); + std::vector dist(lens.begin() + hlit, lens.end()); + if (!litlen_.build(lit)) return false; + if (!dist_.build(dist)) return false; + dynamic_ready_ = true; + fixed_ready_ = false; + return true; +} + +// 1 = parsed, 0 = need more input, -1 = corrupt. +int GzipDecoder::ParseHeader() { + if (staging_.size() - bytepos_ < 10) return 0; + const char* d = staging_.data() + bytepos_; + if ((unsigned char)d[0] != 0x1f || (unsigned char)d[1] != 0x8b) { + return -1; + } + if ((unsigned char)d[2] != 0x08) return -1; // CM must be deflate + unsigned flg = (unsigned char)d[3]; + if (flg & 0xe0) return -1; // reserved bits + std::size_t pos = bytepos_ + 10; + if (flg & 0x04) { + if (staging_.size() - pos < 2) return 0; + uint16_t xlen = (uint16_t)((unsigned char)staging_[pos] | + ((unsigned char)staging_[pos + 1] << 8)); + pos += 2; + if (staging_.size() - pos < xlen) return 0; + pos += xlen; + } + if (flg & 0x08) { + std::size_t end = staging_.find('\0', pos); + if (end == std::string::npos) return 0; + if (end - pos > caps::kHeaderLineMax) return -1; + pos = end + 1; + } + if (flg & 0x10) { + std::size_t end = staging_.find('\0', pos); + if (end == std::string::npos) return 0; + if (end - pos > caps::kHeaderLineMax) return -1; + pos = end + 1; + } + if (flg & 0x02) { + if (staging_.size() - pos < 2) return 0; + pos += 2; // FHCRC noted but not verified (header integrity only) + } + bytepos_ = pos; + bitpos_ = pos * 8; + header_done_ = true; + return 1; +} + +CodecStatus GzipDecoder::Inflate(char* out, std::size_t cap, + std::size_t& produced, bool finish) { + produced = 0; + BitReader br(staging_.data(), staging_.size(), bitpos_); + auto emit = [&](char b) -> bool { + if (produced >= cap) return false; + if (total_out_ >= caps::kDecompressMaxAbsolute) return false; + out[produced++] = b; + crc_ = Crc32Update(crc_, &b, 1); + ++isize_; + ++total_out_; + window_.push_back(b); + if (window_.size() > 36864) window_.erase(0, window_.size() - 32768); + return true; + }; + for (;;) { + if (!in_block_) { + if (!br.need(3)) { + bitpos_ = br.bitpos(); + return finish ? CodecStatus::Error : CodecStatus::Ok; + } + std::size_t snap = br.bitpos(); + block_final_ = br.take(1) != 0; + block_type_ = (int)br.take(2); + if (block_type_ == 3) return CodecStatus::Error; + if (block_type_ == 2) { + if (!BuildDynamic(br)) { + bitpos_ = snap; // truncated or corrupt header: retry/end + return finish ? CodecStatus::Error : CodecStatus::Ok; + } + } else if (block_type_ == 1) { + if (!BuildFixed()) return CodecStatus::Error; + } else { + stored_left_ = 0; + } + match_stage_ = MatchStage::None; + match_len_ = 0; + match_dist_ = 0; + in_block_ = true; + } + if (block_type_ == 0) { + // Stored: align, LEN/NLEN, then raw bytes. + std::size_t skip = (8 - (br.bitpos() % 8)) % 8; + if (stored_left_ == 0 && match_stage_ == MatchStage::None) { + if (!br.need(skip + 32)) { + bitpos_ = br.bitpos(); + return finish ? CodecStatus::Error : CodecStatus::Ok; + } + if (skip > 0) br.take(skip); + uint32_t len = br.take(16); + uint32_t nlen = br.take(16); + if (((len ^ nlen) & 0xffff) != 0xffff) return CodecStatus::Error; + stored_left_ = len; + match_stage_ = MatchStage::Copying; + if (len == 0) match_stage_ = MatchStage::None; + } + while (stored_left_ > 0) { + if (produced >= cap || total_out_ >= caps::kDecompressMaxAbsolute) { + bitpos_ = br.bitpos(); + if (total_out_ >= caps::kDecompressMaxAbsolute) { + return CodecStatus::Error; // bomb guard + } + return CodecStatus::NeedOutput; + } + if (!br.need(8)) { + bitpos_ = br.bitpos(); + return finish ? CodecStatus::Error : CodecStatus::Ok; + } + char b = (char)br.take(8); + --stored_left_; + emit(b); + } + match_stage_ = MatchStage::None; + in_block_ = false; + if (block_final_) { + bitpos_ = br.bitpos(); + stream_end_ = true; + return CodecStatus::Done; + } + continue; + } + // Huffman-coded block. + for (;;) { + if (match_stage_ == MatchStage::None) { + int sym = litlen_.decode(br); + if (sym == -2) { + bitpos_ = br.bitpos(); + return finish ? CodecStatus::Error : CodecStatus::Ok; + } + if (sym < 0 || sym > 285) return CodecStatus::Error; + if (sym < 256) { + if (produced >= cap || + total_out_ >= caps::kDecompressMaxAbsolute) { + bitpos_ = br.bitpos(); + // Rewind one symbol: the bitpos already passed it. The symbol + // is a literal, recompute is impossible, so emit-or-fail here: + // cap-full with a decoded literal means output stall. Since the + // literal byte is known, stash it via match state. + match_stage_ = MatchStage::Copying; + match_len_ = 1; + // Stash the literal in match_dist_ (flagged, low byte). + match_dist_ = (uint32_t)(unsigned char)sym | 0x80000000u; + if (total_out_ >= caps::kDecompressMaxAbsolute) { + return CodecStatus::Error; + } + return CodecStatus::NeedOutput; + } + emit((char)sym); + continue; + } + if (sym == 256) { + in_block_ = false; + if (block_final_) { + bitpos_ = br.bitpos(); + stream_end_ = true; + return CodecStatus::Done; + } + break; // next block header + } + int li = sym - 257; + match_len_ = kLenBase[li]; + len_extra_ = kLenExtra[li]; + match_stage_ = MatchStage::LenExtra; + } + if (match_stage_ == MatchStage::LenExtra) { + if (len_extra_ > 0) { + if (!br.need((std::size_t)len_extra_)) { + bitpos_ = br.bitpos(); + return finish ? CodecStatus::Error : CodecStatus::Ok; + } + match_len_ += br.take((std::size_t)len_extra_); + } + match_stage_ = MatchStage::DistSym; + } + if (match_stage_ == MatchStage::DistSym) { + int dsym = dist_.decode(br); + if (dsym == -2) { + bitpos_ = br.bitpos(); + return finish ? CodecStatus::Error : CodecStatus::Ok; + } + if (dsym < 0 || dsym > 29) return CodecStatus::Error; + match_dist_ = kDistBase[dsym]; + dist_extra_ = kDistExtra[dsym]; + match_stage_ = MatchStage::DistExtra; + } + if (match_stage_ == MatchStage::DistExtra) { + if (dist_extra_ > 0) { + if (!br.need((std::size_t)dist_extra_)) { + bitpos_ = br.bitpos(); + return finish ? CodecStatus::Error : CodecStatus::Ok; + } + match_dist_ += br.take((std::size_t)dist_extra_); + } + if (match_dist_ == 0 || match_dist_ > window_.size()) { + return CodecStatus::Error; + } + match_stage_ = MatchStage::Copying; + } + if (match_stage_ == MatchStage::Copying) { + if (match_dist_ & 0x80000000u) { + // Stashed literal (cap stall path above). + char b = (char)(match_dist_ & 0xff); + match_dist_ = 0; + match_len_ = 0; + match_stage_ = MatchStage::None; + if (!emit(b)) { + bitpos_ = br.bitpos(); + return CodecStatus::NeedOutput; + } + continue; + } + while (match_len_ > 0) { + if (produced >= cap || + total_out_ >= caps::kDecompressMaxAbsolute) { + bitpos_ = br.bitpos(); + if (total_out_ >= caps::kDecompressMaxAbsolute) { + return CodecStatus::Error; + } + return CodecStatus::NeedOutput; + } + char b = window_[window_.size() - match_dist_]; + --match_len_; + emit(b); + } + match_stage_ = MatchStage::None; + } + } + } +} + +CodecResult GzipDecoder::feed(const char* in, std::size_t n, char* out, + std::size_t cap, bool finish) { + CodecResult r; + if (done_) { + r.status = CodecStatus::Error; + return r; + } + if (n > 0 && in != nullptr) { + if (staging_.size() + n > caps::kBodyMaxCeiling) { + r.status = CodecStatus::Error; + return r; + } + staging_.append(in, n); + total_in_ += n; + } + r.in_used = n; + if (!header_done_) { + int ph = ParseHeader(); + if (ph < 0 || (ph == 0 && finish)) { + r.status = CodecStatus::Error; + return r; + } + if (ph == 0) { + r.status = CodecStatus::Ok; + return r; + } + } + if (!stream_end_) { + std::size_t produced = 0; + CodecStatus st = Inflate(out, cap, produced, finish); + r.out_used = produced; + if (st == CodecStatus::Error) { + r.status = CodecStatus::Error; + return r; + } + if (st == CodecStatus::NeedOutput) { + r.status = CodecStatus::NeedOutput; + return r; + } + // Ok means need more input; Done falls through to the trailer. + if (st == CodecStatus::Ok) { + r.status = CodecStatus::Ok; + return r; + } + } + // Trailer: 8 bytes after the final block (byte aligned). + std::size_t tpos = (bitpos_ + 7) / 8; + if (staging_.size() - tpos < 8) { + r.status = finish ? CodecStatus::Error : CodecStatus::Ok; + return r; + } + const char* d = staging_.data() + tpos; + uint32_t want_crc = (uint32_t)(unsigned char)d[0] | + ((uint32_t)(unsigned char)d[1] << 8) | + ((uint32_t)(unsigned char)d[2] << 16) | + ((uint32_t)(unsigned char)d[3] << 24); + uint32_t want_size = (uint32_t)(unsigned char)d[4] | + ((uint32_t)(unsigned char)d[5] << 8) | + ((uint32_t)(unsigned char)d[6] << 16) | + ((uint32_t)(unsigned char)d[7] << 24); + if ((crc_ ^ 0xffffffffu) != want_crc || isize_ != want_size) { + r.status = CodecStatus::Error; + return r; + } + if (total_out_ > caps::kDecompressMaxAbsolute || + (total_in_ > 0 && + total_out_ > total_in_ * caps::kDecompressMaxRatio)) { + r.status = CodecStatus::Error; + return r; + } + if (staging_.size() - tpos != 8) { + // Trailing bytes: multi-member gzip is out of v1 scope. + r.status = CodecStatus::Error; + return r; + } + done_ = true; + r.status = CodecStatus::Done; + return r; +} + +} // namespace molo::runtime::compress diff --git a/src/runtime/compress/gzip.h b/src/runtime/compress/gzip.h new file mode 100644 index 0000000..3453133 --- /dev/null +++ b/src/runtime/compress/gzip.h @@ -0,0 +1,141 @@ +// Spec: plans/01-runtime.md 6.5, RFC 1952. In-house gzip. The encoder runs +// greedy LZ77 plus fixed-Huffman DEFLATE blocks: real compression, +// streaming, bounded. The decoder implements full inflate (stored, fixed +// and dynamic Huffman) plus gzip framing, CRC32 and ISIZE checks, and the +// bomb guard. +#pragma once + +#include +#include +#include +#include +#include + +#include "compressor.h" + +namespace molo::runtime::compress { + +uint32_t Crc32Update(uint32_t crc, const char* data, std::size_t len); +inline uint32_t Crc32(const char* data, std::size_t len) { + return Crc32Update(0xffffffffu, data, len) ^ 0xffffffffu; +} + +class GzipEncoder : public IEncoder { + public: + GzipEncoder(); + CodecResult feed(const char* in, std::size_t n, char* out, std::size_t cap, + bool finish) override; + void reset() override; + const char* name() const override { return "gzip"; } + + private: + void PutBit(int bit); + void PutBitsLsb(uint32_t bits, int count); + void PutCode(uint32_t code, int len); // Huffman codes go MSB-first + void PutLitLen(int sym); + void PutDist(int sym); + void FlushBits(); + void EncodeBlock(const char* data, std::size_t start, std::size_t len, + bool last); + static int Hash3(const char* p); + + bool header_done_ = false; + bool trailer_done_ = false; + bool finished_ = false; + bool final_emitted_ = false; + std::string pending_in_; + std::string hist_; // last 32 KiB of encoded input for matches + std::string stage_; + uint32_t crc_ = 0xffffffffu; + uint32_t isize_ = 0; + uint32_t acc_ = 0; + int have_ = 0; + // LZ77 hash chains over the current view (hist_ + block). + std::vector head_; + std::vector prev_; +}; + +class GzipDecoder : public IDecoder { + public: + GzipDecoder() = default; + CodecResult feed(const char* in, std::size_t n, char* out, std::size_t cap, + bool finish) override; + void reset() override; + const char* name() const override { return "gzip"; } + + private: + class BitReader; + struct HuffmanTable { + // Canonical decode: count[len], offset[len], symbols in code order. + std::array count{}; + std::array offset{}; + std::vector symbols; + bool build(const std::vector& lengths); + // Symbol, or -1 invalid, or -2 truncated (resume from same bitpos). + int decode(BitReader& br) const; + }; + + class BitReader { + public: + BitReader(const char* data, std::size_t len, std::size_t bitpos) + : data_(data), len_(len), bitpos_(bitpos) { + bytepos_ = bitpos_ / 8; + std::size_t skip = bitpos_ % 8; + if (skip > 0 && bytepos_ < len_) { + acc_ = (uint64_t)((unsigned char)data_[bytepos_++] >> skip); + have_ = 8 - skip; + } + } + bool need(std::size_t k); + uint32_t take(std::size_t k); // caller ensures need(k), k <= 24 + void rewind(std::size_t bp); // restore a snapshot bit position + std::size_t bitpos() const { return bitpos_; } + std::size_t bits_left() const { + return len_ * 8 > bitpos_ ? len_ * 8 - bitpos_ : 0; + } + + private: + const char* data_; + std::size_t len_; + std::size_t bitpos_; + uint64_t acc_ = 0; + std::size_t have_ = 0; + std::size_t bytepos_ = 0; + }; + + enum class MatchStage { None, LenExtra, DistSym, DistExtra, Copying }; + + int ParseHeader(); // 1 = parsed, 0 = need more, -1 = corrupt + CodecStatus Inflate(char* out, std::size_t cap, std::size_t& produced, + bool finish); + bool BuildFixed(); + bool BuildDynamic(BitReader& br); + + std::string staging_; + std::size_t bytepos_ = 0; // header/trailer cursor + std::size_t bitpos_ = 0; // inflate cursor (absolute bits in staging_) + bool header_done_ = false; + bool stream_end_ = false; + bool done_ = false; + uint32_t crc_ = 0xffffffffu; + uint32_t isize_ = 0; + std::size_t total_out_ = 0; + std::size_t total_in_ = 0; + std::string window_; // last 32 KiB of output for matches + HuffmanTable litlen_; + HuffmanTable dist_; + bool fixed_ready_ = false; + bool dynamic_ready_ = false; + // Persistent block/match state across feed calls. + bool in_block_ = false; + bool block_final_ = false; + int block_type_ = 0; + std::size_t stored_left_ = 0; + MatchStage match_stage_ = MatchStage::None; + uint32_t match_len_ = 0; + uint32_t match_dist_ = 0; + int len_extra_ = 0; + int dist_extra_ = 0; +}; + +} // namespace molo::runtime::compress diff --git a/src/runtime/compress/negotiate.cpp b/src/runtime/compress/negotiate.cpp new file mode 100644 index 0000000..11b3fd8 --- /dev/null +++ b/src/runtime/compress/negotiate.cpp @@ -0,0 +1,167 @@ +// Spec: plans/01-runtime.md 6.5. Accept-Encoding negotiation. +#include "negotiate.h" + +#include + +#include "runtime/caps.h" + +namespace molo::runtime::compress { + +namespace { +std::string Lower(std::string_view s) { + std::string out(s); + for (char& c : out) c = (char)tolower((unsigned char)c); + return out; +} + +std::string Trim(const std::string& s) { + std::size_t b = 0; + while (b < s.size() && (s[b] == ' ' || s[b] == '\t')) ++b; + std::size_t e = s.size(); + while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\t')) --e; + return s.substr(b, e - b); +} + +// Parses qvalue 0..1 into 0..1000. Returns -1 when malformed. +int ParseQ(const std::string& s) { + if (s.empty() || s[0] != 'q' || s.size() < 3 || s[1] != '=') return -1; + std::string v = s.substr(2); + if (v == "1" || v == "1." || v == "1.0" || v == "1.00" || v == "1.000") { + return 1000; + } + if (v == "0" || v == "0." || v == "0.0" || v == "0.00" || v == "0.000") { + return 0; + } + if (v.size() < 2 || v[0] != '0' || v[1] != '.') return -1; + std::string frac = v.substr(2); + if (frac.empty() || frac.size() > 3) return -1; + for (char c : frac) { + if (c < '0' || c > '9') return -1; + } + while (frac.size() < 3) frac.push_back('0'); + return (frac[0] - '0') * 100 + (frac[1] - '0') * 10 + (frac[2] - '0'); +} + +struct Offers { + int br = -1; // -1 = absent + int gzip = -1; + int wildcard = -1; +}; + +Offers ParseAcceptEncoding(std::string_view header) { + Offers o; + std::string h(header); + std::size_t pos = 0; + while (pos <= h.size()) { + std::size_t comma = h.find(',', pos); + if (comma == std::string::npos) comma = h.size(); + std::string item = Trim(h.substr(pos, comma - pos)); + pos = comma + 1; + if (item.empty()) continue; + std::size_t semi = item.find(';'); + std::string token = Lower(Trim(item.substr(0, semi))); + int q = 1000; + if (semi != std::string::npos) { + std::string param = Trim(item.substr(semi + 1)); + // Only the first parameter matters; it should be q. + std::size_t semi2 = param.find(';'); + param = Trim(param.substr(0, semi2)); + int parsed = ParseQ(Lower(param)); + if (parsed >= 0) q = parsed; + } + if (token == "br") { + if (o.br < 0) o.br = q; + } else if (token == "gzip" || token == "x-gzip") { + if (o.gzip < 0) o.gzip = q; + } else if (token == "*") { + if (o.wildcard < 0) o.wildcard = q; + } + } + return o; +} +} // namespace + +bool Negotiator::IsCompressible(std::string_view content_type) { + std::string ct = Lower(content_type); + std::size_t semi = ct.find(';'); + if (semi != std::string::npos) ct = Trim(ct.substr(0, semi)); + if (ct.compare(0, 5, "text/") == 0) return true; + if (ct == "application/json" || ct == "application/xml" || + ct == "application/javascript" || ct == "application/x-javascript" || + ct == "application/ecmascript" || ct == "application/wasm" || + ct == "image/svg+xml") { + return true; + } + // Structured syntax suffixes. + if (ct.size() > 5 && + (ct.compare(ct.size() - 5, 5, "+json") == 0 || + ct.compare(ct.size() - 4, 4, "+xml") == 0)) { + return true; + } + return false; +} + +Decision Negotiator::Negotiate(std::string_view accept_encoding, + std::string_view content_type, + std::size_t body_len, bool has_content_encoding, + bool is_head, int status) { + Decision d{"identity"}; + if (is_head || has_content_encoding) return d; + if (status == 204 || status == 304 || (status >= 100 && status < 200)) { + return d; + } + if (body_len < caps::kCompressThreshold) return d; + if (!IsCompressible(content_type)) return d; + Offers o = ParseAcceptEncoding(accept_encoding); + int br_q = o.br >= 0 ? o.br : o.wildcard; + int gzip_q = o.gzip >= 0 ? o.gzip : o.wildcard; + if (br_q <= 0 && gzip_q <= 0) return d; + // br wins ties. + if (br_q >= gzip_q && br_q > 0) { + d.encoding = "br"; + return d; + } + if (gzip_q > 0) { + d.encoding = "gzip"; + return d; + } + return d; +} + +Precompressed Negotiator::SelectPrecompressed(std::string_view accept_encoding, + bool has_gz, bool has_br) { + Precompressed p; + if (!has_gz && !has_br) return p; + Offers o = ParseAcceptEncoding(accept_encoding); + int br_q = o.br >= 0 ? o.br : o.wildcard; + int gzip_q = o.gzip >= 0 ? o.gzip : o.wildcard; + if (has_br && br_q > 0 && br_q >= gzip_q) { + p.found = true; + p.suffix = ".br"; + p.encoding = "br"; + return p; + } + if (has_gz && gzip_q > 0) { + p.found = true; + p.suffix = ".gz"; + p.encoding = "gzip"; + return p; + } + // Fallthrough: br available but gzip ranked higher is handled above; if + // only br exists and gzip outranks it, still serve br when acceptable. + if (has_br && br_q > 0 && !has_gz) { + p.found = true; + p.suffix = ".br"; + p.encoding = "br"; + return p; + } + if (has_gz && gzip_q > 0 && !has_br) { + p.found = true; + p.suffix = ".gz"; + p.encoding = "gzip"; + return p; + } + return p; +} + +} // namespace molo::runtime::compress diff --git a/src/runtime/compress/negotiate.h b/src/runtime/compress/negotiate.h new file mode 100644 index 0000000..5756071 --- /dev/null +++ b/src/runtime/compress/negotiate.h @@ -0,0 +1,33 @@ +// Spec: plans/01-runtime.md 6.5. Accept-Encoding negotiation: br over gzip +// on tied q-values, 1024-byte threshold, content-type allowlist, Vary. +// Never compresses 204/304, HEAD, or already-encoded bodies. +#pragma once + +#include +#include + +namespace molo::runtime::compress { + +struct Decision { + std::string encoding; // "br", "gzip", or "identity" +}; + +struct Precompressed { + bool found = false; + std::string suffix; // ".br" or ".gz" + std::string encoding; // "br" or "gzip" +}; + +class Negotiator { + public: + static Decision Negotiate(std::string_view accept_encoding, + std::string_view content_type, std::size_t body_len, + bool has_content_encoding, bool is_head, + int status); + static bool IsCompressible(std::string_view content_type); + static Precompressed SelectPrecompressed(std::string_view accept_encoding, + bool has_gz, bool has_br); + static const char* VaryValue() { return "Accept-Encoding"; } +}; + +} // namespace molo::runtime::compress diff --git a/src/runtime/cookie/cookie.cpp b/src/runtime/cookie/cookie.cpp new file mode 100644 index 0000000..fe3fb7b --- /dev/null +++ b/src/runtime/cookie/cookie.cpp @@ -0,0 +1,153 @@ +// Spec: plans/01-runtime.md 6.6, RFC 6265 sections 4.1, 4.2, 5.2, 5.3. +#include "cookie.h" + +#include + +#include "runtime/caps.h" + +namespace molo::runtime::cookie { + +bool IsValidCookieName(std::string_view name) { + if (name.empty() || name.size() > 1024) return false; + for (char c : name) { + unsigned char u = (unsigned char)c; + if (u < 0x21 || u > 0x7e) return false; + switch (c) { + case '(': + case ')': + case '<': + case '>': + case '@': + case ',': + case ';': + case ':': + case '\\': + case '"': + case '/': + case '[': + case ']': + case '?': + case '=': + case '{': + case '}': + case ' ': + case '\t': + return false; + default: + break; + } + } + return true; +} + +bool IsValidCookieValue(std::string_view value, bool& needs_quotes) { + needs_quotes = false; + if (value.size() > (std::size_t)caps::kSetCookieMax) return false; + if (value.empty()) return true; + for (char c : value) { + unsigned char u = (unsigned char)c; + if (u < 0x21 || u == 0x22 || u == 0x2c || u == 0x3b || u == 0x5c || + u == 0x7f) { + if (u == 0x20) { + needs_quotes = true; + continue; + } + return false; + } + } + return true; +} + +CookieError ParseCookieHeader( + std::string_view header, + std::vector>& out) { + std::size_t pos = 0; + auto trim = [](std::string_view s) { + while (!s.empty() && (s.front() == ' ' || s.front() == '\t')) { + s.remove_prefix(1); + } + while (!s.empty() && (s.back() == ' ' || s.back() == '\t')) { + s.remove_suffix(1); + } + return s; + }; + while (pos <= header.size()) { + std::size_t semi = header.find(';', pos); + if (semi == std::string_view::npos) semi = header.size(); + std::string_view pair = trim(header.substr(pos, semi - pos)); + pos = semi + 1; + if (pair.empty()) continue; + std::size_t eq = pair.find('='); + std::string name, value; + if (eq == std::string_view::npos) { + name = std::string(trim(pair)); + } else { + name = std::string(trim(pair.substr(0, eq))); + value = std::string(trim(pair.substr(eq + 1))); + } + if (!IsValidCookieName(name)) return CookieError::BadName; + if (value.size() >= 2 && value.front() == '"' && value.back() == '"') { + value = value.substr(1, value.size() - 2); + } + bool quotes = false; + if (!IsValidCookieValue(value, quotes)) return CookieError::BadValue; + out.emplace_back(std::move(name), std::move(value)); + if (out.size() > caps::kCookieCountMax) return CookieError::TooLarge; + } + return CookieError::Ok; +} + +CookieError SerializeSetCookie(const Cookie& c, std::string& out) { + if (!IsValidCookieName(c.name)) return CookieError::BadName; + bool quotes = false; + if (!IsValidCookieValue(c.value, quotes)) return CookieError::BadValue; + if (c.same_site == SameSite::None && !c.secure) { + return CookieError::BadValue; // None requires Secure + } + out.clear(); + out.append(c.name); + out.push_back('='); + if (quotes) out.push_back('"'); + out.append(c.value); + if (quotes) out.push_back('"'); + if (!c.domain.empty()) { + out.append("; Domain="); + out.append(c.domain); + } + if (!c.path.empty()) { + out.append("; Path="); + out.append(c.path); + } + if (c.max_age >= 0) { + out.append("; Max-Age="); + out.append(std::to_string(c.max_age)); + } + if (!c.expires.empty()) { + out.append("; Expires="); + out.append(c.expires); + } + if (c.secure) out.append("; Secure"); + if (c.http_only) out.append("; HttpOnly"); + if (c.has_same_site) { + out.append("; SameSite="); + switch (c.same_site) { + case SameSite::None: + out.append("None"); + break; + case SameSite::Lax: + out.append("Lax"); + break; + case SameSite::Strict: + out.append("Strict"); + break; + } + } + if (c.partitioned) out.append("; Partitioned"); + if (out.size() > caps::kSetCookieMax) { + out.clear(); + return CookieError::TooLarge; // never silently truncated + } + return CookieError::Ok; +} + +} // namespace molo::runtime::cookie diff --git a/src/runtime/cookie/cookie.h b/src/runtime/cookie/cookie.h new file mode 100644 index 0000000..5d26af9 --- /dev/null +++ b/src/runtime/cookie/cookie.h @@ -0,0 +1,49 @@ +// Spec: plans/01-runtime.md 6.6, RFC 6265. Cookie transport only: parse and +// serialize. Signing/encryption crypto lives in 03-security.md; the ISigner +// interface is in signer.h. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace molo::runtime::cookie { + +enum class SameSite { None, Lax, Strict }; + +struct Cookie { + std::string name; + std::string value; + std::string domain; + std::string path; + long long max_age = -1; // -1 = absent + std::string expires; // already formatted, empty = absent + bool secure = false; + bool http_only = true; // default on per R9 + SameSite same_site = SameSite::Lax; // default per R9 + bool has_same_site = true; + bool partitioned = false; +}; + +enum class CookieError { + Ok, + BadName, + BadValue, + TooLarge, +}; + +// Parses one Cookie request header value into pairs. Lenient about spacing, +// strict about control characters. Stops at the first bad pair. +CookieError ParseCookieHeader(std::string_view header, + std::vector>& out); + +// Serializes one Set-Cookie line (no folding, never joined). +CookieError SerializeSetCookie(const Cookie& c, std::string& out); + +bool IsValidCookieName(std::string_view name); +bool IsValidCookieValue(std::string_view value, bool& needs_quotes); + +} // namespace molo::runtime::cookie diff --git a/src/runtime/cookie/jar.cpp b/src/runtime/cookie/jar.cpp new file mode 100644 index 0000000..e7faf34 --- /dev/null +++ b/src/runtime/cookie/jar.cpp @@ -0,0 +1,62 @@ +// Spec: plans/01-runtime.md 6.6. Cookie jar. +#include "jar.h" + +#include "runtime/caps.h" + +namespace molo::runtime::cookie { + +JarError CookieJar::ParseRequest( + const std::vector& cookie_fields) { + cookies_.clear(); + std::size_t total = 0; + std::string joined; + for (std::size_t i = 0; i < cookie_fields.size(); ++i) { + total += cookie_fields[i].size(); + if (total > caps::kCookieTotalMax) return JarError::TooLarge; + if (i > 0) joined.append("; "); + joined.append(cookie_fields[i]); + } + if (joined.size() > caps::kCookieTotalMax) return JarError::TooLarge; + std::vector> pairs; + CookieError e = ParseCookieHeader(joined, pairs); + if (e == CookieError::TooLarge) return JarError::TooMany; + if (e != CookieError::Ok) return JarError::BadCookie; + if (pairs.size() > caps::kCookieCountMax) return JarError::TooMany; + cookies_ = std::move(pairs); + return JarError::Ok; +} + +std::string CookieJar::Get(std::string_view name) const { + for (const auto& kv : cookies_) { + if (kv.first == name) return kv.second; + } + return ""; +} + +bool CookieJar::VerifySigned(std::string_view name, ISigner& signer, + std::string& payload_out) const { + std::string v = Get(name); + if (v.empty()) return false; + return signer.Verify(v, payload_out); +} + +JarError CookieJar::Set(Cookie c) { + if (secure_context_) c.secure = true; // forced on https + std::string line; + CookieError e = SerializeSetCookie(c, line); + if (e == CookieError::TooLarge) return JarError::TooLarge; + if (e != CookieError::Ok) return JarError::BadCookie; + set_headers_.push_back(std::move(line)); + return JarError::Ok; +} + +JarError CookieJar::SetSigned(std::string name, std::string payload, + ISigner& signer, std::string path) { + Cookie c; + c.name = std::move(name); + c.value = signer.Sign(payload); + c.path = std::move(path); + return Set(std::move(c)); +} + +} // namespace molo::runtime::cookie diff --git a/src/runtime/cookie/jar.h b/src/runtime/cookie/jar.h new file mode 100644 index 0000000..ce6f9e3 --- /dev/null +++ b/src/runtime/cookie/jar.h @@ -0,0 +1,53 @@ +// Spec: plans/01-runtime.md 6.6. Request jar (joins H2-split Cookie fields +// with "; ", caps 50 cookies and 8 KB, overflow is 431) and response list +// (one Set-Cookie per cookie, never joined). Secure forced on https. +#pragma once + +#include +#include +#include +#include + +#include "cookie.h" +#include "signer.h" + +namespace molo::runtime::cookie { + +enum class JarError { + Ok, + TooMany, // maps to 431 + TooLarge, // maps to 431 + BadCookie, // maps to 400 +}; + +class CookieJar { + public: + explicit CookieJar(bool secure_context = false) + : secure_context_(secure_context) {} + + // Parses request Cookie fields. H2 may split one logical Cookie across + // several fields: callers pass them all and the jar joins with "; ". + JarError ParseRequest(const std::vector& cookie_fields); + std::string Get(std::string_view name) const; + bool VerifySigned(std::string_view name, ISigner& signer, + std::string& payload_out) const; + const std::vector>& cookies() const { + return cookies_; + } + void Clear() { cookies_.clear(); } + + // Response side. + JarError Set(Cookie c); // applies Secure default in secure contexts + JarError SetSigned(std::string name, std::string payload, ISigner& signer, + std::string path = "/"); + const std::vector& set_cookie_headers() const { + return set_headers_; + } + + private: + bool secure_context_; + std::vector> cookies_; + std::vector set_headers_; +}; + +} // namespace molo::runtime::cookie diff --git a/src/runtime/cookie/signer.h b/src/runtime/cookie/signer.h new file mode 100644 index 0000000..55a3ee3 --- /dev/null +++ b/src/runtime/cookie/signer.h @@ -0,0 +1,45 @@ +// Spec: plans/01-runtime.md 6.6. Cookie signing interface. Format: +// `v1..`. Real HMAC and key hierarchy +// live in 03-security.md; NullSigner is a deterministic test-only signer, +// clearly marked insecure. +#pragma once + +#include +#include +#include + +#include "runtime/base64.h" + +namespace molo::runtime::cookie { + +class ISigner { + public: + virtual ~ISigner() = default; + virtual std::string Sign(std::string_view payload) = 0; + virtual bool Verify(std::string_view signed_value, + std::string& payload_out) = 0; + virtual const char* name() const = 0; +}; + +// Test-only signer: format-valid, cryptographically void. Never use in prod. +class NullSigner : public ISigner { + public: + std::string Sign(std::string_view payload) override { + return "v1." + Base64Url::Encode(payload) + ".bnVsbA"; + } + bool Verify(std::string_view signed_value, + std::string& payload_out) override { + if (signed_value.compare(0, 3, "v1.") != 0) return false; + std::size_t dot = signed_value.find('.', 3); + if (dot == std::string_view::npos) return false; + if (signed_value.substr(dot + 1) != "bnVsbA") return false; + std::optional p = + Base64Url::Decode(signed_value.substr(3, dot - 3)); + if (!p.has_value()) return false; + payload_out = std::move(*p); + return true; + } + const char* name() const override { return "null-insecure-test-only"; } +}; + +} // namespace molo::runtime::cookie diff --git a/src/runtime/event_loop.cpp b/src/runtime/event_loop.cpp new file mode 100644 index 0000000..879637f --- /dev/null +++ b/src/runtime/event_loop.cpp @@ -0,0 +1,137 @@ +// Spec: plans/01-runtime.md 6.1. One EventLoop per thread. +#include "event_loop.h" + +#include "caps.h" +#include "poller_epoll.h" +#include "poller_iocp.h" +#include "poller_kqueue.h" + +namespace molo::runtime { + +std::unique_ptr MakePoller() { +#if defined(__linux__) + auto p = std::make_unique(); + if (p->valid()) return p; +#elif defined(_WIN32) + auto p = std::make_unique(); + if (p->valid()) return p; +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) || defined(__DragonFly__) + auto p = std::make_unique(); + if (p->valid()) return p; +#endif + // Last resort: try each backend in turn. + { + auto p = std::make_unique(); + if (p->valid()) return p; + } + { + auto p = std::make_unique(); + if (p->valid()) return p; + } + { + auto p = std::make_unique(); + if (p->valid()) return p; + } + return std::make_unique(); +} + +EventLoop::EventLoop() + : poller_(MakePoller()), + pool_(std::make_unique(caps::kThreadPoolThreadsDefault, + caps::kThreadPoolQueueCap)), + timers_(caps::kTimerGranularityMs, caps::kTimerWheelSlots) {} + +EventLoop::~EventLoop() { stop(); } + +bool EventLoop::defer(std::function fn) { + if (!fn) return false; + { + std::lock_guard lock(mutex_); + if (deferred_.size() >= caps::kDeferredQueueCap) { + ++deferred_dropped_; + return false; + } + deferred_.push_back(std::move(fn)); + } + poller_->wake(); + return true; +} + +TimerWheel::TimerId EventLoop::schedule_after(std::chrono::milliseconds delay, + std::function fn) { + // Called from the loop thread or via defer; the wheel itself is not locked. + // Cross-thread callers must route through defer. This direct path serves + // the loop thread and tests that own the loop. + return timers_.add(delay, std::move(fn)); +} + +bool EventLoop::cancel_timer(TimerWheel::TimerId id) { + return timers_.cancel(id); +} + +void EventLoop::run() { + loop_thread_ = std::this_thread::get_id(); + while (run_once(-1)) { + } +} + +bool EventLoop::run_once(int timeout_ms) { + { + std::lock_guard lock(mutex_); + if (stop_) return false; + } + std::deque> work; + { + std::lock_guard lock(mutex_); + work.swap(deferred_); + } + while (!work.empty()) { + auto fn = std::move(work.front()); + work.pop_front(); + fn(); + } + timers_.run_due(); + int wait_ms = timers_.next_expiry_ms(); + if (timeout_ms >= 0 && (wait_ms < 0 || timeout_ms < wait_ms)) { + wait_ms = timeout_ms; + } + poller_->wait(wait_ms); + { + std::lock_guard lock(mutex_); + if (stop_) return false; + } + return true; +} + +void EventLoop::stop() { + { + std::lock_guard lock(mutex_); + stop_ = true; + } + if (poller_) poller_->wake(); +} + +bool EventLoop::stopped() const { + std::lock_guard lock(mutex_); + return stop_; +} + +bool EventLoop::in_loop_thread() const { + return std::this_thread::get_id() == loop_thread_; +} + +void EventLoop::post(Task task) { + if (!task.valid()) return; + if (in_loop_thread()) { + Detach(std::move(task)); + return; + } + auto holder = std::make_shared>(std::move(task)); + defer([holder] { + Task t = std::move(*holder); + Detach(std::move(t)); + }); +} + +} // namespace molo::runtime diff --git a/src/runtime/event_loop.h b/src/runtime/event_loop.h new file mode 100644 index 0000000..7d53804 --- /dev/null +++ b/src/runtime/event_loop.h @@ -0,0 +1,98 @@ +// Spec: plans/01-runtime.md 6.1. One EventLoop per thread. Connections are +// pinned to their loop; cross-loop contact goes through Channel. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "executor.h" +#include "poller.h" +#include "task.h" +#include "timer_wheel.h" + +namespace molo::runtime { + +class EventLoop { + public: + EventLoop(); + ~EventLoop(); + EventLoop(const EventLoop&) = delete; + EventLoop& operator=(const EventLoop&) = delete; + + IPoller& poller() { return *poller_; } + ThreadPool& pool() { return *pool_; } + TimerWheel& timers() { return timers_; } + + // Thread-safe. Drops (returns false) past the deferred cap. + bool defer(std::function fn); + TimerWheel::TimerId schedule_after(std::chrono::milliseconds delay, + std::function fn); + bool cancel_timer(TimerWheel::TimerId id); + + void run(); + // One iteration. Returns false once stop() was requested. + bool run_once(int timeout_ms); + void stop(); + bool stopped() const; + bool in_loop_thread() const; + + // Awaitable: resume after at least d on this loop. + auto sleep(std::chrono::milliseconds d); + // Awaitable: resume on the next loop iteration. + auto yield(); + // Start a coroutine on this loop (thread-safe). + void post(Task task); + + std::size_t deferred_dropped() const { return deferred_dropped_; } + + private: + std::unique_ptr poller_; + std::unique_ptr pool_; + TimerWheel timers_; + mutable std::mutex mutex_; + std::deque> deferred_; + std::size_t deferred_dropped_ = 0; + bool stop_ = false; + std::thread::id loop_thread_; +}; + +class SleepAwaitable { + public: + SleepAwaitable(EventLoop& loop, std::chrono::milliseconds d) + : loop_(loop), delay_(d) {} + bool await_ready() const noexcept { return delay_.count() <= 0; } + void await_suspend(std::coroutine_handle<> h) { + loop_.schedule_after(delay_, [h] { h.resume(); }); + } + void await_resume() noexcept {} + + private: + EventLoop& loop_; + std::chrono::milliseconds delay_; +}; + +class YieldAwaitable { + public: + explicit YieldAwaitable(EventLoop& loop) : loop_(loop) {} + bool await_ready() const noexcept { return false; } + void await_suspend(std::coroutine_handle<> h) { + if (!loop_.defer([h] { h.resume(); })) h.resume(); + } + void await_resume() noexcept {} + + private: + EventLoop& loop_; +}; + +inline auto EventLoop::sleep(std::chrono::milliseconds d) { + return SleepAwaitable(*this, d); +} + +inline auto EventLoop::yield() { return YieldAwaitable(*this); } + +} // namespace molo::runtime diff --git a/src/runtime/executor.cpp b/src/runtime/executor.cpp new file mode 100644 index 0000000..6a26d09 --- /dev/null +++ b/src/runtime/executor.cpp @@ -0,0 +1,76 @@ +// Spec: plans/01-runtime.md 6.1. Bounded pool for blocking work. +#include "executor.h" + +#include "caps.h" + +namespace molo::runtime { + +ThreadPool::ThreadPool(std::size_t threads, std::size_t queue_cap) + : queue_cap_(queue_cap) { + if (threads == 0) threads = 1; + if (threads > caps::kThreadPoolThreadsCeiling) { + threads = caps::kThreadPoolThreadsCeiling; + } + if (queue_cap_ == 0) queue_cap_ = 1; + for (std::size_t i = 0; i < threads; ++i) { + workers_.emplace_back([this] { Worker(); }); + } +} + +ThreadPool::~ThreadPool() { shutdown(); } + +bool ThreadPool::try_submit(Job job) { + if (!job) return false; + { + std::lock_guard lock(mutex_); + if (stop_) return false; + if (queue_.size() >= queue_cap_) return false; + queue_.push_back(std::move(job)); + } + cv_.notify_one(); + return true; +} + +void ThreadPool::shutdown() { + { + std::lock_guard lock(mutex_); + if (stop_) { + // Fall through to join below; workers may still need joining. + } + stop_ = true; + } + cv_.notify_all(); + for (auto& w : workers_) { + if (w.joinable()) w.join(); + } + workers_.clear(); +} + +std::size_t ThreadPool::pending() const { + std::lock_guard lock(mutex_); + return queue_.size(); +} + +bool ThreadPool::stopped() const { + std::lock_guard lock(mutex_); + return stop_; +} + +void ThreadPool::Worker() { + for (;;) { + Job job; + { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return stop_ || !queue_.empty(); }); + if (queue_.empty()) { + if (stop_) return; + continue; + } + job = std::move(queue_.front()); + queue_.pop_front(); + } + job(); + } +} + +} // namespace molo::runtime diff --git a/src/runtime/executor.h b/src/runtime/executor.h new file mode 100644 index 0000000..9e1eb59 --- /dev/null +++ b/src/runtime/executor.h @@ -0,0 +1,44 @@ +// Spec: plans/01-runtime.md 6.1. Bounded pool for blocking work. Default 4 +// threads, queue cap 1024 jobs. Overflow returns false (caller maps to 503 +// with Retry-After); the pool never grows silently. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::runtime { + +class ThreadPool { + public: + using Job = std::function; + + explicit ThreadPool(std::size_t threads = 4, + std::size_t queue_cap = 1024); + ~ThreadPool(); + ThreadPool(const ThreadPool&) = delete; + ThreadPool& operator=(const ThreadPool&) = delete; + + // False when the queue is full or the pool is stopped. + bool try_submit(Job job); + void shutdown(); + std::size_t pending() const; + std::size_t threads() const { return workers_.size(); } + bool stopped() const; + + private: + void Worker(); + + mutable std::mutex mutex_; + std::condition_variable cv_; + std::deque queue_; + std::size_t queue_cap_; + std::vector workers_; + bool stop_ = false; +}; + +} // namespace molo::runtime diff --git a/src/runtime/http/connection.cpp b/src/runtime/http/connection.cpp new file mode 100644 index 0000000..d8cdedc --- /dev/null +++ b/src/runtime/http/connection.cpp @@ -0,0 +1,672 @@ +// Spec: plans/01-runtime.md 6.2. Connection and request lifecycle. +#include "connection.h" + +#include +#include +#ifdef __linux__ +#include +#endif + +#include "runtime/observability_hooks.h" +#include "v1/serializer.h" +#include "v2/connection.h" + +namespace molo::runtime::http { + +namespace { +std::size_t SaturateToSize(uint64_t v) { + return v > (uint64_t)SIZE_MAX ? SIZE_MAX : (std::size_t)v; +} +} // namespace + +std::shared_ptr Connection::Create( + EventLoop& loop, std::unique_ptr transport, + std::unique_ptr tls, Handler handler, + ConnectionConfig config) { + if (!transport || !handler) return nullptr; + if (config.body_cap > caps::kBodyMaxCeiling) { + config.body_cap = caps::kBodyMaxCeiling; + } + auto c = std::shared_ptr( + new Connection(loop, std::move(transport), std::move(tls), + std::move(handler), config)); + c->waiter_ = std::make_unique(loop, c->transport_->fd()); + return c; +} + +Connection::Connection(EventLoop& loop, std::unique_ptr transport, + std::unique_ptr tls, Handler handler, + ConnectionConfig config) + : loop_(loop), + transport_(std::move(transport)), + tls_(std::move(tls)), + handler_(std::move(handler)), + config_(config) {} + +Connection::~Connection() { close(); } + +std::string Connection::protocol() const { + if (protocol_ == Protocol::H2PriorKnowledge) return "HTTP/2"; + if (protocol_ == Protocol::Http1) return "HTTP/1.1"; + return "detecting"; +} + +bool Connection::is_tls() const { return tls_ && tls_->is_tls(); } + +std::string Connection::peer() const { + return transport_ ? transport_->peer() : ""; +} + +void Connection::start() { + auto self = shared_from_this(); + obs::OnAccept(transport_->peer()); + MOLO_RUNTIME_COUNT("runtime.accept"); + loop_.post(Runner(std::move(self))); +} + +Task Connection::Runner(std::shared_ptr self) { + co_await self->Run(); +} + +void Connection::drain() { + // Thread-safe: the flag flip and waiter poke run on the loop thread so a + // parked coroutine never resumes on a foreign thread. + std::weak_ptr weak = weak_from_this(); + loop_.defer([weak] { + auto self = weak.lock(); + if (!self || self->closed_) return; + self->draining_ = true; + if (self->active_h2_) { + self->active_h2_->drain(); + } else if (self->waiter_) { + self->waiter_->Wake(); + } + }); +} + +void Connection::InterruptRead() { + if (waiter_) waiter_->Wake(); +} + +void Connection::close() { + // Pin via weak_ptr: shared_from_this() would throw in the destructor. + // In the dtor the pin is null, which is safe because Run (the only parked + // coroutine) holds a shared ref and is therefore already finished. + std::shared_ptr pin = weak_from_this().lock(); + if (closed_) return; + closed_ = true; + if (waiter_) waiter_->cancel(); + if (transport_) { + obs::OnClose(transport_->peer(), transport_->bytes_read(), + transport_->bytes_written()); + transport_->close(); + } +} + +Task Connection::PumpRead( + std::chrono::milliseconds timeout) { + if (closed_) co_return ReadResult::Closed; + // Drain interrupts idle waits but lets the in-flight request finish. + if (draining_ && !in_request_) co_return ReadResult::Closed; + auto deadline = std::chrono::steady_clock::now() + timeout; + char tmp[caps::kReadChunkSize]; + for (;;) { + if (closed_) co_return ReadResult::Closed; + if (tls_ && tls_->is_tls() && tls_done_ && !raw_.empty()) { + // Drain stashed ciphertext through decrypt first. + char plain[caps::kReadChunkSize]; + std::size_t used = 0; + ssize_t p = tls_->decrypt(raw_.data(), raw_.size(), plain, + sizeof(plain), used); + if (p < 0) co_return ReadResult::Error; + if (p > 0 || used > 0) { + in_.append(plain, (std::size_t)(p > 0 ? p : 0)); + raw_.erase(0, used); + if (p > 0) co_return ReadResult::Ok; + } + } + ssize_t n = transport_->recv(tmp, sizeof(tmp)); + if (n > 0) { + if (tls_ && tls_->is_tls() && tls_done_) { + raw_.append(tmp, (std::size_t)n); + continue; // loop back into the decrypt drain above + } + in_.append(tmp, (std::size_t)n); + co_return ReadResult::Ok; + } + if (n == 0) co_return ReadResult::Eof; + if (errno != EAGAIN && errno != EWOULDBLOCK) { + co_return ReadResult::Error; + } + auto now = std::chrono::steady_clock::now(); + if (now >= deadline) co_return ReadResult::Timeout; + auto left = + std::chrono::duration_cast(deadline - now); + bool fired = false; + TimerWheel::TimerId timer = loop_.schedule_after(left, [this, &fired] { + fired = true; + if (waiter_) waiter_->Wake(); + }); + bool alive = co_await waiter_->readable(); + loop_.cancel_timer(timer); + if (!alive || closed_) co_return ReadResult::Closed; + if (draining_ && !in_request_) co_return ReadResult::Closed; + if (fired) { + // One last non-blocking attempt before declaring the timeout. + n = transport_->recv(tmp, sizeof(tmp)); + if (n > 0) { + if (tls_ && tls_->is_tls() && tls_done_) { + raw_.append(tmp, (std::size_t)n); + continue; + } + in_.append(tmp, (std::size_t)n); + co_return ReadResult::Ok; + } + co_return ReadResult::Timeout; + } + // Woken without data or timeout: drain poke or readiness race. Surface + // it so the caller re-checks exit conditions instead of sleeping past + // them. No data is lost: the next call tries recv first. + co_return ReadResult::Interrupted; + } +} + +Task Connection::Flush(std::chrono::milliseconds timeout) { + std::string send_buf; + if (tls_ && tls_->is_tls() && tls_done_) { + send_buf.resize(out_.size() + 512); + ssize_t n = tls_->encrypt(out_.data(), out_.size(), send_buf.data(), + send_buf.size()); + if (n < 0) co_return false; + send_buf.resize((std::size_t)n); + out_.clear(); + } else { + send_buf.swap(out_); + } + std::size_t off = 0; + while (off < send_buf.size()) { + if (closed_) co_return false; + ssize_t n = transport_->send(send_buf.data() + off, + send_buf.size() - off); + if (n > 0) { + off += (std::size_t)n; + continue; + } + if (n == 0) co_return false; + if (errno != EAGAIN && errno != EWOULDBLOCK) co_return false; + bool fired = false; + TimerWheel::TimerId timer = 0; + if (timeout.count() > 0) { + timer = loop_.schedule_after(timeout, [this, &fired] { + fired = true; + if (waiter_) waiter_->Wake(); + }); + } + bool alive = co_await waiter_->writable(); + if (timer != 0) loop_.cancel_timer(timer); + if (!alive || closed_) co_return false; + if (fired) co_return false; + } + co_return true; +} + +Task Connection::AwaitWritable(std::chrono::milliseconds timeout) { + bool fired = false; + TimerWheel::TimerId timer = 0; + if (timeout.count() > 0) { + timer = loop_.schedule_after(timeout, [this, &fired] { + fired = true; + if (waiter_) waiter_->Wake(); + }); + } + bool alive = co_await waiter_->writable(); + if (timer != 0) loop_.cancel_timer(timer); + if (!alive || closed_) co_return false; + co_return !fired; +} + +Task Connection::SendFileBody(FileBody fb, + std::chrono::milliseconds timeout) { + struct Closer { + int fd; + ~Closer() { + if (fd >= 0) ::close(fd); + } + } closer{fb.fd}; + if (!ClampFileBody(fb)) co_return false; + uint64_t start = fb.offset; + uint64_t length = fb.length; + bool use_tls = tls_ && tls_->is_tls() && tls_done_; +#ifdef __linux__ + if (!use_tls) { + // Zero-copy path: one sendfile call per 64 KB chunk with writable waits + // for backpressure. The socket is non-blocking, so EAGAIN just parks. + off_t off = (off_t)start; + uint64_t left = length; + while (left > 0) { + if (closed_) co_return false; + std::size_t n = left > caps::kFileChunkSize + ? caps::kFileChunkSize + : (std::size_t)left; + ssize_t w = ::sendfile(transport_->fd(), fb.fd, &off, n); + if (w > 0) { + left -= (uint64_t)w; + continue; + } + if (w == 0) co_return false; // truncated under us: fail closed + if (errno != EAGAIN && errno != EWOULDBLOCK) co_return false; + if (!co_await AwaitWritable(timeout)) co_return false; + } + co_return true; + } +#else + (void)use_tls; +#endif + // TLS or non-Linux: bounded pread streaming through the outbox, one Flush + // per chunk so the write ceiling always holds. + uint64_t pos = start; + uint64_t left = length; + char buf[caps::kFileChunkSize]; + while (left > 0) { + if (closed_) co_return false; + std::size_t n = + left > sizeof(buf) ? sizeof(buf) : (std::size_t)left; + ssize_t r = ::pread(fb.fd, buf, n, (off_t)pos); + if (r <= 0) co_return false; + pos += (uint64_t)r; + left -= (uint64_t)r; + out_.append(buf, (std::size_t)r); + if (!co_await Flush(timeout)) co_return false; + } + co_return true; +} + +Task Connection::DoHandshake() { + if (!tls_ || !tls_->is_tls()) { + tls_done_ = true; + co_return true; + } + auto deadline = + std::chrono::steady_clock::now() + config_.tls_handshake_timeout; + for (;;) { + if (closed_) co_return false; + const char* in_ptr = raw_.data(); + std::size_t in_len = raw_.size(); + std::size_t before = in_len; + std::string hs_out; + tls::HandshakeState st = + tls_->handshake_step(in_ptr, in_len, hs_out); + raw_.erase(0, before - in_len); + if (!hs_out.empty()) { + out_.append(hs_out); + auto now = std::chrono::steady_clock::now(); + if (now >= deadline) co_return false; + auto left = + std::chrono::duration_cast(deadline - now); + if (!co_await Flush(left)) co_return false; + } + if (st == tls::HandshakeState::Done) { + tls_done_ = true; + co_return true; + } + if (st == tls::HandshakeState::Failed) co_return false; + auto now = std::chrono::steady_clock::now(); + if (now >= deadline) co_return false; + auto left = + std::chrono::duration_cast(deadline - now); + // Handshake bytes bypass the plaintext buffer: read raw directly. + char tmp[caps::kReadChunkSize]; + for (;;) { + ssize_t n = transport_->recv(tmp, sizeof(tmp)); + if (n > 0) { + raw_.append(tmp, (std::size_t)n); + break; + } + if (n == 0) co_return false; + if (errno != EAGAIN && errno != EWOULDBLOCK) co_return false; + auto hs_now = std::chrono::steady_clock::now(); + if (hs_now >= deadline) co_return false; + auto hs_left = std::chrono::duration_cast( + deadline - hs_now); + bool fired = false; + TimerWheel::TimerId timer = + loop_.schedule_after(hs_left, [this, &fired] { + fired = true; + if (waiter_) waiter_->Wake(); + }); + bool alive = co_await waiter_->readable(); + loop_.cancel_timer(timer); + if (!alive || closed_ || fired) co_return false; + } + } +} + +bool Connection::IsH2cUpgrade(const RequestHead& head) const { + if (!config_.allow_h2c_upgrade) return false; + if (head.version != "HTTP/1.1") return false; + std::string conn = ToLower(head.headers.get("connection")); + if (conn.find("upgrade") == std::string::npos) return false; + std::string up = ToLower(head.headers.get("upgrade")); + if (up.find("h2c") == std::string::npos) return false; + return head.headers.has("http2-settings"); +} + +Task Connection::SendError(int status, bool close_after) { + ResponseHead head; + head.status = status; + std::string body = std::to_string(status) + " " + StatusReason(status) + "\n"; + head.headers.set("content-type", "text/plain; charset=utf-8"); + out_.append(v1::Serializer::SerializeHead(head, "HTTP/1.1", false, + body.size(), false, close_after)); + out_.append(body); + co_await Flush(config_.idle_timeout); + if (close_after) close(); +} + +Task Connection::Run() { + auto self = shared_from_this(); + if (!co_await DoHandshake()) { + close(); + co_return; + } + if (tls_ && tls_->is_tls()) { + protocol_ = ProtocolDetector::from_alpn(tls_->alpn_protocol()); + } else { + ProtocolDetector detector; + std::size_t det_fed = 0; + auto start = std::chrono::steady_clock::now(); + auto idle_deadline = start + config_.idle_timeout; + auto head_deadline = start + config_.header_timeout; + bool det_got_any = false; + for (;;) { + if (det_fed < in_.size()) { + // Feed only new bytes; the detector keeps match progress. + protocol_ = + detector.feed(in_.data() + det_fed, in_.size() - det_fed); + det_fed = in_.size(); + if (protocol_ != Protocol::NeedMore) break; + } + // First byte races the idle timeout (silent close); completing + // detection races the header timeout (408). + auto limit = det_got_any ? head_deadline : idle_deadline; + auto now = std::chrono::steady_clock::now(); + if (now >= limit) { + if (!det_got_any) { + close(); + co_return; + } + co_await SendError(408, true); + co_return; + } + auto left = + std::chrono::duration_cast(limit - now); + ReadResult r = co_await PumpRead(left); + if (r == ReadResult::Ok) { + det_got_any = true; + continue; + } + if (r == ReadResult::Interrupted) continue; + if (r == ReadResult::Timeout) { + if (!det_got_any) { + close(); + co_return; + } + co_await SendError(408, true); + co_return; + } + close(); + co_return; + } + } + if (protocol_ == Protocol::H2PriorKnowledge) { + bool expect_magic = !(tls_ && tls_->is_tls()); + co_await RunH2(expect_magic, false, RequestHead{}, ""); + co_return; + } + co_await RunH1(); +} + +Task Connection::RunH1() { + auto self = shared_from_this(); + v1::Parser parser(config_.body_cap); + while (!closed_ && requests_ < config_.max_requests) { + if (draining_) { + close(); + co_return; + } + parser.reset(); + auto head_deadline = + std::chrono::steady_clock::now() + config_.header_timeout; + auto body_deadline = head_deadline; + bool body_phase = false; + v1::ParseResult pr = v1::ParseResult::NeedMore; + bool got_any = false; + bool head_seen = false; + for (;;) { + if (!in_.empty()) { + // Bytes already here (detection or pipelining) count as activity: + // the idle timeout must not apply. + got_any = true; + in_request_ = true; + // The parser owns its pending buffer; hand over everything. + pr = parser.feed(in_.data(), in_.size()); + in_.clear(); + if (!head_seen && parser.head_done()) { + head_seen = true; + body_phase = true; + body_deadline = + std::chrono::steady_clock::now() + config_.body_timeout; + if (parser.expect_continue() && + pr == v1::ParseResult::NeedMore) { + // Head is in, body still coming: release the client. + out_.append("HTTP/1.1 100 Continue\r\n\r\n"); + if (!co_await Flush(config_.idle_timeout)) { + close(); + co_return; + } + } + } + if (pr != v1::ParseResult::NeedMore) break; + } + auto now = std::chrono::steady_clock::now(); + auto limit = body_phase ? body_deadline : head_deadline; + if (!got_any) limit = std::chrono::steady_clock::now() + + config_.idle_timeout; + if (now >= limit) { + if (!got_any) { + // Idle keep-alive expiry: silent close, no error page. + close(); + co_return; + } + co_await SendError(408, true); + co_return; + } + auto left = + std::chrono::duration_cast(limit - now); + ReadResult r = co_await PumpRead(left); + if (r == ReadResult::Ok) { + got_any = true; + in_request_ = true; + continue; + } + if (r == ReadResult::Interrupted) continue; + if (r == ReadResult::Eof || r == ReadResult::Closed) { + close(); + co_return; + } + if (r == ReadResult::Timeout) { + if (!got_any) { + close(); + co_return; + } + co_await SendError(408, true); + co_return; + } + close(); + co_return; + } + // Pipelined bytes after this message return to the input buffer. + in_ = parser.take_pending(); + if (pr == v1::ParseResult::Error400) { + obs::OnParseError(400); + co_await SendError(400, true); + co_return; + } + if (pr == v1::ParseResult::Error413) { + obs::OnParseError(413); + co_await SendError(413, true); + co_return; + } + if (pr == v1::ParseResult::Error431) { + obs::OnParseError(431); + co_await SendError(431, true); + co_return; + } + if (pr == v1::ParseResult::Error501) { + obs::OnParseError(501); + co_await SendError(501, true); + co_return; + } + if (pr != v1::ParseResult::Complete) { + close(); + co_return; + } + const RequestHead& head = parser.head(); + if (IsH2cUpgrade(head)) { + std::string body = parser.body(); + out_.append( + "HTTP/1.1 101 Switching Protocols\r\nconnection: Upgrade\r\n" + "upgrade: h2c\r\ncontent-length: 0\r\n\r\n"); + if (!co_await Flush(config_.idle_timeout)) { + close(); + co_return; + } + ++requests_; + protocol_ = Protocol::H2PriorKnowledge; + co_await RunH2(false, true, head, body); + co_return; + } + FullRequest req; + req.head = head; + req.body = parser.body(); + req.trailers = parser.trailers(); + req.is_tls = tls_ && tls_->is_tls(); + req.peer = transport_->peer(); + req.protocol = "HTTP/1.1"; + bool keep = parser.keep_alive() && !draining_; + // Handler soft budget: the timer closes the transport so stuck handlers + // fail fast; the result is discarded when over budget. + budget_exceeded_ = false; + TimerWheel::TimerId budget = loop_.schedule_after( + config_.handler_budget, [self] { + self->budget_exceeded_ = true; + if (self->waiter_) self->waiter_->Wake(); + if (self->transport_) self->transport_->close(); + }); + FullResponse resp; + bool handler_ok = true; + try { + resp = co_await handler_(req); + } catch (...) { + handler_ok = false; + } + loop_.cancel_timer(budget); + if (closed_ || budget_exceeded_) { + close(); + co_return; + } + if (!handler_ok) { + co_await SendError(500, true); + co_return; + } + ++requests_; + bool head_only = (head.method == "HEAD"); + bool no_body_status = resp.head.status == 204 || resp.head.status == 304 || + (resp.head.status >= 100 && resp.head.status < 200); + bool have_file = resp.file.has_value() && resp.file->fd >= 0; + if (have_file && !resp.body.empty()) { + // Contract violation: file and string body at once. Fail closed. + ::close(resp.file->fd); + resp.file.reset(); + co_await SendError(500, true); + co_return; + } + uint64_t file_len = 0; + bool send_file = false; + if (have_file) { + // Clamp before serializing so Content-Length always matches the bytes. + if (!ClampFileBody(*resp.file)) { + ::close(resp.file->fd); + resp.file.reset(); + co_await SendError(500, true); + co_return; + } + file_len = resp.file->length; + if (head_only || no_body_status) { + ::close(resp.file->fd); // HEAD keeps the length, drops the bytes + resp.file.reset(); + } else { + send_file = true; + } + } + std::size_t body_len = + have_file ? SaturateToSize(file_len) : resp.body.size(); + out_.append(v1::Serializer::SerializeHead(resp.head, head.version, false, + body_len, head_only, !keep)); + if (send_file) { + FileBody fb = *resp.file; + resp.file.reset(); + if (out_.size() > config_.write_ceiling) { + ::close(fb.fd); + close(); + co_return; + } + if (!co_await Flush(config_.idle_timeout)) { + ::close(fb.fd); + close(); + co_return; + } + if (!co_await SendFileBody(fb, config_.idle_timeout)) { + close(); + co_return; + } + } else { + if (!head_only) out_.append(resp.body); + if (out_.size() > config_.write_ceiling) { + close(); + co_return; + } + if (!co_await Flush(config_.idle_timeout)) { + close(); + co_return; + } + } + in_request_ = false; + if (!keep || requests_ >= config_.max_requests) { + close(); + co_return; + } + } + close(); +} + +Task Connection::RunH2(bool expect_magic, bool from_upgrade, + const RequestHead& upgrade_head, + const std::string& upgrade_body) { + auto self = shared_from_this(); + v2::H2Config cfg; + cfg.body_cap = config_.body_cap; + cfg.idle_timeout = config_.idle_timeout; + cfg.handler_budget = config_.handler_budget; + auto h2 = std::make_shared(*this, cfg); + set_in_request(true); + set_active_h2(h2.get()); + if (draining_) h2->drain(); + co_await h2->Run(expect_magic, from_upgrade, upgrade_head, upgrade_body); + requests_ += h2->requests_served(); + set_active_h2(nullptr); + set_in_request(false); + close(); +} + +} // namespace molo::runtime::http diff --git a/src/runtime/http/connection.h b/src/runtime/http/connection.h new file mode 100644 index 0000000..2cf7bb7 --- /dev/null +++ b/src/runtime/http/connection.h @@ -0,0 +1,118 @@ +// Spec: plans/01-runtime.md 6.2. Connection and request lifecycle: accept, +// TLS handshake, protocol detect, head parse, body read, dispatch, handler, +// serialize, keep-alive or close. H1 persistent by default, H2 via GOAWAY on +// drain. Timeouts: header 10 s, body 30 s, idle 75 s, handler budget 30 s. +#pragma once + +#include +#include +#include +#include +#include + +#include "runtime/caps.h" +#include "runtime/event_loop.h" +#include "runtime/task.h" +#include "runtime/tls_handle.h" +#include "runtime/transport.h" +#include "fd_waiter.h" +#include "message.h" +#include "protocol_detect.h" +#include "v1/parser.h" + +namespace molo::runtime::http { + +namespace v2 { +class H2Connection; +} // namespace v2 + +using Handler = std::function(const FullRequest&)>; + +struct ConnectionConfig { + std::chrono::milliseconds header_timeout = caps::kHeaderTimeout; + std::chrono::milliseconds body_timeout = caps::kBodyTimeout; + std::chrono::milliseconds idle_timeout = caps::kIdleKeepAliveTimeout; + std::chrono::milliseconds handler_budget = caps::kHandlerBudget; + std::chrono::milliseconds tls_handshake_timeout = caps::kTlsHandshakeTimeout; + std::size_t body_cap = caps::kBodyMaxDefault; + std::size_t max_requests = caps::kMaxRequestsPerH1Connection; + std::size_t high_watermark = caps::kWriteHighWatermark; + std::size_t write_ceiling = caps::kWriteBufferCeiling; + bool allow_h2c_upgrade = true; +}; + +class Connection : public std::enable_shared_from_this { + public: + static std::shared_ptr Create( + EventLoop& loop, std::unique_ptr transport, + std::unique_ptr tls, Handler handler, + ConnectionConfig config = ConnectionConfig{}); + + ~Connection(); + + void start(); + // Graceful: finish the in-flight request, then GOAWAY (H2) or + // Connection: close (H1), then hard close. + void drain(); + void close(); + + bool closed() const { return closed_; } + bool draining() const { return draining_; } + std::size_t requests_served() const { return requests_; } + std::string protocol() const; + + // Shared with H2Connection, which runs on this connection's transport. + // Interrupted means woken without data (drain poke or readiness race): + // the caller re-checks its exit conditions and retries. + enum class ReadResult { Ok, Eof, Error, Timeout, Closed, Interrupted }; + Task PumpRead(std::chrono::milliseconds timeout); + Task Flush(std::chrono::milliseconds timeout); + EventLoop& loop() { return loop_; } + Transport& transport() { return *transport_; } + Handler& handler() { return handler_; } + std::string& inbox() { return in_; } + std::string& outbox() { return out_; } + void set_in_request(bool v) { in_request_ = v; } + bool is_tls() const; + std::string peer() const; + // Pokes a parked PumpRead/Flush so the owner re-checks exit conditions. + void InterruptRead(); + void set_active_h2(v2::H2Connection* h2) { active_h2_ = h2; } + + private: + Connection(EventLoop& loop, std::unique_ptr transport, + std::unique_ptr tls, Handler handler, + ConnectionConfig config); + + Task Run(); + static Task Runner(std::shared_ptr self); + Task DoHandshake(); + Task RunH1(); + Task RunH2(bool expect_magic, bool from_upgrade, + const RequestHead& upgrade_head, const std::string& upgrade_body); + Task SendError(int status, bool close_after); + // Sends a file-backed body (takes fd ownership, always closes it). + Task SendFileBody(FileBody fb, std::chrono::milliseconds timeout); + Task AwaitWritable(std::chrono::milliseconds timeout); + bool IsH2cUpgrade(const RequestHead& head) const; + + EventLoop& loop_; + std::unique_ptr transport_; + std::unique_ptr tls_; + std::unique_ptr waiter_; + Handler handler_; + ConnectionConfig config_; + std::string raw_; // bytes from transport, pre-TLS + std::string in_; // plaintext bytes awaiting parse + std::string out_; // plaintext bytes awaiting send + bool closed_ = false; + bool draining_ = false; + bool tls_done_ = false; + bool budget_exceeded_ = false; + bool in_request_ = false; + v2::H2Connection* active_h2_ = nullptr; + std::size_t requests_ = 0; + Protocol protocol_ = Protocol::NeedMore; +}; + +} // namespace molo::runtime::http diff --git a/src/runtime/http/fd_waiter.h b/src/runtime/http/fd_waiter.h new file mode 100644 index 0000000..e9aa06d --- /dev/null +++ b/src/runtime/http/fd_waiter.h @@ -0,0 +1,127 @@ +// One fd readiness waiter for connection coroutines. A connection parks at +// most one reader and one writer; close() resumes both with an error so no +// coroutine leaks parked on a dead fd. +#pragma once + +#include + +#include "../event_loop.h" +#include "../poller.h" + +namespace molo::runtime::http { + +class FdWaiter { + public: + FdWaiter(EventLoop& loop, int fd) : loop_(loop), fd_(fd) {} + ~FdWaiter() { cancel(); } + + FdWaiter(const FdWaiter&) = delete; + FdWaiter& operator=(const FdWaiter&) = delete; + + struct Awaitable { + FdWaiter* self; + uint32_t events; + std::coroutine_handle<>* slot; + bool await_ready() const noexcept { return self->cancelled_; } + void await_suspend(std::coroutine_handle<> h) { + *slot = h; + self->Arm(events); + } + bool await_resume() const noexcept { return !self->cancelled_; } + }; + + Awaitable readable() { return Awaitable{this, IPoller::kRead, &read_h_}; } + Awaitable writable() { return Awaitable{this, IPoller::kWrite, &write_h_}; } + + void cancel() { + if (cancelled_) return; + cancelled_ = true; + if (fd_ >= 0) loop_.poller().remove_fd(fd_); + // Null slots before resuming: a resume may destroy this waiter. + auto rh = read_h_; + read_h_ = nullptr; + auto wh = write_h_; + write_h_ = nullptr; + if (rh) rh.resume(); + if (wh) wh.resume(); + } + + // Resume parked waiters without cancelling: used by timeouts. The waiter + // stays usable; callers check their own fired flag. + void Wake() { + if (cancelled_) return; + if (registered_) { + loop_.poller().remove_fd(fd_); + registered_ = false; + } + armed_ = 0; + auto rh = read_h_; + read_h_ = nullptr; + auto wh = write_h_; + write_h_ = nullptr; + if (rh) rh.resume(); + if (wh) wh.resume(); + } + + bool cancelled() const { return cancelled_; } + + private: + void Arm(uint32_t events) { + if (cancelled_) return; + uint32_t want = armed_ | events; + if (!registered_) { + bool ok = loop_.poller().add_fd( + fd_, want, [this](int, uint32_t ev) { OnEvent(ev); }); + if (!ok) { + cancel(); + return; + } + registered_ = true; + } else if (want != armed_) { + loop_.poller().modify_fd(fd_, want); + } + armed_ = want; + } + + void OnEvent(uint32_t ev) { + // Error acts as readiness; the syscall reports the real cause. + bool want_r = (ev & (IPoller::kRead | IPoller::kError)) && read_h_; + bool want_w = (ev & (IPoller::kWrite | IPoller::kError)) && write_h_; + std::coroutine_handle<> rh = nullptr; + std::coroutine_handle<> wh = nullptr; + if (want_r) { + rh = read_h_; + read_h_ = nullptr; + armed_ &= ~IPoller::kRead; + } + if (want_w) { + wh = write_h_; + write_h_ = nullptr; + armed_ &= ~IPoller::kWrite; + } + Refresh(); + // No member access after the first resume: it may destroy this waiter. + if (rh) rh.resume(); + if (wh) wh.resume(); + } + + void Refresh() { + if (!registered_) return; + if (armed_ == 0) { + loop_.poller().remove_fd(fd_); + registered_ = false; + return; + } + loop_.poller().modify_fd(fd_, armed_); + } + + EventLoop& loop_; + int fd_; + bool cancelled_ = false; + bool registered_ = false; + uint32_t armed_ = 0; + std::coroutine_handle<> read_h_ = nullptr; + std::coroutine_handle<> write_h_ = nullptr; +}; + +} // namespace molo::runtime::http diff --git a/src/runtime/http/listener.cpp b/src/runtime/http/listener.cpp new file mode 100644 index 0000000..a3c6941 --- /dev/null +++ b/src/runtime/http/listener.cpp @@ -0,0 +1,175 @@ +// Spec: plans/01-runtime.md 6.2. TCP listener. +#include "listener.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "runtime/caps.h" +#include "runtime/event_loop.h" +#include "runtime/http/connection.h" +#include "runtime/observability_hooks.h" + +namespace molo::runtime::http { + +Listener::Listener(EventLoop& loop) : loop_(loop) {} + +Listener::~Listener() { stop(); } + +bool Listener::bind(const std::string& host, uint16_t port, bool reuse_port) { + if (fd_ >= 0) return false; + int fd = socket(AF_INET6, SOCK_STREAM | SOCK_CLOEXEC, 0); + bool v6 = true; + if (fd < 0) { + fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + v6 = false; + } + if (fd < 0) return false; + int one = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); +#if defined(SO_REUSEPORT) + if (reuse_port) setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); +#else + (void)reuse_port; +#endif + if (v6) { + int off = 0; + setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &off, sizeof(off)); + } + int flags = fcntl(fd, F_GETFL, 0); + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + if (v6) { + struct sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_port = htons(port); + if (host.empty() || host == "*" || host == "::") { + addr.sin6_addr = in6addr_any; + } else if (inet_pton(AF_INET6, host.c_str(), &addr.sin6_addr) != 1) { + struct in_addr v4; + if (inet_pton(AF_INET, host.c_str(), &v4) == 1) { + addr.sin6_addr.s6_addr[10] = 0xff; + addr.sin6_addr.s6_addr[11] = 0xff; + memcpy(&addr.sin6_addr.s6_addr[12], &v4, 4); + } else { + close(fd); + return false; + } + } + if (::bind(fd, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + close(fd); + return false; + } + } else { + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + if (host.empty() || host == "*") { + addr.sin_addr.s_addr = htonl(INADDR_ANY); + } else if (inet_pton(AF_INET, host.c_str(), &addr.sin_addr) != 1) { + close(fd); + return false; + } + if (::bind(fd, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + close(fd); + return false; + } + } + if (listen(fd, 1024) != 0) { + close(fd); + return false; + } + if (port == 0) { + struct sockaddr_in6 addr; + socklen_t len = sizeof(addr); + if (getsockname(fd, (struct sockaddr*)&addr, &len) == 0) { + port_ = ntohs(addr.sin6_port); + } + } else { + port_ = port; + } + bool ok = loop_.poller().add_fd( + fd, IPoller::kRead, [this](int, uint32_t) { OnReadable(); }); + if (!ok) { + close(fd); + return false; + } + fd_ = fd; + return true; +} + +void Listener::stop() { + if (fd_ >= 0) { + loop_.poller().remove_fd(fd_); + close(fd_); + fd_ = -1; + } +} + +void Listener::track(std::shared_ptr conn) { + tracked_.push_back(std::move(conn)); +} + +std::size_t Listener::LiveCount() { + std::size_t n = 0; + std::vector> kept; + kept.reserve(tracked_.size()); + for (auto& w : tracked_) { + if (auto s = w.lock()) { + if (!s->closed()) { + ++n; + kept.push_back(std::move(w)); + } + } + } + tracked_.swap(kept); + return n; +} + +void Listener::OnReadable() { + for (int i = 0; i < 64; ++i) { + struct sockaddr_in6 addr; + socklen_t len = sizeof(addr); + int cfd = + accept4(fd_, (struct sockaddr*)&addr, &len, SOCK_CLOEXEC | SOCK_NONBLOCK); + if (cfd < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) return; + if (errno == EINTR) continue; + return; + } + int one = 1; + setsockopt(cfd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + char host[INET6_ADDRSTRLEN] = {0}; + uint16_t cport = 0; + if (addr.sin6_family == AF_INET6) { + inet_ntop(AF_INET6, &addr.sin6_addr, host, sizeof(host)); + cport = ntohs(addr.sin6_port); + } else if (addr.sin6_family == AF_INET) { + struct sockaddr_in* v4 = (struct sockaddr_in*)&addr; + inet_ntop(AF_INET, &v4->sin_addr, host, sizeof(host)); + cport = ntohs(v4->sin_port); + } + std::string peer = std::string(host) + ":" + std::to_string(cport); + std::size_t cap = max_connections_; + if (cap > caps::kMaxConnectionsCeiling) cap = caps::kMaxConnectionsCeiling; + if (LiveCount() >= cap) { + ++refused_; + MOLO_RUNTIME_COUNT("runtime.accept_refused"); + close(cfd); + continue; + } + if (accept_cb_) { + accept_cb_(cfd, peer); + } else { + close(cfd); + } + } +} + +} // namespace molo::runtime::http diff --git a/src/runtime/http/listener.h b/src/runtime/http/listener.h new file mode 100644 index 0000000..d3f6876 --- /dev/null +++ b/src/runtime/http/listener.h @@ -0,0 +1,53 @@ +// Spec: plans/01-runtime.md 6.2. TCP listener: bind, SO_REUSEPORT where +// available, non-blocking accept loop, per-process connection cap. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace molo::runtime { + +class EventLoop; + +namespace http { +class Connection; + +class Listener { + public: + using AcceptCallback = + std::function; + + explicit Listener(EventLoop& loop); + ~Listener(); + Listener(const Listener&) = delete; + Listener& operator=(const Listener&) = delete; + + bool bind(const std::string& host, uint16_t port, bool reuse_port = true); + void on_accept(AcceptCallback cb) { accept_cb_ = std::move(cb); } + void set_max_connections(std::size_t n) { max_connections_ = n; } + // Graceful drain step 1: stop accepting, keep existing connections. + void stop(); + bool listening() const { return fd_ >= 0; } + uint16_t port() const { return port_; } + std::size_t refused_connections() const { return refused_; } + void track(std::shared_ptr conn); + + private: + void OnReadable(); + std::size_t LiveCount(); + + EventLoop& loop_; + int fd_ = -1; + uint16_t port_ = 0; + AcceptCallback accept_cb_; + std::size_t max_connections_ = 16384; + std::size_t refused_ = 0; + std::vector> tracked_; +}; + +} // namespace http +} // namespace molo::runtime diff --git a/src/runtime/http/message.cpp b/src/runtime/http/message.cpp new file mode 100644 index 0000000..ca226b5 --- /dev/null +++ b/src/runtime/http/message.cpp @@ -0,0 +1,147 @@ +// Spec: plans/01-runtime.md, RFC 9110. Semantic message layer. +#include "message.h" + +#include + +#include +#include + +namespace molo::runtime::http { + +bool ClampFileBody(FileBody& fb) { + if (fb.fd < 0) return false; + struct stat st; + if (::fstat(fb.fd, &st) != 0 || st.st_size < 0) return false; + uint64_t size = (uint64_t)st.st_size; + uint64_t start = fb.offset > size ? size : fb.offset; + uint64_t avail = size - start; + fb.offset = start; + fb.length = fb.length > avail ? avail : fb.length; + return true; +} + +std::string ToLower(std::string_view s) { + std::string out(s); + for (char& c : out) c = (char)tolower((unsigned char)c); + return out; +} + +bool EqCaseInsensitive(std::string_view a, std::string_view b) { + if (a.size() != b.size()) return false; + for (std::size_t i = 0; i < a.size(); ++i) { + if (tolower((unsigned char)a[i]) != tolower((unsigned char)b[i])) { + return false; + } + } + return true; +} + +std::string Trim(std::string_view s) { + std::size_t b = 0; + while (b < s.size() && (s[b] == ' ' || s[b] == '\t')) ++b; + std::size_t e = s.size(); + while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\t')) --e; + return std::string(s.substr(b, e - b)); +} + +void Headers::add(std::string name, std::string value) { + bytes_ += name.size() + value.size() + 4; + entries_.emplace_back(std::move(name), std::move(value)); +} + +std::string Headers::get(std::string_view name) const { + for (const auto& kv : entries_) { + if (EqCaseInsensitive(kv.first, name)) return kv.second; + } + return ""; +} + +std::vector Headers::get_all(std::string_view name) const { + std::vector out; + for (const auto& kv : entries_) { + if (EqCaseInsensitive(kv.first, name)) out.push_back(kv.second); + } + return out; +} + +void Headers::set(std::string name, std::string value) { + remove(name); + add(std::move(name), std::move(value)); +} + +bool Headers::has(std::string_view name) const { + for (const auto& kv : entries_) { + if (EqCaseInsensitive(kv.first, name)) return true; + } + return false; +} + +void Headers::remove(std::string_view name) { + std::vector> kept; + kept.reserve(entries_.size()); + bytes_ = 0; + for (auto& kv : entries_) { + if (EqCaseInsensitive(kv.first, name)) continue; + bytes_ += kv.first.size() + kv.second.size() + 4; + kept.push_back(std::move(kv)); + } + entries_ = std::move(kept); +} + +std::string StatusReason(int status) { + switch (status) { + case 100: return "Continue"; + case 101: return "Switching Protocols"; + case 103: return "Early Hints"; + case 200: return "OK"; + case 201: return "Created"; + case 204: return "No Content"; + case 206: return "Partial Content"; + case 301: return "Moved Permanently"; + case 302: return "Found"; + case 303: return "See Other"; + case 304: return "Not Modified"; + case 307: return "Temporary Redirect"; + case 308: return "Permanent Redirect"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 408: return "Request Timeout"; + case 411: return "Length Required"; + case 413: return "Content Too Large"; + case 414: return "URI Too Long"; + case 416: return "Range Not Satisfiable"; + case 426: return "Upgrade Required"; + case 429: return "Too Many Requests"; + case 431: return "Request Header Fields Too Large"; + case 500: return "Internal Server Error"; + case 501: return "Not Implemented"; + case 502: return "Bad Gateway"; + case 503: return "Service Unavailable"; + default: return "Unknown"; + } +} + +std::optional ParseContentLength(const Headers& h, + bool& conflict) { + conflict = false; + std::optional value; + for (const std::string& raw : h.get_all("content-length")) { + std::string v = Trim(raw); + if (v.empty()) return std::nullopt; + std::size_t n = 0; + for (char c : v) { + if (c < '0' || c > '9') return std::nullopt; + std::size_t d = (std::size_t)(c - '0'); + if (n > (SIZE_MAX - d) / 10) return std::nullopt; + n = n * 10 + d; + } + if (value.has_value() && *value != n) conflict = true; + value = n; + } + return value; +} + +} // namespace molo::runtime::http diff --git a/src/runtime/http/message.h b/src/runtime/http/message.h new file mode 100644 index 0000000..e3d2e47 --- /dev/null +++ b/src/runtime/http/message.h @@ -0,0 +1,91 @@ +// Spec: plans/01-runtime.md 6.x, RFC 9110. One version-independent semantic +// layer shared by H1 and H2. Handlers never see framing differences. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::runtime::http { + +std::string ToLower(std::string_view s); +bool EqCaseInsensitive(std::string_view a, std::string_view b); +std::string Trim(std::string_view s); + +class Headers { + public: + void add(std::string name, std::string value); + std::string get(std::string_view name) const; + std::vector get_all(std::string_view name) const; + void set(std::string name, std::string value); + bool has(std::string_view name) const; + void remove(std::string_view name); + std::size_t count() const { return entries_.size(); } + std::size_t bytes() const { return bytes_; } + const std::vector>& entries() const { + return entries_; + } + void clear() { + entries_.clear(); + bytes_ = 0; + } + + private: + std::vector> entries_; + std::size_t bytes_ = 0; +}; + +struct RequestHead { + std::string method; + std::string target; + std::string version = "HTTP/1.1"; + Headers headers; +}; + +struct ResponseHead { + int status = 200; + Headers headers; +}; + +// Integration seam for 05-request-routing.md: the full semantic exchange. +// Today Connection fills these directly; routing builds its Request/Response +// views on top without touching framing. +struct FullRequest { + RequestHead head; + std::string body; + Headers trailers; + bool is_tls = false; + std::string peer; + std::string protocol; // "HTTP/1.1" or "HTTP/2" +}; + +// File-backed body (plans/01-runtime.md 6.7): the handler transfers ownership +// of a seekable fd to the Connection, which sends [offset, offset+length) +// via sendfile on plain H1 or bounded 64 KB streaming otherwise, then closes +// the fd. The Connection clamps length to the file size and closes the fd on +// every path, including HEAD, 204/304, errors, and resets. +struct FileBody { + int fd = -1; + uint64_t offset = 0; + uint64_t length = 0; +}; + +struct FullResponse { + ResponseHead head; + std::string body; // used only when file is not set + std::optional file; // when set, body must be empty +}; + +// Clamps fb.offset/length into the file size in place. False when the fd is +// not a stat-able file; then fb is untouched. +bool ClampFileBody(FileBody& fb); + +std::string StatusReason(int status); +// Parses a single Content-Length value. Nullopt when missing or invalid. +std::optional ParseContentLength(const Headers& h, bool& conflict); + +} // namespace molo::runtime::http diff --git a/src/runtime/http/protocol_detect.cpp b/src/runtime/http/protocol_detect.cpp new file mode 100644 index 0000000..4ef236f --- /dev/null +++ b/src/runtime/http/protocol_detect.cpp @@ -0,0 +1,41 @@ +// Spec: plans/01-runtime.md 6.4. Protocol detection. +#include "protocol_detect.h" + +namespace molo::runtime::http { + +void ProtocolDetector::reset() { + matched_ = 0; + decided_ = false; + decision_ = Protocol::NeedMore; +} + +Protocol ProtocolDetector::feed(const char* data, std::size_t len) { + if (decided_) return decision_; + for (std::size_t i = 0; i < len; ++i) { + if (matched_ >= kH2Preface.size()) { + decided_ = true; + decision_ = Protocol::H2PriorKnowledge; + return decision_; + } + if (data[i] == kH2Preface[matched_]) { + ++matched_; + if (matched_ >= kH2Preface.size()) { + decided_ = true; + decision_ = Protocol::H2PriorKnowledge; + return decision_; + } + continue; + } + decided_ = true; + decision_ = Protocol::Http1; + return decision_; + } + return Protocol::NeedMore; +} + +Protocol ProtocolDetector::from_alpn(std::string_view alpn) { + if (alpn == "h2") return Protocol::H2PriorKnowledge; + return Protocol::Http1; +} + +} // namespace molo::runtime::http diff --git a/src/runtime/http/protocol_detect.h b/src/runtime/http/protocol_detect.h new file mode 100644 index 0000000..b854a97 --- /dev/null +++ b/src/runtime/http/protocol_detect.h @@ -0,0 +1,30 @@ +// Spec: plans/01-runtime.md 6.4. Protocol detection: ALPN wiring point from +// the TLS handle, h2c prior knowledge (PRI preface), H1 Upgrade to h2c. +#pragma once + +#include +#include + +namespace molo::runtime::http { + +enum class Protocol { NeedMore, Http1, H2PriorKnowledge }; + +// Exact 24-byte client connection preface, RFC 9113 section 3.4. +inline constexpr std::string_view kH2Preface = + "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; + +class ProtocolDetector { + public: + ProtocolDetector() = default; + void reset(); + Protocol feed(const char* data, std::size_t len); + // ALPN wiring point: the TLS handle reports "h2" or "http/1.1". + static Protocol from_alpn(std::string_view alpn); + + private: + std::size_t matched_ = 0; + bool decided_ = false; + Protocol decision_ = Protocol::NeedMore; +}; + +} // namespace molo::runtime::http diff --git a/src/runtime/http/v1/parser.cpp b/src/runtime/http/v1/parser.cpp new file mode 100644 index 0000000..115967c --- /dev/null +++ b/src/runtime/http/v1/parser.cpp @@ -0,0 +1,425 @@ +// Spec: plans/01-runtime.md 6.3, RFC 9112. Strict incremental parser. +#include "parser.h" + +#include + +#include "runtime/caps.h" + +namespace molo::runtime::http::v1 { + +Parser::Parser(std::size_t body_cap) : body_cap_(body_cap) {} + +void Parser::reset() { + state_ = State::RequestLine; + head_ = RequestHead{}; + trailers_ = Headers{}; + body_.clear(); + pending_.clear(); + consumed_ = 0; + content_length_ = 0; + chunk_left_ = 0; + header_bytes_ = 0; + chunked_ = false; + keep_alive_ = true; + expect_continue_ = false; + trailers_section_ = false; +} + +bool Parser::IsTokenChar(char c) { + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9')) { + return true; + } + switch (c) { + case '!': + case '#': + case '$': + case '%': + case '&': + case '\'': + case '*': + case '+': + case '-': + case '.': + case '^': + case '_': + case '`': + case '|': + case '~': + return true; + default: + return false; + } +} + +bool Parser::IsMethodChar(char c) { + return (c >= 'A' && c <= 'Z') || c == '-' || c == '_'; +} + +bool Parser::TakeLine(std::string& out) { + std::size_t pos = pending_.find("\r\n"); + if (pos == std::string::npos) { + if (pending_.size() > caps::kHeaderLineMax) { + // Overlong line without terminator: caller maps to 431. + return false; + } + return false; + } + if (pos > caps::kHeaderLineMax) return false; + out.assign(pending_.data(), pos); + pending_.erase(0, pos + 2); + consumed_ += pos + 2; + return true; +} + +ParseResult Parser::feed(const char* data, std::size_t len) { + pending_.append(data, len); + for (;;) { + State before = state_; + ParseResult r = ParseResult::NeedMore; + switch (state_) { + case State::RequestLine: + r = FeedRequestLine(); + break; + case State::Headers: + r = FeedHeaders(); + break; + case State::BodyIdentity: + r = FeedBodyIdentity(); + break; + case State::ChunkSize: + r = FeedChunkSize(); + break; + case State::ChunkData: + r = FeedChunkData(); + break; + case State::ChunkDataEnd: + r = FeedChunkDataEnd(); + break; + case State::Trailers: + r = FeedTrailers(); + break; + case State::Done: + return ParseResult::Complete; + case State::Failed: + return ParseResult::Error400; + } + // Terminal results return at once; NeedMore loops only on progress. + if (r != ParseResult::NeedMore || state_ == before) return r; + } +} + +ParseResult Parser::FeedRequestLine() { + std::string line; + if (!TakeLine(line)) { + if (pending_.size() > caps::kHeaderLineMax) { + state_ = State::Failed; + return ParseResult::Error431; + } + return ParseResult::NeedMore; + } + // METHOD SP TARGET SP VERSION + std::size_t sp1 = line.find(' '); + if (sp1 == std::string::npos || sp1 == 0) { + state_ = State::Failed; + return ParseResult::Error400; + } + std::size_t sp2 = line.find(' ', sp1 + 1); + if (sp2 == std::string::npos || sp2 + 1 >= line.size()) { + state_ = State::Failed; + return ParseResult::Error400; + } + std::string method = line.substr(0, sp1); + std::string target = line.substr(sp1 + 1, sp2 - sp1 - 1); + std::string version = line.substr(sp2 + 1); + for (char c : method) { + if (!IsMethodChar(c)) { + state_ = State::Failed; + return ParseResult::Error400; + } + } + if (target.empty() || target.size() > caps::kUriMax) { + state_ = State::Failed; + return target.size() > caps::kUriMax ? ParseResult::Error431 + : ParseResult::Error400; + } + for (char c : target) { + unsigned char u = (unsigned char)c; + if (u <= 0x20 || u == 0x7f) { + state_ = State::Failed; + return ParseResult::Error400; + } + } + if (version != "HTTP/1.0" && version != "HTTP/1.1") { + state_ = State::Failed; + return ParseResult::Error400; + } + head_.method = method; + head_.target = target; + head_.version = version; + keep_alive_ = (version == "HTTP/1.1"); + header_bytes_ = 0; + state_ = State::Headers; + return ParseResult::NeedMore; +} + +ParseResult Parser::FeedHeaders() { + for (;;) { + std::string line; + std::size_t before = pending_.size(); + if (!TakeLine(line)) { + if (pending_.size() > caps::kHeaderLineMax || + before > caps::kHeaderLineMax) { + state_ = State::Failed; + return ParseResult::Error431; + } + return ParseResult::NeedMore; + } + if (line.empty()) { + return BeginBody(); + } + if (line[0] == ' ' || line[0] == '\t') { + // obs-fold rejected per R5. + state_ = State::Failed; + return ParseResult::Error400; + } + std::size_t colon = line.find(':'); + if (colon == std::string::npos || colon == 0) { + state_ = State::Failed; + return ParseResult::Error400; + } + std::string name = line.substr(0, colon); + for (char c : name) { + if (!IsTokenChar(c)) { + state_ = State::Failed; + return ParseResult::Error400; + } + } + std::string value = Trim(line.substr(colon + 1)); + for (char c : value) { + unsigned char u = (unsigned char)c; + if (u < 0x20 && u != '\t') { + state_ = State::Failed; + return ParseResult::Error400; + } + if (u == 0x7f) { + state_ = State::Failed; + return ParseResult::Error400; + } + } + header_bytes_ += name.size() + value.size() + 4; + if (header_bytes_ > caps::kHeaderBlockMax) { + state_ = State::Failed; + return ParseResult::Error431; + } + head_.headers.add(ToLower(name), value); + if (head_.headers.count() > caps::kHeaderFieldMax) { + state_ = State::Failed; + return ParseResult::Error431; + } + } +} + +ParseResult Parser::BeginBody() { + if (!head_.headers.has("host")) { + state_ = State::Failed; + return ParseResult::Error400; + } + std::string conn = ToLower(head_.headers.get("connection")); + if (conn.find("close") != std::string::npos) keep_alive_ = false; + if (head_.version == "HTTP/1.0" && + conn.find("keep-alive") != std::string::npos) { + keep_alive_ = true; + } + std::string expect = ToLower(head_.headers.get("expect")); + expect_continue_ = (expect == "100-continue"); + if (!expect.empty() && !expect_continue_) { + state_ = State::Failed; + return ParseResult::Error400; + } + bool has_cl = head_.headers.has("content-length"); + bool has_te = head_.headers.has("transfer-encoding"); + if (has_cl && has_te) { + // Smuggling defense: never accept both. + state_ = State::Failed; + return ParseResult::Error400; + } + if (has_te) { + std::string te = ToLower(head_.headers.get("transfer-encoding")); + // Only chunked (possibly last of a list we otherwise reject). + bool chunked = false; + std::size_t pos = 0; + while (pos <= te.size()) { + std::size_t comma = te.find(',', pos); + if (comma == std::string::npos) comma = te.size(); + std::string coding = Trim(te.substr(pos, comma - pos)); + if (coding == "chunked") { + chunked = true; + } else if (!coding.empty() && coding != "identity") { + state_ = State::Failed; + return ParseResult::Error501; + } + pos = comma + 1; + } + if (!chunked) { + // Identity only: no body framing. + state_ = State::Done; + return ParseResult::Complete; + } + chunked_ = true; + state_ = State::ChunkSize; + return ParseResult::NeedMore; + } + if (has_cl) { + bool conflict = false; + std::optional len = + ParseContentLength(head_.headers, conflict); + if (!len.has_value() || conflict) { + state_ = State::Failed; + return ParseResult::Error400; + } + content_length_ = *len; + if (content_length_ > body_cap_) { + state_ = State::Failed; + return ParseResult::Error413; + } + if (content_length_ == 0) { + state_ = State::Done; + return ParseResult::Complete; + } + state_ = State::BodyIdentity; + return ParseResult::NeedMore; + } + state_ = State::Done; + return ParseResult::Complete; +} + +ParseResult Parser::FeedBodyIdentity() { + std::size_t want = content_length_ - body_.size(); + std::size_t have = pending_.size() < want ? pending_.size() : want; + body_.append(pending_.data(), have); + pending_.erase(0, have); + consumed_ += have; + if (body_.size() >= content_length_) { + state_ = State::Done; + return ParseResult::Complete; + } + return ParseResult::NeedMore; +} + +ParseResult Parser::FeedChunkSize() { + std::string line; + if (!TakeLine(line)) { + if (pending_.size() > caps::kHeaderLineMax) { + state_ = State::Failed; + return ParseResult::Error400; + } + return ParseResult::NeedMore; + } + // Strip extensions. + std::size_t semi = line.find(';'); + std::string num = semi == std::string::npos ? line : line.substr(0, semi); + num = Trim(num); + if (num.empty() || num.size() > 16) { + state_ = State::Failed; + return ParseResult::Error400; + } + std::size_t size = 0; + for (char c : num) { + unsigned v; + if (c >= '0' && c <= '9') v = (unsigned)(c - '0'); + else if (c >= 'a' && c <= 'f') v = (unsigned)(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') v = (unsigned)(c - 'A' + 10); + else { + state_ = State::Failed; + return ParseResult::Error400; + } + if (size > (SIZE_MAX - v) / 16) { + state_ = State::Failed; + return ParseResult::Error400; + } + size = size * 16 + v; + } + if (size == 0) { + trailers_section_ = true; + state_ = State::Trailers; + return ParseResult::NeedMore; + } + if (body_.size() + size > body_cap_) { + state_ = State::Failed; + return ParseResult::Error413; + } + chunk_left_ = size; + state_ = State::ChunkData; + return ParseResult::NeedMore; +} + +ParseResult Parser::FeedChunkData() { + std::size_t have = pending_.size() < chunk_left_ ? pending_.size() + : chunk_left_; + body_.append(pending_.data(), have); + pending_.erase(0, have); + consumed_ += have; + chunk_left_ -= have; + if (chunk_left_ > 0) return ParseResult::NeedMore; + state_ = State::ChunkDataEnd; + return ParseResult::NeedMore; +} + +ParseResult Parser::FeedChunkDataEnd() { + if (pending_.size() < 2) return ParseResult::NeedMore; + if (pending_[0] != '\r' || pending_[1] != '\n') { + state_ = State::Failed; + return ParseResult::Error400; + } + pending_.erase(0, 2); + consumed_ += 2; + state_ = State::ChunkSize; + return ParseResult::NeedMore; +} + +ParseResult Parser::FeedTrailers() { + for (;;) { + std::string line; + if (!TakeLine(line)) { + if (pending_.size() > caps::kHeaderLineMax) { + state_ = State::Failed; + return ParseResult::Error400; + } + return ParseResult::NeedMore; + } + if (line.empty()) { + state_ = State::Done; + trailers_section_ = false; + return ParseResult::Complete; + } + if (line[0] == ' ' || line[0] == '\t') { + state_ = State::Failed; + return ParseResult::Error400; + } + std::size_t colon = line.find(':'); + if (colon == std::string::npos || colon == 0) { + state_ = State::Failed; + return ParseResult::Error400; + } + std::string name = line.substr(0, colon); + for (char c : name) { + if (!IsTokenChar(c)) { + state_ = State::Failed; + return ParseResult::Error400; + } + } + header_bytes_ += name.size() + line.size(); + if (header_bytes_ > caps::kHeaderBlockMax) { + state_ = State::Failed; + return ParseResult::Error431; + } + trailers_.add(ToLower(name), Trim(line.substr(colon + 1))); + if (head_.headers.count() + trailers_.count() > caps::kHeaderFieldMax) { + state_ = State::Failed; + return ParseResult::Error431; + } + } +} + +} // namespace molo::runtime::http::v1 diff --git a/src/runtime/http/v1/parser.h b/src/runtime/http/v1/parser.h new file mode 100644 index 0000000..d772ae5 --- /dev/null +++ b/src/runtime/http/v1/parser.h @@ -0,0 +1,92 @@ +// Spec: plans/01-runtime.md 6.3, RFC 9112 sections 2, 5, 6, 7. Strict +// incremental request parser. Rejects ambiguous framing with 400 and close, +// requires Host, accepts chunked, answers unknown transfer coding with 501. +#pragma once + +#include +#include + +#include "../message.h" + +namespace molo::runtime::http::v1 { + +enum class ParseResult { + NeedMore, + Complete, + Error400, + Error413, + Error431, + Error501, +}; + +class Parser { + public: + explicit Parser(std::size_t body_cap = 10 * 1024 * 1024); + void reset(); + + // Feeds bytes. Consumed prefix length is tracked in consumed(); the caller + // keeps the buffer and erases the consumed prefix after Complete. + ParseResult feed(const char* data, std::size_t len); + std::size_t consumed() const { return consumed_; } + + const RequestHead& head() const { return head_; } + const std::string& body() const { return body_; } + const Headers& trailers() const { return trailers_; } + bool keep_alive() const { return keep_alive_; } + bool expect_continue() const { return expect_continue_; } + bool head_done() const { + return state_ == State::BodyIdentity || state_ == State::ChunkSize || + state_ == State::ChunkData || state_ == State::ChunkDataEnd || + state_ == State::Trailers || state_ == State::Done; + } + // Bytes fed but not part of this message (pipelined next request). + std::string take_pending() { + std::string out = std::move(pending_); + pending_.clear(); + return out; + } + + private: + enum class State { + RequestLine, + Headers, + BodyIdentity, + ChunkSize, + ChunkData, + ChunkDataEnd, + Trailers, + Done, + Failed, + }; + + ParseResult FeedRequestLine(); + ParseResult FeedHeaders(); + ParseResult FeedBodyIdentity(); + ParseResult FeedChunkSize(); + ParseResult FeedChunkData(); + ParseResult FeedChunkDataEnd(); + ParseResult FeedTrailers(); + ParseResult BeginBody(); + // Returns line without CRLF and consumes it, or nullopt when incomplete. + bool TakeLine(std::string& out); + + static bool IsTokenChar(char c); + static bool IsMethodChar(char c); + + State state_ = State::RequestLine; + RequestHead head_; + Headers trailers_; + std::string body_; + std::string pending_; + std::size_t consumed_ = 0; + std::size_t body_cap_; + std::size_t content_length_ = 0; + std::size_t chunk_left_ = 0; + std::size_t header_bytes_ = 0; + bool chunked_ = false; + bool keep_alive_ = true; + bool expect_continue_ = false; + bool trailers_section_ = false; +}; + +} // namespace molo::runtime::http::v1 diff --git a/src/runtime/http/v1/serializer.cpp b/src/runtime/http/v1/serializer.cpp new file mode 100644 index 0000000..33220b8 --- /dev/null +++ b/src/runtime/http/v1/serializer.cpp @@ -0,0 +1,83 @@ +// Spec: plans/01-runtime.md 6.3, RFC 9112. Response serializer. +#include "serializer.h" + +#include +#include + +namespace molo::runtime::http::v1 { + +std::string Serializer::HttpDateNow() { + std::time_t t = std::time(nullptr); + struct tm tmv; + gmtime_r(&t, &tmv); + char buf[40]; + strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", &tmv); + return std::string(buf); +} + +std::string Serializer::SerializeHead(const ResponseHead& head, + std::string_view version, bool chunked, + std::size_t content_length, + bool head_only, bool close) { + std::string out; + out.reserve(512); + out.append(version); + out.push_back(' '); + out.append(std::to_string(head.status)); + out.push_back(' '); + out.append(StatusReason(head.status)); + out.append("\r\n"); + bool has_date = false; + bool has_length = false; + bool has_te = false; + bool has_connection = false; + for (const auto& kv : head.headers.entries()) { + if (EqCaseInsensitive(kv.first, "date")) has_date = true; + if (EqCaseInsensitive(kv.first, "content-length")) has_length = true; + if (EqCaseInsensitive(kv.first, "transfer-encoding")) has_te = true; + if (EqCaseInsensitive(kv.first, "connection")) has_connection = true; + out.append(kv.first); + out.append(": "); + out.append(kv.second); + out.append("\r\n"); + } + // RFC 9110 section 8.6: HEAD responses carry the Content-Length the + // matching GET would send. Only 204/304/1xx never carry framing. + // head_only is informational here: the caller suppresses the body bytes. + (void)head_only; + bool never_body = head.status == 204 || head.status == 304 || + (head.status >= 100 && head.status < 200); + if (!has_date) { + out.append("date: "); + out.append(HttpDateNow()); + out.append("\r\n"); + } + if (!never_body && !has_te && !has_length) { + if (chunked) { + out.append("transfer-encoding: chunked\r\n"); + } else { + out.append("content-length: "); + out.append(std::to_string(content_length)); + out.append("\r\n"); + } + } + if (!has_connection && (close || version == "HTTP/1.0")) { + out.append("connection: close\r\n"); + } + out.append("\r\n"); + return out; +} + +std::string Serializer::Chunk(const char* data, std::size_t len) { + if (len == 0) return ""; + char hex[24]; + snprintf(hex, sizeof(hex), "%zx\r\n", len); + std::string out(hex); + out.append(data, len); + out.append("\r\n"); + return out; +} + +std::string Serializer::LastChunk() { return "0\r\n\r\n"; } + +} // namespace molo::runtime::http::v1 diff --git a/src/runtime/http/v1/serializer.h b/src/runtime/http/v1/serializer.h new file mode 100644 index 0000000..1f7a8bc --- /dev/null +++ b/src/runtime/http/v1/serializer.h @@ -0,0 +1,28 @@ +// Spec: plans/01-runtime.md 6.3, RFC 9112 sections 6, 7. Response serializer: +// automatic Content-Length or chunked, HEAD sends head only, 204/304 never +// carry a body. +#pragma once + +#include +#include +#include + +#include "../message.h" + +namespace molo::runtime::http::v1 { + +class Serializer { + public: + // Builds the status line plus head fields plus terminating CRLF. Adds + // Content-Length (identity) or Transfer-Encoding: chunked, plus Date when + // the head lacks it. head_only covers HEAD requests. + static std::string SerializeHead(const ResponseHead& head, + std::string_view version, bool chunked, + std::size_t content_length, bool head_only, + bool close); + static std::string Chunk(const char* data, std::size_t len); + static std::string LastChunk(); + static std::string HttpDateNow(); +}; + +} // namespace molo::runtime::http::v1 diff --git a/src/runtime/http/v2/connection.cpp b/src/runtime/http/v2/connection.cpp new file mode 100644 index 0000000..9b6a956 --- /dev/null +++ b/src/runtime/http/v2/connection.cpp @@ -0,0 +1,962 @@ +// Spec: plans/01-runtime.md 6.4, RFC 9113. H2 connection engine. +#include "connection.h" + +#include + +#include "runtime/base64.h" +#include "runtime/event_loop.h" +#include "runtime/http/connection.h" +#include "runtime/http/protocol_detect.h" +#include "runtime/observability_hooks.h" +#include "runtime/task.h" + +namespace molo::runtime::http::v2 { + +namespace { +constexpr uint8_t kData = 0x0; +constexpr uint8_t kHeaders = 0x1; +constexpr uint8_t kPriority = 0x2; +constexpr uint8_t kRst = 0x3; +constexpr uint8_t kSettings = 0x4; +constexpr uint8_t kPushPromise = 0x5; +constexpr uint8_t kPing = 0x6; +constexpr uint8_t kGoaway = 0x7; +constexpr uint8_t kWindowUpdate = 0x8; +constexpr uint8_t kContinuation = 0x9; + +constexpr uint8_t kFlagEndStream = 0x1; +constexpr uint8_t kFlagAck = 0x1; +constexpr uint8_t kFlagEndHeaders = 0x4; +constexpr uint8_t kFlagPadded = 0x8; +constexpr uint8_t kFlagPriority = 0x20; + +constexpr uint32_t kNoError = 0x0; +constexpr uint32_t kProtocolError = 0x1; +constexpr uint32_t kInternalError = 0x2; +constexpr uint32_t kFlowControlError = 0x3; +constexpr uint32_t kStreamClosed = 0x5; +constexpr uint32_t kFrameSizeError = 0x6; +constexpr uint32_t kRefusedStream = 0x7; +constexpr uint32_t kCancel = 0x8; +constexpr uint32_t kCompressionError = 0x9; + +bool IsH2HopHeader(std::string_view name) { + return EqCaseInsensitive(name, "connection") || + EqCaseInsensitive(name, "keep-alive") || + EqCaseInsensitive(name, "proxy-connection") || + EqCaseInsensitive(name, "transfer-encoding") || + EqCaseInsensitive(name, "upgrade") || + EqCaseInsensitive(name, "http2-settings"); +} +} // namespace + +H2Connection::H2Connection(Connection& conn, H2Config config) + : conn_(conn), config_(config) { + if (config_.max_concurrent_streams > caps::kH2MaxConcurrentStreamsCeiling) { + config_.max_concurrent_streams = caps::kH2MaxConcurrentStreamsCeiling; + } + local_.max_concurrent_streams = + (uint32_t)config_.max_concurrent_streams; + decoder_.SetMaxTableSize(local_.header_table_size); +} + +void H2Connection::drain() { + draining_ = true; + SendGoaway(kNoError); + conn_.InterruptRead(); // unpark the read loop for a prompt exit +} + +std::string H2Connection::FrameBytes(uint8_t type, uint8_t flags, + uint32_t stream_id, const char* payload, + std::size_t len) { + std::string out; + out.resize(9 + len); + out[0] = (char)((len >> 16) & 0xff); + out[1] = (char)((len >> 8) & 0xff); + out[2] = (char)(len & 0xff); + out[3] = (char)type; + out[4] = (char)flags; + out[5] = (char)((stream_id >> 24) & 0x7f); + out[6] = (char)((stream_id >> 16) & 0xff); + out[7] = (char)((stream_id >> 8) & 0xff); + out[8] = (char)(stream_id & 0xff); + if (len > 0) out.replace(9, len, payload, len); + return out; +} + +void H2Connection::EnqueueControl(uint32_t stream_id, + const std::string& frame_bytes) { + OutItem item; + item.stream_id = stream_id; + item.bytes = frame_bytes; + item.is_data = false; + item.end_stream = false; + out_queued_bytes_ += item.bytes.size(); + out_queue_.push_back(std::move(item)); + KickWriter(); +} + +void H2Connection::EnqueueData(uint32_t stream_id, std::string payload, + bool end_stream) { + OutItem item; + item.stream_id = stream_id; + item.bytes = std::move(payload); + item.is_data = true; + item.end_stream = end_stream; + out_queued_bytes_ += item.bytes.size(); + out_queue_.push_back(std::move(item)); + KickWriter(); +} + +void H2Connection::KickWriter() { + if (writer_active_ || conn_.closed()) return; + writer_active_ = true; + conn_.loop().post(WriterTask(weak_from_this())); +} + +void H2Connection::WakeWriter() { + if (window_waiter_) { + auto h = window_waiter_; + window_waiter_ = nullptr; + h.resume(); + } +} + +void H2Connection::SendSettingsAck() { + EnqueueControl(0, FrameBytes(kSettings, kFlagAck, 0, nullptr, 0)); +} + +void H2Connection::SendPingAck(const char* payload) { + EnqueueControl(0, FrameBytes(kPing, kFlagAck, 0, payload, 8)); +} + +void H2Connection::SendRst(uint32_t stream_id, uint32_t code) { + char p[4] = {(char)(code >> 24), (char)((code >> 16) & 0xff), + (char)((code >> 8) & 0xff), (char)(code & 0xff)}; + EnqueueControl(stream_id, FrameBytes(kRst, 0, stream_id, p, 4)); +} + +void H2Connection::SendGoaway(uint32_t code) { + if (goaway_sent_) return; + goaway_sent_ = true; + char p[8]; + p[0] = (char)((max_stream_id_ >> 24) & 0x7f); + p[1] = (char)((max_stream_id_ >> 16) & 0xff); + p[2] = (char)((max_stream_id_ >> 8) & 0xff); + p[3] = (char)(max_stream_id_ & 0xff); + p[4] = (char)(code >> 24); + p[5] = (char)((code >> 16) & 0xff); + p[6] = (char)((code >> 8) & 0xff); + p[7] = (char)(code & 0xff); + EnqueueControl(0, FrameBytes(kGoaway, 0, 0, p, 8)); +} + +void H2Connection::SendWindowUpdate(uint32_t stream_id, uint32_t increment) { + if (increment == 0) return; + char p[4] = {(char)((increment >> 24) & 0x7f), + (char)((increment >> 16) & 0xff), + (char)((increment >> 8) & 0xff), (char)(increment & 0xff)}; + EnqueueControl(stream_id, FrameBytes(kWindowUpdate, 0, stream_id, p, 4)); +} + +void H2Connection::Fatal(uint32_t code) { + if (fatal_) return; + fatal_ = true; + close_after_flush_ = true; + SendGoaway(code); + WakeWriter(); +} + +Task H2Connection::Ensure(std::size_t n) { + std::string& in = conn_.inbox(); + while (in.size() < n) { + if (conn_.closed() || fatal_) co_return false; + Connection::ReadResult r = co_await conn_.PumpRead(config_.idle_timeout); + if (r == Connection::ReadResult::Ok) continue; + if (r == Connection::ReadResult::Interrupted) { + // Drain poke or race: return so the read loop re-checks its exit + // conditions. The loop must verify byte counts before parsing. + co_return true; + } + if (r == Connection::ReadResult::Timeout) { + if (!streams_.empty()) continue; // in-flight work: keep waiting + SendGoaway(kNoError); // idle: graceful close + co_return false; + } + co_return false; + } + co_return true; +} + +Task H2Connection::Run(bool expect_magic, bool from_upgrade, + const RequestHead& upgrade_head, + const std::string& upgrade_body) { + auto self = shared_from_this(); + if (expect_magic) { + for (;;) { + if (!co_await Ensure(kH2Preface.size())) co_return; + if (conn_.closed() || fatal_) co_return; + if (conn_.inbox().size() >= kH2Preface.size()) break; + // Interrupted with a short buffer: re-park. + } + std::string& in = conn_.inbox(); + if (in.compare(0, kH2Preface.size(), kH2Preface) != 0) { + Fatal(kProtocolError); + } else { + in.erase(0, kH2Preface.size()); + } + if (fatal_) { + read_done_ = true; + WakeWriter(); + co_return; + } + } + // Server preface. + { + std::string payload = EncodeSettingsPayload(local_); + conn_.outbox().append( + FrameBytes(kSettings, 0, 0, payload.data(), payload.size())); + if (!co_await conn_.Flush(config_.idle_timeout)) co_return; + } + if (from_upgrade) { + if (!ApplyUpgrade(upgrade_head, upgrade_body)) { + read_done_ = true; + WakeWriter(); + co_return; + } + co_await ReadLoop(false); + } else { + co_await ReadLoop(true); + } + read_done_ = true; + WakeWriter(); + // Join the writer so GOAWAY is delivered before the socket closes. + if (writer_active_ || !out_queue_.empty()) { + struct JoinAwait { + H2Connection* self; + bool await_ready() const noexcept { return false; } + void await_suspend(std::coroutine_handle<> h) { + self->writer_done_waiter_ = h; + H2Connection* h2 = self; + h2->conn_.loop().schedule_after(std::chrono::milliseconds(5000), + [h2] { + if (h2->writer_done_waiter_) { + auto w = + h2->writer_done_waiter_; + h2->writer_done_waiter_ = + nullptr; + w.resume(); + } + }); + } + void await_resume() noexcept {} + }; + KickWriter(); + co_await JoinAwait{this}; + writer_done_waiter_ = nullptr; + } +} + +Task H2Connection::ReadLoop(bool first_must_be_settings) { + auto self = shared_from_this(); + bool first = first_must_be_settings; + std::string& in = conn_.inbox(); + for (;;) { + if (conn_.closed() || fatal_) co_return; + if ((peer_goaway_ || draining_) && streams_.empty()) co_return; + if (!co_await Ensure(9)) co_return; + if (conn_.closed() || fatal_) co_return; + if ((peer_goaway_ || draining_) && streams_.empty()) co_return; + if (in.size() < 9) continue; // interrupted: re-check, then re-park + Frame fr; + fr.length = ((uint32_t)(unsigned char)in[0] << 16) | + ((uint32_t)(unsigned char)in[1] << 8) | + (uint32_t)(unsigned char)in[2]; + fr.type = (unsigned char)in[3]; + fr.flags = (unsigned char)in[4]; + fr.stream_id = ((uint32_t)((unsigned char)in[5] & 0x7f) << 24) | + ((uint32_t)(unsigned char)in[6] << 16) | + ((uint32_t)(unsigned char)in[7] << 8) | + (uint32_t)(unsigned char)in[8]; + if (fr.length > local_.max_frame_size) { + Fatal(kFrameSizeError); + co_return; + } + if (!co_await Ensure(9 + fr.length)) co_return; + if (conn_.closed() || fatal_) co_return; + if ((peer_goaway_ || draining_) && streams_.empty()) co_return; + if (in.size() < 9 + fr.length) continue; // interrupted: re-park + std::string payload(in.data() + 9, fr.length); + in.erase(0, 9 + fr.length); + if (first) { + first = false; + if (fr.type != kSettings) { + Fatal(kProtocolError); + co_return; + } + } + if (!ProcessFrame(fr, payload.data())) co_return; + } +} + +bool H2Connection::ProcessFrame(const Frame& fr, const char* payload) { + if (cont_expected_ && fr.type != kContinuation) { + Fatal(kProtocolError); + return false; + } + switch (fr.type) { + case kData: + return OnData(fr, payload); + case kHeaders: + return OnHeaders(fr, payload); + case kPriority: + return OnPriority(fr, payload); + case kRst: + return OnRst(fr, payload); + case kSettings: + return OnSettings(fr, payload); + case kPushPromise: + Fatal(kProtocolError); // server push is gone; never accept it + return false; + case kPing: + return OnPing(fr, payload); + case kGoaway: + return OnGoaway(fr, payload); + case kWindowUpdate: + return OnWindowUpdate(fr, payload); + case kContinuation: + return OnContinuation(fr, payload); + default: + return true; // unknown types are ignored + } +} + +bool H2Connection::OnSettings(const Frame& fr, const char* payload) { + if (fr.stream_id != 0) { + Fatal(kProtocolError); + return false; + } + if (fr.flags & kFlagAck) { + if (fr.length != 0) { + Fatal(kFrameSizeError); + return false; + } + return true; + } + uint32_t old_window = peer_.initial_window_size; + Settings next = peer_; + SettingsError e = DecodeSettingsPayload(payload, fr.length, next); + if (e == SettingsError::ProtocolError) { + Fatal(kProtocolError); + return false; + } + if (e == SettingsError::FlowControlError) { + Fatal(kFlowControlError); + return false; + } + if (e == SettingsError::CompressionError) { + Fatal(kCompressionError); + return false; + } + peer_ = next; + encoder_.SetMaxTableSize(peer_.header_table_size); + if (peer_.initial_window_size != old_window) { + int64_t delta = + (int64_t)peer_.initial_window_size - (int64_t)old_window; + if (!flow_.AdjustAllStreams(delta, peer_.initial_window_size)) { + Fatal(kFlowControlError); + return false; + } + } + SendSettingsAck(); + return true; +} + +bool H2Connection::OnHeaders(const Frame& fr, const char* payload) { + if (fr.stream_id == 0) { + Fatal(kProtocolError); + return false; + } + std::size_t off = 0; + std::size_t len = fr.length; + if (fr.flags & kFlagPadded) { + if (len < 1) { + Fatal(kProtocolError); + return false; + } + std::size_t pad = (unsigned char)payload[0]; + if (pad + 1 > len) { + Fatal(kProtocolError); + return false; + } + off = 1; + len -= 1 + pad; + } + if (fr.flags & kFlagPriority) { + if (len < 5) { + Fatal(kProtocolError); + return false; + } + off += 5; + len -= 5; + } + bool end_stream = (fr.flags & kFlagEndStream) != 0; + if (!(fr.flags & kFlagEndHeaders)) { + cont_expected_ = true; + cont_stream_ = fr.stream_id; + cont_buf_.assign(payload + off, len); + cont_end_stream_ = end_stream; + return true; + } + return FinishHeaders(fr.stream_id, std::string(payload + off, len), + end_stream); +} + +bool H2Connection::FinishHeaders(uint32_t stream_id, + const std::string& block, bool end_stream) { + std::vector fields; + HpackError e = decoder_.Decode(block.data(), block.size(), fields); + if (e != HpackError::Ok) { + Fatal(kCompressionError); + return false; + } + auto it = streams_.find(stream_id); + if (it == streams_.end()) { + if ((stream_id % 2) == 0) { + Fatal(kProtocolError); + return false; + } + if (stream_id <= max_stream_id_) { + SendRst(stream_id, kStreamClosed); + return true; + } + if (goaway_sent_ || peer_goaway_ || draining_) return true; + if (streams_.size() >= config_.max_concurrent_streams) { + max_stream_id_ = stream_id; + SendRst(stream_id, kRefusedStream); + return true; + } + max_stream_id_ = stream_id; + streams_.emplace(stream_id, H2Stream(stream_id, config_.body_cap)); + flow_.AddStream(stream_id, peer_.initial_window_size); + stream_in_window_[stream_id] = (int64_t)local_.initial_window_size; + it = streams_.find(stream_id); + } + StreamOutcome oc = it->second.OnHeaders(fields, end_stream); + if (oc.event == StreamEvent::LocalReset) { + SendRst(stream_id, oc.rst_code); + EraseStream(stream_id); + return true; + } + if (oc.event == StreamEvent::MessageComplete) Dispatch(stream_id); + return true; +} + +bool H2Connection::OnContinuation(const Frame& fr, const char* payload) { + if (!cont_expected_ || fr.stream_id != cont_stream_) { + Fatal(kProtocolError); + return false; + } + if (cont_buf_.size() + fr.length > caps::kH2HeaderListMax) { + Fatal(kCompressionError); + return false; + } + cont_buf_.append(payload, fr.length); + if (!(fr.flags & kFlagEndHeaders)) return true; + cont_expected_ = false; + std::string block = std::move(cont_buf_); + cont_buf_.clear(); + return FinishHeaders(fr.stream_id, block, cont_end_stream_); +} + +bool H2Connection::OnData(const Frame& fr, const char* payload) { + if (fr.stream_id == 0) { + Fatal(kProtocolError); + return false; + } + std::size_t off = 0; + std::size_t len = fr.length; + if (fr.flags & kFlagPadded) { + if (len < 1) { + Fatal(kProtocolError); + return false; + } + std::size_t pad = (unsigned char)payload[0]; + if (pad + 1 > len) { + Fatal(kProtocolError); + return false; + } + off = 1; + len -= 1 + pad; + } + bool end_stream = (fr.flags & kFlagEndStream) != 0; + auto it = streams_.find(fr.stream_id); + if (it == streams_.end()) { + if (fr.stream_id <= max_stream_id_) { + SendRst(fr.stream_id, kStreamClosed); + return true; + } + Fatal(kProtocolError); + return false; + } + if ((int64_t)fr.length > conn_in_window_) { + Fatal(kFlowControlError); + return false; + } + auto win = stream_in_window_.find(fr.stream_id); + if (win == stream_in_window_.end() || + (int64_t)fr.length > win->second) { + Fatal(kFlowControlError); + return false; + } + conn_in_window_ -= (int64_t)fr.length; + win->second -= (int64_t)fr.length; + flow_.ConsumedConnection(fr.length); + flow_.ConsumedStream(fr.stream_id, fr.length); + // Immediate refund keeps the peer flowing; bounded by frame caps. + SendWindowUpdate(0, (uint32_t)flow_.TakeConnectionRefund()); + SendWindowUpdate(fr.stream_id, + (uint32_t)flow_.TakeStreamRefund(fr.stream_id)); + conn_in_window_ += (int64_t)fr.length; + win->second += (int64_t)fr.length; + StreamOutcome oc = + it->second.OnData(payload + off, len, end_stream); + if (oc.event == StreamEvent::LocalReset) { + SendRst(fr.stream_id, oc.rst_code); + EraseStream(fr.stream_id); + return true; + } + if (oc.event == StreamEvent::MessageComplete) { + Dispatch(fr.stream_id); + } + return true; +} + +bool H2Connection::OnRst(const Frame& fr, const char* payload) { + if (fr.stream_id == 0) { + Fatal(kProtocolError); + return false; + } + if (fr.length != 4) { + Fatal(kFrameSizeError); + return false; + } + auto it = streams_.find(fr.stream_id); + if (it == streams_.end()) return true; // idle or closed: ignore + uint32_t code = ((uint32_t)(unsigned char)payload[0] << 24) | + ((uint32_t)(unsigned char)payload[1] << 16) | + ((uint32_t)(unsigned char)payload[2] << 8) | + (uint32_t)(unsigned char)payload[3]; + it->second.OnRst(code); + EraseStream(fr.stream_id); + return true; +} + +bool H2Connection::OnPing(const Frame& fr, const char* payload) { + if (fr.stream_id != 0) { + Fatal(kProtocolError); + return false; + } + if (fr.length != 8) { + Fatal(kFrameSizeError); + return false; + } + if (fr.flags & kFlagAck) return true; + SendPingAck(payload); + return true; +} + +bool H2Connection::OnGoaway(const Frame& fr, const char* payload) { + if (fr.stream_id != 0) { + Fatal(kProtocolError); + return false; + } + if (fr.length < 8) { + Fatal(kFrameSizeError); + return false; + } + (void)payload; + peer_goaway_ = true; + return true; +} + +bool H2Connection::OnWindowUpdate(const Frame& fr, const char* payload) { + if (fr.length != 4) { + Fatal(kFrameSizeError); + return false; + } + uint32_t inc = ((uint32_t)((unsigned char)payload[0] & 0x7f) << 24) | + ((uint32_t)(unsigned char)payload[1] << 16) | + ((uint32_t)(unsigned char)payload[2] << 8) | + (uint32_t)(unsigned char)payload[3]; + if (inc == 0) { + Fatal(kFlowControlError); + return false; + } + if (fr.stream_id == 0) { + flow_.OnConnectionWindowUpdate(inc); + WakeWriter(); + return true; + } + auto it = streams_.find(fr.stream_id); + if (it == streams_.end()) { + if (fr.stream_id > max_stream_id_) { + Fatal(kProtocolError); + return false; + } + return true; // closed: ignore + } + StreamOutcome oc = it->second.OnWindowUpdate(inc); + if (oc.event == StreamEvent::LocalReset) { + SendRst(fr.stream_id, oc.rst_code); + EraseStream(fr.stream_id); + return true; + } + flow_.OnStreamWindowUpdate(fr.stream_id, inc); + WakeWriter(); + return true; +} + +bool H2Connection::OnPriority(const Frame& fr, const char* payload) { + (void)payload; + if (fr.stream_id == 0) { + Fatal(kProtocolError); + return false; + } + if (fr.length != 5) { + Fatal(kFrameSizeError); + return false; + } + return true; // accepted and ignored per plan open question 2 +} + +void H2Connection::EraseStream(uint32_t stream_id) { + streams_.erase(stream_id); + flow_.RemoveStream(stream_id); + stream_in_window_.erase(stream_id); + if ((draining_ || peer_goaway_) && streams_.empty()) { + conn_.InterruptRead(); // let the read loop notice the drain is done + } +} + +void H2Connection::Dispatch(uint32_t stream_id) { + auto it = streams_.find(stream_id); + if (it == streams_.end()) return; + FullRequest req; + req.head = it->second.head(); + req.body = it->second.body(); + req.trailers = it->second.trailers(); + req.is_tls = conn_.is_tls(); + req.peer = conn_.peer(); + req.protocol = "HTTP/2"; + ++requests_; + MOLO_RUNTIME_COUNT("runtime.h2.requests"); + conn_.loop().post(StreamTask(weak_from_this(), stream_id, std::move(req))); +} + +Task H2Connection::StreamTask(std::weak_ptr weak, + uint32_t stream_id, FullRequest req) { + auto self = weak.lock(); + if (!self) co_return; + TimerWheel::TimerId budget = self->conn_.loop().schedule_after( + self->config_.handler_budget, [weak, stream_id] { + if (auto s = weak.lock()) { + auto it = s->streams_.find(stream_id); + if (it != s->streams_.end()) { + s->SendRst(stream_id, kCancel); + s->EraseStream(stream_id); + } + } + }); + FullResponse resp; + bool ok = true; + try { + resp = co_await self->conn_.handler()(req); + } catch (...) { + ok = false; + } + self = weak.lock(); + if (!self) co_return; + self->conn_.loop().cancel_timer(budget); + if (!ok) { + resp.head.status = 500; + resp.head.headers.clear(); + resp.head.headers.set("content-type", "text/plain; charset=utf-8"); + resp.body = "500 Internal Server Error\n"; + } + std::optional fb; + if (resp.file.has_value()) { + fb = resp.file; + resp.file.reset(); + } + if (fb.has_value() && fb->fd >= 0 && !resp.body.empty()) { + // Contract violation: file and string body at once. Fail closed. + ::close(fb->fd); + fb.reset(); + resp.head.status = 500; + resp.head.headers.clear(); + resp.head.headers.set("content-type", "text/plain; charset=utf-8"); + resp.body = "500 Internal Server Error\n"; + } + bool file_follows = fb.has_value() && fb->fd >= 0; + bool stream_file = self->CompleteResponse(stream_id, std::move(resp), + file_follows); + if (stream_file && file_follows) { + co_await FileBodyTask(weak, stream_id, *fb); + } else if (file_follows) { + ::close(fb->fd); // HEAD, reset, or no-body status: drop the bytes + } +} + +Task H2Connection::FileBodyTask(std::weak_ptr weak, + uint32_t stream_id, FileBody fb) { + struct Closer { + int fd; + ~Closer() { + if (fd >= 0) ::close(fd); + } + } closer{fb.fd}; + if (!ClampFileBody(fb)) { + if (auto s = weak.lock()) s->SendRst(stream_id, kInternalError); + co_return; + } + auto self = weak.lock(); + if (!self) co_return; + if (fb.length == 0) { + // Empty file still needs the terminal flag. + if (self->streams_.find(stream_id) != self->streams_.end()) { + self->EnqueueData(stream_id, std::string(), true); + } + co_return; + } + // Framing forbids sendfile here, so stream bounded 64 KB DATA items with + // queue backpressure: park on yield until the writer drains below the + // high watermark. + uint64_t pos = fb.offset; + uint64_t left = fb.length; + std::string chunk; + chunk.resize(caps::kFileChunkSize); + while (left > 0) { + self = weak.lock(); + if (!self || self->conn_.closed()) co_return; + if (self->streams_.find(stream_id) == self->streams_.end()) co_return; + std::size_t n = + left > chunk.size() ? chunk.size() : (std::size_t)left; + ssize_t r = ::pread(fb.fd, chunk.data(), n, (off_t)pos); + if (r <= 0) { + self->SendRst(stream_id, kInternalError); + co_return; + } + pos += (uint64_t)r; + left -= (uint64_t)r; + self->EnqueueData(stream_id, chunk.substr(0, (std::size_t)r), + left == 0); + while (self && !self->conn_.closed() && + self->out_queued_bytes_ > caps::kWriteHighWatermark && + self->streams_.find(stream_id) != self->streams_.end()) { + co_await self->conn_.loop().yield(); + self = weak.lock(); + } + } +} + +bool H2Connection::CompleteResponse(uint32_t stream_id, FullResponse resp, + bool file_follows) { + auto it = streams_.find(stream_id); + if (it == streams_.end()) return false; // reset or timed out: drop + if (it->second.state() == StreamState::Closed) { + EraseStream(stream_id); + return false; + } + std::vector fields; + fields.push_back({":status", std::to_string(resp.head.status)}); + for (const auto& kv : resp.head.headers.entries()) { + if (IsH2HopHeader(kv.first)) continue; + if (!kv.first.empty() && kv.first[0] == ':') continue; + fields.push_back({kv.first, kv.second}); + } + std::string block = encoder_.Encode(fields); + bool head_only = (it->second.head().method == "HEAD"); + bool no_body = head_only || resp.head.status == 204 || + resp.head.status == 304 || + (resp.head.status >= 100 && resp.head.status < 200) || + (!file_follows && resp.body.empty()); + std::size_t maxf = peer_.max_frame_size; + if (maxf == 0) maxf = caps::kH2MaxFrameDefault; + std::size_t off = 0; + bool first = true; + // HEADERS plus CONTINUATIONs as one control item. + std::string head_frames; + while (off < block.size()) { + std::size_t n = block.size() - off; + if (n > maxf) n = maxf; + bool last = (off + n >= block.size()); + uint8_t flags = 0; + if (last) flags |= kFlagEndHeaders; + if (last && no_body) flags |= kFlagEndStream; + head_frames.append(FrameBytes(first ? kHeaders : kContinuation, flags, + stream_id, block.data() + off, n)); + off += n; + first = false; + } + if (block.empty()) { + uint8_t flags = kFlagEndHeaders; + if (no_body) flags |= kFlagEndStream; + head_frames.append(FrameBytes(kHeaders, flags, stream_id, nullptr, 0)); + } + OutItem ctl; + ctl.stream_id = stream_id; + ctl.bytes = std::move(head_frames); + ctl.is_data = false; + ctl.end_stream = no_body; + out_queued_bytes_ += ctl.bytes.size(); + out_queue_.push_back(std::move(ctl)); + if (no_body) { + KickWriter(); + return false; + } + if (file_follows) { + KickWriter(); // HEADERS flush while the file streams behind + return true; + } + EnqueueData(stream_id, std::move(resp.body), true); + return false; +} + +Task H2Connection::WriterTask(std::weak_ptr weak) { + auto self = weak.lock(); + if (!self) co_return; + struct WindowAwait { + H2Connection* h2; + bool await_ready() const noexcept { return false; } + void await_suspend(std::coroutine_handle<> h) { h2->window_waiter_ = h; } + void await_resume() noexcept {} + }; + while (self && !self->out_queue_.empty() && !self->conn_.closed()) { + OutItem& item = self->out_queue_.front(); + if (!item.is_data) { + self->conn_.outbox().append(item.bytes); + bool ok = co_await self->conn_.Flush(self->config_.idle_timeout); + self = weak.lock(); + if (!self) co_return; + if (!ok) break; + bool end = item.end_stream; + uint32_t sid = item.stream_id; + self->UnqueueBytes(item.bytes.size() - item.offset); + self->out_queue_.pop_front(); + auto it = self->streams_.find(sid); + if (it != self->streams_.end()) { + it->second.MarkResponseHeadersSent(end); + if (end) self->EraseStream(sid); + } + continue; + } + // DATA with flow control. + auto it = self->streams_.find(item.stream_id); + if (it == self->streams_.end()) { + self->UnqueueBytes(item.bytes.size() - item.offset); + self->out_queue_.pop_front(); // reset meanwhile: drop + continue; + } + std::size_t remaining = item.bytes.size() - item.offset; + int64_t cw = self->flow_.ConnectionAvailable(); + int64_t sw = self->flow_.StreamAvailable(item.stream_id); + if (cw <= 0 || sw <= 0) { + if (self->read_done_) { + self->UnqueueBytes(item.bytes.size() - item.offset); + self->out_queue_.pop_front(); // no more updates coming + continue; + } + co_await WindowAwait{self.get()}; + self = weak.lock(); + if (!self) co_return; + continue; + } + std::size_t n = remaining; + if ((int64_t)n > cw) n = (std::size_t)cw; + if ((int64_t)n > sw) n = (std::size_t)sw; + std::size_t maxf = self->peer_.max_frame_size; + if (maxf == 0) maxf = caps::kH2MaxFrameDefault; + if (n > maxf) n = maxf; + self->flow_.TakeConnection(n); + self->flow_.TakeStream(item.stream_id, n); + bool last = (item.offset + n >= item.bytes.size()); + bool end = last && item.end_stream; + uint8_t flags = end ? kFlagEndStream : 0; + self->conn_.outbox().append(FrameBytes(kData, flags, item.stream_id, + item.bytes.data() + item.offset, + n)); + bool ok = co_await self->conn_.Flush(self->config_.idle_timeout); + self = weak.lock(); + if (!self) co_return; + if (!ok) break; + item.offset += n; + self->UnqueueBytes(n); + auto jt = self->streams_.find(item.stream_id); + if (jt != self->streams_.end()) { + jt->second.MarkResponseDataSent(end); + if (end) self->EraseStream(item.stream_id); + } + if (last) self->out_queue_.pop_front(); + } + if (!self) co_return; + self->writer_active_ = false; + self->window_waiter_ = nullptr; + if (self->writer_done_waiter_) { + auto h = self->writer_done_waiter_; + self->writer_done_waiter_ = nullptr; + h.resume(); + } +} + +bool H2Connection::ApplyUpgrade(const RequestHead& head, + const std::string& body) { + std::string encoded = head.headers.get("http2-settings"); + std::optional raw = Base64Url::Decode(encoded); + if (!raw.has_value()) { + Fatal(kProtocolError); + return false; + } + Settings next = peer_; + SettingsError e = DecodeSettingsPayload(raw->data(), raw->size(), next); + if (e != SettingsError::Ok) { + Fatal(kProtocolError); + return false; + } + peer_ = next; + encoder_.SetMaxTableSize(peer_.header_table_size); + // Stream 1 carries the upgraded request, half-closed (remote). + std::vector fields; + fields.push_back({":method", head.method}); + fields.push_back({":scheme", conn_.is_tls() ? "https" : "http"}); + std::string authority = head.headers.get("host"); + if (authority.empty()) authority = head.headers.get(":authority"); + fields.push_back({":authority", authority}); + fields.push_back({":path", head.target}); + for (const auto& kv : head.headers.entries()) { + if (IsH2HopHeader(kv.first)) continue; + if (EqCaseInsensitive(kv.first, "host")) continue; + fields.push_back({kv.first, kv.second}); + } + streams_.emplace(1, H2Stream(1, config_.body_cap)); + max_stream_id_ = 1; + flow_.AddStream(1, peer_.initial_window_size); + stream_in_window_[1] = (int64_t)local_.initial_window_size; + auto it = streams_.find(1); + StreamOutcome oc = it->second.OnHeaders(fields, body.empty()); + if (oc.event == StreamEvent::LocalReset) { + Fatal(kProtocolError); + return false; + } + if (!body.empty()) { + oc = it->second.OnData(body.data(), body.size(), true); + if (oc.event != StreamEvent::MessageComplete) { + Fatal(kProtocolError); + return false; + } + } + if (oc.event == StreamEvent::MessageComplete) Dispatch(1); + return true; +} + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/http/v2/connection.h b/src/runtime/http/v2/connection.h new file mode 100644 index 0000000..71c167c --- /dev/null +++ b/src/runtime/http/v2/connection.h @@ -0,0 +1,152 @@ +// Spec: plans/01-runtime.md 6.4, RFC 9113. H2 framing, settings handshake, +// HPACK, stream multiplexing, dual flow control, GOAWAY/RST/PING. No server +// push. PRIORITY accepted and ignored. Runs on the owning Connection's +// transport, waiter and buffers; the Connection keeps lifecycle ownership. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "runtime/caps.h" +#include "runtime/http/message.h" +#include "runtime/task.h" +#include "flow_control.h" +#include "hpack.h" +#include "settings.h" +#include "stream.h" + +namespace molo::runtime { +class EventLoop; +namespace tls { +class ITlsHandle; +} +namespace http { +class Connection; +struct FullRequest; +struct FullResponse; +} // namespace http +} // namespace molo::runtime + +namespace molo::runtime::http::v2 { + +struct H2Config { + std::size_t body_cap = caps::kBodyMaxDefault; + std::size_t max_concurrent_streams = caps::kH2MaxConcurrentStreamsDefault; + std::chrono::milliseconds idle_timeout = caps::kIdleKeepAliveTimeout; + std::chrono::milliseconds handler_budget = caps::kHandlerBudget; +}; + +class H2Connection : public std::enable_shared_from_this { + public: + H2Connection(Connection& conn, H2Config config = H2Config{}); + ~H2Connection() = default; + + Task Run(bool expect_magic, bool from_upgrade, + const RequestHead& upgrade_head, + const std::string& upgrade_body); + void drain(); + std::size_t requests_served() const { return requests_; } + + private: + struct Frame { + uint32_t length = 0; + uint8_t type = 0; + uint8_t flags = 0; + uint32_t stream_id = 0; + }; + + struct OutItem { + uint32_t stream_id = 0; + std::string bytes; // payload for DATA, full frame(s) otherwise + bool is_data = false; + bool end_stream = false; + std::size_t offset = 0; + }; + + static Task StreamTask(std::weak_ptr weak, + uint32_t stream_id, FullRequest req); + static Task WriterTask(std::weak_ptr weak); + static Task FileBodyTask(std::weak_ptr weak, + uint32_t stream_id, FileBody fb); + + Task Ensure(std::size_t n); + Task ReadLoop(bool first_must_be_settings); + bool ProcessFrame(const Frame& fr, const char* payload); + // Sends HEADERS. file_follows means a file body streams next: the stream + // stays open unless HEAD or a no-body status applies. Returns true when + // the caller must stream the file body (via FileBodyTask). + bool CompleteResponse(uint32_t stream_id, FullResponse resp, + bool file_follows); + + void EnqueueControl(uint32_t stream_id, const std::string& frame_bytes); + void EnqueueData(uint32_t stream_id, std::string payload, bool end_stream); + void KickWriter(); + void WakeWriter(); + void SendSettingsAck(); + void SendPingAck(const char* payload); + void SendRst(uint32_t stream_id, uint32_t code); + void SendGoaway(uint32_t code); + void SendWindowUpdate(uint32_t stream_id, uint32_t increment); + void Fatal(uint32_t code); + static std::string FrameBytes(uint8_t type, uint8_t flags, + uint32_t stream_id, const char* payload, + std::size_t len); + + bool OnSettings(const Frame& fr, const char* payload); + bool OnHeaders(const Frame& fr, const char* payload); + bool OnData(const Frame& fr, const char* payload); + bool OnRst(const Frame& fr, const char* payload); + bool OnPing(const Frame& fr, const char* payload); + bool OnGoaway(const Frame& fr, const char* payload); + bool OnWindowUpdate(const Frame& fr, const char* payload); + bool OnPriority(const Frame& fr, const char* payload); + bool OnContinuation(const Frame& fr, const char* payload); + bool FinishHeaders(uint32_t stream_id, const std::string& block, + bool end_stream); + void EraseStream(uint32_t stream_id); + void Dispatch(uint32_t stream_id); + bool ApplyUpgrade(const RequestHead& head, const std::string& body); + + Connection& conn_; + H2Config config_; + Settings local_; + Settings peer_; + HpackEncoder encoder_; + HpackDecoder decoder_; + FlowController flow_; + std::unordered_map streams_; + int64_t conn_in_window_ = 65535; + std::unordered_map stream_in_window_; + std::deque out_queue_; + // Payload bytes currently queued; bounds file-body streaming. + std::size_t out_queued_bytes_ = 0; + void UnqueueBytes(std::size_t n) { + out_queued_bytes_ -= + n > out_queued_bytes_ ? out_queued_bytes_ : n; + } + bool writer_active_ = false; + bool read_done_ = false; + bool close_after_flush_ = false; + bool draining_ = false; + bool goaway_sent_ = false; + bool peer_goaway_ = false; + bool fatal_ = false; + uint32_t max_stream_id_ = 0; + std::size_t requests_ = 0; + // CONTINUATION reassembly state. + bool cont_expected_ = false; + uint32_t cont_stream_ = 0; + std::string cont_buf_; + bool cont_end_stream_ = false; + std::coroutine_handle<> window_waiter_ = nullptr; + std::coroutine_handle<> writer_done_waiter_ = nullptr; +}; + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/http/v2/flow_control.cpp b/src/runtime/http/v2/flow_control.cpp new file mode 100644 index 0000000..d20887b --- /dev/null +++ b/src/runtime/http/v2/flow_control.cpp @@ -0,0 +1,82 @@ +// Spec: plans/01-runtime.md 6.4, RFC 9113 section 6.9. Flow control. +#include "flow_control.h" + +namespace molo::runtime::http::v2 { + +namespace { +constexpr int64_t kWindowMax = 0x7fffffffLL; +} + +FlowController::FlowController(uint32_t initial_window) + : conn_out_((int64_t)initial_window) {} + +bool FlowController::TakeConnection(std::size_t n) { + if ((int64_t)n > conn_out_) return false; + conn_out_ -= (int64_t)n; + return true; +} + +bool FlowController::TakeStream(uint32_t stream_id, std::size_t n) { + auto it = stream_out_.find(stream_id); + if (it == stream_out_.end()) return false; + if ((int64_t)n > it->second) return false; + it->second -= (int64_t)n; + return true; +} + +void FlowController::OnConnectionWindowUpdate(uint32_t increment) { + conn_out_ += (int64_t)increment; + if (conn_out_ > kWindowMax) conn_out_ = kWindowMax; +} + +void FlowController::OnStreamWindowUpdate(uint32_t stream_id, + uint32_t increment) { + auto it = stream_out_.find(stream_id); + if (it == stream_out_.end()) return; + it->second += (int64_t)increment; + if (it->second > kWindowMax) it->second = kWindowMax; +} + +bool FlowController::AdjustAllStreams(int64_t delta, uint32_t new_initial) { + (void)new_initial; + for (const auto& kv : stream_out_) { + if (kv.second + delta > kWindowMax) return false; + } + for (auto& kv : stream_out_) kv.second += delta; + return true; +} + +void FlowController::AddStream(uint32_t stream_id, uint32_t initial_window) { + stream_out_[stream_id] = (int64_t)initial_window; +} + +void FlowController::RemoveStream(uint32_t stream_id) { + stream_out_.erase(stream_id); + stream_in_used_.erase(stream_id); +} + +int64_t FlowController::StreamAvailable(uint32_t stream_id) const { + auto it = stream_out_.find(stream_id); + if (it == stream_out_.end()) return 0; + return it->second; +} + +void FlowController::ConsumedStream(uint32_t stream_id, std::size_t n) { + stream_in_used_[stream_id] += n; +} + +std::size_t FlowController::TakeConnectionRefund() { + std::size_t n = conn_in_used_; + conn_in_used_ = 0; + return n; +} + +std::size_t FlowController::TakeStreamRefund(uint32_t stream_id) { + auto it = stream_in_used_.find(stream_id); + if (it == stream_in_used_.end()) return 0; + std::size_t n = it->second; + it->second = 0; + return n; +} + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/http/v2/flow_control.h b/src/runtime/http/v2/flow_control.h new file mode 100644 index 0000000..d40f843 --- /dev/null +++ b/src/runtime/http/v2/flow_control.h @@ -0,0 +1,43 @@ +// Spec: plans/01-runtime.md 6.4, RFC 9113 section 6.9. Connection-level plus +// stream-level flow control accounting. Windows are signed: a SETTINGS +// reduction may push a stream window negative, which only blocks sends. +#pragma once + +#include +#include +#include + +namespace molo::runtime::http::v2 { + +class FlowController { + public: + explicit FlowController(uint32_t initial_window = 65535); + + // Outbound (peer windows): take returns false when the window is too + // small; the caller waits for WINDOW_UPDATE. + bool TakeConnection(std::size_t n); + bool TakeStream(uint32_t stream_id, std::size_t n); + void OnConnectionWindowUpdate(uint32_t increment); + void OnStreamWindowUpdate(uint32_t stream_id, uint32_t increment); + // Applies an INITIAL_WINDOW_SIZE change to every live stream. False when + // a window would exceed 2^31-1 (FLOW_CONTROL_ERROR). + bool AdjustAllStreams(int64_t delta, uint32_t new_initial); + void AddStream(uint32_t stream_id, uint32_t initial_window); + void RemoveStream(uint32_t stream_id); + int64_t ConnectionAvailable() const { return conn_out_; } + int64_t StreamAvailable(uint32_t stream_id) const; + + // Inbound: track consumed bytes; the connection emits WINDOW_UPDATE. + void ConsumedConnection(std::size_t n) { conn_in_used_ += n; } + void ConsumedStream(uint32_t stream_id, std::size_t n); + std::size_t TakeConnectionRefund(); + std::size_t TakeStreamRefund(uint32_t stream_id); + + private: + int64_t conn_out_; + std::unordered_map stream_out_; + std::size_t conn_in_used_ = 0; + std::unordered_map stream_in_used_; +}; + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/http/v2/hpack.cpp b/src/runtime/http/v2/hpack.cpp new file mode 100644 index 0000000..a2c0a07 --- /dev/null +++ b/src/runtime/http/v2/hpack.cpp @@ -0,0 +1,344 @@ +// Spec: plans/01-runtime.md 6.4, RFC 7541. HPACK codec. +#include "hpack.h" + +#include "runtime/caps.h" + +namespace molo::runtime::http::v2 { +namespace { + +struct StaticEntry { + const char* name; + const char* value; +}; + +const uint32_t kHuffmanCodes[][2] = { +#include "huffman_table.inc" +}; + +const StaticEntry kStaticTable[] = { +#include "static_table.inc" +}; + +constexpr std::size_t kStaticCount = + sizeof(kStaticTable) / sizeof(kStaticTable[0]); // 62 with slot 0 unused + +} // namespace + +std::string HuffmanCodec::Encode(std::string_view in) { + std::string out; + uint64_t acc = 0; + int bits = 0; + auto push = [&](uint32_t code, int len) { + acc = (acc << len) | code; + bits += len; + while (bits >= 8) { + bits -= 8; + out.push_back((char)((acc >> bits) & 0xff)); + } + }; + for (unsigned char c : in) { + push(kHuffmanCodes[c][0], (int)kHuffmanCodes[c][1]); + } + if (bits > 0) { + // Pad with ones (EOS prefix) per RFC 7541 section 5.2. + acc = (acc << (8 - bits)) | ((1u << (8 - bits)) - 1u); + out.push_back((char)(acc & 0xff)); + } + return out; +} + +namespace { + +struct TrieNode { + int child[2] = {-1, -1}; + int symbol = -1; // 0..256, or -1 for interior +}; + +const std::vector& DecodeTrie() { + static const std::vector trie = [] { + std::vector t(1); + for (int s = 0; s < 257; ++s) { + uint32_t code = kHuffmanCodes[s][0]; + int nbits = (int)kHuffmanCodes[s][1]; + int node = 0; + for (int i = nbits - 1; i >= 0; --i) { + int bit = (code >> i) & 1; + if (t[(std::size_t)node].child[bit] < 0) { + t[(std::size_t)node].child[bit] = (int)t.size(); + t.emplace_back(); + } + node = t[(std::size_t)node].child[bit]; + } + t[(std::size_t)node].symbol = s; + } + return t; + }(); + return trie; +} + +} // namespace + +std::optional HuffmanCodec::Decode(const char* data, + std::size_t len) { + const std::vector& trie = DecodeTrie(); + std::string out; + int node = 0; + int depth = 0; // bits since the last emitted symbol + bool ones_only = true; // partial path is all ones (EOS prefix) + for (std::size_t i = 0; i < len; ++i) { + unsigned char c = (unsigned char)data[i]; + for (int b = 7; b >= 0; --b) { + int bit = (c >> b) & 1; + int next = trie[(std::size_t)node].child[bit]; + if (next < 0) return std::nullopt; + node = next; + ++depth; + ones_only = ones_only && (bit == 1); + int sym = trie[(std::size_t)node].symbol; + if (sym >= 0) { + if (sym == 256) return std::nullopt; // EOS in content + out.push_back((char)sym); + node = 0; + depth = 0; + ones_only = true; + } + } + } + // End state: exact symbol boundary, or an all-ones EOS-prefix padding of + // at most 7 bits. This matches the reference COMPLETE flag semantics. + if (node == 0) return out; + if (ones_only && depth <= 7) return out; + return std::nullopt; +} + +HpackEncoder::HpackEncoder(std::size_t max_table_size) + : max_table_size_(max_table_size) {} + +void HpackEncoder::SetMaxTableSize(std::size_t n) { max_table_size_ = n; } + +namespace { + +void EncodeInt(std::string& out, uint32_t value, uint8_t prefix_bits, + uint8_t prefix) { + uint8_t max = (uint8_t)((1u << prefix_bits) - 1u); + if (value < max) { + out.push_back((char)(prefix | (uint8_t)value)); + return; + } + out.push_back((char)(prefix | max)); + value -= max; + while (value >= 128) { + out.push_back((char)((value & 0x7f) | 0x80)); + value >>= 7; + } + out.push_back((char)value); +} + +void EncodeString(std::string& out, const std::string& s) { + // Literal without Huffman: smaller code, always valid. Huffman saving is + // an encoder optimization, not an interop requirement. + EncodeInt(out, (uint32_t)s.size(), 7, 0x00); + out.append(s); +} + +} // namespace + +std::string HpackEncoder::Encode(const std::vector& fields) { + (void)max_table_size_; + std::string out; + for (const auto& f : fields) { + int full = -1; + int name_only = -1; + for (std::size_t i = 1; i < kStaticCount; ++i) { + if (kStaticTable[i].name == f.name) { + if (name_only < 0) name_only = (int)i; + if (kStaticTable[i].value == f.value) { + full = (int)i; + break; + } + } + } + if (full >= 0) { + EncodeInt(out, (uint32_t)full, 7, 0x80); // indexed field + continue; + } + if (name_only >= 0) { + // Literal without indexing, indexed name. + EncodeInt(out, (uint32_t)name_only, 4, 0x00); + EncodeString(out, f.value); + continue; + } + // Literal without indexing, new name. + out.push_back((char)0x00); + EncodeString(out, f.name); + EncodeString(out, f.value); + } + return out; +} + +HpackDecoder::HpackDecoder(std::size_t max_table_size, + std::size_t max_header_list) + : max_table_size_(max_table_size), max_header_list_(max_header_list) {} + +void HpackDecoder::SetMaxTableSize(std::size_t n) { + max_table_size_ = n; + EvictTo(n); +} + +HpackError HpackDecoder::DecodeInt(const char*& p, const char* end, + uint8_t prefix_bits, uint32_t& out) { + if (p >= end) return HpackError::Truncated; + uint8_t max = (uint8_t)((1u << prefix_bits) - 1u); + uint32_t v = (uint32_t)((unsigned char)*p & max); + ++p; + if (v < max) { + out = v; + return HpackError::Ok; + } + uint32_t m = 0; + for (;;) { + if (p >= end) return HpackError::Truncated; + uint8_t b = (unsigned char)*p++; + if (m >= 28 && (b & 0x7f) >= (1u << (32 - m))) { + return HpackError::BadInteger; + } + v += (uint32_t)(b & 0x7f) << m; + if ((b & 0x80) == 0) break; + m += 7; + if (m >= 35) return HpackError::BadInteger; + } + out = v; + return HpackError::Ok; +} + +HpackError HpackDecoder::DecodeString(const char*& p, const char* end, + std::string& out) { + if (p >= end) return HpackError::Truncated; + bool huffman = ((unsigned char)*p & 0x80) != 0; + uint32_t len = 0; + HpackError e = DecodeInt(p, end, 7, len); + if (e != HpackError::Ok) return e; + if ((std::size_t)(end - p) < len) return HpackError::Truncated; + if (huffman) { + std::optional s = HuffmanCodec::Decode(p, len); + if (!s.has_value()) return HpackError::BadHuffman; + out = std::move(*s); + } else { + out.assign(p, len); + } + p += len; + return HpackError::Ok; +} + +HpackError HpackDecoder::Lookup(uint32_t index, HpackField& out) const { + if (index == 0) return HpackError::BadIndex; + if (index < kStaticCount) { + out.name = kStaticTable[index].name; + out.value = kStaticTable[index].value; + return HpackError::Ok; + } + uint32_t dyn = index - (uint32_t)kStaticCount; + if (dyn >= dynamic_.size()) return HpackError::BadIndex; + out = dynamic_[dyn]; + return HpackError::Ok; +} + +void HpackDecoder::EvictTo(std::size_t limit) { + while (table_bytes_ > limit && !dynamic_.empty()) { + const HpackField& back = dynamic_.back(); + table_bytes_ -= back.name.size() + back.value.size() + kEntryOverhead; + dynamic_.pop_back(); + } +} + +void HpackDecoder::Insert(HpackField f) { + std::size_t sz = f.name.size() + f.value.size() + kEntryOverhead; + if (sz > max_table_size_) { + dynamic_.clear(); + table_bytes_ = 0; + return; + } + EvictTo(max_table_size_ - sz); + table_bytes_ += sz; + dynamic_.push_front(std::move(f)); +} + +HpackError HpackDecoder::Decode(const char* data, std::size_t len, + std::vector& out) { + const char* p = data; + const char* end = data + len; + std::size_t list_bytes = 0; + while (p < end) { + uint8_t b = (unsigned char)*p; + if (b & 0x80) { + // Indexed field. + uint32_t index = 0; + HpackError e = DecodeInt(p, end, 7, index); + if (e != HpackError::Ok) return e; + HpackField f; + e = Lookup(index, f); + if (e != HpackError::Ok) return e; + list_bytes += f.name.size() + f.value.size() + kEntryOverhead; + if (list_bytes > max_header_list_) { + return HpackError::HeaderListTooLarge; + } + out.push_back(std::move(f)); + } else if ((b & 0xc0) == 0x40) { + // Literal with incremental indexing. + uint32_t index = 0; + HpackError e = DecodeInt(p, end, 6, index); + if (e != HpackError::Ok) return e; + HpackField f; + if (index == 0) { + e = DecodeString(p, end, f.name); + if (e != HpackError::Ok) return e; + } else { + HpackField t; + e = Lookup(index, t); + if (e != HpackError::Ok) return e; + f.name = std::move(t.name); + } + e = DecodeString(p, end, f.value); + if (e != HpackError::Ok) return e; + list_bytes += f.name.size() + f.value.size() + kEntryOverhead; + if (list_bytes > max_header_list_) { + return HpackError::HeaderListTooLarge; + } + out.push_back(f); + Insert(std::move(f)); + } else if ((b & 0xe0) == 0x20) { + // Table size update. + uint32_t size = 0; + HpackError e = DecodeInt(p, end, 5, size); + if (e != HpackError::Ok) return e; + if (size > max_table_size_) return HpackError::TableSizeChanged; + EvictTo(size); + } else { + // Literal without indexing (0x00) or never indexed (0x10). + uint32_t index = 0; + HpackError e = DecodeInt(p, end, 4, index); + if (e != HpackError::Ok) return e; + HpackField f; + if (index == 0) { + e = DecodeString(p, end, f.name); + if (e != HpackError::Ok) return e; + } else { + HpackField t; + e = Lookup(index, t); + if (e != HpackError::Ok) return e; + f.name = std::move(t.name); + } + e = DecodeString(p, end, f.value); + if (e != HpackError::Ok) return e; + list_bytes += f.name.size() + f.value.size() + kEntryOverhead; + if (list_bytes > max_header_list_) { + return HpackError::HeaderListTooLarge; + } + out.push_back(std::move(f)); + } + if (out.size() > caps::kHeaderFieldMax) return HpackError::TooManyFields; + } + return HpackError::Ok; +} + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/http/v2/hpack.h b/src/runtime/http/v2/hpack.h new file mode 100644 index 0000000..97b7f67 --- /dev/null +++ b/src/runtime/http/v2/hpack.h @@ -0,0 +1,76 @@ +// Spec: plans/01-runtime.md 6.4, RFC 7541. HPACK field compression: static +// plus dynamic table, Huffman string codec. The encoder emits indexed fields +// for static hits and literals without indexing otherwise (no dynamic table +// inserts); the decoder implements the full representation set including +// incremental indexing and table size updates for interop. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::runtime::http::v2 { + +struct HpackField { + std::string name; + std::string value; +}; + +enum class HpackError { + Ok, + Truncated, + BadIndex, + BadInteger, + BadHuffman, + TableSizeChanged, + HeaderListTooLarge, + TooManyFields, +}; + +// Huffman string codec over the RFC 7541 Appendix B table. +class HuffmanCodec { + public: + static std::string Encode(std::string_view in); + // Nullopt on invalid encoding (bad padding, EOS symbol, overlong). + static std::optional Decode(const char* data, std::size_t len); +}; + +class HpackEncoder { + public: + explicit HpackEncoder(std::size_t max_table_size = 4096); + std::string Encode(const std::vector& fields); + void SetMaxTableSize(std::size_t n); + + private: + std::size_t max_table_size_; +}; + +class HpackDecoder { + public: + explicit HpackDecoder(std::size_t max_table_size = 4096, + std::size_t max_header_list = 65536); + HpackError Decode(const char* data, std::size_t len, + std::vector& out); + void SetMaxTableSize(std::size_t n); + std::size_t table_size() const { return table_bytes_; } + + private: + static constexpr std::size_t kEntryOverhead = 32; + HpackError DecodeInt(const char*& p, const char* end, uint8_t prefix_bits, + uint32_t& out); + HpackError DecodeString(const char*& p, const char* end, std::string& out); + HpackError Lookup(uint32_t index, HpackField& out) const; + void Insert(HpackField f); + void EvictTo(std::size_t limit); + + std::size_t max_table_size_; + std::size_t max_header_list_; + std::size_t table_bytes_ = 0; + std::deque dynamic_; +}; + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/http/v2/huffman_table.inc b/src/runtime/http/v2/huffman_table.inc new file mode 100644 index 0000000..c54e654 --- /dev/null +++ b/src/runtime/http/v2/huffman_table.inc @@ -0,0 +1,259 @@ +// Generated from RFC 7541 Appendix B, cross-checked against python-hpack. +// Entry: {code (LSB aligned), bit length}. Index 256 is EOS (encode-only guard). +{0x1ff8u, 13u}, // 0 +{0x7fffd8u, 23u}, // 1 +{0xfffffe2u, 28u}, // 2 +{0xfffffe3u, 28u}, // 3 +{0xfffffe4u, 28u}, // 4 +{0xfffffe5u, 28u}, // 5 +{0xfffffe6u, 28u}, // 6 +{0xfffffe7u, 28u}, // 7 +{0xfffffe8u, 28u}, // 8 +{0xffffeau, 24u}, // 9 +{0x3ffffffcu, 30u}, // 10 +{0xfffffe9u, 28u}, // 11 +{0xfffffeau, 28u}, // 12 +{0x3ffffffdu, 30u}, // 13 +{0xfffffebu, 28u}, // 14 +{0xfffffecu, 28u}, // 15 +{0xfffffedu, 28u}, // 16 +{0xfffffeeu, 28u}, // 17 +{0xfffffefu, 28u}, // 18 +{0xffffff0u, 28u}, // 19 +{0xffffff1u, 28u}, // 20 +{0xffffff2u, 28u}, // 21 +{0x3ffffffeu, 30u}, // 22 +{0xffffff3u, 28u}, // 23 +{0xffffff4u, 28u}, // 24 +{0xffffff5u, 28u}, // 25 +{0xffffff6u, 28u}, // 26 +{0xffffff7u, 28u}, // 27 +{0xffffff8u, 28u}, // 28 +{0xffffff9u, 28u}, // 29 +{0xffffffau, 28u}, // 30 +{0xffffffbu, 28u}, // 31 +{0x14u, 6u}, // 32 +{0x3f8u, 10u}, // 33 +{0x3f9u, 10u}, // 34 +{0xffau, 12u}, // 35 +{0x1ff9u, 13u}, // 36 +{0x15u, 6u}, // 37 +{0xf8u, 8u}, // 38 +{0x7fau, 11u}, // 39 +{0x3fau, 10u}, // 40 +{0x3fbu, 10u}, // 41 +{0xf9u, 8u}, // 42 +{0x7fbu, 11u}, // 43 +{0xfau, 8u}, // 44 +{0x16u, 6u}, // 45 +{0x17u, 6u}, // 46 +{0x18u, 6u}, // 47 +{0x0u, 5u}, // 48 +{0x1u, 5u}, // 49 +{0x2u, 5u}, // 50 +{0x19u, 6u}, // 51 +{0x1au, 6u}, // 52 +{0x1bu, 6u}, // 53 +{0x1cu, 6u}, // 54 +{0x1du, 6u}, // 55 +{0x1eu, 6u}, // 56 +{0x1fu, 6u}, // 57 +{0x5cu, 7u}, // 58 +{0xfbu, 8u}, // 59 +{0x7ffcu, 15u}, // 60 +{0x20u, 6u}, // 61 +{0xffbu, 12u}, // 62 +{0x3fcu, 10u}, // 63 +{0x1ffau, 13u}, // 64 +{0x21u, 6u}, // 65 +{0x5du, 7u}, // 66 +{0x5eu, 7u}, // 67 +{0x5fu, 7u}, // 68 +{0x60u, 7u}, // 69 +{0x61u, 7u}, // 70 +{0x62u, 7u}, // 71 +{0x63u, 7u}, // 72 +{0x64u, 7u}, // 73 +{0x65u, 7u}, // 74 +{0x66u, 7u}, // 75 +{0x67u, 7u}, // 76 +{0x68u, 7u}, // 77 +{0x69u, 7u}, // 78 +{0x6au, 7u}, // 79 +{0x6bu, 7u}, // 80 +{0x6cu, 7u}, // 81 +{0x6du, 7u}, // 82 +{0x6eu, 7u}, // 83 +{0x6fu, 7u}, // 84 +{0x70u, 7u}, // 85 +{0x71u, 7u}, // 86 +{0x72u, 7u}, // 87 +{0xfcu, 8u}, // 88 +{0x73u, 7u}, // 89 +{0xfdu, 8u}, // 90 +{0x1ffbu, 13u}, // 91 +{0x7fff0u, 19u}, // 92 +{0x1ffcu, 13u}, // 93 +{0x3ffcu, 14u}, // 94 +{0x22u, 6u}, // 95 +{0x7ffdu, 15u}, // 96 +{0x3u, 5u}, // 97 +{0x23u, 6u}, // 98 +{0x4u, 5u}, // 99 +{0x24u, 6u}, // 100 +{0x5u, 5u}, // 101 +{0x25u, 6u}, // 102 +{0x26u, 6u}, // 103 +{0x27u, 6u}, // 104 +{0x6u, 5u}, // 105 +{0x74u, 7u}, // 106 +{0x75u, 7u}, // 107 +{0x28u, 6u}, // 108 +{0x29u, 6u}, // 109 +{0x2au, 6u}, // 110 +{0x7u, 5u}, // 111 +{0x2bu, 6u}, // 112 +{0x76u, 7u}, // 113 +{0x2cu, 6u}, // 114 +{0x8u, 5u}, // 115 +{0x9u, 5u}, // 116 +{0x2du, 6u}, // 117 +{0x77u, 7u}, // 118 +{0x78u, 7u}, // 119 +{0x79u, 7u}, // 120 +{0x7au, 7u}, // 121 +{0x7bu, 7u}, // 122 +{0x7ffeu, 15u}, // 123 +{0x7fcu, 11u}, // 124 +{0x3ffdu, 14u}, // 125 +{0x1ffdu, 13u}, // 126 +{0xffffffcu, 28u}, // 127 +{0xfffe6u, 20u}, // 128 +{0x3fffd2u, 22u}, // 129 +{0xfffe7u, 20u}, // 130 +{0xfffe8u, 20u}, // 131 +{0x3fffd3u, 22u}, // 132 +{0x3fffd4u, 22u}, // 133 +{0x3fffd5u, 22u}, // 134 +{0x7fffd9u, 23u}, // 135 +{0x3fffd6u, 22u}, // 136 +{0x7fffdau, 23u}, // 137 +{0x7fffdbu, 23u}, // 138 +{0x7fffdcu, 23u}, // 139 +{0x7fffddu, 23u}, // 140 +{0x7fffdeu, 23u}, // 141 +{0xffffebu, 24u}, // 142 +{0x7fffdfu, 23u}, // 143 +{0xffffecu, 24u}, // 144 +{0xffffedu, 24u}, // 145 +{0x3fffd7u, 22u}, // 146 +{0x7fffe0u, 23u}, // 147 +{0xffffeeu, 24u}, // 148 +{0x7fffe1u, 23u}, // 149 +{0x7fffe2u, 23u}, // 150 +{0x7fffe3u, 23u}, // 151 +{0x7fffe4u, 23u}, // 152 +{0x1fffdcu, 21u}, // 153 +{0x3fffd8u, 22u}, // 154 +{0x7fffe5u, 23u}, // 155 +{0x3fffd9u, 22u}, // 156 +{0x7fffe6u, 23u}, // 157 +{0x7fffe7u, 23u}, // 158 +{0xffffefu, 24u}, // 159 +{0x3fffdau, 22u}, // 160 +{0x1fffddu, 21u}, // 161 +{0xfffe9u, 20u}, // 162 +{0x3fffdbu, 22u}, // 163 +{0x3fffdcu, 22u}, // 164 +{0x7fffe8u, 23u}, // 165 +{0x7fffe9u, 23u}, // 166 +{0x1fffdeu, 21u}, // 167 +{0x7fffeau, 23u}, // 168 +{0x3fffddu, 22u}, // 169 +{0x3fffdeu, 22u}, // 170 +{0xfffff0u, 24u}, // 171 +{0x1fffdfu, 21u}, // 172 +{0x3fffdfu, 22u}, // 173 +{0x7fffebu, 23u}, // 174 +{0x7fffecu, 23u}, // 175 +{0x1fffe0u, 21u}, // 176 +{0x1fffe1u, 21u}, // 177 +{0x3fffe0u, 22u}, // 178 +{0x1fffe2u, 21u}, // 179 +{0x7fffedu, 23u}, // 180 +{0x3fffe1u, 22u}, // 181 +{0x7fffeeu, 23u}, // 182 +{0x7fffefu, 23u}, // 183 +{0xfffeau, 20u}, // 184 +{0x3fffe2u, 22u}, // 185 +{0x3fffe3u, 22u}, // 186 +{0x3fffe4u, 22u}, // 187 +{0x7ffff0u, 23u}, // 188 +{0x3fffe5u, 22u}, // 189 +{0x3fffe6u, 22u}, // 190 +{0x7ffff1u, 23u}, // 191 +{0x3ffffe0u, 26u}, // 192 +{0x3ffffe1u, 26u}, // 193 +{0xfffebu, 20u}, // 194 +{0x7fff1u, 19u}, // 195 +{0x3fffe7u, 22u}, // 196 +{0x7ffff2u, 23u}, // 197 +{0x3fffe8u, 22u}, // 198 +{0x1ffffecu, 25u}, // 199 +{0x3ffffe2u, 26u}, // 200 +{0x3ffffe3u, 26u}, // 201 +{0x3ffffe4u, 26u}, // 202 +{0x7ffffdeu, 27u}, // 203 +{0x7ffffdfu, 27u}, // 204 +{0x3ffffe5u, 26u}, // 205 +{0xfffff1u, 24u}, // 206 +{0x1ffffedu, 25u}, // 207 +{0x7fff2u, 19u}, // 208 +{0x1fffe3u, 21u}, // 209 +{0x3ffffe6u, 26u}, // 210 +{0x7ffffe0u, 27u}, // 211 +{0x7ffffe1u, 27u}, // 212 +{0x3ffffe7u, 26u}, // 213 +{0x7ffffe2u, 27u}, // 214 +{0xfffff2u, 24u}, // 215 +{0x1fffe4u, 21u}, // 216 +{0x1fffe5u, 21u}, // 217 +{0x3ffffe8u, 26u}, // 218 +{0x3ffffe9u, 26u}, // 219 +{0xffffffdu, 28u}, // 220 +{0x7ffffe3u, 27u}, // 221 +{0x7ffffe4u, 27u}, // 222 +{0x7ffffe5u, 27u}, // 223 +{0xfffecu, 20u}, // 224 +{0xfffff3u, 24u}, // 225 +{0xfffedu, 20u}, // 226 +{0x1fffe6u, 21u}, // 227 +{0x3fffe9u, 22u}, // 228 +{0x1fffe7u, 21u}, // 229 +{0x1fffe8u, 21u}, // 230 +{0x7ffff3u, 23u}, // 231 +{0x3fffeau, 22u}, // 232 +{0x3fffebu, 22u}, // 233 +{0x1ffffeeu, 25u}, // 234 +{0x1ffffefu, 25u}, // 235 +{0xfffff4u, 24u}, // 236 +{0xfffff5u, 24u}, // 237 +{0x3ffffeau, 26u}, // 238 +{0x7ffff4u, 23u}, // 239 +{0x3ffffebu, 26u}, // 240 +{0x7ffffe6u, 27u}, // 241 +{0x3ffffecu, 26u}, // 242 +{0x3ffffedu, 26u}, // 243 +{0x7ffffe7u, 27u}, // 244 +{0x7ffffe8u, 27u}, // 245 +{0x7ffffe9u, 27u}, // 246 +{0x7ffffeau, 27u}, // 247 +{0x7ffffebu, 27u}, // 248 +{0xffffffeu, 28u}, // 249 +{0x7ffffecu, 27u}, // 250 +{0x7ffffedu, 27u}, // 251 +{0x7ffffeeu, 27u}, // 252 +{0x7ffffefu, 27u}, // 253 +{0x7fffff0u, 27u}, // 254 +{0x3ffffeeu, 26u}, // 255 +{0x3fffffffu, 30u}, // 256 diff --git a/src/runtime/http/v2/settings.h b/src/runtime/http/v2/settings.h new file mode 100644 index 0000000..778beda --- /dev/null +++ b/src/runtime/http/v2/settings.h @@ -0,0 +1,102 @@ +// Spec: plans/01-runtime.md 6.4, RFC 9113 sections 6.5, 10. SETTINGS frame +// payload codec with validation. No server push: ENABLE_PUSH must be 0. +#pragma once + +#include +#include +#include +#include + +#include "runtime/caps.h" + +namespace molo::runtime::http::v2 { + +struct Settings { + uint32_t header_table_size = 4096; + bool enable_push = false; // MUST stay false; push is gone in RFC 9113. + uint32_t max_concurrent_streams = caps::kH2MaxConcurrentStreamsDefault; + uint32_t initial_window_size = 65535; + uint32_t max_frame_size = 16384; + uint32_t max_header_list_size = 65536; +}; + +enum class SettingsError { + Ok, + ProtocolError, // bad length, ENABLE_PUSH=1, bad MAX_FRAME_SIZE + FlowControlError, // bad INITIAL_WINDOW_SIZE + CompressionError, // bad HEADER_TABLE_SIZE change discipline +}; + +// Decodes a SETTINGS payload into out (applies peer values, clamped to our +// ceilings). Returns the connection error to raise, if any. +inline SettingsError DecodeSettingsPayload(const char* data, std::size_t len, + Settings& out) { + if (len % 6 != 0) return SettingsError::ProtocolError; + for (std::size_t i = 0; i < len; i += 6) { + uint16_t id = ((uint16_t)(unsigned char)data[i] << 8) | + (uint16_t)(unsigned char)data[i + 1]; + uint32_t v = ((uint32_t)(unsigned char)data[i + 2] << 24) | + ((uint32_t)(unsigned char)data[i + 3] << 16) | + ((uint32_t)(unsigned char)data[i + 4] << 8) | + (uint32_t)(unsigned char)data[i + 5]; + switch (id) { + case 0x1: + if (v > caps::kH2HeaderTableCeiling) return SettingsError::CompressionError; + out.header_table_size = v; + break; + case 0x2: + // Deprecated push knob. Real clients (hyper-h2 included) still send + // ENABLE_PUSH=1; accept and ignore it. We never push, so the value + // is meaningless to us either way. + out.enable_push = false; + break; + case 0x3: + out.max_concurrent_streams = v; + if (out.max_concurrent_streams > caps::kH2MaxConcurrentStreamsCeiling) { + out.max_concurrent_streams = caps::kH2MaxConcurrentStreamsCeiling; + } + break; + case 0x4: + if (v > 0x7fffffffu) return SettingsError::FlowControlError; + out.initial_window_size = v; + break; + case 0x5: + if (v < 16384 || v > caps::kH2MaxFrameCeiling) { + return SettingsError::ProtocolError; + } + out.max_frame_size = v; + break; + case 0x6: + out.max_header_list_size = v; + if (out.max_header_list_size > caps::kH2HeaderListMax) { + out.max_header_list_size = caps::kH2HeaderListMax; + } + break; + default: + // Unknown identifiers are ignored. + break; + } + } + return SettingsError::Ok; +} + +// Encodes our SETTINGS for the server preface. +inline std::string EncodeSettingsPayload(const Settings& s) { + std::string out; + auto param = [&out](uint16_t id, uint32_t v) { + out.push_back((char)(id >> 8)); + out.push_back((char)(id & 0xff)); + out.push_back((char)(v >> 24)); + out.push_back((char)((v >> 16) & 0xff)); + out.push_back((char)((v >> 8) & 0xff)); + out.push_back((char)(v & 0xff)); + }; + param(0x1, s.header_table_size); + param(0x3, s.max_concurrent_streams); + param(0x4, s.initial_window_size); + param(0x5, s.max_frame_size); + param(0x6, s.max_header_list_size); + return out; +} + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/http/v2/static_table.inc b/src/runtime/http/v2/static_table.inc new file mode 100644 index 0000000..0243b26 --- /dev/null +++ b/src/runtime/http/v2/static_table.inc @@ -0,0 +1,64 @@ +// HPACK static table, RFC 7541 Appendix A, cross-checked against python-hpack. +// Index 0 unused; entries 1..61. +{"", ""}, +{":authority", ""}, +{":method", "GET"}, +{":method", "POST"}, +{":path", "/"}, +{":path", "/index.html"}, +{":scheme", "http"}, +{":scheme", "https"}, +{":status", "200"}, +{":status", "204"}, +{":status", "206"}, +{":status", "304"}, +{":status", "400"}, +{":status", "404"}, +{":status", "500"}, +{"accept-charset", ""}, +{"accept-encoding", "gzip, deflate"}, +{"accept-language", ""}, +{"accept-ranges", ""}, +{"accept", ""}, +{"access-control-allow-origin", ""}, +{"age", ""}, +{"allow", ""}, +{"authorization", ""}, +{"cache-control", ""}, +{"content-disposition", ""}, +{"content-encoding", ""}, +{"content-language", ""}, +{"content-length", ""}, +{"content-location", ""}, +{"content-range", ""}, +{"content-type", ""}, +{"cookie", ""}, +{"date", ""}, +{"etag", ""}, +{"expect", ""}, +{"expires", ""}, +{"from", ""}, +{"host", ""}, +{"if-match", ""}, +{"if-modified-since", ""}, +{"if-none-match", ""}, +{"if-range", ""}, +{"if-unmodified-since", ""}, +{"last-modified", ""}, +{"link", ""}, +{"location", ""}, +{"max-forwards", ""}, +{"proxy-authenticate", ""}, +{"proxy-authorization", ""}, +{"range", ""}, +{"referer", ""}, +{"refresh", ""}, +{"retry-after", ""}, +{"server", ""}, +{"set-cookie", ""}, +{"strict-transport-security", ""}, +{"transfer-encoding", ""}, +{"user-agent", ""}, +{"vary", ""}, +{"via", ""}, +{"www-authenticate", ""}, diff --git a/src/runtime/http/v2/stream.cpp b/src/runtime/http/v2/stream.cpp new file mode 100644 index 0000000..ad156de --- /dev/null +++ b/src/runtime/http/v2/stream.cpp @@ -0,0 +1,145 @@ +// Spec: plans/01-runtime.md 6.4, RFC 9113 section 5. Stream states. +#include "stream.h" + +namespace molo::runtime::http::v2 { + +namespace { +constexpr uint32_t kNoError = 0x0; +constexpr uint32_t kProtocolError = 0x1; +constexpr uint32_t kCancel = 0x8; +constexpr uint32_t kRefusedStream = 0x7; +} // namespace + +H2Stream::H2Stream(uint32_t id, std::size_t body_cap) + : id_(id), body_cap_(body_cap) {} + +StreamOutcome H2Stream::OnHeaders(const std::vector& fields, + bool end_stream) { + if (state_ == StreamState::Idle) { + // First HEADERS: build the request head from pseudo-headers. + std::string method, scheme, authority, path; + for (const auto& f : fields) { + if (f.name.empty() || f.name[0] != ':') { + head_.headers.add(f.name, f.value); + continue; + } + if (f.name == ":method") { + if (!method.empty()) return {StreamEvent::LocalReset, kProtocolError}; + method = f.value; + } else if (f.name == ":scheme") { + if (!scheme.empty()) return {StreamEvent::LocalReset, kProtocolError}; + scheme = f.value; + } else if (f.name == ":authority") { + if (!authority.empty()) { + return {StreamEvent::LocalReset, kProtocolError}; + } + authority = f.value; + } else if (f.name == ":path") { + if (!path.empty()) return {StreamEvent::LocalReset, kProtocolError}; + path = f.value; + } else { + return {StreamEvent::LocalReset, kProtocolError}; + } + } + if (method.empty() || scheme.empty() || path.empty()) { + return {StreamEvent::LocalReset, kProtocolError}; + } + if (method == "CONNECT") { + // v1: plain tunnels are out of scope. + return {StreamEvent::LocalReset, kRefusedStream}; + } + head_.method = method; + head_.target = path; + head_.version = "HTTP/2"; + if (!authority.empty()) head_.headers.add("host", authority); + head_complete_ = true; + if (end_stream) { + state_ = StreamState::HalfClosedRemote; + return CompleteMessage(); + } + state_ = StreamState::Open; + return {StreamEvent::HeadersComplete, kNoError}; + } + if (state_ == StreamState::Open) { + // Trailers (second HEADERS must carry only regular fields). + for (const auto& f : fields) { + if (!f.name.empty() && f.name[0] == ':') { + return {StreamEvent::LocalReset, kProtocolError}; + } + trailers_.add(f.name, f.value); + } + trailers_phase_ = true; + if (end_stream) { + state_ = StreamState::HalfClosedRemote; + return CompleteMessage(); + } + return {StreamEvent::HeadersComplete, kNoError}; + } + if (state_ == StreamState::HalfClosedRemote) { + return {StreamEvent::LocalReset, kProtocolError}; + } + return {StreamEvent::LocalReset, kProtocolError}; +} + +StreamOutcome H2Stream::OnData(const char* data, std::size_t len, + bool end_stream) { + if (state_ != StreamState::Open) { + return {StreamEvent::LocalReset, kProtocolError}; + } + if (body_.size() + len > body_cap_) { + return {StreamEvent::LocalReset, kCancel}; + } + body_.append(data, len); + if (end_stream) { + state_ = StreamState::HalfClosedRemote; + return CompleteMessage(); + } + return {StreamEvent::None, kNoError}; +} + +StreamOutcome H2Stream::OnRst(uint32_t code) { + peer_code_ = code; + state_ = StreamState::Closed; + return {StreamEvent::ResetByPeer, kNoError}; +} + +StreamOutcome H2Stream::OnWindowUpdate(uint32_t increment) { + if (increment == 0) return {StreamEvent::LocalReset, kProtocolError}; + if (state_ == StreamState::Idle) { + return {StreamEvent::LocalReset, kProtocolError}; + } + return {StreamEvent::None, kNoError}; +} + +void H2Stream::MarkResponseHeadersSent(bool end_stream) { + if (state_ == StreamState::HalfClosedRemote) { + state_ = end_stream ? StreamState::Closed : StreamState::HalfClosedLocal; + } else if (state_ == StreamState::Open) { + state_ = + end_stream ? StreamState::HalfClosedLocal : StreamState::Open; + } else if (state_ == StreamState::HalfClosedLocal && end_stream) { + state_ = StreamState::Closed; + } +} + +void H2Stream::MarkResponseDataSent(bool end_stream) { + if (end_stream) { + if (state_ == StreamState::HalfClosedRemote || + state_ == StreamState::HalfClosedLocal) { + state_ = StreamState::Closed; + } else if (state_ == StreamState::Open) { + state_ = StreamState::HalfClosedLocal; + } + } +} + +StreamOutcome H2Stream::LocalClose(uint32_t code) { + state_ = StreamState::Closed; + return {StreamEvent::LocalReset, code}; +} + +StreamOutcome H2Stream::CompleteMessage() { + return {StreamEvent::MessageComplete, kNoError}; +} + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/http/v2/stream.h b/src/runtime/http/v2/stream.h new file mode 100644 index 0000000..966d585 --- /dev/null +++ b/src/runtime/http/v2/stream.h @@ -0,0 +1,81 @@ +// Spec: plans/01-runtime.md 6.4, RFC 9113 section 5. Server-side stream state +// machine: idle, reserved, open, half-closed, closed. RST_STREAM on handler +// cancel or caps. +#pragma once + +#include +#include +#include +#include + +#include "runtime/http/message.h" + +#include "hpack.h" + +namespace molo::runtime::http::v2 { + +enum class StreamState { + Idle, + ReservedLocal, + ReservedRemote, + Open, + HalfClosedLocal, + HalfClosedRemote, + Closed, +}; + +enum class StreamEvent { + None, + // HEADERS completed the request head; body may follow via DATA. + HeadersComplete, + // END_STREAM seen: the full request is ready for dispatch. + MessageComplete, + ResetByPeer, + // Local error with the RST code the connection must send. + LocalReset, +}; + +struct StreamOutcome { + StreamEvent event = StreamEvent::None; + uint32_t rst_code = 0; +}; + +class H2Stream { + public: + explicit H2Stream(uint32_t id, std::size_t body_cap); + + uint32_t id() const { return id_; } + StreamState state() const { return state_; } + const RequestHead& head() const { return head_; } + const std::string& body() const { return body_; } + const Headers& trailers() const { return trailers_; } + bool head_complete() const { return head_complete_; } + + // HEADERS frame payload (one decoded header list, CONTINUATION already + // reassembled by the connection). end_stream reflects END_STREAM. + StreamOutcome OnHeaders(const std::vector& fields, + bool end_stream); + StreamOutcome OnData(const char* data, std::size_t len, bool end_stream); + StreamOutcome OnRst(uint32_t code); + StreamOutcome OnWindowUpdate(uint32_t increment); + + // Response path: HEADERS then DATA then close. + void MarkResponseHeadersSent(bool end_stream); + void MarkResponseDataSent(bool end_stream); + StreamOutcome LocalClose(uint32_t code); // send RST_STREAM + + private: + StreamOutcome CompleteMessage(); + + uint32_t id_; + std::size_t body_cap_; + StreamState state_ = StreamState::Idle; + RequestHead head_; + std::string body_; + Headers trailers_; + bool head_complete_ = false; + bool trailers_phase_ = false; + uint32_t peer_code_ = 0; +}; + +} // namespace molo::runtime::http::v2 diff --git a/src/runtime/observability_hooks.h b/src/runtime/observability_hooks.h new file mode 100644 index 0000000..96bc5a2 --- /dev/null +++ b/src/runtime/observability_hooks.h @@ -0,0 +1,39 @@ +// Spec: plans/01-runtime.md 6.x, integration with 02-observability.md. +// +// STUB: zero-cost hook points. Accept, parse, compress and static hit/miss +// call these. They compile to nothing today; 02-observability.md replaces +// this header with real MOLO_LOG_*/counter/span wiring behind identical +// names, keeping disabled builds at near-zero overhead. +#pragma once + +#include +#include + +#define MOLO_RUNTIME_COUNT(name) \ + ::molo::runtime::obs::Counter((name), 1) +#define MOLO_RUNTIME_COUNT_N(name, n) \ + ::molo::runtime::obs::Counter((name), (n)) +#define MOLO_RUNTIME_GAUGE(name, value) \ + ::molo::runtime::obs::Gauge((name), (value)) +#define MOLO_RUNTIME_SPAN(name) \ + ::molo::runtime::obs::NullSpan _molo_span_##__LINE__((name)) + +namespace molo::runtime::obs { + +struct NullSpan { + explicit NullSpan(std::string_view) {} + ~NullSpan() = default; + void end() {} +}; + +inline void Counter(std::string_view, uint64_t) {} +inline void Gauge(std::string_view, int64_t) {} + +inline void OnAccept(std::string_view) {} +inline void OnClose(std::string_view, uint64_t, uint64_t) {} +inline void OnParseError(int) {} +inline void OnCompress(std::string_view, std::size_t, std::size_t) {} +inline void OnStaticHit(std::string_view) {} +inline void OnStaticMiss(std::string_view) {} + +} // namespace molo::runtime::obs diff --git a/src/runtime/poller.h b/src/runtime/poller.h new file mode 100644 index 0000000..9e14b5d --- /dev/null +++ b/src/runtime/poller.h @@ -0,0 +1,36 @@ +// Spec: plans/01-runtime.md 6.1. One Poller interface hiding epoll, kqueue +// and IOCP. Default is level-triggered for safety; edge-triggered stays +// inside a Poller implementation with drain-to-EAGAIN. +#pragma once + +#include +#include +#include +#include + +namespace molo::runtime { + +class IPoller { + public: + static constexpr uint32_t kRead = 1u << 0; + static constexpr uint32_t kWrite = 1u << 1; + static constexpr uint32_t kError = 1u << 2; + + using Callback = std::function; + + virtual ~IPoller() = default; + virtual bool add_fd(int fd, uint32_t events, Callback cb) = 0; + virtual bool modify_fd(int fd, uint32_t events) = 0; + virtual void remove_fd(int fd) = 0; + // Blocks up to timeout_ms, dispatches callbacks, returns dispatched count. + virtual int wait(int timeout_ms) = 0; + // Thread-safe: wakes a thread blocked in wait(). + virtual void wake() = 0; + virtual std::size_t watched() const = 0; +}; + +// Best Poller for the build platform. Never null; falls back to a stub that +// reports failure when no backend exists (should not happen on supported OS). +std::unique_ptr MakePoller(); + +} // namespace molo::runtime diff --git a/src/runtime/poller_epoll.cpp b/src/runtime/poller_epoll.cpp new file mode 100644 index 0000000..7c2333b --- /dev/null +++ b/src/runtime/poller_epoll.cpp @@ -0,0 +1,148 @@ +// Spec: plans/01-runtime.md 6.1. Linux epoll backend, level-triggered. +#include "poller_epoll.h" + +#include +#include +#include +#include +#include +#include + +#include "caps.h" + +namespace molo::runtime { + +EpollPoller::EpollPoller() : batch_(caps::kPollerBatchMax) { +#if defined(__linux__) + epoll_fd_ = epoll_create1(EPOLL_CLOEXEC); + if (epoll_fd_ < 0) return; + wake_fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (wake_fd_ < 0) { + close(epoll_fd_); + epoll_fd_ = -1; + return; + } + ::epoll_event ev; + memset(&ev, 0, sizeof(ev)); + ev.events = EPOLLIN; + ev.data.fd = wake_fd_; + if (epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, wake_fd_, &ev) != 0) { + close(wake_fd_); + close(epoll_fd_); + wake_fd_ = -1; + epoll_fd_ = -1; + } +#endif +} + +EpollPoller::~EpollPoller() { + if (wake_fd_ >= 0) close(wake_fd_); + if (epoll_fd_ >= 0) close(epoll_fd_); +} + +uint32_t EpollPoller::ToEpoll(uint32_t events) { + uint32_t out = 0; + if (events & kRead) out |= EPOLLIN; + if (events & kWrite) out |= EPOLLOUT; + out |= EPOLLERR | EPOLLHUP | EPOLLRDHUP; + return out; +} + +bool EpollPoller::add_fd(int fd, uint32_t events, Callback cb) { +#if defined(__linux__) + if (epoll_fd_ < 0 || fd < 0 || !cb) return false; + if (callbacks_.size() >= caps::kPollerFdsCeiling) return false; + if (callbacks_.find(fd) != callbacks_.end()) return false; + ::epoll_event ev; + memset(&ev, 0, sizeof(ev)); + ev.events = ToEpoll(events); + ev.data.fd = fd; + if (epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &ev) != 0) return false; + callbacks_.emplace(fd, std::move(cb)); + return true; +#else + (void)fd; + (void)events; + (void)cb; + return false; +#endif +} + +bool EpollPoller::modify_fd(int fd, uint32_t events) { +#if defined(__linux__) + if (epoll_fd_ < 0) return false; + auto it = callbacks_.find(fd); + if (it == callbacks_.end()) return false; + ::epoll_event ev; + memset(&ev, 0, sizeof(ev)); + ev.events = ToEpoll(events); + ev.data.fd = fd; + return epoll_ctl(epoll_fd_, EPOLL_CTL_MOD, fd, &ev) == 0; +#else + (void)fd; + (void)events; + return false; +#endif +} + +void EpollPoller::remove_fd(int fd) { + callbacks_.erase(fd); +#if defined(__linux__) + if (epoll_fd_ >= 0 && fd >= 0) { + epoll_ctl(epoll_fd_, EPOLL_CTL_DEL, fd, nullptr); + } +#endif +} + +void EpollPoller::DrainWake() { +#if defined(__linux__) + uint64_t v = 0; + while (read(wake_fd_, &v, sizeof(v)) == (ssize_t)sizeof(v)) { + if (v == 0) break; + } +#endif +} + +int EpollPoller::wait(int timeout_ms) { +#if defined(__linux__) + if (epoll_fd_ < 0) return -1; + if (timeout_ms < -1) timeout_ms = -1; + int n = epoll_wait(epoll_fd_, batch_.data(), (int)batch_.size(), timeout_ms); + if (n < 0) { + if (errno == EINTR) return 0; + return -1; + } + int dispatched = 0; + for (int i = 0; i < n; ++i) { + int fd = batch_[i].data.fd; + if (fd == wake_fd_) { + DrainWake(); + continue; + } + auto it = callbacks_.find(fd); + if (it == callbacks_.end()) continue; + uint32_t ev = 0; + if (batch_[i].events & EPOLLIN) ev |= kRead; + if (batch_[i].events & EPOLLOUT) ev |= kWrite; + if (batch_[i].events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) ev |= kError; + Callback cb = it->second; + cb(fd, ev); + ++dispatched; + } + return dispatched; +#else + (void)timeout_ms; + return -1; +#endif +} + +void EpollPoller::wake() { +#if defined(__linux__) + if (wake_fd_ < 0) return; + uint64_t v = 1; + ssize_t r = write(wake_fd_, &v, sizeof(v)); + (void)r; +#endif +} + +} // namespace molo::runtime diff --git a/src/runtime/poller_epoll.h b/src/runtime/poller_epoll.h new file mode 100644 index 0000000..ef63682 --- /dev/null +++ b/src/runtime/poller_epoll.h @@ -0,0 +1,52 @@ +// Spec: plans/01-runtime.md 6.1. Linux epoll backend, level-triggered. +#pragma once + +#include +#include +#include +#include + +#if defined(__linux__) +#include +#else +struct epoll_event { + uint32_t events; + union { + void* ptr; + int fd; + uint32_t u32; + uint64_t u64; + } data; +}; +#endif + +#include "poller.h" + +namespace molo::runtime { + +class EpollPoller : public IPoller { + public: + EpollPoller(); + ~EpollPoller() override; + EpollPoller(const EpollPoller&) = delete; + EpollPoller& operator=(const EpollPoller&) = delete; + + bool valid() const { return epoll_fd_ >= 0; } + bool add_fd(int fd, uint32_t events, Callback cb) override; + bool modify_fd(int fd, uint32_t events) override; + void remove_fd(int fd) override; + int wait(int timeout_ms) override; + void wake() override; + std::size_t watched() const override { return callbacks_.size(); } + + private: + static uint32_t ToEpoll(uint32_t events); + void DrainWake(); + + int epoll_fd_ = -1; + int wake_fd_ = -1; + std::unordered_map callbacks_; + std::vector<::epoll_event> batch_; +}; + +} // namespace molo::runtime diff --git a/src/runtime/poller_iocp.cpp b/src/runtime/poller_iocp.cpp new file mode 100644 index 0000000..1f03dc0 --- /dev/null +++ b/src/runtime/poller_iocp.cpp @@ -0,0 +1,130 @@ +// Spec: plans/01-runtime.md 6.1. Windows backend. v1 uses select() based +// readiness (level-triggered, same contract as epoll/kqueue here) inside the +// IOCP file slot; a native overlapped IOCP completion path is the documented +// upgrade once Windows CI exists. Non-Windows builds get an inert stub. +#include "poller_iocp.h" + +#include "caps.h" + +#if defined(_WIN32) +#include +#endif + +namespace molo::runtime { + +IocpPoller::IocpPoller() { +#if defined(_WIN32) + port_ = CreateIoCompletionPort(INVALID_HANDLE_VALUE, nullptr, 0, 1); +#endif +} + +IocpPoller::~IocpPoller() { +#if defined(_WIN32) + if (port_) CloseHandle((HANDLE)port_); +#endif +} + +bool IocpPoller::valid() const { +#if defined(_WIN32) + return port_ != nullptr; +#else + return false; +#endif +} + +bool IocpPoller::add_fd(int fd, uint32_t events, Callback cb) { +#if defined(_WIN32) + if (!port_ || fd < 0 || !cb) return false; + if (callbacks_.size() >= caps::kPollerFdsCeiling) return false; + if (callbacks_.find(fd) != callbacks_.end()) return false; + callbacks_.emplace(fd, std::move(cb)); + masks_.emplace(fd, events); + return true; +#else + (void)fd; + (void)events; + (void)cb; + return false; +#endif +} + +bool IocpPoller::modify_fd(int fd, uint32_t events) { +#if defined(_WIN32) + auto it = callbacks_.find(fd); + if (it == callbacks_.end()) return false; + masks_[fd] = events; + return true; +#else + (void)fd; + (void)events; + return false; +#endif +} + +void IocpPoller::remove_fd(int fd) { + callbacks_.erase(fd); + masks_.erase(fd); +} + +int IocpPoller::wait(int timeout_ms) { +#if defined(_WIN32) + if (!port_) return -1; + fd_set rfds, wfds, efds; + FD_ZERO(&rfds); + FD_ZERO(&wfds); + FD_ZERO(&efds); + for (const auto& kv : masks_) { + SOCKET s = (SOCKET)kv.first; + if (kv.second & kRead) FD_SET(s, &rfds); + if (kv.second & kWrite) FD_SET(s, &wfds); + FD_SET(s, &efds); + } + struct timeval tv; + tv.tv_sec = timeout_ms < 0 ? 0 : timeout_ms / 1000; + tv.tv_usec = timeout_ms < 0 ? 0 : (timeout_ms % 1000) * 1000; + int n = select(0, &rfds, &wfds, &efds, timeout_ms < 0 ? nullptr : &tv); + if (n == SOCKET_ERROR) { + if (WSAGetLastError() == WSAEINTR) return 0; + PostQueuedCompletionStatus((HANDLE)port_, 0, 0, nullptr); + DWORD bytes = 0; + ULONG_PTR key = 0; + OVERLAPPED* ov = nullptr; + if (GetQueuedCompletionStatus((HANDLE)port_, &bytes, &key, &ov, 0)) { + return 0; + } + return -1; + } + DWORD bytes = 0; + ULONG_PTR key = 0; + OVERLAPPED* ov = nullptr; + while (GetQueuedCompletionStatus((HANDLE)port_, &bytes, &key, &ov, 0)) { + } + int dispatched = 0; + std::size_t budget = caps::kPollerBatchMax; + for (const auto& kv : callbacks_) { + if (budget == 0) break; + SOCKET s = (SOCKET)kv.first; + uint32_t ev = 0; + if (FD_ISSET(s, &rfds)) ev |= kRead; + if (FD_ISSET(s, &wfds)) ev |= kWrite; + if (FD_ISSET(s, &efds)) ev |= kError; + if (ev == 0) continue; + Callback cb = kv.second; + cb(kv.first, ev); + ++dispatched; + --budget; + } + return dispatched; +#else + (void)timeout_ms; + return -1; +#endif +} + +void IocpPoller::wake() { +#if defined(_WIN32) + if (port_) PostQueuedCompletionStatus((HANDLE)port_, 0, 0, nullptr); +#endif +} + +} // namespace molo::runtime diff --git a/src/runtime/poller_iocp.h b/src/runtime/poller_iocp.h new file mode 100644 index 0000000..07db636 --- /dev/null +++ b/src/runtime/poller_iocp.h @@ -0,0 +1,39 @@ +// Spec: plans/01-runtime.md 6.1. IOCP backend for Windows. On other +// platforms this compiles to a stub that reports unsupported; MakePoller +// never selects it there. +#pragma once + +#include +#include +#include + +#include "poller.h" + +namespace molo::runtime { + +class IocpPoller : public IPoller { + public: + IocpPoller(); + ~IocpPoller() override; + IocpPoller(const IocpPoller&) = delete; + IocpPoller& operator=(const IocpPoller&) = delete; + + bool valid() const; + bool add_fd(int fd, uint32_t events, Callback cb) override; + bool modify_fd(int fd, uint32_t events) override; + void remove_fd(int fd) override; + int wait(int timeout_ms) override; + void wake() override; + std::size_t watched() const override { return callbacks_.size(); } + + private: +#if defined(_WIN32) + void* port_ = nullptr; +#else + int port_ = -1; +#endif + std::unordered_map callbacks_; + std::unordered_map masks_; +}; + +} // namespace molo::runtime diff --git a/src/runtime/poller_kqueue.cpp b/src/runtime/poller_kqueue.cpp new file mode 100644 index 0000000..d268cd4 --- /dev/null +++ b/src/runtime/poller_kqueue.cpp @@ -0,0 +1,169 @@ +// Spec: plans/01-runtime.md 6.1. kqueue backend for BSD/macOS. +#include "poller_kqueue.h" + +#include +#include +#include +#include + +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) || defined(__DragonFly__) +#include +#define MOLO_HAS_KQUEUE 1 +#endif + +#include "caps.h" + +namespace molo::runtime { + +KqueuePoller::KqueuePoller() { +#if defined(MOLO_HAS_KQUEUE) + kq_ = kqueue(); + if (kq_ < 0) return; + if (pipe(wake_pipe_) != 0) { + close(kq_); + kq_ = -1; + return; + } + fcntl(wake_pipe_[0], F_SETFD, FD_CLOEXEC); + fcntl(wake_pipe_[1], F_SETFD, FD_CLOEXEC); + int flags = fcntl(wake_pipe_[0], F_GETFL, 0); + fcntl(wake_pipe_[0], F_SETFL, flags | O_NONBLOCK); + flags = fcntl(wake_pipe_[1], F_GETFL, 0); + fcntl(wake_pipe_[1], F_SETFL, flags | O_NONBLOCK); + struct kevent ev; + EV_SET(&ev, (uintptr_t)wake_pipe_[0], EVFILT_READ, EV_ADD, 0, 0, nullptr); + if (kevent(kq_, &ev, 1, nullptr, 0, nullptr) != 0) { + close(wake_pipe_[0]); + close(wake_pipe_[1]); + close(kq_); + kq_ = -1; + wake_pipe_[0] = wake_pipe_[1] = -1; + } +#endif +} + +KqueuePoller::~KqueuePoller() { + if (wake_pipe_[0] >= 0) close(wake_pipe_[0]); + if (wake_pipe_[1] >= 0) close(wake_pipe_[1]); + if (kq_ >= 0) close(kq_); +} + +#if defined(MOLO_HAS_KQUEUE) +static void ApplyMask(int kq, int fd, uint32_t old_mask, uint32_t new_mask) { + struct kevent evs[2]; + int n = 0; + bool old_r = (old_mask & IPoller::kRead) != 0; + bool new_r = (new_mask & IPoller::kRead) != 0; + bool old_w = (old_mask & IPoller::kWrite) != 0; + bool new_w = (new_mask & IPoller::kWrite) != 0; + if (new_r && !old_r) { + EV_SET(&evs[n++], (uintptr_t)fd, EVFILT_READ, EV_ADD, 0, 0, nullptr); + } else if (!new_r && old_r) { + EV_SET(&evs[n++], (uintptr_t)fd, EVFILT_READ, EV_DELETE, 0, 0, nullptr); + } + if (new_w && !old_w) { + EV_SET(&evs[n++], (uintptr_t)fd, EVFILT_WRITE, EV_ADD, 0, 0, nullptr); + } else if (!new_w && old_w) { + EV_SET(&evs[n++], (uintptr_t)fd, EVFILT_WRITE, EV_DELETE, 0, 0, + nullptr); + } + if (n > 0) kevent(kq, evs, n, nullptr, 0, nullptr); +} +#endif + +bool KqueuePoller::add_fd(int fd, uint32_t events, Callback cb) { +#if defined(MOLO_HAS_KQUEUE) + if (kq_ < 0 || fd < 0 || !cb) return false; + if (callbacks_.size() >= caps::kPollerFdsCeiling) return false; + if (callbacks_.find(fd) != callbacks_.end()) return false; + ApplyMask(kq_, fd, 0, events); + callbacks_.emplace(fd, std::move(cb)); + masks_.emplace(fd, events); + return true; +#else + (void)fd; + (void)events; + (void)cb; + return false; +#endif +} + +bool KqueuePoller::modify_fd(int fd, uint32_t events) { +#if defined(MOLO_HAS_KQUEUE) + if (kq_ < 0) return false; + auto it = callbacks_.find(fd); + if (it == callbacks_.end()) return false; + ApplyMask(kq_, fd, masks_[fd], events); + masks_[fd] = events; + return true; +#else + (void)fd; + (void)events; + return false; +#endif +} + +void KqueuePoller::remove_fd(int fd) { +#if defined(MOLO_HAS_KQUEUE) + auto it = masks_.find(fd); + if (it != masks_.end()) { + if (kq_ >= 0) ApplyMask(kq_, fd, it->second, 0); + masks_.erase(it); + } +#endif + callbacks_.erase(fd); +} + +int KqueuePoller::wait(int timeout_ms) { +#if defined(MOLO_HAS_KQUEUE) + if (kq_ < 0) return -1; + struct kevent evs[caps::kPollerBatchMax]; + struct timespec ts; + struct timespec* tsp = nullptr; + if (timeout_ms >= 0) { + ts.tv_sec = timeout_ms / 1000; + ts.tv_nsec = (long)(timeout_ms % 1000) * 1000000L; + tsp = &ts; + } + int n = kevent(kq_, nullptr, 0, evs, caps::kPollerBatchMax, tsp); + if (n < 0) { + if (errno == EINTR) return 0; + return -1; + } + int dispatched = 0; + for (int i = 0; i < n; ++i) { + int fd = (int)evs[i].ident; + if (fd == wake_pipe_[0]) { + char buf[64]; + while (read(wake_pipe_[0], buf, sizeof(buf)) > 0) { + } + continue; + } + auto it = callbacks_.find(fd); + if (it == callbacks_.end()) continue; + uint32_t ev = 0; + if (evs[i].filter == EVFILT_READ) ev |= kRead; + if (evs[i].filter == EVFILT_WRITE) ev |= kWrite; + if (evs[i].flags & (EV_ERROR | EV_EOF)) ev |= kError; + Callback cb = it->second; + cb(fd, ev); + ++dispatched; + } + return dispatched; +#else + (void)timeout_ms; + return -1; +#endif +} + +void KqueuePoller::wake() { +#if defined(MOLO_HAS_KQUEUE) + if (wake_pipe_[1] < 0) return; + char c = 1; + ssize_t r = write(wake_pipe_[1], &c, 1); + (void)r; +#endif +} + +} // namespace molo::runtime diff --git a/src/runtime/poller_kqueue.h b/src/runtime/poller_kqueue.h new file mode 100644 index 0000000..97bf125 --- /dev/null +++ b/src/runtime/poller_kqueue.h @@ -0,0 +1,36 @@ +// Spec: plans/01-runtime.md 6.1. kqueue backend for BSD/macOS. On other +// platforms this compiles to a stub that reports unsupported; MakePoller +// never selects it there. +#pragma once + +#include +#include +#include + +#include "poller.h" + +namespace molo::runtime { + +class KqueuePoller : public IPoller { + public: + KqueuePoller(); + ~KqueuePoller() override; + KqueuePoller(const KqueuePoller&) = delete; + KqueuePoller& operator=(const KqueuePoller&) = delete; + + bool valid() const { return kq_ >= 0; } + bool add_fd(int fd, uint32_t events, Callback cb) override; + bool modify_fd(int fd, uint32_t events) override; + void remove_fd(int fd) override; + int wait(int timeout_ms) override; + void wake() override; + std::size_t watched() const override { return callbacks_.size(); } + + private: + int kq_ = -1; + int wake_pipe_[2] = {-1, -1}; + std::unordered_map callbacks_; + std::unordered_map masks_; +}; + +} // namespace molo::runtime diff --git a/src/runtime/runtime.mk b/src/runtime/runtime.mk new file mode 100644 index 0000000..1552948 --- /dev/null +++ b/src/runtime/runtime.mk @@ -0,0 +1,94 @@ +# Runtime part (plans/01-runtime.md): reactor, HTTP/1.1 + HTTP/2, compression, +# cookies, static files. Outputs only under build/. +RUNTIME_SRCS := \ + src/runtime/poller_epoll.cpp \ + src/runtime/poller_kqueue.cpp \ + src/runtime/poller_iocp.cpp \ + src/runtime/timer_wheel.cpp \ + src/runtime/executor.cpp \ + src/runtime/event_loop.cpp \ + src/runtime/signal_handler.cpp \ + src/runtime/transport_tcp.cpp \ + src/runtime/http/message.cpp \ + src/runtime/http/protocol_detect.cpp \ + src/runtime/http/connection.cpp \ + src/runtime/http/listener.cpp \ + src/runtime/http/v1/parser.cpp \ + src/runtime/http/v1/serializer.cpp \ + src/runtime/http/v2/hpack.cpp \ + src/runtime/http/v2/flow_control.cpp \ + src/runtime/http/v2/stream.cpp \ + src/runtime/http/v2/connection.cpp \ + src/runtime/compress/gzip.cpp \ + src/runtime/compress/brotli.cpp \ + src/runtime/compress/negotiate.cpp \ + src/runtime/cookie/cookie.cpp \ + src/runtime/cookie/jar.cpp \ + src/runtime/static/path_guard.cpp \ + src/runtime/static/asset_provider.cpp \ + src/runtime/static/disk_provider.cpp \ + src/runtime/static/embedded_provider.cpp \ + src/runtime/static/etag.cpp \ + src/runtime/static/file_server.cpp + +RUNTIME_TEST_SRCS := \ + tests/src/runtime/main.cpp \ + tests/src/runtime/test_task.cpp \ + tests/src/runtime/test_timer.cpp \ + tests/src/runtime/test_executor.cpp \ + tests/src/runtime/test_event_loop.cpp \ + tests/src/runtime/test_channel_strand.cpp \ + tests/src/runtime/test_base64.cpp \ + tests/src/runtime/http/test_message.cpp \ + tests/src/runtime/http/test_protocol_detect.cpp \ + tests/src/runtime/http/v1/test_parser.cpp \ + tests/src/runtime/http/v1/test_serializer.cpp \ + tests/src/runtime/http/v2/test_hpack.cpp \ + tests/src/runtime/http/v2/test_settings.cpp \ + tests/src/runtime/http/v2/test_flow.cpp \ + tests/src/runtime/http/v2/test_stream.cpp \ + tests/src/runtime/http/v2/test_h2connection.cpp \ + tests/src/runtime/http/test_integration.cpp \ + tests/src/runtime/compress/test_gzip.cpp \ + tests/src/runtime/compress/test_brotli.cpp \ + tests/src/runtime/compress/test_negotiate.cpp \ + tests/src/runtime/cookie/test_cookie.cpp \ + tests/src/runtime/static/test_path_guard.cpp \ + tests/src/runtime/static/test_file_server.cpp + +RUNTIME_OBJS := $(RUNTIME_SRCS:src/runtime/%.cpp=build/obj/runtime/%.o) +RUNTIME_TEST_OBJS := $(RUNTIME_TEST_SRCS:tests/%.cpp=build/obj/runtime-tests/%.o) +RUNTIME_LIB := build/lib/libruntime.a +RUNTIME_TESTS := build/bin/runtime_tests +RUNTIME_INTEROP_SERVER := build/bin/runtime_interop_server + +MOLO_ALL += $(RUNTIME_LIB) +MOLO_TEST_BINS += $(RUNTIME_TESTS) +MOLO_INTEROP += runtime-interop + +$(RUNTIME_LIB): $(RUNTIME_OBJS) + mkdir -p $(dir $@) + $(AR) rcs $@ $^ + +$(RUNTIME_TESTS): $(RUNTIME_TEST_OBJS) $(RUNTIME_LIB) + mkdir -p $(dir $@) + $(CXX) $(MOLO_CXXFLAGS) -o $@ $(RUNTIME_TEST_OBJS) $(RUNTIME_LIB) + +$(RUNTIME_INTEROP_SERVER): tests/src/runtime/interop/test_server.cpp $(RUNTIME_LIB) + mkdir -p $(dir $@) + $(CXX) $(MOLO_CXXFLAGS) -Itests/src/runtime -o $@ tests/src/runtime/interop/test_server.cpp $(RUNTIME_LIB) + +build/obj/runtime/%.o: src/runtime/%.cpp + mkdir -p $(dir $@) + $(CXX) $(MOLO_CXXFLAGS) -MMD -MP -c $< -o $@ + +build/obj/runtime-tests/%.o: tests/%.cpp + mkdir -p $(dir $@) + $(CXX) $(MOLO_CXXFLAGS) -MMD -MP -Itests/src/runtime -c $< -o $@ + +-include $(RUNTIME_OBJS:.o=.d) $(RUNTIME_TEST_OBJS:.o=.d) + +runtime-interop: $(RUNTIME_INTEROP_SERVER) + python3 tests/src/runtime/interop/interop.py $(RUNTIME_INTEROP_SERVER) + +.PHONY: runtime-interop diff --git a/src/runtime/signal_handler.cpp b/src/runtime/signal_handler.cpp new file mode 100644 index 0000000..5e11e85 --- /dev/null +++ b/src/runtime/signal_handler.cpp @@ -0,0 +1,111 @@ +// Spec: plans/01-runtime.md 6.2. Signal delivery on the loop thread. +#include "signal_handler.h" + +#include +#include +#include +#include +#include + +#include "event_loop.h" + +#if defined(__linux__) +#include +#endif + +namespace molo::runtime { + +namespace { +#if !defined(__linux__) +int g_pipe_write = -1; +void PipeHandler(int signo) { + if (g_pipe_write >= 0) { + char c = (char)signo; + ssize_t r = write(g_pipe_write, &c, 1); + (void)r; + } +} +#endif +} // namespace + +SignalHandler::SignalHandler(EventLoop& loop) : loop_(loop) {} + +SignalHandler::~SignalHandler() { uninstall(); } + +bool SignalHandler::install(Callback cb) { + if (installed_ || !cb) return false; + cb_ = std::move(cb); +#if defined(__linux__) + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGINT); + sigaddset(&mask, SIGTERM); + sigaddset(&mask, SIGHUP); + if (pthread_sigmask(SIG_BLOCK, &mask, &old_mask_) != 0) return false; + mask_saved_ = true; + fd_ = signalfd(-1, &mask, SFD_NONBLOCK | SFD_CLOEXEC); + if (fd_ < 0) return false; +#else + if (pipe(pipe_fds_) != 0) return false; + fcntl(pipe_fds_[0], F_SETFD, FD_CLOEXEC); + fcntl(pipe_fds_[1], F_SETFD, FD_CLOEXEC); + fd_ = pipe_fds_[0]; + g_pipe_write = pipe_fds_[1]; + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = PipeHandler; + sigemptyset(&sa.sa_mask); + sigaction(SIGINT, &sa, nullptr); + sigaction(SIGTERM, &sa, nullptr); + sigaction(SIGHUP, &sa, nullptr); +#endif + bool ok = loop_.poller().add_fd(fd_, IPoller::kRead, + [this](int, uint32_t) { OnReadable(); }); + if (!ok) { + uninstall(); + return false; + } + installed_ = true; + return true; +} + +void SignalHandler::uninstall() { + if (installed_) { + loop_.poller().remove_fd(fd_); + installed_ = false; + } +#if defined(__linux__) + if (fd_ >= 0) { + close(fd_); + fd_ = -1; + } + if (mask_saved_) { + pthread_sigmask(SIG_SETMASK, &old_mask_, nullptr); + mask_saved_ = false; + } +#else + if (pipe_fds_[0] >= 0) close(pipe_fds_[0]); + if (pipe_fds_[1] >= 0) close(pipe_fds_[1]); + pipe_fds_[0] = pipe_fds_[1] = -1; + g_pipe_write = -1; + fd_ = -1; +#endif + cb_ = nullptr; +} + +void SignalHandler::OnReadable() { +#if defined(__linux__) + struct signalfd_siginfo info; + while (read(fd_, &info, sizeof(info)) == (ssize_t)sizeof(info)) { + if (cb_) cb_((int)info.ssi_signo); + } +#else + char buf[32]; + ssize_t n = read(fd_, buf, sizeof(buf)); + for (ssize_t i = 0; i < n; ++i) { + if (cb_) cb_((int)buf[i]); + } +#endif +} + +} // namespace molo::runtime diff --git a/src/runtime/signal_handler.h b/src/runtime/signal_handler.h new file mode 100644 index 0000000..4352e7e --- /dev/null +++ b/src/runtime/signal_handler.h @@ -0,0 +1,45 @@ +// Spec: plans/01-runtime.md 6.2. SIGINT/SIGTERM start drain, SIGHUP reloads +// config where safe, SIGTERM during drain shortens the deadline to 5 s. The +// policy lives in the caller; this class only delivers the signal number on +// the loop thread. +#pragma once + +#include + +#if defined(__linux__) +#include +#endif + +namespace molo::runtime { + +class EventLoop; + +class SignalHandler { + public: + using Callback = std::function; + + explicit SignalHandler(EventLoop& loop); + ~SignalHandler(); + SignalHandler(const SignalHandler&) = delete; + SignalHandler& operator=(const SignalHandler&) = delete; + + bool install(Callback cb); + void uninstall(); + + private: + void OnReadable(); + + EventLoop& loop_; + Callback cb_; + int fd_ = -1; + bool installed_ = false; +#if defined(__linux__) + sigset_t old_mask_; + bool mask_saved_ = false; +#endif +#if !defined(__linux__) + int pipe_fds_[2] = {-1, -1}; +#endif +}; + +} // namespace molo::runtime diff --git a/src/runtime/static/asset_provider.cpp b/src/runtime/static/asset_provider.cpp new file mode 100644 index 0000000..dbce17e --- /dev/null +++ b/src/runtime/static/asset_provider.cpp @@ -0,0 +1,75 @@ +// Spec: plans/01-runtime.md 6.7. Asset RAII. +#include "asset_provider.h" + +#include + +#include + +namespace molo::runtime::static_ { + +Asset::~Asset() { + if (fd_ >= 0) close(fd_); +} + +Asset::Asset(Asset&& other) noexcept + : valid_(other.valid_), + meta_(other.meta_), + fd_(other.fd_), + data_(other.data_), + size_(other.size_) { + other.valid_ = false; + other.fd_ = -1; + other.data_ = nullptr; + other.size_ = 0; +} + +Asset& Asset::operator=(Asset&& other) noexcept { + if (this != &other) { + if (fd_ >= 0) close(fd_); + valid_ = other.valid_; + meta_ = other.meta_; + fd_ = other.fd_; + data_ = other.data_; + size_ = other.size_; + other.valid_ = false; + other.fd_ = -1; + other.data_ = nullptr; + other.size_ = 0; + } + return *this; +} + +int Asset::release() { + int fd = fd_; + fd_ = -1; + valid_ = false; + data_ = nullptr; + size_ = 0; + return fd; +} + +Asset Asset::ForFile(int fd, AssetMeta meta) { + Asset a; + a.valid_ = true; + a.meta_ = meta; + a.fd_ = fd; + return a; +} + +Asset Asset::ForMemory(const char* data, std::size_t size, AssetMeta meta) { + Asset a; + a.valid_ = true; + a.meta_ = meta; + a.data_ = data; + a.size_ = size; + return a; +} + +Asset Asset::ForDir(AssetMeta meta) { + Asset a; + a.valid_ = true; + a.meta_ = meta; + return a; +} + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/asset_provider.h b/src/runtime/static/asset_provider.h new file mode 100644 index 0000000..d6eb4d0 --- /dev/null +++ b/src/runtime/static/asset_provider.h @@ -0,0 +1,59 @@ +// Spec: plans/01-runtime.md 6.7. Asset provider interface: DiskProvider +// (root jailed) and EmbeddedProvider (from the 09-cli-ops.md embedder) share +// one FileServer logic. +#pragma once + +#include +#include +#include +#include + +namespace molo::runtime::static_ { + +struct AssetMeta { + uint64_t length = 0; + int64_t mtime = 0; // unix seconds + bool is_file = false; + bool is_dir = false; +}; + +// Move-only open asset. Disk assets carry an fd for sendfile; embedded +// assets carry a memory view that stays valid while the provider lives. +class Asset { + public: + Asset() = default; + ~Asset(); + Asset(Asset&& other) noexcept; + Asset& operator=(Asset&& other) noexcept; + Asset(const Asset&) = delete; + Asset& operator=(const Asset&) = delete; + + bool valid() const { return valid_; } + AssetMeta meta() const { return meta_; } + int fd() const { return fd_; } // -1 for memory assets + const char* data() const { return data_; } + std::size_t size() const { return size_; } + // Hands the fd to the caller (for FileBody); this asset becomes invalid. + int release(); + + static Asset ForFile(int fd, AssetMeta meta); + static Asset ForMemory(const char* data, std::size_t size, AssetMeta meta); + static Asset ForDir(AssetMeta meta); + + private: + bool valid_ = false; + AssetMeta meta_; + int fd_ = -1; + const char* data_ = nullptr; + std::size_t size_ = 0; +}; + +class IAssetProvider { + public: + virtual ~IAssetProvider() = default; + // rel is a guard-normalized relative path (see path_guard.h). + virtual std::optional open(std::string_view rel) = 0; + virtual const char* name() const = 0; +}; + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/disk_provider.cpp b/src/runtime/static/disk_provider.cpp new file mode 100644 index 0000000..a679604 --- /dev/null +++ b/src/runtime/static/disk_provider.cpp @@ -0,0 +1,84 @@ +// Spec: plans/01-runtime.md 6.7. Jailed disk provider. +#include "disk_provider.h" + +#include +#include +#include +#include + +namespace molo::runtime::static_ { + +DiskProvider::DiskProvider(std::string root, bool follow_symlinks, + bool allow_dotfiles) + : root_(std::move(root)), + follow_symlinks_(follow_symlinks), + allow_dotfiles_(allow_dotfiles) { + root_fd_ = ::open(root_.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); +} + +DiskProvider::~DiskProvider() { + if (root_fd_ >= 0) close(root_fd_); +} + +std::optional DiskProvider::open(std::string_view rel) { + if (root_fd_ < 0) return std::nullopt; + std::string path(rel); + if (path.empty()) return std::nullopt; + if (path.size() > 4096) return std::nullopt; + if (path[0] == '/') return std::nullopt; + if (path.find('\0') != std::string::npos) return std::nullopt; + if (!allow_dotfiles_) { + // Any dotfile segment denies the request. + std::size_t i = 0; + while (i <= path.size()) { + std::size_t slash = path.find('/', i); + if (slash == std::string::npos) slash = path.size(); + if (slash > i && path[i] == '.') return std::nullopt; + i = slash + 1; + } + } + int fd = openat(root_fd_, path.c_str(), + O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd < 0 && follow_symlinks_) { + // Resolve through realpath and verify containment (v1: small TOCTOU + // window, documented; strict deployments keep follow_symlinks off). + std::string full = root_ + "/" + path; + char* resolved = realpath(full.c_str(), nullptr); + if (resolved == nullptr) return std::nullopt; + std::string r(resolved); + free(resolved); + char* root_resolved = realpath(root_.c_str(), nullptr); + if (root_resolved == nullptr) return std::nullopt; + std::string rr(root_resolved); + free(root_resolved); + if (r != rr && (r.size() <= rr.size() || + r.compare(0, rr.size(), rr) != 0 || + r[rr.size()] != '/')) { + return std::nullopt; + } + fd = ::open(r.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) return std::nullopt; + } + if (fd < 0) return std::nullopt; + struct stat st; + if (fstat(fd, &st) != 0) { + close(fd); + return std::nullopt; + } + AssetMeta meta; + meta.mtime = (int64_t)st.st_mtime; + if (S_ISDIR(st.st_mode)) { + close(fd); + meta.is_dir = true; + return Asset::ForDir(meta); + } + if (!S_ISREG(st.st_mode)) { + close(fd); + return std::nullopt; + } + meta.length = (uint64_t)st.st_size; + meta.is_file = true; + return Asset::ForFile(fd, meta); +} + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/disk_provider.h b/src/runtime/static/disk_provider.h new file mode 100644 index 0000000..a46ea72 --- /dev/null +++ b/src/runtime/static/disk_provider.h @@ -0,0 +1,31 @@ +// Spec: plans/01-runtime.md 6.7. Disk provider: root jailed via openat, +// O_NOFOLLOW unless follow_symlinks is set (then realpath containment is +// verified), dotfiles denied by default. +#pragma once + +#include + +#include "asset_provider.h" + +namespace molo::runtime::static_ { + +class DiskProvider : public IAssetProvider { + public: + explicit DiskProvider(std::string root, bool follow_symlinks = false, + bool allow_dotfiles = false); + ~DiskProvider() override; + DiskProvider(const DiskProvider&) = delete; + DiskProvider& operator=(const DiskProvider&) = delete; + + bool valid() const { return root_fd_ >= 0; } + std::optional open(std::string_view rel) override; + const char* name() const override { return "disk"; } + + private: + std::string root_; + bool follow_symlinks_; + bool allow_dotfiles_; + int root_fd_ = -1; +}; + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/embedded_provider.cpp b/src/runtime/static/embedded_provider.cpp new file mode 100644 index 0000000..74fca31 --- /dev/null +++ b/src/runtime/static/embedded_provider.cpp @@ -0,0 +1,36 @@ +// Spec: plans/01-runtime.md 6.7. Embedded provider. +#include "embedded_provider.h" + +namespace molo::runtime::static_ { + +void EmbeddedProvider::Register(std::string rel, std::vector bytes, + int64_t mtime) { + EmbeddedAsset a; + a.bytes = std::move(bytes); + a.mtime = mtime; + std::lock_guard lock(mutex_); + assets_[std::move(rel)] = std::move(a); +} + +bool EmbeddedProvider::Remove(std::string_view rel) { + std::lock_guard lock(mutex_); + return assets_.erase(std::string(rel)) > 0; +} + +std::optional EmbeddedProvider::open(std::string_view rel) { + // NOTE: the returned memory view borrows the registry entry. Callers must + // not Register/Remove concurrently with serving. Registration happens at + // startup (embedder output); serving is read-only after that. + std::lock_guard lock(mutex_); + auto it = assets_.find(std::string(rel)); + if (it == assets_.end()) return std::nullopt; + const EmbeddedAsset& a = it->second; + AssetMeta meta; + meta.length = a.bytes.size(); + meta.mtime = a.mtime; + meta.is_file = true; + const char* data = a.bytes.empty() ? "" : a.bytes.data(); + return Asset::ForMemory(data, a.bytes.size(), meta); +} + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/embedded_provider.h b/src/runtime/static/embedded_provider.h new file mode 100644 index 0000000..071138d --- /dev/null +++ b/src/runtime/static/embedded_provider.h @@ -0,0 +1,35 @@ +// Spec: plans/01-runtime.md 6.7. Embedded provider: serves assets registered +// in memory. The 09-cli-ops.md asset embedder generates the registration +// calls; the registry API here is the stable target. +#pragma once + +#include +#include +#include +#include +#include + +#include "asset_provider.h" + +namespace molo::runtime::static_ { + +struct EmbeddedAsset { + std::vector bytes; + int64_t mtime = 0; +}; + +class EmbeddedProvider : public IAssetProvider { + public: + EmbeddedProvider() = default; + + void Register(std::string rel, std::vector bytes, int64_t mtime); + bool Remove(std::string_view rel); + std::optional open(std::string_view rel) override; + const char* name() const override { return "embedded"; } + + private: + mutable std::mutex mutex_; + std::unordered_map assets_; +}; + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/etag.cpp b/src/runtime/static/etag.cpp new file mode 100644 index 0000000..16864b9 --- /dev/null +++ b/src/runtime/static/etag.cpp @@ -0,0 +1,129 @@ +// Spec: plans/01-runtime.md 6.7, RFC 9110. Validators. +#include "etag.h" + +#include +#include +#include + +namespace molo::runtime::static_ { + +std::string Etag::StrongForDisk(uint64_t size, int64_t mtime) { + char buf[64]; + snprintf(buf, sizeof(buf), "\"%llx-%llx\"", (unsigned long long)size, + (unsigned long long)mtime); + return std::string(buf); +} + +std::string Etag::StrongForEmbedded(uint64_t len, uint64_t hash) { + char buf[64]; + snprintf(buf, sizeof(buf), "\"%llx-%llx\"", (unsigned long long)len, + (unsigned long long)hash); + return std::string(buf); +} + +uint64_t Etag::Fnva1a(const char* data, std::size_t len) { + uint64_t h = 1469598103934665603ull; + for (std::size_t i = 0; i < len; ++i) { + h ^= (unsigned char)data[i]; + h *= 1099511628211ull; + } + return h; +} + +bool Etag::Matches(std::string_view if_none_match, std::string_view etag) { + std::size_t pos = 0; + auto trim = [](std::string_view s) { + while (!s.empty() && (s.front() == ' ' || s.front() == '\t')) { + s.remove_prefix(1); + } + while (!s.empty() && (s.back() == ' ' || s.back() == '\t')) { + s.remove_suffix(1); + } + return s; + }; + while (pos <= if_none_match.size()) { + std::size_t comma = if_none_match.find(',', pos); + if (comma == std::string_view::npos) comma = if_none_match.size(); + std::string_view cand = trim(if_none_match.substr(pos, comma - pos)); + pos = comma + 1; + if (cand == "*") return true; + // Weak comparison: strip W/ from both sides. + if (cand.compare(0, 2, "W/") == 0) cand.remove_prefix(2); + std::string_view mine = etag; + if (mine.compare(0, 2, "W/") == 0) mine.remove_prefix(2); + if (!cand.empty() && cand == mine) return true; + } + return false; +} + +bool Etag::RangeMatches(std::string_view if_range, std::string_view etag) { + std::string_view v = if_range; + while (!v.empty() && (v.front() == ' ' || v.front() == '\t')) { + v.remove_prefix(1); + } + while (!v.empty() && (v.back() == ' ' || v.back() == '\t')) { + v.remove_suffix(1); + } + if (v.empty() || v.compare(0, 2, "W/") == 0) return false; + return v == etag; +} + +std::string Etag::HttpDate(int64_t unix_seconds) { + std::time_t t = (std::time_t)unix_seconds; + struct tm tmv; + gmtime_r(&t, &tmv); + char buf[40]; + strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", &tmv); + return std::string(buf); +} + +int64_t Etag::ParseHttpDate(std::string_view s) { + // IMF-fixdate only: "Sun, 06 Nov 1994 08:49:37 GMT". + if (s.size() != 29) return -1; + if (s[3] != ',' || s[4] != ' ' || s[7] != ' ' || s[11] != ' ' || + s[16] != ' ' || s[19] != ':' || s[22] != ':' || s[25] != ' ') { + return -1; + } + if (s.compare(26, 3, "GMT") != 0) return -1; + auto digits = [&](std::size_t at, int n) -> int { + int v = 0; + for (int i = 0; i < n; ++i) { + char c = s[at + (std::size_t)i]; + if (c < '0' || c > '9') return -1; + v = v * 10 + (c - '0'); + } + return v; + }; + int day = digits(5, 2); + int year = digits(12, 4); + int hour = digits(17, 2); + int min = digits(20, 2); + int sec = digits(23, 2); + if (day < 1 || day > 31 || year < 1970 || hour > 23 || min > 59 || + sec > 60) { + return -1; + } + static const char* kMonths[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + int mon = -1; + for (int i = 0; i < 12; ++i) { + if (s.compare(8, 3, kMonths[i]) == 0) { + mon = i; + break; + } + } + if (mon < 0) return -1; + struct tm tmv; + memset(&tmv, 0, sizeof(tmv)); + tmv.tm_mday = day; + tmv.tm_mon = mon; + tmv.tm_year = year - 1900; + tmv.tm_hour = hour; + tmv.tm_min = min; + tmv.tm_sec = sec; + std::time_t t = timegm(&tmv); + if (t == (std::time_t)-1) return -1; + return (int64_t)t; +} + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/etag.h b/src/runtime/static/etag.h new file mode 100644 index 0000000..06e75fa --- /dev/null +++ b/src/runtime/static/etag.h @@ -0,0 +1,26 @@ +// Spec: plans/01-runtime.md 6.7, RFC 9110 sections 8.8, 13. Validators: +// strong ETags plus Last-Modified, If-None-Match precedence over +// If-Modified-Since, If-Range. +#pragma once + +#include +#include +#include + +namespace molo::runtime::static_ { + +class Etag { + public: + static std::string StrongForDisk(uint64_t size, int64_t mtime); + static std::string StrongForEmbedded(uint64_t len, uint64_t hash); + static uint64_t Fnva1a(const char* data, std::size_t len); + // If-None-Match comparison: handles "*", lists, and W/ weak prefixes. + static bool Matches(std::string_view if_none_match, std::string_view etag); + // If-Range: strong comparison only (no weak match, no lists). + static bool RangeMatches(std::string_view if_range, std::string_view etag); + static std::string HttpDate(int64_t unix_seconds); + // Parses IMF-fixdate. Returns -1 when unparseable. + static int64_t ParseHttpDate(std::string_view s); +}; + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/file_server.cpp b/src/runtime/static/file_server.cpp new file mode 100644 index 0000000..ac0489c --- /dev/null +++ b/src/runtime/static/file_server.cpp @@ -0,0 +1,280 @@ +// Spec: plans/01-runtime.md 6.7. File server. +#include "file_server.h" + +#include "etag.h" +#include "path_guard.h" +#include "runtime/compress/negotiate.h" +#include "runtime/observability_hooks.h" + +namespace molo::runtime::static_ { + +FileServer::FileServer(IAssetProvider& provider, FileServerConfig config) + : provider_(provider), config_(std::move(config)) {} + +FileOutcome FileServer::Error(int status) { + FileOutcome o; + o.status = status; + o.head.status = status; + o.head.headers.set("content-type", "text/plain; charset=utf-8"); + return o; +} + +std::string FileServer::ContentTypeFor(std::string_view rel) { + std::size_t dot = rel.rfind('.'); + std::string_view ext = + dot == std::string_view::npos ? std::string_view{} : rel.substr(dot); + if (ext == ".html" || ext == ".htm") return "text/html; charset=utf-8"; + if (ext == ".css") return "text/css; charset=utf-8"; + if (ext == ".js" || ext == ".mjs") { + return "text/javascript; charset=utf-8"; + } + if (ext == ".json" || ext == ".map") return "application/json"; + if (ext == ".xml") return "application/xml"; + if (ext == ".svg") return "image/svg+xml"; + if (ext == ".txt" || ext == ".md" || ext == ".csv") { + return "text/plain; charset=utf-8"; + } + if (ext == ".wasm") return "application/wasm"; + if (ext == ".png") return "image/png"; + if (ext == ".jpg" || ext == ".jpeg") return "image/jpeg"; + if (ext == ".gif") return "image/gif"; + if (ext == ".webp") return "image/webp"; + if (ext == ".ico") return "image/x-icon"; + if (ext == ".woff") return "font/woff"; + if (ext == ".woff2") return "font/woff2"; + if (ext == ".ttf") return "font/ttf"; + if (ext == ".pdf") return "application/pdf"; + if (ext == ".zip") return "application/zip"; + return "application/octet-stream"; +} + +std::string FileServer::PickCacheControl(std::string_view rel) { + if (rel.size() >= 5 && + (rel.compare(rel.size() - 5, 5, ".html") == 0 || + rel.compare(rel.size() - 4, 4, ".htm") == 0)) { + return config_.cache_control_html; + } + return config_.cache_control_versioned; +} + +FileOutcome FileServer::Handle(const http::RequestHead& req) { + if (req.method != "GET" && req.method != "HEAD" && req.method != "OPTIONS") { + FileOutcome o = Error(405); + o.head.headers.set("allow", "GET, HEAD, OPTIONS"); + return o; + } + if (req.method == "OPTIONS") { + FileOutcome o; + o.status = 204; + o.head.status = 204; + o.head.headers.set("allow", "GET, HEAD, OPTIONS"); + o.head_only = true; + return o; + } + std::optional rel = NormalizeUrlPath(req.target); + if (!rel.has_value() || rel->empty()) { + obs::OnStaticMiss(req.target); + return Error(404); + } + std::optional asset = provider_.open(*rel); + if (!asset.has_value() || !asset->valid()) { + obs::OnStaticMiss(req.target); + return Error(404); + } + if (asset->meta().is_dir) { + // Directory index is off by default. + if (!config_.directory_index) return Error(403); + return Error(403); // indexing on is still 403 in v1 (no renderer) + } + if (!asset->meta().is_file) { + obs::OnStaticMiss(req.target); + return Error(404); + } + obs::OnStaticHit(req.target); + AssetMeta meta = asset->meta(); + // Precompressed variants: probe .br/.gz siblings by negotiation. + std::string content_encoding; + std::string etag_suffix; + { + std::string ae = req.headers.get("accept-encoding"); + // Probe in preference order without reading bodies. + std::optional br_asset; + std::optional gz_asset; + if (!ae.empty()) { + br_asset = provider_.open(*rel + ".br"); + if (br_asset.has_value() && !br_asset->meta().is_file) { + br_asset.reset(); + } + gz_asset = provider_.open(*rel + ".gz"); + if (gz_asset.has_value() && !gz_asset->meta().is_file) { + gz_asset.reset(); + } + } + compress::Precompressed pre = + compress::Negotiator::SelectPrecompressed( + ae, gz_asset.has_value(), br_asset.has_value()); + if (pre.found && pre.suffix == ".br") { + asset = std::move(br_asset); + meta = asset->meta(); + content_encoding = "br"; + etag_suffix = "-br"; + } else if (pre.found && pre.suffix == ".gz") { + asset = std::move(gz_asset); + meta = asset->meta(); + content_encoding = "gzip"; + etag_suffix = "-gz"; + } + } + std::string etag; + if (asset->fd() >= 0) { + etag = Etag::StrongForDisk(meta.length, meta.mtime); + } else { + uint64_t h = Etag::Fnva1a(asset->data(), asset->size()); + etag = Etag::StrongForEmbedded(meta.length, h); + } + if (!etag_suffix.empty()) { + // Variant suffix goes inside the quotes. + etag = etag.substr(0, etag.size() - 1) + etag_suffix + "\""; + } + std::string last_mod = Etag::HttpDate(meta.mtime); + // Conditionals: If-None-Match takes precedence over If-Modified-Since. + std::string inm = req.headers.get("if-none-match"); + if (!inm.empty()) { + if (Etag::Matches(inm, etag)) { + FileOutcome o; + o.status = 304; + o.head.status = 304; + o.head.headers.set("etag", etag); + o.head.headers.set("last-modified", last_mod); + o.head.headers.set("cache-control", PickCacheControl(*rel)); + o.head_only = true; + return o; + } + } else { + std::string ims = req.headers.get("if-modified-since"); + if (!ims.empty()) { + int64_t since = Etag::ParseHttpDate(ims); + if (since >= 0 && meta.mtime <= since) { + FileOutcome o; + o.status = 304; + o.head.status = 304; + o.head.headers.set("etag", etag); + o.head.headers.set("last-modified", last_mod); + o.head.headers.set("cache-control", PickCacheControl(*rel)); + o.head_only = true; + return o; + } + } + } + FileOutcome o; + o.head.headers.set("etag", etag); + o.head.headers.set("last-modified", last_mod); + o.head.headers.set("cache-control", PickCacheControl(*rel)); + o.head.headers.set("accept-ranges", "bytes"); + o.head.headers.set("content-type", ContentTypeFor(*rel)); + if (!content_encoding.empty()) { + o.head.headers.set("content-encoding", content_encoding); + o.head.headers.set("vary", compress::Negotiator::VaryValue()); + } + // Single range support. + std::string range = req.headers.get("range"); + uint64_t offset = 0; + uint64_t length = meta.length; + bool partial = false; + if (!range.empty()) { + std::string if_range = req.headers.get("if-range"); + bool range_ok = true; + if (!if_range.empty()) { + // Strong ETag compare, else treat as date. + if (if_range.find('"') != std::string::npos || + if_range.compare(0, 2, "W/") == 0) { + range_ok = Etag::RangeMatches(if_range, etag); + } else { + int64_t d = Etag::ParseHttpDate(if_range); + range_ok = (d >= 0 && meta.mtime <= d); + } + } + if (range_ok) { + // bytes=FIRST-LAST, bytes=FIRST-, bytes=-SUFFIX. Multiple ranges + // give 416 in v1 (documented gap). + if (range.find(',') != std::string::npos) { + FileOutcome e = Error(416); + e.head.headers.set("content-range", + "bytes */" + std::to_string(meta.length)); + return e; + } + std::string v = range; + while (!v.empty() && (v.front() == ' ' || v.front() == '\t')) { + v.erase(v.begin()); + } + bool bad = true; + if (v.compare(0, 6, "bytes=") == 0) { + std::string spec = v.substr(6); + std::size_t dash = spec.find('-'); + if (dash != std::string::npos) { + std::string a = spec.substr(0, dash); + std::string b = spec.substr(dash + 1); + auto parse_num = [](const std::string& s, uint64_t& n) { + if (s.empty() || s.size() > 20) return false; + n = 0; + for (char c : s) { + if (c < '0' || c > '9') return false; + n = n * 10 + (uint64_t)(c - '0'); + } + return true; + }; + if (a.empty()) { + uint64_t suffix = 0; + if (parse_num(b, suffix) && suffix > 0) { + if (suffix > meta.length) suffix = meta.length; + offset = meta.length - suffix; + length = suffix; + bad = false; + } + } else { + uint64_t first = 0, last = 0; + bool nums_ok = parse_num(a, first); + if (nums_ok) { + if (b.empty()) { + last = meta.length == 0 ? 0 : meta.length - 1; + } else { + nums_ok = parse_num(b, last); + } + } + if (nums_ok && first < meta.length && meta.length > 0) { + if (last >= meta.length) last = meta.length - 1; + if (last >= first) { + offset = first; + length = last - first + 1; + bad = false; + } + } + } + } + } + if (bad) { + FileOutcome e = Error(416); + e.head.headers.set("content-range", + "bytes */" + std::to_string(meta.length)); + return e; + } + partial = true; + } + } + o.status = partial ? 206 : 200; + o.head.status = o.status; + if (partial) { + o.head.headers.set("content-range", + "bytes " + std::to_string(offset) + "-" + + std::to_string(offset + length - 1) + "/" + + std::to_string(meta.length)); + } + o.head.headers.set("content-length", std::to_string(length)); + o.asset = std::move(asset); + o.offset = offset; + o.length = length; + o.head_only = (req.method == "HEAD"); + return o; +} + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/file_server.h b/src/runtime/static/file_server.h new file mode 100644 index 0000000..d80cd65 --- /dev/null +++ b/src/runtime/static/file_server.h @@ -0,0 +1,45 @@ +// Spec: plans/01-runtime.md 6.7. One FileServer over disk and embedded +// providers: traversal guard, conditional requests, single ranges, +// cache headers, precompressed variants. Only GET, HEAD, OPTIONS. +#pragma once + +#include +#include +#include + +#include "asset_provider.h" +#include "runtime/http/message.h" + +namespace molo::runtime::static_ { + +struct FileServerConfig { + std::string cache_control_versioned = "public, max-age=3600"; + std::string cache_control_html = "no-cache"; + bool directory_index = false; // off: directories give 403 +}; + +struct FileOutcome { + int status = 500; + http::ResponseHead head; + std::optional asset; // body source for 200/206 + uint64_t offset = 0; + uint64_t length = 0; + bool head_only = false; +}; + +class FileServer { + public: + FileServer(IAssetProvider& provider, FileServerConfig config = {}); + + FileOutcome Handle(const http::RequestHead& req); + static std::string ContentTypeFor(std::string_view rel); + + private: + FileOutcome Error(int status); + std::string PickCacheControl(std::string_view rel); + + IAssetProvider& provider_; + FileServerConfig config_; +}; + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/path_guard.cpp b/src/runtime/static/path_guard.cpp new file mode 100644 index 0000000..e8fb38a --- /dev/null +++ b/src/runtime/static/path_guard.cpp @@ -0,0 +1,69 @@ +// Spec: plans/01-runtime.md 6.7. Traversal guard. +#include "path_guard.h" + +#include + +#include "runtime/caps.h" + +namespace molo::runtime::static_ { + +namespace { +int HexVal(char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} +} // namespace + +std::optional NormalizeUrlPath(std::string_view target) { + // Strip query and fragment. + std::size_t q = target.find_first_of("?#"); + std::string_view path = target.substr(0, q); + if (path.empty() || path[0] != '/') return std::nullopt; + if (path.size() > caps::kStaticPathMax) return std::nullopt; + // Percent-decode (single pass; %25 stays literal, no double decode). + std::string decoded; + decoded.reserve(path.size()); + for (std::size_t i = 0; i < path.size(); ++i) { + char c = path[i]; + if (c == '%') { + if (i + 2 >= path.size()) return std::nullopt; + int hi = HexVal(path[i + 1]); + int lo = HexVal(path[i + 2]); + if (hi < 0 || lo < 0) return std::nullopt; + decoded.push_back((char)((hi << 4) | lo)); + i += 2; + continue; + } + decoded.push_back(c); + } + for (char c : decoded) { + if (c == '\0') return std::nullopt; + if (c == '\\') return std::nullopt; + } + // Lexical normalize over segments. + std::vector segs; + std::size_t i = 0; + while (i <= decoded.size()) { + std::size_t slash = decoded.find('/', i); + if (slash == std::string::npos) slash = decoded.size(); + std::string_view seg(decoded.data() + i, slash - i); + i = slash + 1; + if (seg.empty() || seg == ".") continue; + if (seg == "..") { + if (segs.empty()) return std::nullopt; // escape above root + segs.pop_back(); + continue; + } + segs.emplace_back(seg); + } + std::string out; + for (std::size_t k = 0; k < segs.size(); ++k) { + if (k > 0) out.push_back('/'); + out.append(segs[k]); + } + return out; +} + +} // namespace molo::runtime::static_ diff --git a/src/runtime/static/path_guard.h b/src/runtime/static/path_guard.h new file mode 100644 index 0000000..e82ab40 --- /dev/null +++ b/src/runtime/static/path_guard.h @@ -0,0 +1,16 @@ +// Spec: plans/01-runtime.md 6.7. Traversal guard: percent-decode, lexical +// normalize, reject NUL, backslash, and `..` escapes above root. Symlink +// policy is enforced by DiskProvider at open time. +#pragma once + +#include +#include +#include + +namespace molo::runtime::static_ { + +// Returns the normalized in-root relative path (no leading slash), or +// nullopt when the request escapes or is malformed. +std::optional NormalizeUrlPath(std::string_view target); + +} // namespace molo::runtime::static_ diff --git a/src/runtime/strand.h b/src/runtime/strand.h new file mode 100644 index 0000000..a8b8da9 --- /dev/null +++ b/src/runtime/strand.h @@ -0,0 +1,56 @@ +// Spec: plans/01-runtime.md section 3. Per-connection serial queue. +#pragma once + +#include +#include +#include + +#include "event_loop.h" + +namespace molo::runtime { + +class Strand { + public: + explicit Strand(EventLoop& loop) : loop_(loop) {} + ~Strand() = default; + Strand(const Strand&) = delete; + Strand& operator=(const Strand&) = delete; + + void post(std::function fn) { + if (!fn) return; + bool kick = false; + { + std::lock_guard lock(mutex_); + queue_.push_back(std::move(fn)); + if (!active_) { + active_ = true; + kick = true; + } + } + if (kick) loop_.defer([this] { Drain(); }); + } + + private: + void Drain() { + for (;;) { + std::function fn; + { + std::lock_guard lock(mutex_); + if (queue_.empty()) { + active_ = false; + return; + } + fn = std::move(queue_.front()); + queue_.pop_front(); + } + fn(); + } + } + + EventLoop& loop_; + std::mutex mutex_; + std::deque> queue_; + bool active_ = false; +}; + +} // namespace molo::runtime diff --git a/src/runtime/task.h b/src/runtime/task.h new file mode 100644 index 0000000..8d78d1e --- /dev/null +++ b/src/runtime/task.h @@ -0,0 +1,216 @@ +// Spec: plans/01-runtime.md 6.1. Lazy coroutine Task with symmetric +// transfer. Handlers are `Task handle(Request)`. Never hold a lock +// across co_await. Blocking work goes to ThreadPool, never the loop thread. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::runtime { + +template +using Generator = std::generator; + +struct Cancelled : std::exception { + const char* what() const noexcept override { return "cancelled"; } +}; + +// Checked by handlers at await points. Cancellation is cooperative. +class CancellationToken { + public: + CancellationToken() : state_(std::make_shared>(false)) {} + void cancel() const { state_->store(true, std::memory_order_relaxed); } + bool is_cancelled() const { return state_->load(std::memory_order_relaxed); } + void throw_if_cancelled() const { + if (is_cancelled()) throw Cancelled{}; + } + + private: + std::shared_ptr> state_; +}; + +template +class Task { + public: + struct promise_type { + std::coroutine_handle<> continuation_ = std::noop_coroutine(); + std::variant result_; + bool destroy_on_complete_ = false; + + Task get_return_object() { + return Task{std::coroutine_handle::from_promise(*this)}; + } + std::suspend_always initial_suspend() noexcept { return {}; } + struct FinalAwaiter { + bool await_ready() noexcept { return false; } + std::coroutine_handle<> await_suspend( + std::coroutine_handle h) noexcept { + promise_type& p = h.promise(); + if (p.destroy_on_complete_) { + h.destroy(); + return std::noop_coroutine(); + } + return p.continuation_; + } + void await_resume() noexcept {} + }; + FinalAwaiter final_suspend() noexcept { return {}; } + void unhandled_exception() noexcept { + result_.template emplace<2>(std::current_exception()); + } + template + void return_value(U&& v) { + result_.template emplace<1>(std::forward(v)); + } + }; + + using Handle = std::coroutine_handle; + + Task() = default; + explicit Task(Handle h) : handle_(h) {} + Task(Task&& other) noexcept : handle_(other.handle_) { + other.handle_ = nullptr; + } + Task& operator=(Task&& other) noexcept { + if (this != &other) { + destroy(); + handle_ = other.handle_; + other.handle_ = nullptr; + } + return *this; + } + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + ~Task() { destroy(); } + + bool valid() const { return handle_ != nullptr; } + bool done() const { return !handle_ || handle_.done(); } + Handle release() { + Handle h = handle_; + handle_ = nullptr; + return h; + } + void start() { + if (handle_ && !handle_.done()) handle_.resume(); + } + + bool await_ready() const noexcept { return !handle_ || handle_.done(); } + std::coroutine_handle<> await_suspend( + std::coroutine_handle<> awaiting) noexcept { + handle_.promise().continuation_ = awaiting; + return handle_; + } + T await_resume() { + auto& r = handle_.promise().result_; + if (std::holds_alternative(r)) { + std::rethrow_exception(std::get<2>(r)); + } + return std::move(std::get<1>(r)); + } + + private: + void destroy() { + if (handle_) { + handle_.destroy(); + handle_ = nullptr; + } + } + Handle handle_ = nullptr; +}; + +template <> +class Task { + public: + struct promise_type { + std::coroutine_handle<> continuation_ = std::noop_coroutine(); + std::exception_ptr error_; + bool destroy_on_complete_ = false; + + Task get_return_object() { + return Task{std::coroutine_handle::from_promise(*this)}; + } + std::suspend_always initial_suspend() noexcept { return {}; } + struct FinalAwaiter { + bool await_ready() noexcept { return false; } + std::coroutine_handle<> await_suspend( + std::coroutine_handle h) noexcept { + promise_type& p = h.promise(); + if (p.destroy_on_complete_) { + h.destroy(); + return std::noop_coroutine(); + } + return p.continuation_; + } + void await_resume() noexcept {} + }; + FinalAwaiter final_suspend() noexcept { return {}; } + void unhandled_exception() noexcept { error_ = std::current_exception(); } + void return_void() noexcept {} + }; + + using Handle = std::coroutine_handle; + + Task() = default; + explicit Task(Handle h) : handle_(h) {} + Task(Task&& other) noexcept : handle_(other.handle_) { + other.handle_ = nullptr; + } + Task& operator=(Task&& other) noexcept { + if (this != &other) { + destroy(); + handle_ = other.handle_; + other.handle_ = nullptr; + } + return *this; + } + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + ~Task() { destroy(); } + + bool valid() const { return handle_ != nullptr; } + bool done() const { return !handle_ || handle_.done(); } + Handle release() { + Handle h = handle_; + handle_ = nullptr; + return h; + } + void start() { + if (handle_ && !handle_.done()) handle_.resume(); + } + + bool await_ready() const noexcept { return !handle_ || handle_.done(); } + std::coroutine_handle<> await_suspend( + std::coroutine_handle<> awaiting) noexcept { + handle_.promise().continuation_ = awaiting; + return handle_; + } + void await_resume() { + if (handle_.promise().error_) { + std::rethrow_exception(handle_.promise().error_); + } + } + + private: + void destroy() { + if (handle_) { + handle_.destroy(); + handle_ = nullptr; + } + } + Handle handle_ = nullptr; +}; + +// Fire and forget. The frame destroys itself on completion. +inline void Detach(Task&& task) { + if (!task.valid()) return; + auto h = task.release(); + h.promise().destroy_on_complete_ = true; + h.resume(); +} + +} // namespace molo::runtime diff --git a/src/runtime/timer_wheel.cpp b/src/runtime/timer_wheel.cpp new file mode 100644 index 0000000..267ee1e --- /dev/null +++ b/src/runtime/timer_wheel.cpp @@ -0,0 +1,132 @@ +// Spec: plans/01-runtime.md 6.1. Hashed timing wheel. +#include "timer_wheel.h" + +#include "caps.h" + +namespace molo::runtime { + +TimerWheel::TimerWheel(int granularity_ms, std::size_t slots) + : granularity_ms_(granularity_ms > 0 ? granularity_ms : 1), + slots_(slots > 0 ? slots : 1), + wheel_(slots_) { + start_tick_ = NowTick(); + last_tick_ = start_tick_; +} + +int64_t TimerWheel::NowTick() const { + auto now = std::chrono::steady_clock::now().time_since_epoch(); + return std::chrono::duration_cast(now).count() / + granularity_ms_; +} + +TimerWheel::TimerId TimerWheel::add(std::chrono::milliseconds delay, + Callback cb) { + if (!cb) return 0; + if (live_ >= caps::kTimersPerLoopCap) return 0; + int64_t now = NowTick(); + int64_t wait_ticks = delay.count() / granularity_ms_; + int64_t at = now + wait_ticks; + auto e = std::make_shared(); + e->id = next_id_++; + if (next_id_ == 0) next_id_ = 1; + e->tick = at; + e->cb = std::move(cb); + int64_t delta = at - now; + if (delta < (int64_t)slots_) { + wheel_[(std::size_t)(at % (int64_t)slots_)].push_back(e); + } else { + overflow_.push_back(e); + } + ++live_; + return e->id; +} + +bool TimerWheel::cancel(TimerId id) { + if (id == 0) return false; + for (auto& slot : wheel_) { + for (auto& e : slot) { + if (e->id == id && !e->cancelled) { + e->cancelled = true; + e->cb = nullptr; + --live_; + return true; + } + } + } + for (auto& e : overflow_) { + if (e->id == id && !e->cancelled) { + e->cancelled = true; + e->cb = nullptr; + --live_; + return true; + } + } + return false; +} + +void TimerWheel::Cascade(int64_t now) { + for (auto it = overflow_.begin(); it != overflow_.end();) { + EntryPtr e = *it; + if (e->cancelled) { + it = overflow_.erase(it); + continue; + } + int64_t delta = e->tick - now; + if (delta < (int64_t)slots_) { + wheel_[(std::size_t)(e->tick % (int64_t)slots_)].push_back(e); + it = overflow_.erase(it); + } else { + ++it; + } + } +} + +int TimerWheel::next_expiry_ms() const { + if (live_ == 0) return -1; + int64_t now = NowTick(); + int64_t best = -1; + for (const auto& slot : wheel_) { + for (const auto& e : slot) { + if (e->cancelled) continue; + int64_t d = e->tick - now; + if (d < 0) d = 0; + if (best < 0 || d < best) best = d; + } + } + for (const auto& e : overflow_) { + if (e->cancelled) continue; + int64_t d = e->tick - now; + if (d < 0) d = 0; + if (best < 0 || d < best) best = d; + } + if (best < 0) return -1; + return (int)(best * granularity_ms_); +} + +void TimerWheel::run_due() { + int64_t now = NowTick(); + Cascade(now); + int64_t from = last_tick_ + 1; + if (from < now - (int64_t)slots_) from = now - (int64_t)slots_; + for (int64_t t = from; t <= now; ++t) { + std::size_t idx = + (std::size_t)((t % (int64_t)slots_ + (int64_t)slots_) % (int64_t)slots_); + auto& slot = wheel_[idx]; + for (auto it = slot.begin(); it != slot.end();) { + EntryPtr e = *it; + if (e->cancelled || e->tick > now) { + if (e->cancelled) it = slot.erase(it); + else ++it; + continue; + } + it = slot.erase(it); + --live_; + Callback cb = std::move(e->cb); + e->cb = nullptr; + if (cb) cb(); + } + } + last_tick_ = now; +} + +} // namespace molo::runtime diff --git a/src/runtime/timer_wheel.h b/src/runtime/timer_wheel.h new file mode 100644 index 0000000..97e9860 --- /dev/null +++ b/src/runtime/timer_wheel.h @@ -0,0 +1,56 @@ +// Spec: plans/01-runtime.md 6.1. Hashed timing wheel, 10 ms granularity, +// cap 65536 timers per loop. Single-threaded: only the owning loop touches it. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::runtime { + +class TimerWheel { + public: + using TimerId = uint64_t; + using Callback = std::function; + + explicit TimerWheel(int granularity_ms = 10, + std::size_t slots = 1024); + ~TimerWheel() = default; + TimerWheel(const TimerWheel&) = delete; + TimerWheel& operator=(const TimerWheel&) = delete; + + // Returns 0 when the per-loop cap is hit. + TimerId add(std::chrono::milliseconds delay, Callback cb); + bool cancel(TimerId id); + // Milliseconds until the next expiry, -1 when empty. + int next_expiry_ms() const; + void run_due(); + std::size_t size() const { return live_; } + + private: + struct Entry { + TimerId id = 0; + int64_t tick = 0; + Callback cb; + bool cancelled = false; + }; + using EntryPtr = std::shared_ptr; + + int64_t NowTick() const; + void Cascade(int64_t now); + + int granularity_ms_; + std::size_t slots_; + int64_t start_tick_; + int64_t last_tick_; + std::vector> wheel_; + std::list overflow_; + TimerId next_id_ = 1; + std::size_t live_ = 0; +}; + +} // namespace molo::runtime diff --git a/src/runtime/tls_handle.h b/src/runtime/tls_handle.h new file mode 100644 index 0000000..2e2aacc --- /dev/null +++ b/src/runtime/tls_handle.h @@ -0,0 +1,74 @@ +// Spec: plans/01-runtime.md execution step 10, integration with +// 03-security.md (TlsContext/TlsAcceptor over wolfSSL, TLS 1.3 only, ALPN +// h2 and http/1.1, handshake timeout 5 s). +// +// STUB: wolfSSL is not installed in this environment, so this header carries +// the stable interface plus a passthrough StubTlsHandle used by tests and the +// cleartext path. The real handle implementing this interface arrives with +// 03-security.md; Connection code below already routes every byte through it. +#pragma once + +#include +#include +#include +#include + +namespace molo::runtime::tls { + +enum class HandshakeState { NeedRead, NeedWrite, Done, Failed }; + +// Non-blocking TLS session bound to one connection. All methods run on the +// owning loop thread. +class ITlsHandle { + public: + virtual ~ITlsHandle() = default; + // Advance the handshake using bytes already pulled from the transport. + // Consumes from `in`, appends handshake output to the caller's out buffer. + virtual HandshakeState handshake_step(const char*& in, std::size_t& in_len, + std::string& out) = 0; + // Negotiated ALPN protocol: "h2" or "http/1.1". Valid after Done. + virtual std::string alpn_protocol() const = 0; + virtual bool is_tls() const = 0; + // Application data. Return bytes produced, or -1 on fatal error. + virtual ssize_t decrypt(const char* in, std::size_t n, char* out, + std::size_t cap, std::size_t& in_used) = 0; + virtual ssize_t encrypt(const char* in, std::size_t n, char* out, + std::size_t cap) = 0; +}; + +// STUB: cleartext passthrough. handshake completes at once, ALPN is fixed at +// construction ("http/1.1" by default, tests use "h2" for the ALPN path). +class StubTlsHandle : public ITlsHandle { + public: + explicit StubTlsHandle(std::string alpn = "http/1.1", bool is_tls = false) + : alpn_(std::move(alpn)), is_tls_(is_tls) {} + + HandshakeState handshake_step(const char*& in, std::size_t& in_len, + std::string& out) override { + (void)in; + (void)in_len; + (void)out; + return HandshakeState::Done; + } + std::string alpn_protocol() const override { return alpn_; } + bool is_tls() const override { return is_tls_; } + ssize_t decrypt(const char* in, std::size_t n, char* out, std::size_t cap, + std::size_t& in_used) override { + std::size_t k = n < cap ? n : cap; + if (k > 0) memcpy(out, in, k); + in_used = k; + return (ssize_t)k; + } + ssize_t encrypt(const char* in, std::size_t n, char* out, + std::size_t cap) override { + std::size_t k = n < cap ? n : cap; + if (k > 0) memcpy(out, in, k); + return (ssize_t)k; + } + + private: + std::string alpn_; + bool is_tls_; +}; + +} // namespace molo::runtime::tls diff --git a/src/runtime/transport.h b/src/runtime/transport.h new file mode 100644 index 0000000..fc3f812 --- /dev/null +++ b/src/runtime/transport.h @@ -0,0 +1,27 @@ +// Spec: plans/01-runtime.md 6.8. Transport abstraction: TCP today, a future +// QUIC transport reuses the semantic Request/Response. H2 logic never touches +// sockets directly, only this interface. +#pragma once + +#include +#include +#include + +namespace molo::runtime { + +class Transport { + public: + virtual ~Transport() = default; + virtual int fd() const = 0; + // Non-blocking. Returns bytes, 0 on orderly EOF, -1 with errno EAGAIN for + // would-block, -1 with other errno on error. + virtual ssize_t recv(char* buf, std::size_t len) = 0; + virtual ssize_t send(const char* buf, std::size_t len) = 0; + virtual void close() = 0; + virtual bool closed() const = 0; + virtual std::string peer() const = 0; + virtual uint64_t bytes_read() const = 0; + virtual uint64_t bytes_written() const = 0; +}; + +} // namespace molo::runtime diff --git a/src/runtime/transport_tcp.cpp b/src/runtime/transport_tcp.cpp new file mode 100644 index 0000000..e4c1125 --- /dev/null +++ b/src/runtime/transport_tcp.cpp @@ -0,0 +1,40 @@ +// Spec: plans/01-runtime.md 6.8. TCP transport over a non-blocking socket. +#include "transport_tcp.h" + +#include +#include +#include + +namespace molo::runtime { + +TcpTransport::TcpTransport(int fd, std::string peer) + : fd_(fd), peer_(std::move(peer)) {} + +TcpTransport::~TcpTransport() { close(); } + +ssize_t TcpTransport::recv(char* buf, std::size_t len) { + if (fd_ < 0) return -1; + ssize_t n = ::recv(fd_, buf, len, 0); + if (n > 0) bytes_read_ += (uint64_t)n; + return n; +} + +ssize_t TcpTransport::send(const char* buf, std::size_t len) { + if (fd_ < 0) return -1; +#if defined(MSG_NOSIGNAL) + ssize_t n = ::send(fd_, buf, len, MSG_NOSIGNAL); +#else + ssize_t n = ::send(fd_, buf, len, 0); +#endif + if (n > 0) bytes_written_ += (uint64_t)n; + return n; +} + +void TcpTransport::close() { + if (fd_ >= 0) { + ::close(fd_); + fd_ = -1; + } +} + +} // namespace molo::runtime diff --git a/src/runtime/transport_tcp.h b/src/runtime/transport_tcp.h new file mode 100644 index 0000000..9180543 --- /dev/null +++ b/src/runtime/transport_tcp.h @@ -0,0 +1,34 @@ +// Spec: plans/01-runtime.md 6.8. TCP transport over a non-blocking socket. +#pragma once + +#include +#include + +#include "transport.h" + +namespace molo::runtime { + +class TcpTransport : public Transport { + public: + explicit TcpTransport(int fd, std::string peer = ""); + ~TcpTransport() override; + TcpTransport(const TcpTransport&) = delete; + TcpTransport& operator=(const TcpTransport&) = delete; + + int fd() const override { return fd_; } + ssize_t recv(char* buf, std::size_t len) override; + ssize_t send(const char* buf, std::size_t len) override; + void close() override; + bool closed() const override { return fd_ < 0; } + std::string peer() const override { return peer_; } + uint64_t bytes_read() const override { return bytes_read_; } + uint64_t bytes_written() const override { return bytes_written_; } + + private: + int fd_; + std::string peer_; + uint64_t bytes_read_ = 0; + uint64_t bytes_written_ = 0; +}; + +} // namespace molo::runtime diff --git a/src/security/auth/Authenticator.cpp b/src/security/auth/Authenticator.cpp new file mode 100644 index 0000000..af435ff --- /dev/null +++ b/src/security/auth/Authenticator.cpp @@ -0,0 +1,108 @@ +#include "Authenticator.h" + +#include "../crypto/Base64.h" +#include "../crypto/PasswordHasher.h" +#include "../crypto/SecureRandom.h" +#include "../guard/DenialSink.h" +#include "../guard/HttpMessageView.h" + +namespace molo::security::auth { + +namespace { +molo::security::guard::RateLimiter::Policy login_policy() { + molo::security::guard::RateLimiter::Policy p; + p.per_ip = {Authenticator::kLoginPerMinPerIp, 1, 10}; + p.per_user = {Authenticator::kLoginPerMinPerIp, 1, 10}; + return p; +} +} // namespace + +Authenticator::Authenticator(std::shared_ptr users, + molo::security::guard::IDenialSink& sink) + : users_(std::move(users)), sink_(sink), login_limiter_(login_policy(), nullptr) {} + +void Authenticator::set_clock(molo::security::guard::RateLimiter::ClockSec clock) { + login_limiter_.set_clock(std::move(clock)); +} + +BasicCredentials Authenticator::parse_basic(const molo::security::guard::HttpRequestView& req) { + BasicCredentials out; + std::string h = req.header("authorization"); + const char* prefix = "Basic "; + if (h.compare(0, 6, prefix) != 0) return out; + std::string b64 = h.substr(6); + if (b64.empty() || b64.size() > 4096) return out; + std::string raw; + if (!molo::security::crypto::Base64::decode(b64, raw)) return out; + auto colon = raw.find(':'); + if (colon == std::string::npos || colon == 0) return out; + std::string user = raw.substr(0, colon); + std::string pass = raw.substr(colon + 1); + molo::security::crypto::SecureRandom::zeroize(raw); + if (user.size() > kMaxUsernameBytes || pass.size() > kMaxPasswordBytes) { + molo::security::crypto::SecureRandom::zeroize(pass); + return out; + } + out.username = user; + out.password = pass; + molo::security::crypto::SecureRandom::zeroize(pass); + out.present = true; + return out; +} + +Authenticator::LoginResult Authenticator::login( + const BasicCredentials& creds, const std::string& peer_ip, + molo::security::session::Session& session, const molo::security::guard::HttpRequestView& req) { + if (!users_) return LoginResult::NoStore; + if (!creds.present) { + sink_.deny({"auth", "auth.missing_credentials", "no basic credentials", peer_ip, req.path}); + return LoginResult::BadCredentials; + } + molo::security::guard::HttpRequestView rl = req; + rl.peer_ip = peer_ip; + auto verdict = login_limiter_.check(rl, creds.username); + if (!verdict.allowed) { + sink_.deny({"auth", "auth.login_rate_limited", "too many attempts", peer_ip, req.path}); + return LoginResult::RateLimited; + } + IUserStore::Account acct; + // Constant-shape path: always run a verify-shaped comparison even on + // unknown users (dummy hash) to avoid user enumeration via timing. + static const char* kDummy = + "$argon2id$v=19$m=8,t=1,p=1$ZHVtbXlzYWx0ZHVtbXk$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + bool known = users_->find(creds.username, acct); + const std::string& hash = (known && !acct.argon2id_hash.empty()) ? acct.argon2id_hash : kDummy; + bool ok = molo::security::crypto::PasswordHasher::verify(creds.password, hash); + if (!known || acct.argon2id_hash.empty() || !ok) { + sink_.deny({"auth", "auth.bad_credentials", "login failed", peer_ip, req.path}); + return (known && acct.argon2id_hash.empty()) ? LoginResult::Locked + : LoginResult::BadCredentials; + } + if (molo::security::crypto::PasswordHasher::needs_rehash( + hash, molo::security::crypto::PasswordHasher::owasp_defaults())) { + std::string fresh; + if (molo::security::crypto::PasswordHasher::hash_password( + creds.password, molo::security::crypto::PasswordHasher::owasp_defaults(), fresh)) { + users_->store_rehash(creds.username, fresh); + molo::security::crypto::SecureRandom::zeroize(fresh); + sink_.audit({"auth", "auth.rehashed", "work factor rise applied", peer_ip, req.path}); + } + } + session.user_id = acct.user_id; + session.privilege = molo::security::session::Session::Privilege::User; + session.last_reauth_at = req.now_unix_sec; + sink_.audit({"auth", "auth.login", "login ok", peer_ip, req.path}); + return LoginResult::Ok; +} + +bool Authenticator::reauth_fresh(const molo::security::session::Session& s, std::uint64_t now) { + if (s.privilege == molo::security::session::Session::Privilege::Anonymous) return false; + if (now < s.last_reauth_at) return true; + return now - s.last_reauth_at <= kReauthWindowSec; +} + +void Authenticator::mark_reauth(molo::security::session::Session& s, std::uint64_t now) { + s.last_reauth_at = now; +} + +} // namespace molo::security::auth diff --git a/src/security/auth/Authenticator.h b/src/security/auth/Authenticator.h new file mode 100644 index 0000000..f369375 --- /dev/null +++ b/src/security/auth/Authenticator.h @@ -0,0 +1,80 @@ +#pragma once + +// Authentication: Basic credential parsing, login and logout against an +// abstract user store, re-auth gate for high-risk actions. Password +// verification is blocking (Argon2id) and must run off the event loop; +// this class is pure logic so the runtime can dispatch it freely. + +#include +#include +#include +#include + +#include "../guard/RateLimiter.h" +#include "../session/Session.h" +#include "Permission.h" + +namespace molo::security::guard { +struct HttpRequestView; +struct HttpResponseView; +class IDenialSink; +} // namespace molo::security::guard + +namespace molo::security::auth { + +// App-provided credential backend. Implementations compare in +// constant time on the hash path and never log secrets. +class IUserStore { +public: + virtual ~IUserStore() = default; + struct Account { + std::string user_id; + std::string argon2id_hash; // encoded form, empty means locked + std::vector roles; + }; + // False when unknown. Out param untouched on miss. + virtual bool find(const std::string& username, Account& out) = 0; + // Persist a re-hashed password (work factor rise). Best effort. + virtual bool store_rehash(const std::string& username, const std::string& encoded) = 0; +}; + +struct BasicCredentials { + std::string username; + std::string password; + bool present = false; +}; + +class Authenticator { +public: + static constexpr std::uint32_t kLoginPerMinPerIp = 10; + static constexpr std::uint64_t kReauthWindowSec = 10u * 60u; + static constexpr std::size_t kMaxUsernameBytes = 256; + static constexpr std::size_t kMaxPasswordBytes = 1024; + + Authenticator(std::shared_ptr users, molo::security::guard::IDenialSink& sink); + + // Parses "Authorization: Basic base64(user:pass)" once. Malformed or + // missing header yields present == false, never throws. + static BasicCredentials parse_basic(const molo::security::guard::HttpRequestView& req); + + enum class LoginResult : std::uint8_t { Ok, BadCredentials, RateLimited, Locked, NoStore }; + + // Verifies credentials, fills session identity. Caller rotates the + // session afterwards via SessionManager::set_privilege. + LoginResult login(const BasicCredentials& creds, const std::string& peer_ip, + molo::security::session::Session& session, + const molo::security::guard::HttpRequestView& req); + + // True when the session re-authed within the last 10 minutes. + static bool reauth_fresh(const molo::security::session::Session& s, std::uint64_t now); + static void mark_reauth(molo::security::session::Session& s, std::uint64_t now); + + void set_clock(molo::security::guard::RateLimiter::ClockSec clock); + +private: + std::shared_ptr users_; + molo::security::guard::IDenialSink& sink_; + molo::security::guard::RateLimiter login_limiter_; +}; + +} // namespace molo::security::auth diff --git a/src/security/auth/Guard.cpp b/src/security/auth/Guard.cpp new file mode 100644 index 0000000..a97d695 --- /dev/null +++ b/src/security/auth/Guard.cpp @@ -0,0 +1,56 @@ +#include "Guard.h" + +#include "../guard/DenialSink.h" +#include "../guard/HttpMessageView.h" + +namespace molo::security::auth { + +Guard::Verdict Guard::check(const molo::security::session::Session& s, + const std::vector& roles, + const std::vector& required) { + Verdict v; + if (required.size() > kMaxRequired || roles.size() > kMaxRoles) { + v.code = "auth.forbidden"; + return v; + } + bool anonymous = (s.privilege == molo::security::session::Session::Privilege::Anonymous); + if (anonymous) { + v.http_status = 401; + v.code = "auth.anonymous"; + return v; + } + for (const auto& need : required) { + bool granted = false; + for (const auto& role : roles) { + for (const auto& g : role.grants) { + if (g == need) { + granted = true; + break; + } + } + if (granted) break; + } + if (!granted) { + v.code = "auth.forbidden"; + return v; + } + } + v.allowed = true; + v.http_status = 200; + v.code = "auth.allowed"; + return v; +} + +Guard::Verdict Guard::check_logged(const molo::security::session::Session& s, + const std::vector& roles, + const std::vector& required, + const molo::security::guard::HttpRequestView& req, + molo::security::guard::IDenialSink& sink) { + Verdict v = check(s, roles, required); + if (!v.allowed) { + sink.deny({"guard", v.code, "route guard rejected", req.peer_ip, req.path}); + } + return v; +} + +} // namespace molo::security::auth diff --git a/src/security/auth/Guard.h b/src/security/auth/Guard.h new file mode 100644 index 0000000..8b9b201 --- /dev/null +++ b/src/security/auth/Guard.h @@ -0,0 +1,44 @@ +#pragma once + +// Route guard: deny-by-default permission check over a session plus the +// caller's role set. Single check() entry; no inline authz in handlers. + +#include +#include + +#include "../session/Session.h" +#include "Permission.h" + +namespace molo::security::guard { +class IDenialSink; +struct HttpRequestView; +} // namespace molo::security::guard + +namespace molo::security::auth { + +class Guard { +public: + static constexpr std::size_t kMaxRoles = 32; + static constexpr std::size_t kMaxRequired = 32; + + struct Verdict { + bool allowed = false; + int http_status = 403; // 401 when anonymous, 403 when forbidden + std::string code; // "auth.anonymous" or "auth.forbidden" + }; + + // required: every permission must be granted (AND). roles: the + // authenticated principal's roles (empty when anonymous). + static Verdict check(const molo::security::session::Session& s, + const std::vector& roles, + const std::vector& required); + + // Same but emits a denial on reject (needs request context for logs). + static Verdict check_logged(const molo::security::session::Session& s, + const std::vector& roles, + const std::vector& required, + const molo::security::guard::HttpRequestView& req, + molo::security::guard::IDenialSink& sink); +}; + +} // namespace molo::security::auth diff --git a/src/security/auth/Permission.h b/src/security/auth/Permission.h new file mode 100644 index 0000000..29a6848 --- /dev/null +++ b/src/security/auth/Permission.h @@ -0,0 +1,36 @@ +#pragma once + +// Permission objects. Permissions are plain names ("doc.read"); roles +// bundle them. Route guards consume these; deny is the default. + +#include +#include + +namespace molo::security::auth { + +struct Permission { + std::string name; + bool operator==(const Permission& o) const { return name == o.name; } +}; + +struct Role { + std::string name; + std::vector grants; +}; + +// Audit patch: permission names enforced by the audit query path via +// Guard::check (audit.read today; audit.query and rights.admin reserved for +// the query service and the rights-request console). +inline const Permission kPermAuditRead{"audit.read"}; +inline const Permission kPermAuditQuery{"audit.query"}; +inline const Permission kPermRightsAdmin{"rights.admin"}; + +namespace perms { +// Well-known permission names. Apps extend with their own strings. +inline const char* kRead = "read"; +inline const char* kWrite = "write"; +inline const char* kAdmin = "admin"; +inline const char* kUpload = "upload.write"; +} // namespace perms + +} // namespace molo::security::auth diff --git a/src/security/crypto/AesGcm.cpp b/src/security/crypto/AesGcm.cpp new file mode 100644 index 0000000..634a963 --- /dev/null +++ b/src/security/crypto/AesGcm.cpp @@ -0,0 +1,237 @@ +#include "AesGcm.h" + +#include "SecureRandom.h" + +namespace molo::security::crypto { + +namespace { + +constexpr std::uint8_t kSbox[256] = { + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16}; + +constexpr std::uint8_t kRcon[15] = {0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, + 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d}; + +std::uint8_t xtime(std::uint8_t x) { return std::uint8_t((x << 1) ^ (x & 0x80 ? 0x1b : 0x00)); } + +// GF(2^128) multiply for GHASH, big-endian blocks, R = 0xe1 prefix. +void gcm_mul(const std::uint8_t x[16], const std::uint8_t y[16], std::uint8_t out[16]) { + std::uint8_t z[16] = {0}; + std::uint8_t v[16]; + for (int i = 0; i < 16; ++i) v[i] = y[i]; + for (int i = 0; i < 128; ++i) { + if (x[i / 8] & (0x80 >> (i % 8))) { + for (int k = 0; k < 16; ++k) z[k] ^= v[k]; + } + bool lsb = (v[15] & 1) != 0; + for (int k = 15; k > 0; --k) v[k] = std::uint8_t((v[k] >> 1) | (v[k - 1] << 7)); + v[0] >>= 1; + if (lsb) v[0] ^= 0xe1; + } + for (int i = 0; i < 16; ++i) out[i] = z[i]; +} + +void ghash(const std::uint8_t h[16], const std::uint8_t* aad, std::size_t aad_len, + const std::uint8_t* ct, std::size_t ct_len, std::uint8_t out[16]) { + std::uint8_t y[16] = {0}; + std::uint8_t blk[16]; + auto feed = [&](const std::uint8_t* p, std::size_t n) { + std::size_t done = 0; + while (done < n) { + std::size_t take = n - done; + if (take > 16) take = 16; + for (std::size_t i = 0; i < 16; ++i) blk[i] = i < take ? p[done + i] : 0; + for (int k = 0; k < 16; ++k) y[k] ^= blk[k]; + gcm_mul(y, h, y); + done += take; + } + }; + if (aad_len > 0) feed(aad, aad_len); + if (ct_len > 0) feed(ct, ct_len); + for (int i = 0; i < 8; ++i) { + blk[i] = std::uint8_t((std::uint64_t(aad_len) * 8) >> (56 - i * 8)); + blk[8 + i] = std::uint8_t((std::uint64_t(ct_len) * 8) >> (56 - i * 8)); + } + for (int k = 0; k < 16; ++k) y[k] ^= blk[k]; + gcm_mul(y, h, y); + for (int i = 0; i < 16; ++i) out[i] = y[i]; +} + +void inc32(std::uint8_t ctr[16]) { + std::uint32_t v = (std::uint32_t(ctr[12]) << 24) | (std::uint32_t(ctr[13]) << 16) | + (std::uint32_t(ctr[14]) << 8) | std::uint32_t(ctr[15]); + ++v; + ctr[12] = std::uint8_t(v >> 24); + ctr[13] = std::uint8_t(v >> 16); + ctr[14] = std::uint8_t(v >> 8); + ctr[15] = std::uint8_t(v); +} + +} // namespace + +void AesGcm::expand_key(const std::uint8_t key[32], RoundKeys& rk) { + std::uint8_t w[240]; + for (int i = 0; i < 32; ++i) w[i] = key[i]; + int nk = 8; + for (int i = nk; i < 60; ++i) { + std::uint8_t t[4] = {w[(i - 1) * 4], w[(i - 1) * 4 + 1], w[(i - 1) * 4 + 2], + w[(i - 1) * 4 + 3]}; + if (i % nk == 0) { + std::uint8_t u = t[0]; + t[0] = t[1]; + t[1] = t[2]; + t[2] = t[3]; + t[3] = u; + t[0] = kSbox[t[0]]; + t[1] = kSbox[t[1]]; + t[2] = kSbox[t[2]]; + t[3] = kSbox[t[3]]; + t[0] ^= kRcon[i / nk]; + } else if (i % nk == 4) { + t[0] = kSbox[t[0]]; + t[1] = kSbox[t[1]]; + t[2] = kSbox[t[2]]; + t[3] = kSbox[t[3]]; + } + for (int k = 0; k < 4; ++k) w[i * 4 + k] = w[(i - nk) * 4 + k] ^ t[k]; + } + for (int r = 0; r < 15; ++r) + for (int i = 0; i < 16; ++i) rk.k[r][i] = w[r * 16 + i]; + SecureRandom::zeroize(w, sizeof w); +} + +void AesGcm::encrypt_block(const RoundKeys& rk, const std::uint8_t in[16], std::uint8_t out[16]) { + std::uint8_t s[16]; + for (int i = 0; i < 16; ++i) s[i] = in[i] ^ rk.k[0][i]; + for (int round = 1; round <= 14; ++round) { + for (int i = 0; i < 16; ++i) s[i] = kSbox[s[i]]; + // ShiftRows (column-major state). + std::uint8_t t; + t = s[1]; + s[1] = s[5]; + s[5] = s[9]; + s[9] = s[13]; + s[13] = t; + t = s[2]; + s[2] = s[10]; + s[10] = t; + t = s[6]; + s[6] = s[14]; + s[14] = t; + t = s[3]; + s[3] = s[15]; + s[15] = s[11]; + s[11] = s[7]; + s[7] = t; + if (round != 14) { + for (int c = 0; c < 4; ++c) { + std::uint8_t a0 = s[c * 4], a1 = s[c * 4 + 1], a2 = s[c * 4 + 2], + a3 = s[c * 4 + 3]; + std::uint8_t x0 = xtime(a0), x1 = xtime(a1), x2 = xtime(a2), x3 = xtime(a3); + s[c * 4] = std::uint8_t(x0 ^ x1 ^ a1 ^ a2 ^ a3); + s[c * 4 + 1] = std::uint8_t(a0 ^ x1 ^ x2 ^ a2 ^ a3); + s[c * 4 + 2] = std::uint8_t(a0 ^ a1 ^ x2 ^ x3 ^ a3); + s[c * 4 + 3] = std::uint8_t(x0 ^ a0 ^ a1 ^ a2 ^ x3); + } + } + for (int i = 0; i < 16; ++i) s[i] ^= rk.k[round][i]; + } + for (int i = 0; i < 16; ++i) out[i] = s[i]; +} + +bool AesGcm::seal(const std::uint8_t key[kKeyBytes], const std::uint8_t nonce[kNonceBytes], + const std::uint8_t* aad, std::size_t aad_len, const std::uint8_t* plain, + std::size_t plain_len, std::uint8_t* out, std::uint8_t tag[kTagBytes]) { + if (key == nullptr || nonce == nullptr || tag == nullptr) return false; + if (plain_len > kMaxPlainBytes || aad_len > kMaxAadBytes) return false; + if ((plain_len > 0 && (plain == nullptr || out == nullptr))) return false; + if (aad_len > 0 && aad == nullptr) return false; + RoundKeys rk{}; + expand_key(key, rk); + std::uint8_t h[16], zero[16] = {0}; + encrypt_block(rk, zero, h); + std::uint8_t j0[16] = {0}; + for (int i = 0; i < 12; ++i) j0[i] = nonce[i]; + j0[15] = 1; + std::uint8_t ctr[16]; + for (int i = 0; i < 16; ++i) ctr[i] = j0[i]; + std::uint8_t stream[16]; + std::size_t done = 0; + while (done < plain_len) { + inc32(ctr); + encrypt_block(rk, ctr, stream); + std::size_t take = plain_len - done; + if (take > 16) take = 16; + for (std::size_t i = 0; i < take; ++i) out[done + i] = plain[done + i] ^ stream[i]; + done += take; + } + std::uint8_t s[16]; + ghash(h, aad, aad_len, out, plain_len, s); + std::uint8_t ej0[16]; + encrypt_block(rk, j0, ej0); + for (int i = 0; i < 16; ++i) tag[i] = s[i] ^ ej0[i]; + SecureRandom::zeroize(&rk, sizeof rk); + SecureRandom::zeroize(h, sizeof h); + SecureRandom::zeroize(stream, sizeof stream); + return true; +} + +bool AesGcm::open(const std::uint8_t key[kKeyBytes], const std::uint8_t nonce[kNonceBytes], + const std::uint8_t* aad, std::size_t aad_len, const std::uint8_t* cipher, + std::size_t cipher_len, const std::uint8_t tag[kTagBytes], std::uint8_t* out) { + if (key == nullptr || nonce == nullptr || tag == nullptr) return false; + if (cipher_len > kMaxPlainBytes || aad_len > kMaxAadBytes) return false; + if ((cipher_len > 0 && (cipher == nullptr || out == nullptr))) return false; + if (aad_len > 0 && aad == nullptr) return false; + RoundKeys rk{}; + expand_key(key, rk); + std::uint8_t h[16], zero[16] = {0}; + encrypt_block(rk, zero, h); + std::uint8_t j0[16] = {0}; + for (int i = 0; i < 12; ++i) j0[i] = nonce[i]; + j0[15] = 1; + std::uint8_t s[16]; + ghash(h, aad, aad_len, cipher, cipher_len, s); + std::uint8_t ej0[16]; + encrypt_block(rk, j0, ej0); + std::uint8_t expect[16]; + for (int i = 0; i < 16; ++i) expect[i] = s[i] ^ ej0[i]; + bool ok = SecureRandom::constant_time_equal(expect, tag, 16); + SecureRandom::zeroize(expect, sizeof expect); + if (!ok) { + SecureRandom::zeroize(&rk, sizeof rk); + return false; + } + std::uint8_t ctr[16]; + for (int i = 0; i < 16; ++i) ctr[i] = j0[i]; + std::uint8_t stream[16]; + std::size_t done = 0; + while (done < cipher_len) { + inc32(ctr); + encrypt_block(rk, ctr, stream); + std::size_t take = cipher_len - done; + if (take > 16) take = 16; + for (std::size_t i = 0; i < take; ++i) out[done + i] = cipher[done + i] ^ stream[i]; + done += take; + } + SecureRandom::zeroize(&rk, sizeof rk); + SecureRandom::zeroize(stream, sizeof stream); + return true; +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/AesGcm.h b/src/security/crypto/AesGcm.h new file mode 100644 index 0000000..a88e3ab --- /dev/null +++ b/src/security/crypto/AesGcm.h @@ -0,0 +1,37 @@ +#pragma once + +// AES-256-GCM AEAD (FIPS 197 + SP 800-38D), in-house. +// 256-bit key, 96-bit nonce. Caps: plaintext 16 MiB, AAD 64 KiB. + +#include +#include +#include + +namespace molo::security::crypto { + +class AesGcm { +public: + static constexpr std::size_t kKeyBytes = 32; + static constexpr std::size_t kNonceBytes = 12; + static constexpr std::size_t kTagBytes = 16; + static constexpr std::size_t kMaxPlainBytes = 16u * 1024u * 1024u; + static constexpr std::size_t kMaxAadBytes = 64u * 1024u; + + static bool seal(const std::uint8_t key[kKeyBytes], const std::uint8_t nonce[kNonceBytes], + const std::uint8_t* aad, std::size_t aad_len, const std::uint8_t* plain, + std::size_t plain_len, std::uint8_t* out, std::uint8_t tag[kTagBytes]); + + // Returns false and leaves out untouched when the tag does not verify. + static bool open(const std::uint8_t key[kKeyBytes], const std::uint8_t nonce[kNonceBytes], + const std::uint8_t* aad, std::size_t aad_len, const std::uint8_t* cipher, + std::size_t cipher_len, const std::uint8_t tag[kTagBytes], std::uint8_t* out); + +private: + struct RoundKeys { + std::uint8_t k[15][16]; + }; + static void expand_key(const std::uint8_t key[32], RoundKeys& rk); + static void encrypt_block(const RoundKeys& rk, const std::uint8_t in[16], std::uint8_t out[16]); +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/Base64.cpp b/src/security/crypto/Base64.cpp new file mode 100644 index 0000000..0a4ce30 --- /dev/null +++ b/src/security/crypto/Base64.cpp @@ -0,0 +1,125 @@ +#include "Base64.h" + +namespace molo::security::crypto { + +namespace { + +constexpr char kEnc[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +constexpr char kEncUrl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +int table_val(char c, bool url) { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (!url && c == '+') return 62; + if (!url && c == '/') return 63; + if (url && c == '-') return 62; + if (url && c == '_') return 63; + return -1; +} + +std::string encode_impl(const std::uint8_t* data, std::size_t len, const char* alpha, bool pad) { + std::string out; + out.reserve(((len + 2) / 3) * 4); + std::size_t i = 0; + while (i + 3 <= len) { + std::uint32_t n = (std::uint32_t(data[i]) << 16) | (std::uint32_t(data[i + 1]) << 8) | + std::uint32_t(data[i + 2]); + out.push_back(alpha[(n >> 18) & 63]); + out.push_back(alpha[(n >> 12) & 63]); + out.push_back(alpha[(n >> 6) & 63]); + out.push_back(alpha[n & 63]); + i += 3; + } + std::size_t rem = len - i; + if (rem == 1) { + std::uint32_t n = std::uint32_t(data[i]) << 16; + out.push_back(alpha[(n >> 18) & 63]); + out.push_back(alpha[(n >> 12) & 63]); + if (pad) { + out.push_back('='); + out.push_back('='); + } + } else if (rem == 2) { + std::uint32_t n = (std::uint32_t(data[i]) << 16) | (std::uint32_t(data[i + 1]) << 8); + out.push_back(alpha[(n >> 18) & 63]); + out.push_back(alpha[(n >> 12) & 63]); + out.push_back(alpha[(n >> 6) & 63]); + if (pad) out.push_back('='); + } + return out; +} + +bool decode_impl(const std::string& text, std::string& out, bool url, bool require_pad) { + out.clear(); + if (text.empty()) return true; + if (require_pad && (text.size() % 4 != 0)) return false; + if (url && (text.size() % 4 == 1)) return false; + out.reserve((text.size() / 4) * 3 + 3); + std::size_t i = 0; + auto take = [&](std::uint32_t& n, int& valid) -> bool { + n = 0; + valid = 0; + for (int k = 0; k < 4 && i < text.size(); ++k, ++i) { + char c = text[i]; + if (!url && require_pad && c == '=') { + // Padding only valid at the very tail. + for (std::size_t j = i; j < text.size(); ++j) + if (text[j] != '=') return false; + std::size_t pads = text.size() - i; + if (pads > 2 || k < 2) return false; + i = text.size(); + return true; + } + int v = table_val(c, url); + if (v < 0) return false; + n = (n << 6) | std::uint32_t(v); + ++valid; + } + return true; + }; + while (i < text.size()) { + std::uint32_t n = 0; + int valid = 0; + if (!take(n, valid)) return false; + if (valid == 4) { + out.push_back(char((n >> 16) & 0xFF)); + out.push_back(char((n >> 8) & 0xFF)); + out.push_back(char(n & 0xFF)); + } else if (valid == 3) { + out.push_back(char((n >> 10) & 0xFF)); + out.push_back(char((n >> 2) & 0xFF)); + if ((n & 3) != 0) return false; + } else if (valid == 2) { + out.push_back(char((n >> 4) & 0xFF)); + if ((n & 15) != 0) return false; + } else if (valid != 0) { + return false; + } + } + return true; +} + +} // namespace + +std::string Base64::encode(const std::uint8_t* data, std::size_t len) { + if (len > kMaxInputBytes || (len > 0 && data == nullptr)) return {}; + return encode_impl(data, len, kEnc, true); +} + +bool Base64::decode(const std::string& text, std::string& out) { + if (text.size() > ((kMaxInputBytes + 2) / 3) * 4 + 4) return false; + return decode_impl(text, out, false, true); +} + +std::string Base64::encode_url(const std::uint8_t* data, std::size_t len) { + if (len > kMaxInputBytes || (len > 0 && data == nullptr)) return {}; + return encode_impl(data, len, kEncUrl, false); +} + +bool Base64::decode_url(const std::string& text, std::string& out) { + if (text.size() > ((kMaxInputBytes + 2) / 3) * 4 + 4) return false; + return decode_impl(text, out, true, false); +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/Base64.h b/src/security/crypto/Base64.h new file mode 100644 index 0000000..f7a6ab1 --- /dev/null +++ b/src/security/crypto/Base64.h @@ -0,0 +1,32 @@ +#pragma once + +// Base64 and base64url codec (RFC 4648). Header-only, no allocations +// beyond the returned string. Caps: input 16 MiB max. + +#include +#include +#include + +namespace molo::security::crypto { + +class Base64 { +public: + static constexpr std::size_t kMaxInputBytes = 16u * 1024u * 1024u; + + // Standard alphabet with '=' padding. + static std::string encode(const std::uint8_t* data, std::size_t len); + static std::string encode(const std::string& raw) { + return encode(reinterpret_cast(raw.data()), raw.size()); + } + // Returns false on any invalid character or bad padding. + static bool decode(const std::string& text, std::string& out); + + // URL-safe alphabet, no padding. + static std::string encode_url(const std::uint8_t* data, std::size_t len); + static std::string encode_url(const std::string& raw) { + return encode_url(reinterpret_cast(raw.data()), raw.size()); + } + static bool decode_url(const std::string& text, std::string& out); +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/Blake2b.cpp b/src/security/crypto/Blake2b.cpp new file mode 100644 index 0000000..b3f3990 --- /dev/null +++ b/src/security/crypto/Blake2b.cpp @@ -0,0 +1,120 @@ +#include "Blake2b.h" + +namespace molo::security::crypto { + +namespace { + +constexpr std::uint64_t kIv[8] = {0x6a09e667f3bcc908ull, 0xbb67ae8584caa73bull, + 0x3c6ef372fe94f82bull, 0xa54ff53a5f1d36f1ull, + 0x510e527fade682d1ull, 0x9b05688c2b3e6c1full, + 0x1f83d9abfb41bd6bull, 0x5be0cd19137e2179ull}; + +constexpr std::uint8_t kSigma[12][16] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3}, + {11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4}, + {7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8}, + {9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13}, + {2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9}, + {12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11}, + {13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10}, + {6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5}, + {10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3}}; + +constexpr std::uint64_t rotr64(std::uint64_t x, int n) { return (x >> n) | (x << (64 - n)); } + +constexpr std::uint64_t load64le(const std::uint8_t* p) { + std::uint64_t v = 0; + for (int i = 0; i < 8; ++i) v |= std::uint64_t(p[i]) << (i * 8); + return v; +} + +#define G2(v, a, b, c, d, x, y) \ + v[a] = v[a] + v[b] + x; \ + v[d] = rotr64(v[d] ^ v[a], 32); \ + v[c] = v[c] + v[d]; \ + v[b] = rotr64(v[b] ^ v[c], 24); \ + v[a] = v[a] + v[b] + y; \ + v[d] = rotr64(v[d] ^ v[a], 16); \ + v[c] = v[c] + v[d]; \ + v[b] = rotr64(v[b] ^ v[c], 63) + +} // namespace + +Blake2b::Blake2b(std::size_t digest_len) : digest_len_(digest_len) { + if (digest_len_ < 1) digest_len_ = 1; + if (digest_len_ > kMaxDigest) digest_len_ = kMaxDigest; + std::uint8_t params[64] = {0}; + params[0] = std::uint8_t(digest_len_); + params[2] = 1; // fanout + params[3] = 1; // depth + for (int i = 0; i < 8; ++i) h_[i] = kIv[i] ^ load64le(params + i * 8); +} + +void Blake2b::compress(const std::uint8_t* block) { + std::uint64_t m[16]; + for (int i = 0; i < 16; ++i) m[i] = load64le(block + i * 8); + std::uint64_t v[16]; + for (int i = 0; i < 8; ++i) { + v[i] = h_[i]; + v[i + 8] = kIv[i]; + } + v[12] ^= t0_; + v[13] ^= t1_; + if (finalized_) v[14] = ~v[14]; + for (int r = 0; r < 12; ++r) { + const std::uint8_t* s = kSigma[r]; + G2(v, 0, 4, 8, 12, m[s[0]], m[s[1]]); + G2(v, 1, 5, 9, 13, m[s[2]], m[s[3]]); + G2(v, 2, 6, 10, 14, m[s[4]], m[s[5]]); + G2(v, 3, 7, 11, 15, m[s[6]], m[s[7]]); + G2(v, 0, 5, 10, 15, m[s[8]], m[s[9]]); + G2(v, 1, 6, 11, 12, m[s[10]], m[s[11]]); + G2(v, 2, 7, 8, 13, m[s[12]], m[s[13]]); + G2(v, 3, 4, 9, 14, m[s[14]], m[s[15]]); + } + for (int i = 0; i < 8; ++i) h_[i] ^= v[i] ^ v[i + 8]; +} + +void Blake2b::update(const std::uint8_t* data, std::size_t len) { + if (finalized_) return; + if (len > 0 && data == nullptr) return; + while (len > 0) { + if (buf_used_ == kBlockBytes) { + t0_ += kBlockBytes; + if (t0_ < kBlockBytes) ++t1_; + compress(buf_.data()); + buf_used_ = 0; + } + std::size_t take = kBlockBytes - buf_used_; + if (take > len) take = len; + for (std::size_t i = 0; i < take; ++i) buf_[buf_used_ + i] = data[i]; + buf_used_ += take; + data += take; + len -= take; + } +} + +std::vector Blake2b::final() { + if (!finalized_) { + t0_ += buf_used_; + if (t0_ < buf_used_) ++t1_; + finalized_ = true; + for (std::size_t i = buf_used_; i < kBlockBytes; ++i) buf_[i] = 0; + compress(buf_.data()); + } + std::vector out(digest_len_); + for (std::size_t i = 0; i < digest_len_; ++i) out[i] = std::uint8_t(h_[i / 8] >> ((i % 8) * 8)); + return out; +} + +std::vector Blake2b::hash(const std::uint8_t* data, std::size_t len, + std::size_t digest_len) { + Blake2b b(digest_len); + b.update(data, len); + return b.final(); +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/Blake2b.h b/src/security/crypto/Blake2b.h new file mode 100644 index 0000000..f685cbc --- /dev/null +++ b/src/security/crypto/Blake2b.h @@ -0,0 +1,41 @@ +#pragma once + +// BLAKE2b (RFC 7693), in-house. Unkeyed, digest 1..64 bytes. +// Exists to back Argon2id; general hashing uses SHA-256. + +#include +#include +#include +#include +#include + +namespace molo::security::crypto { + +class Blake2b { +public: + static constexpr std::size_t kMaxDigest = 64; + static constexpr std::size_t kBlockBytes = 128; + + explicit Blake2b(std::size_t digest_len = 64); + void update(const std::uint8_t* data, std::size_t len); + void update(const std::string& s) { + update(reinterpret_cast(s.data()), s.size()); + } + // Returns digest_len bytes. Instance is spent after final(). + std::vector final(); + + static std::vector hash(const std::uint8_t* data, std::size_t len, + std::size_t digest_len = 64); + +private: + void compress(const std::uint8_t* block); + + std::array h_{}; + std::array buf_{}; + std::uint64_t t0_ = 0, t1_ = 0; + std::size_t buf_used_ = 0; + std::size_t digest_len_ = 64; + bool finalized_ = false; +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/ChaCha20Poly1305.cpp b/src/security/crypto/ChaCha20Poly1305.cpp new file mode 100644 index 0000000..561c21f --- /dev/null +++ b/src/security/crypto/ChaCha20Poly1305.cpp @@ -0,0 +1,330 @@ +#include "ChaCha20Poly1305.h" + +#include + +#include "SecureRandom.h" + +namespace molo::security::crypto { + +namespace { + +constexpr std::uint32_t rotl32(std::uint32_t x, int n) { return (x << n) | (x >> (32 - n)); } + +constexpr std::uint32_t load32le(const std::uint8_t* p) { + return std::uint32_t(p[0]) | (std::uint32_t(p[1]) << 8) | (std::uint32_t(p[2]) << 16) | + (std::uint32_t(p[3]) << 24); +} + +void store32le(std::uint8_t* p, std::uint32_t v) { + p[0] = std::uint8_t(v); + p[1] = std::uint8_t(v >> 8); + p[2] = std::uint8_t(v >> 16); + p[3] = std::uint8_t(v >> 24); +} + +void store64le(std::uint8_t* p, std::uint64_t v) { + for (int i = 0; i < 8; ++i) p[i] = std::uint8_t(v >> (i * 8)); +} + +#define QR(x, a, b, c, d) \ + x[a] += x[b]; \ + x[d] ^= x[a]; \ + x[d] = rotl32(x[d], 16); \ + x[c] += x[d]; \ + x[b] ^= x[c]; \ + x[b] = rotl32(x[b], 12); \ + x[a] += x[b]; \ + x[d] ^= x[a]; \ + x[d] = rotl32(x[d], 8); \ + x[c] += x[d]; \ + x[b] ^= x[c]; \ + x[b] = rotl32(x[b], 7) + +// Poly1305 with 26-bit limbs (5 limbs), public domain style. +class Poly1305 { +public: + explicit Poly1305(const std::uint8_t key[32]) { + r_[0] = load32le(key) & 0x3ffffff; + r_[1] = (load32le(key + 3) >> 2) & 0x3ffff03; + r_[2] = (load32le(key + 6) >> 4) & 0x3ffc0ff; + r_[3] = (load32le(key + 9) >> 6) & 0x3f03fff; + r_[4] = (load32le(key + 12) >> 8) & 0x00fffff; + s_[0] = load32le(key + 16); + s_[1] = load32le(key + 20); + s_[2] = load32le(key + 24); + s_[3] = load32le(key + 28); + for (int i = 0; i < 5; ++i) h_[i] = 0; + leftover_ = 0; + } + + void update(const std::uint8_t* data, std::size_t len) { + if (leftover_ > 0) { + std::size_t take = 16 - leftover_; + if (take > len) take = len; + for (std::size_t i = 0; i < take; ++i) buf_[leftover_ + i] = data[i]; + leftover_ += take; + data += take; + len -= take; + if (leftover_ == 16) { + process_block(buf_, false); + leftover_ = 0; + } + } + while (len >= 16) { + process_block(data, false); + data += 16; + len -= 16; + } + if (len > 0) { + for (std::size_t i = 0; i < len; ++i) buf_[i] = data[i]; + leftover_ = len; + } + } + + void finish(std::uint8_t tag[16]) { + if (leftover_ > 0) { + buf_[leftover_] = 1; + for (std::size_t i = leftover_ + 1; i < 16; ++i) buf_[i] = 0; + process_block(buf_, true); + } + // Full carry. + std::uint32_t c = h_[1] >> 26; + h_[1] &= 0x3ffffff; + std::uint64_t h0 = std::uint64_t(h_[0]) + (c * 5); + c = std::uint32_t(h0 >> 26); + h_[0] = std::uint32_t(h0) & 0x3ffffff; + h_[1] += c; + c = h_[2] >> 26; + h_[2] &= 0x3ffffff; + h_[3] += c; + c = h_[3] >> 26; + h_[3] &= 0x3ffffff; + h_[4] += c; + c = h_[4] >> 26; + h_[4] &= 0x3ffffff; + h_[0] += c * 5; + c = h_[0] >> 26; + h_[0] &= 0x3ffffff; + h_[1] += c; + // Compute h + -p. + std::uint32_t g[5]; + g[0] = h_[0] + 5; + g[1] = h_[1]; + g[2] = h_[2]; + g[3] = h_[3]; + g[4] = h_[4]; + c = g[0] >> 26; + g[0] &= 0x3ffffff; + g[1] += c; + c = g[1] >> 26; + g[1] &= 0x3ffffff; + g[2] += c; + c = g[2] >> 26; + g[2] &= 0x3ffffff; + g[3] += c; + c = g[3] >> 26; + g[3] &= 0x3ffffff; + g[4] += c; + // Select h - p when h >= p, else h. Constant-time select. + std::uint32_t mask = (g[4] >> 26) - 1; // 0 when h >= p, all-ones when h < p + g[4] &= 0x3ffffff; // drop bit 130 from the reduced candidate + h_[0] = (g[0] & ~mask) | (h_[0] & mask); + h_[1] = (g[1] & ~mask) | (h_[1] & mask); + h_[2] = (g[2] & ~mask) | (h_[2] & mask); + h_[3] = (g[3] & ~mask) | (h_[3] & mask); + h_[4] = (g[4] & ~mask) | (h_[4] & mask); + // h as 128-bit little endian. Limbs cover bits h0: 0-25, + // h1: 26-51, h2: 52-77, h3: 78-103, h4: 104-129. + std::uint64_t f; + f = std::uint64_t(h_[0]) | (std::uint64_t(h_[1]) << 26); + store32le(tag, std::uint32_t(f)); // bits 0-31 + f = (std::uint64_t(h_[1]) >> 6) | (std::uint64_t(h_[2]) << 20); + store32le(tag + 4, std::uint32_t(f)); // bits 32-63 + f = (std::uint64_t(h_[2]) >> 12) | (std::uint64_t(h_[3]) << 14); + store32le(tag + 8, std::uint32_t(f)); // bits 64-95 + f = (std::uint64_t(h_[3]) >> 18) | (std::uint64_t(h_[4]) << 8); + store32le(tag + 12, std::uint32_t(f)); // bits 96-127 + // Add s. + std::uint64_t carry = 0; + for (int i = 0; i < 4; ++i) { + carry += std::uint64_t(load32le(tag + i * 4)) + s_[i]; + store32le(tag + i * 4, std::uint32_t(carry)); + carry >>= 32; + } + } + +private: + void process_block(const std::uint8_t m[16], bool is_final) { + std::uint32_t t0 = load32le(m); + std::uint32_t t1 = load32le(m + 4); + std::uint32_t t2 = load32le(m + 8); + std::uint32_t t3 = load32le(m + 12); + // Split the 128-bit block into 26-bit limbs. Full blocks add the + // implicit 2^128 bit; final partial blocks already carry the 0x01 + // terminator byte inside m (placed by finish()). + std::uint64_t lo = std::uint64_t(t0) | (std::uint64_t(t1) << 32); + std::uint64_t hi = std::uint64_t(t2) | (std::uint64_t(t3) << 32); + h_[0] += std::uint32_t(lo & 0x3ffffff); + h_[1] += std::uint32_t((lo >> 26) & 0x3ffffff); + h_[2] += std::uint32_t(((lo >> 52) | (hi << 12)) & 0x3ffffff); + h_[3] += std::uint32_t((hi >> 14) & 0x3ffffff); + h_[4] += std::uint32_t((hi >> 40) | (is_final ? 0u : (1u << 24))); + // h *= r + std::uint64_t d0 = std::uint64_t(h_[0]) * r_[0] + std::uint64_t(h_[1]) * (r_[4] * 5) + + std::uint64_t(h_[2]) * (r_[3] * 5) + + std::uint64_t(h_[3]) * (r_[2] * 5) + + std::uint64_t(h_[4]) * (r_[1] * 5); + std::uint64_t d1 = std::uint64_t(h_[0]) * r_[1] + std::uint64_t(h_[1]) * r_[0] + + std::uint64_t(h_[2]) * (r_[4] * 5) + + std::uint64_t(h_[3]) * (r_[3] * 5) + + std::uint64_t(h_[4]) * (r_[2] * 5); + std::uint64_t d2 = std::uint64_t(h_[0]) * r_[2] + std::uint64_t(h_[1]) * r_[1] + + std::uint64_t(h_[2]) * r_[0] + + std::uint64_t(h_[3]) * (r_[4] * 5) + + std::uint64_t(h_[4]) * (r_[3] * 5); + std::uint64_t d3 = std::uint64_t(h_[0]) * r_[3] + std::uint64_t(h_[1]) * r_[2] + + std::uint64_t(h_[2]) * r_[1] + + std::uint64_t(h_[3]) * r_[0] + + std::uint64_t(h_[4]) * (r_[4] * 5); + std::uint64_t d4 = std::uint64_t(h_[0]) * r_[4] + std::uint64_t(h_[1]) * r_[3] + + std::uint64_t(h_[2]) * r_[2] + + std::uint64_t(h_[3]) * r_[1] + std::uint64_t(h_[4]) * r_[0]; + std::uint32_t c; + c = std::uint32_t(d0 >> 26); + h_[0] = std::uint32_t(d0) & 0x3ffffff; + d1 += c; + c = std::uint32_t(d1 >> 26); + h_[1] = std::uint32_t(d1) & 0x3ffffff; + d2 += c; + c = std::uint32_t(d2 >> 26); + h_[2] = std::uint32_t(d2) & 0x3ffffff; + d3 += c; + c = std::uint32_t(d3 >> 26); + h_[3] = std::uint32_t(d3) & 0x3ffffff; + d4 += c; + c = std::uint32_t(d4 >> 26); + h_[4] = std::uint32_t(d4) & 0x3ffffff; + h_[0] += c * 5; + c = h_[0] >> 26; + h_[0] &= 0x3ffffff; + h_[1] += c; + } + + std::uint32_t r_[5]{}; + std::uint32_t h_[5]{}; + std::uint32_t s_[4]{}; + std::uint8_t buf_[16]{}; + std::size_t leftover_ = 0; +}; + +} // namespace + +void ChaCha20Poly1305::block(const std::uint32_t key_words[8], std::uint32_t counter, + const std::uint8_t nonce[kNonceBytes], std::uint8_t out[64]) { + std::uint32_t x[16] = {0x61707865, 0x3320646e, 0x79622d32, 0x6b206574, + key_words[0], key_words[1], key_words[2], key_words[3], + key_words[4], key_words[5], key_words[6], key_words[7], + counter, load32le(nonce), load32le(nonce + 4), + load32le(nonce + 8)}; + std::uint32_t w[16]; + for (int i = 0; i < 16; ++i) w[i] = x[i]; + for (int i = 0; i < 10; ++i) { + QR(w, 0, 4, 8, 12); + QR(w, 1, 5, 9, 13); + QR(w, 2, 6, 10, 14); + QR(w, 3, 7, 11, 15); + QR(w, 0, 5, 10, 15); + QR(w, 1, 6, 11, 12); + QR(w, 2, 7, 8, 13); + QR(w, 3, 4, 9, 14); + } + for (int i = 0; i < 16; ++i) store32le(out + i * 4, w[i] + x[i]); +} + +bool ChaCha20Poly1305::seal(const std::uint8_t key[kKeyBytes], const std::uint8_t nonce[kNonceBytes], + const std::uint8_t* aad, std::size_t aad_len, const std::uint8_t* plain, + std::size_t plain_len, std::uint8_t* out, std::uint8_t tag[kTagBytes]) { + if (key == nullptr || nonce == nullptr || tag == nullptr) return false; + if (plain_len > kMaxPlainBytes || aad_len > kMaxAadBytes) return false; + if ((plain_len > 0 && (plain == nullptr || out == nullptr))) return false; + if (aad_len > 0 && aad == nullptr) return false; + std::uint32_t kw[8]; + for (int i = 0; i < 8; ++i) kw[i] = load32le(key + i * 4); + std::uint8_t b0[64]; + block(kw, 0, nonce, b0); + Poly1305 mac(b0); + SecureRandom::zeroize(b0, sizeof b0); + std::uint8_t stream[64]; + std::size_t done = 0; + std::uint32_t counter = 1; + while (done < plain_len) { + block(kw, counter, nonce, stream); + ++counter; + std::size_t take = plain_len - done; + if (take > 64) take = 64; + for (std::size_t i = 0; i < take; ++i) out[done + i] = plain[done + i] ^ stream[i]; + done += take; + } + SecureRandom::zeroize(stream, sizeof stream); + if (aad_len > 0) mac.update(aad, aad_len); + std::uint8_t pad[16] = {0}; + if (aad_len % 16 != 0) mac.update(pad, 16 - (aad_len % 16)); + if (plain_len > 0) mac.update(out, plain_len); + if (plain_len % 16 != 0) mac.update(pad, 16 - (plain_len % 16)); + std::uint8_t lens[16]; + store64le(lens, aad_len); + store64le(lens + 8, plain_len); + mac.update(lens, 16); + mac.finish(tag); + SecureRandom::zeroize(kw, sizeof kw); + return true; +} + +bool ChaCha20Poly1305::open(const std::uint8_t key[kKeyBytes], const std::uint8_t nonce[kNonceBytes], + const std::uint8_t* aad, std::size_t aad_len, const std::uint8_t* cipher, + std::size_t cipher_len, const std::uint8_t tag[kTagBytes], + std::uint8_t* out) { + if (key == nullptr || nonce == nullptr || tag == nullptr) return false; + if (cipher_len > kMaxPlainBytes || aad_len > kMaxAadBytes) return false; + if ((cipher_len > 0 && (cipher == nullptr || out == nullptr))) return false; + if (aad_len > 0 && aad == nullptr) return false; + std::uint32_t kw[8]; + for (int i = 0; i < 8; ++i) kw[i] = load32le(key + i * 4); + std::uint8_t b0[64]; + block(kw, 0, nonce, b0); + Poly1305 mac(b0); + SecureRandom::zeroize(b0, sizeof b0); + if (aad_len > 0) mac.update(aad, aad_len); + std::uint8_t pad[16] = {0}; + if (aad_len % 16 != 0) mac.update(pad, 16 - (aad_len % 16)); + if (cipher_len > 0) mac.update(cipher, cipher_len); + if (cipher_len % 16 != 0) mac.update(pad, 16 - (cipher_len % 16)); + std::uint8_t lens[16]; + store64le(lens, aad_len); + store64le(lens + 8, cipher_len); + mac.update(lens, 16); + std::uint8_t expect[16]; + mac.finish(expect); + bool ok = SecureRandom::constant_time_equal(expect, tag, 16); + SecureRandom::zeroize(expect, sizeof expect); + if (!ok) { + SecureRandom::zeroize(kw, sizeof kw); + return false; + } + std::uint8_t stream[64]; + std::size_t done = 0; + std::uint32_t counter = 1; + while (done < cipher_len) { + block(kw, counter, nonce, stream); + ++counter; + std::size_t take = cipher_len - done; + if (take > 64) take = 64; + for (std::size_t i = 0; i < take; ++i) out[done + i] = cipher[done + i] ^ stream[i]; + done += take; + } + SecureRandom::zeroize(stream, sizeof stream); + SecureRandom::zeroize(kw, sizeof kw); + return true; +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/ChaCha20Poly1305.h b/src/security/crypto/ChaCha20Poly1305.h new file mode 100644 index 0000000..973bdb1 --- /dev/null +++ b/src/security/crypto/ChaCha20Poly1305.h @@ -0,0 +1,35 @@ +#pragma once + +// ChaCha20-Poly1305 AEAD (RFC 8439), in-house. +// 256-bit key, 96-bit nonce. Caps: plaintext 16 MiB, AAD 64 KiB. + +#include +#include +#include +#include + +namespace molo::security::crypto { + +class ChaCha20Poly1305 { +public: + static constexpr std::size_t kKeyBytes = 32; + static constexpr std::size_t kNonceBytes = 12; + static constexpr std::size_t kTagBytes = 16; + static constexpr std::size_t kMaxPlainBytes = 16u * 1024u * 1024u; + static constexpr std::size_t kMaxAadBytes = 64u * 1024u; + + static bool seal(const std::uint8_t key[kKeyBytes], const std::uint8_t nonce[kNonceBytes], + const std::uint8_t* aad, std::size_t aad_len, const std::uint8_t* plain, + std::size_t plain_len, std::uint8_t* out, std::uint8_t tag[kTagBytes]); + + // Returns false and leaves out untouched when the tag does not verify. + static bool open(const std::uint8_t key[kKeyBytes], const std::uint8_t nonce[kNonceBytes], + const std::uint8_t* aad, std::size_t aad_len, const std::uint8_t* cipher, + std::size_t cipher_len, const std::uint8_t tag[kTagBytes], std::uint8_t* out); + +private: + static void block(const std::uint32_t key_words[8], std::uint32_t counter, + const std::uint8_t nonce[kNonceBytes], std::uint8_t out[64]); +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/CryptoBox.cpp b/src/security/crypto/CryptoBox.cpp new file mode 100644 index 0000000..b7a0739 --- /dev/null +++ b/src/security/crypto/CryptoBox.cpp @@ -0,0 +1,128 @@ +#include "CryptoBox.h" + +#include + +#include "AesGcm.h" +#include "Base64.h" +#include "ChaCha20Poly1305.h" +#include "KeyHierarchy.h" +#include "SecureRandom.h" + +namespace molo::security::crypto { + +CryptoBox::CryptoBox(KeyHierarchy& keys, Alg alg) : keys_(keys), alg_(alg) {} + +namespace { + +const char* alg_name(CryptoBox::Alg a) { + return a == CryptoBox::Alg::AesGcm ? "aesgcm" : "chacha"; +} + +bool split_envelope(const std::string& env, std::string parts[6]) { + if (env.size() > 4u * 1024u * 1024u + 256) return false; + std::size_t start = 0; + for (int i = 0; i < 6; ++i) { + std::size_t pos = (i == 5) ? std::string::npos : env.find(':', start); + if (i < 5 && pos == std::string::npos) return false; + parts[i] = (i == 5) ? env.substr(start) : env.substr(start, pos - start); + start = pos + 1; + if (parts[i].empty()) return false; + } + return true; +} + +} // namespace + +bool CryptoBox::seal(const std::string& plain, KeyHierarchy::Purpose purpose, std::string& envelope) { + envelope.clear(); + if (plain.size() > kMaxPlainBytes || purpose >= KeyHierarchy::Purpose::Count) return false; + std::uint32_t epoch = keys_.current_epoch(); + if (epoch == 0) return false; + std::uint8_t sub[KeyHierarchy::kSubkeyBytes]; + if (!keys_.subkey(epoch, purpose, sub)) return false; + std::uint8_t nonce[kNonceBytes] = {0}; + if (!test_fixed_nonce_) { + if (!SecureRandom::bytes(nonce, sizeof nonce)) { + SecureRandom::zeroize(sub, sizeof sub); + return false; + } + } + KeyHierarchy::KeyId id{epoch, purpose}; + std::string aad = "v1:" + id.encode() + ":" + alg_name(alg_); + std::vector ct(plain.size()); + std::uint8_t tag[16]; + bool ok = false; + if (alg_ == Alg::AesGcm) { + ok = AesGcm::seal(sub, nonce, reinterpret_cast(aad.data()), aad.size(), + reinterpret_cast(plain.data()), plain.size(), + ct.data(), tag); + } else { + ok = ChaCha20Poly1305::seal(sub, nonce, reinterpret_cast(aad.data()), + aad.size(), reinterpret_cast(plain.data()), + plain.size(), ct.data(), tag); + } + SecureRandom::zeroize(sub, sizeof sub); + if (!ok) { + SecureRandom::zeroize(ct); + return false; + } + envelope = aad + ":" + Base64::encode_url(nonce, sizeof nonce) + ":" + + Base64::encode_url(ct.data(), ct.size()) + ":" + Base64::encode_url(tag, sizeof tag); + SecureRandom::zeroize(ct); + return true; +} + +bool CryptoBox::open(const std::string& envelope, KeyHierarchy::Purpose purpose, std::string& plain) { + plain.clear(); + if (purpose >= KeyHierarchy::Purpose::Count) return false; + std::string parts[6]; + if (!split_envelope(envelope, parts)) return false; + if (parts[0] != "v1") return false; + KeyHierarchy::KeyId id; + if (!KeyHierarchy::KeyId::decode(parts[1], id)) return false; + if (id.purpose != purpose) return false; + Alg alg; + if (parts[2] == "aesgcm") + alg = Alg::AesGcm; + else if (parts[2] == "chacha") + alg = Alg::ChaCha20Poly1305; + else + return false; + std::string nonce_raw, ct_raw, tag_raw; + if (!Base64::decode_url(parts[3], nonce_raw)) return false; + if (!Base64::decode_url(parts[4], ct_raw)) return false; + if (!Base64::decode_url(parts[5], tag_raw)) return false; + if (nonce_raw.size() != kNonceBytes || tag_raw.size() != 16) return false; + if (ct_raw.size() > kMaxPlainBytes) return false; + std::uint8_t sub[32]; + if (!keys_.subkey(id.epoch, purpose, sub)) return false; + std::string aad = parts[0] + ":" + parts[1] + ":" + parts[2]; + std::vector pt(ct_raw.size()); + bool ok = false; + if (alg == Alg::AesGcm) { + ok = AesGcm::open(sub, reinterpret_cast(nonce_raw.data()), + reinterpret_cast(aad.data()), aad.size(), + reinterpret_cast(ct_raw.data()), ct_raw.size(), + reinterpret_cast(tag_raw.data()), pt.data()); + } else { + ok = ChaCha20Poly1305::open(sub, reinterpret_cast(nonce_raw.data()), + reinterpret_cast(aad.data()), aad.size(), + reinterpret_cast(ct_raw.data()), + ct_raw.size(), + reinterpret_cast(tag_raw.data()), pt.data()); + } + SecureRandom::zeroize(sub, sizeof sub); + SecureRandom::zeroize(nonce_raw); + SecureRandom::zeroize(tag_raw); + if (!ok) { + SecureRandom::zeroize(pt); + SecureRandom::zeroize(ct_raw); + return false; + } + plain.assign(reinterpret_cast(pt.data()), pt.size()); + SecureRandom::zeroize(pt); + SecureRandom::zeroize(ct_raw); + return true; +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/CryptoBox.h b/src/security/crypto/CryptoBox.h new file mode 100644 index 0000000..bd3a079 --- /dev/null +++ b/src/security/crypto/CryptoBox.h @@ -0,0 +1,41 @@ +#pragma once + +// Authenticated encryption facade: AES-256-GCM or ChaCha20-Poly1305 +// with key IDs from KeyHierarchy. +// Envelope: "v1:" key_id ":" alg ":" nonce_b64url ":" ct_b64url ":" tag_b64url +// where alg is "aesgcm" or "chacha". AAD binds envelope prefix. + +#include +#include +#include + +#include "KeyHierarchy.h" + +namespace molo::security::crypto { + +class CryptoBox { +public: + enum class Alg : std::uint8_t { AesGcm = 0, ChaCha20Poly1305 }; + + static constexpr std::size_t kMaxPlainBytes = 1u * 1024u * 1024u; + static constexpr std::size_t kNonceBytes = 12; + + explicit CryptoBox(KeyHierarchy& keys, Alg alg = Alg::AesGcm); + + // Encrypts with the current epoch subkey for the purpose. + // Returns false without entropy or over cap. Never throws. + bool seal(const std::string& plain, KeyHierarchy::Purpose purpose, std::string& envelope); + // Decrypts any readable-epoch envelope. Returns false on any error, + // wrong purpose, expired epoch, or tag mismatch. Constant-time tag path. + bool open(const std::string& envelope, KeyHierarchy::Purpose purpose, std::string& plain); + + // Test seam: deterministic nonce (all zero). Production must not call it. + void set_test_fixed_nonce(bool on) { test_fixed_nonce_ = on; } + +private: + KeyHierarchy& keys_; + Alg alg_; + bool test_fixed_nonce_ = false; +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/Hmac.cpp b/src/security/crypto/Hmac.cpp new file mode 100644 index 0000000..9bf548e --- /dev/null +++ b/src/security/crypto/Hmac.cpp @@ -0,0 +1,108 @@ +#include "Hmac.h" + +#include "SecureRandom.h" +#include "Sha256.h" + +namespace molo::security::crypto { + +std::array Hmac::sha256(const std::uint8_t* key, std::size_t key_len, + const std::uint8_t* msg, std::size_t msg_len) { + std::array empty{}; + if (key_len > kMaxKeyBytes || msg_len > kMaxMsgBytes) return empty; + if ((key_len > 0 && key == nullptr) || (msg_len > 0 && msg == nullptr)) return empty; + std::uint8_t kb[64] = {0}; + if (key_len > 64) { + auto h = Sha256::hash(key, key_len); + for (int i = 0; i < 32; ++i) kb[i] = h[i]; + } else { + for (std::size_t i = 0; i < key_len; ++i) kb[i] = key[i]; + } + std::uint8_t ipad[64], opad[64]; + for (int i = 0; i < 64; ++i) { + ipad[i] = kb[i] ^ 0x36; + opad[i] = kb[i] ^ 0x5c; + } + Sha256 inner; + inner.update(ipad, 64); + inner.update(msg, msg_len); + auto ih = inner.final(); + Sha256 outer; + outer.update(opad, 64); + outer.update(ih.data(), 32); + auto out = outer.final(); + SecureRandom::zeroize(kb, sizeof kb); + SecureRandom::zeroize(ipad, sizeof ipad); + SecureRandom::zeroize(opad, sizeof opad); + return out; +} + +std::array Hmac::sha256(const std::string& key, const std::string& msg) { + return sha256(reinterpret_cast(key.data()), key.size(), + reinterpret_cast(msg.data()), msg.size()); +} + +std::string Hmac::hex(const std::array& d) { + static constexpr char kHex[] = "0123456789abcdef"; + std::string out; + out.reserve(64); + for (auto byte : d) { + out.push_back(kHex[byte >> 4]); + out.push_back(kHex[byte & 15]); + } + return out; +} + +bool Hkdf::derive(const std::uint8_t* ikm, std::size_t ikm_len, const std::uint8_t* salt, + std::size_t salt_len, const std::uint8_t* info, std::size_t info_len, + std::uint8_t* out, std::size_t out_len) { + if (out_len == 0 || out_len > kMaxOutBytes || out == nullptr) return false; + if ((ikm_len > 0 && ikm == nullptr) || (salt_len > 0 && salt == nullptr) || + (info_len > 0 && info == nullptr)) + return false; + std::uint8_t zero_salt[32] = {0}; + const std::uint8_t* s = salt_len == 0 ? zero_salt : salt; + std::size_t s_len = salt_len == 0 ? 32 : salt_len; + auto prk = Hmac::sha256(s, s_len, ikm, ikm_len); + std::uint8_t t[32] = {0}; + std::size_t t_len = 0; + std::size_t done = 0; + std::uint8_t counter = 1; + while (done < out_len) { + Sha256 h; + // HMAC with key=prk over (T || info || counter), manual pads. + std::uint8_t kb[64] = {0}; + for (int i = 0; i < 32; ++i) kb[i] = prk[i]; + std::uint8_t ipad[64], opad[64]; + for (int i = 0; i < 64; ++i) { + ipad[i] = kb[i] ^ 0x36; + opad[i] = kb[i] ^ 0x5c; + } + (void)h; + Sha256 inner; + inner.update(ipad, 64); + if (t_len > 0) inner.update(t, t_len); + if (info_len > 0) inner.update(info, info_len); + inner.update(&counter, 1); + auto ih = inner.final(); + Sha256 outer; + outer.update(opad, 64); + outer.update(ih.data(), 32); + auto oh = outer.final(); + for (int i = 0; i < 32; ++i) t[i] = oh[i]; + t_len = 32; + std::size_t take = out_len - done; + if (take > 32) take = 32; + for (std::size_t i = 0; i < take; ++i) out[done + i] = t[i]; + done += take; + ++counter; + if (counter == 0) return false; // 255 block cap exceeded + SecureRandom::zeroize(kb, sizeof kb); + SecureRandom::zeroize(ipad, sizeof ipad); + SecureRandom::zeroize(opad, sizeof opad); + } + SecureRandom::zeroize(t, sizeof t); + SecureRandom::zeroize(const_cast(prk.data()), prk.size()); + return true; +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/Hmac.h b/src/security/crypto/Hmac.h new file mode 100644 index 0000000..89f3f2f --- /dev/null +++ b/src/security/crypto/Hmac.h @@ -0,0 +1,34 @@ +#pragma once + +// HMAC-SHA-256 (RFC 2104) and HKDF-SHA-256 (RFC 5869) for the key hierarchy. + +#include +#include +#include +#include +#include + +namespace molo::security::crypto { + +class Hmac { +public: + static constexpr std::size_t kMaxKeyBytes = 1024; + static constexpr std::size_t kMaxMsgBytes = 16u * 1024u * 1024u; + + static std::array sha256(const std::uint8_t* key, std::size_t key_len, + const std::uint8_t* msg, std::size_t msg_len); + static std::array sha256(const std::string& key, const std::string& msg); + static std::string hex(const std::array& d); +}; + +class Hkdf { +public: + static constexpr std::size_t kMaxOutBytes = 1024; + + // RFC 5869 extract-and-expand with SHA-256. + static bool derive(const std::uint8_t* ikm, std::size_t ikm_len, const std::uint8_t* salt, + std::size_t salt_len, const std::uint8_t* info, std::size_t info_len, + std::uint8_t* out, std::size_t out_len); +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/KeyHierarchy.cpp b/src/security/crypto/KeyHierarchy.cpp new file mode 100644 index 0000000..452390e --- /dev/null +++ b/src/security/crypto/KeyHierarchy.cpp @@ -0,0 +1,144 @@ +#include "KeyHierarchy.h" + +#include + +#include "Hmac.h" +#include "SecureRandom.h" + +namespace molo::security::crypto { + +std::string KeyHierarchy::KeyId::encode() const { + char buf[64]; + // No ':' inside: the ID is one segment of the CryptoBox envelope. + std::snprintf(buf, sizeof buf, "e%up%u", epoch, unsigned(purpose)); + return {buf}; +} + +bool KeyHierarchy::KeyId::decode(const std::string& s, KeyId& out) { + if (s.size() < 4 || s.size() > 32 || s[0] != 'e') return false; + auto pee = s.find('p'); + if (pee == std::string::npos || pee < 2) return false; + unsigned epoch = 0, purpose = 0; + try { + epoch = std::stoul(s.substr(1, pee - 1)); + purpose = std::stoul(s.substr(pee + 1)); + } catch (...) { + return false; + } + if (epoch == 0 || epoch > 0xFFFFFFu) return false; + if (purpose >= unsigned(Purpose::Count)) return false; + out.epoch = epoch; + out.purpose = Purpose(purpose); + return true; +} + +const char* KeyHierarchy::purpose_name(Purpose p) { + switch (p) { + case Purpose::Cookies: return "molo-cookies-v1"; + case Purpose::Sessions: return "molo-sessions-v1"; + case Purpose::Csrf: return "molo-csrf-v1"; + case Purpose::Uploads: return "molo-uploads-v1"; + case Purpose::Data: return "molo-data-v1"; + case Purpose::Count: return "molo-invalid"; + } + return "molo-invalid"; +} + +KeyHierarchy::KeyHierarchy() = default; + +KeyHierarchy::~KeyHierarchy() { + std::lock_guard lock(mu_); + for (std::size_t i = 0; i < kMaxEpochs; ++i) + SecureRandom::zeroize(epochs_[i].master.data(), epochs_[i].master.size()); +} + +bool KeyHierarchy::seed_master(const std::uint8_t* key, std::size_t len) { + if (key == nullptr || len != kMasterKeyBytes) return false; + std::lock_guard lock(mu_); + for (std::size_t i = 0; i < kMaxEpochs; ++i) + SecureRandom::zeroize(epochs_[i].master.data(), epochs_[i].master.size()); + epoch_count_ = 1; + next_epoch_ = 2; + epochs_[0].epoch = 1; + epochs_[0].retired_at = 0; + for (std::size_t i = 0; i < kMasterKeyBytes; ++i) epochs_[0].master[i] = key[i]; + return true; +} + +bool KeyHierarchy::seed_random() { + std::uint8_t k[kMasterKeyBytes]; + if (!SecureRandom::bytes(k, sizeof k)) return false; + bool ok = seed_master(k, sizeof k); + SecureRandom::zeroize(k, sizeof k); + return ok; +} + +bool KeyHierarchy::rotate(std::uint64_t now_unix_sec) { + std::uint8_t fresh[kMasterKeyBytes]; + if (!SecureRandom::bytes(fresh, sizeof fresh)) return false; + std::lock_guard lock(mu_); + if (epoch_count_ == 0) { + SecureRandom::zeroize(fresh, sizeof fresh); + return false; + } + if (now_unix_sec < last_now_) now_unix_sec = last_now_; + last_now_ = now_unix_sec; + // Retire current. + for (std::size_t i = 0; i < epoch_count_; ++i) { + if (epochs_[i].retired_at == 0) epochs_[i].retired_at = now_unix_sec; + } + // Drop epochs past the overlap window or over the hard cap. + EpochKeys kept[kMaxEpochs]{}; + std::size_t n = 0; + for (std::size_t i = 0; i < epoch_count_ && n < kMaxEpochs - 1; ++i) { + std::uint64_t retired = epochs_[i].retired_at; + if (retired != 0 && now_unix_sec - retired > kOverlapSeconds) { + SecureRandom::zeroize(epochs_[i].master.data(), epochs_[i].master.size()); + continue; + } + kept[n++] = epochs_[i]; + } + // Shift when full: evict oldest retired (index 0 is oldest). + while (n >= kMaxEpochs) { + SecureRandom::zeroize(kept[0].master.data(), kept[0].master.size()); + for (std::size_t i = 1; i < n; ++i) kept[i - 1] = kept[i]; + --n; + } + kept[n].epoch = next_epoch_++; + kept[n].retired_at = 0; + for (std::size_t i = 0; i < kMasterKeyBytes; ++i) kept[n].master[i] = fresh[i]; + for (std::size_t i = 0; i < kMaxEpochs; ++i) epochs_[i] = kept[i]; + epoch_count_ = n + 1; + SecureRandom::zeroize(fresh, sizeof fresh); + return true; +} + +bool KeyHierarchy::subkey(std::uint32_t epoch, Purpose purpose, + std::uint8_t out[kSubkeyBytes]) const { + if (out == nullptr || purpose >= Purpose::Count) return false; + std::lock_guard lock(mu_); + for (std::size_t i = 0; i < epoch_count_; ++i) { + if (epochs_[i].epoch != epoch) continue; + if (epochs_[i].retired_at != 0 && last_now_ - epochs_[i].retired_at > kOverlapSeconds) + return false; + const char* info = purpose_name(purpose); + std::size_t info_len = 0; + while (info[info_len] != '\0') ++info_len; + std::uint8_t salt[4] = {std::uint8_t(epoch >> 24), std::uint8_t(epoch >> 16), + std::uint8_t(epoch >> 8), std::uint8_t(epoch)}; + return Hkdf::derive(epochs_[i].master.data(), kMasterKeyBytes, salt, sizeof salt, + reinterpret_cast(info), info_len, out, + kSubkeyBytes); + } + return false; +} + +std::uint32_t KeyHierarchy::current_epoch() const { + std::lock_guard lock(mu_); + for (std::size_t i = 0; i < epoch_count_; ++i) { + if (epochs_[i].retired_at == 0) return epochs_[i].epoch; + } + return 0; +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/KeyHierarchy.h b/src/security/crypto/KeyHierarchy.h new file mode 100644 index 0000000..bf463ba --- /dev/null +++ b/src/security/crypto/KeyHierarchy.h @@ -0,0 +1,67 @@ +#pragma once + +// Key hierarchy: one master key, HKDF-derived per-purpose subkeys, +// rotation epochs with a 24 h dual-read window for the previous epoch. +// Purposes: cookies, sessions, CSRF, uploads, general data. + +#include +#include +#include +#include +#include + +namespace molo::security::crypto { + +class KeyHierarchy { +public: + enum class Purpose : std::uint8_t { Cookies = 0, Sessions, Csrf, Uploads, Data, Count }; + + static constexpr std::size_t kMasterKeyBytes = 32; + static constexpr std::size_t kSubkeyBytes = 32; + static constexpr std::size_t kMaxEpochs = 4; // current plus recent, hard cap + static constexpr std::uint64_t kOverlapSeconds = 24u * 3600u; + + struct KeyId { + std::uint32_t epoch = 0; + Purpose purpose = Purpose::Data; + std::string encode() const; // "ep" + static bool decode(const std::string& s, KeyId& out); + }; + + KeyHierarchy(); + ~KeyHierarchy(); + + KeyHierarchy(const KeyHierarchy&) = delete; + KeyHierarchy& operator=(const KeyHierarchy&) = delete; + + // Installs the master key (32 bytes). Returns false on bad length. + bool seed_master(const std::uint8_t* key, std::size_t len); + // Generates a fresh random master key. Returns false without entropy. + bool seed_random(); + + // Starts a new epoch; previous epoch stays readable for 24 h. + // now_unix_sec is injected so tests can freeze time. + bool rotate(std::uint64_t now_unix_sec); + + // Derives the subkey for (epoch, purpose). Fails for unknown or + // expired epochs. Thread-safe. + bool subkey(std::uint32_t epoch, Purpose purpose, std::uint8_t out[kSubkeyBytes]) const; + + std::uint32_t current_epoch() const; + +private: + struct EpochKeys { + std::uint32_t epoch = 0; + std::uint64_t retired_at = 0; // 0 means current + std::array master{}; + }; + static const char* purpose_name(Purpose p); + + mutable std::mutex mu_; + EpochKeys epochs_[kMaxEpochs]{}; + std::size_t epoch_count_ = 0; + std::uint32_t next_epoch_ = 1; + std::uint64_t last_now_ = 0; +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/PasswordHasher.cpp b/src/security/crypto/PasswordHasher.cpp new file mode 100644 index 0000000..b53b2ef --- /dev/null +++ b/src/security/crypto/PasswordHasher.cpp @@ -0,0 +1,442 @@ +#include "PasswordHasher.h" + +#include +#include + +#include "Base64.h" +#include "Blake2b.h" +#include "SecureRandom.h" + +namespace molo::security::crypto { + +namespace { + +constexpr std::uint32_t kSyncPoints = 4; +constexpr std::size_t kBlockWords = 128; +constexpr std::size_t kBlockBytes = 1024; +constexpr std::uint32_t kAddrsInBlock = 128; // 1024 / 8 + +void store32le(std::uint8_t* p, std::uint32_t v) { + p[0] = std::uint8_t(v); + p[1] = std::uint8_t(v >> 8); + p[2] = std::uint8_t(v >> 16); + p[3] = std::uint8_t(v >> 24); +} + +std::uint64_t load64le(const std::uint8_t* p) { + std::uint64_t v = 0; + for (int i = 0; i < 8; ++i) v |= std::uint64_t(p[i]) << (i * 8); + return v; +} + +void store64le(std::uint8_t* p, std::uint64_t v) { + for (int i = 0; i < 8; ++i) p[i] = std::uint8_t(v >> (i * 8)); +} + +constexpr std::uint64_t rotr64(std::uint64_t x, int n) { return (x >> n) | (x << (64 - n)); } + +// Blake2b-style G without the message words (Argon2 round function core). +inline void blamka(std::uint64_t& a, std::uint64_t& b, std::uint64_t& c, std::uint64_t& d) { + auto f = [](std::uint64_t x, std::uint64_t y) { + return x + y + 2 * (x & 0xFFFFFFFFull) * (y & 0xFFFFFFFFull); + }; + a = f(a, b); + d = rotr64(d ^ a, 32); + c = f(c, d); + b = rotr64(b ^ c, 24); + a = f(a, b); + d = rotr64(d ^ a, 16); + c = f(c, d); + b = rotr64(b ^ c, 63); +} + +void round_fn(std::uint64_t v[16]) { + blamka(v[0], v[4], v[8], v[12]); + blamka(v[1], v[5], v[9], v[13]); + blamka(v[2], v[6], v[10], v[14]); + blamka(v[3], v[7], v[11], v[15]); + blamka(v[0], v[5], v[10], v[15]); + blamka(v[1], v[6], v[11], v[12]); + blamka(v[2], v[7], v[8], v[13]); + blamka(v[3], v[4], v[9], v[14]); +} + +struct Block { + std::uint64_t v[kBlockWords]{}; +}; + +// Compression function G: next = R xor P(R) (xor old next when with_xor). +void fill_block(const Block& prev, const Block& ref, Block& next, bool with_xor) { + Block r, z; + for (std::size_t i = 0; i < kBlockWords; ++i) { + r.v[i] = prev.v[i] ^ ref.v[i]; + z.v[i] = r.v[i]; + } + for (std::size_t i = 0; i < 8; ++i) round_fn(z.v + i * 16); + for (std::size_t i = 0; i < 8; ++i) { + std::uint64_t row[16]; + std::size_t k = 0; + for (std::size_t j = 0; j < 8; ++j) { + row[k++] = z.v[2 * i + 16 * j]; + row[k++] = z.v[2 * i + 16 * j + 1]; + } + round_fn(row); + k = 0; + for (std::size_t j = 0; j < 8; ++j) { + z.v[2 * i + 16 * j] = row[k++]; + z.v[2 * i + 16 * j + 1] = row[k++]; + } + } + for (std::size_t i = 0; i < kBlockWords; ++i) { + std::uint64_t w = r.v[i] ^ z.v[i]; + next.v[i] = with_xor ? (next.v[i] ^ w) : w; + } +} + +// Variable-length hash H' from RFC 9106 section 3.3. +std::vector hash_prime(const std::uint8_t* x, std::size_t x_len, std::uint32_t out_len) { + std::vector out; + if (out_len == 0) return out; + out.reserve(out_len); + if (out_len <= 64) { + std::vector in(4 + x_len); + store32le(in.data(), out_len); + if (x_len > 0) std::memcpy(in.data() + 4, x, x_len); + return Blake2b::hash(in.data(), in.size(), out_len); + } + // V1 over (len || X), then chained V blocks emitting 32 bytes each + // while more than 64 bytes remain, then one final block of the rest. + std::vector in(4 + x_len); + store32le(in.data(), out_len); + if (x_len > 0) std::memcpy(in.data() + 4, x, x_len); + std::vector v = Blake2b::hash(in.data(), in.size(), 64); + out.insert(out.end(), v.begin(), v.begin() + 32); + std::uint32_t remaining = out_len - 32; + while (remaining > 64) { + v = Blake2b::hash(v.data(), 64, 64); + out.insert(out.end(), v.begin(), v.begin() + 32); + remaining -= 32; + } + v = Blake2b::hash(v.data(), 64, remaining); + out.insert(out.end(), v.begin(), v.end()); + return out; +} + +void next_addresses(Block& addr, Block& input, const Block& zero) { + input.v[6]++; + Block tmp = addr; + fill_block(zero, input, addr, false); + Block addr_copy = addr; + fill_block(zero, addr_copy, addr, false); + SecureRandom::zeroize(&tmp, sizeof tmp); +} + +// Reference mapping from pseudo-random value to lane position. +std::uint32_t index_alpha(std::uint32_t pass, std::uint32_t slice, std::uint32_t index, + std::uint32_t pseudo_rand, std::uint32_t lane_length, + std::uint32_t segment_length, bool same_lane) { + std::uint32_t ref_area; + if (pass == 0) { + if (slice == 0) { + ref_area = index - 1; + } else if (same_lane) { + ref_area = slice * segment_length + index - 1; + } else { + ref_area = slice * segment_length + (index == 0 ? std::uint32_t(-1) : 0); + } + } else if (same_lane) { + ref_area = lane_length - segment_length + index - 1; + } else { + ref_area = lane_length - segment_length + (index == 0 ? std::uint32_t(-1) : 0); + } + std::uint64_t rel = pseudo_rand; + rel = (rel * rel) >> 32; + rel = ref_area - 1 - ((std::uint64_t(ref_area) * rel) >> 32); + std::uint32_t start = 0; + if (pass != 0) start = (slice == kSyncPoints - 1) ? 0 : (slice + 1) * segment_length; + return std::uint32_t((start + rel) % lane_length); +} + +std::string b64_nopad_encode(const std::uint8_t* d, std::size_t n) { + std::string s = Base64::encode(d, n); + while (!s.empty() && s.back() == '=') s.pop_back(); + return s; +} + +bool b64_nopad_decode(const std::string& s, std::string& out) { + if (s.size() % 4 == 1) return false; + std::string padded = s; + while (padded.size() % 4 != 0) padded.push_back('='); + return Base64::decode(padded, out); +} + +} // namespace + +bool PasswordHasher::valid_params(const Params& p) { + if (p.lanes < 1 || p.lanes > kMaxLanes) return false; + if (p.passes < 1 || p.passes > kMaxPasses) return false; + if (p.hash_len < 4 || p.hash_len > kMaxHashBytes) return false; + if (p.salt_len < 8 || p.salt_len > kMaxSaltBytes) return false; + if (p.memory_kib < 8 * p.lanes) return false; + if (p.memory_kib > kMaxMemoryKib) return false; + return true; +} + +bool PasswordHasher::derive(const std::string& password, const std::string& salt_raw, + const std::string& secret, const std::string& ad, const Params& params, + std::vector& tag_out) { + tag_out.clear(); + if (!valid_params(params)) return false; + if (password.size() > kMaxPasswordBytes) return false; + if (salt_raw.size() != params.salt_len) return false; + if (secret.size() > kMaxPasswordBytes || ad.size() > kMaxPasswordBytes) return false; + + const std::uint32_t lanes = params.lanes; + std::uint32_t memory_blocks = params.memory_kib; + if (memory_blocks < 2 * kSyncPoints * lanes) memory_blocks = 2 * kSyncPoints * lanes; + std::uint32_t segment_length = memory_blocks / (lanes * kSyncPoints); + memory_blocks = segment_length * lanes * kSyncPoints; + const std::uint32_t lane_length = segment_length * kSyncPoints; + + // H: initial 64 bytes. + Blake2b h0(64); + auto le32 = [&](std::uint32_t v) { + std::uint8_t b[4]; + store32le(b, v); + h0.update(b, 4); + }; + le32(lanes); + le32(params.hash_len); + le32(memory_blocks); + le32(params.passes); + le32(kVersion); + le32(2); // Argon2id + le32(std::uint32_t(password.size())); + if (!password.empty()) + h0.update(reinterpret_cast(password.data()), password.size()); + le32(std::uint32_t(salt_raw.size())); + h0.update(reinterpret_cast(salt_raw.data()), salt_raw.size()); + le32(std::uint32_t(secret.size())); + if (!secret.empty()) + h0.update(reinterpret_cast(secret.data()), secret.size()); + le32(std::uint32_t(ad.size())); + if (!ad.empty()) h0.update(reinterpret_cast(ad.data()), ad.size()); + std::vector h_digest = h0.final(); + + std::vector memory; + try { + memory.resize(memory_blocks); + } catch (...) { + return false; + } + + // First two blocks per lane. + for (std::uint32_t lane = 0; lane < lanes; ++lane) { + for (std::uint32_t idx = 0; idx < 2; ++idx) { + std::vector in(72); + std::memcpy(in.data(), h_digest.data(), 64); + store32le(in.data() + 64, idx); + store32le(in.data() + 68, lane); + std::vector blk = hash_prime(in.data(), in.size(), kBlockBytes); + Block& b = memory[lane * lane_length + idx]; + for (std::size_t i = 0; i < kBlockWords; ++i) + b.v[i] = load64le(blk.data() + i * 8); + SecureRandom::zeroize(blk); + } + } + + Block zero{}; + for (std::uint32_t pass = 0; pass < params.passes; ++pass) { + for (std::uint32_t slice = 0; slice < kSyncPoints; ++slice) { + bool indep = (pass == 0 && slice < kSyncPoints / 2); // Argon2id rule + Block addr{}, input{}; + std::uint32_t start_idx = (pass == 0 && slice == 0) ? 2 : 0; + if (indep) { + input.v[0] = pass; + input.v[1] = 0; // lane set per lane below + input.v[2] = slice; + input.v[3] = memory_blocks; + input.v[4] = params.passes; + input.v[5] = 2; // Argon2id + } + for (std::uint32_t lane = 0; lane < lanes; ++lane) { + Block lane_addr = addr; + Block lane_input = input; + if (indep) { + lane_input.v[1] = lane; + // First address block up front: the opening segment + // starts at offset 2 and would otherwise miss the + // modulo trigger and read a zero block. + next_addresses(lane_addr, lane_input, zero); + } + for (std::uint32_t off = start_idx; off < segment_length; ++off) { + std::uint32_t index = slice * segment_length + off; + Block& cur = memory[lane * lane_length + index]; + const Block& prev = + (index == 0) ? memory[lane * lane_length + lane_length - 1] + : memory[lane * lane_length + index - 1]; + std::uint32_t pseudo; + if (indep) { + if (off != start_idx && off % kAddrsInBlock == 0) + next_addresses(lane_addr, lane_input, zero); + pseudo = std::uint32_t(lane_addr.v[off % kAddrsInBlock] & 0xFFFFFFFFull); + // ref lane uses upper half: + std::uint32_t pseudo_hi = + std::uint32_t(lane_addr.v[off % kAddrsInBlock] >> 32); + std::uint32_t ref_lane = pseudo_hi % lanes; + if (pass == 0 && slice == 0) ref_lane = lane; + // index_alpha takes the in-segment offset, not the + // absolute lane index (matches reference vectors). + std::uint32_t ref_index = + index_alpha(pass, slice, off, pseudo, lane_length, segment_length, + ref_lane == lane); + const Block& ref = memory[ref_lane * lane_length + ref_index]; + Block prev_copy = prev; + fill_block(prev_copy, ref, cur, pass != 0); + } else { + pseudo = std::uint32_t(prev.v[0] & 0xFFFFFFFFull); + std::uint32_t ref_lane = std::uint32_t(prev.v[0] >> 32) % lanes; + if (pass == 0 && slice == 0) ref_lane = lane; + std::uint32_t ref_index = + index_alpha(pass, slice, off, pseudo, lane_length, segment_length, + ref_lane == lane); + const Block& ref = memory[ref_lane * lane_length + ref_index]; + Block prev_copy = prev; + fill_block(prev_copy, ref, cur, pass != 0); + } + } + SecureRandom::zeroize(&lane_addr, sizeof lane_addr); + SecureRandom::zeroize(&lane_input, sizeof lane_input); + } + } + } + + // Final block: xor of last column. + Block final{}; + for (std::uint32_t lane = 0; lane < lanes; ++lane) { + const Block& b = memory[lane * lane_length + lane_length - 1]; + for (std::size_t i = 0; i < kBlockWords; ++i) final.v[i] ^= b.v[i]; + } + std::uint8_t final_bytes[kBlockBytes]; + for (std::size_t i = 0; i < kBlockWords; ++i) store64le(final_bytes + i * 8, final.v[i]); + tag_out = hash_prime(final_bytes, sizeof final_bytes, params.hash_len); + SecureRandom::zeroize(&final, sizeof final); + SecureRandom::zeroize(final_bytes, sizeof final_bytes); + SecureRandom::zeroize(memory.data(), memory.size() * sizeof(Block)); + SecureRandom::zeroize(h_digest); + return tag_out.size() == params.hash_len; +} + +std::string PasswordHasher::encode(const Params& params, const std::string& salt_raw, + const std::vector& tag) { + char buf[128]; + std::snprintf(buf, sizeof buf, "$argon2id$v=%u$m=%u,t=%u,p=%u$", kVersion, params.memory_kib, + params.passes, params.lanes); + std::string out = buf; + out += b64_nopad_encode(reinterpret_cast(salt_raw.data()), salt_raw.size()); + out += '$'; + out += b64_nopad_encode(tag.data(), tag.size()); + return out; +} + +bool PasswordHasher::parse(const std::string& encoded, Params& params, std::string& salt_raw, + std::string& hash_raw) { + params = Params{}; + salt_raw.clear(); + hash_raw.clear(); + if (encoded.size() > 1024) return false; + // $argon2id$v=19$m=N,t=N,p=N$salt$hash + const char* prefix = "$argon2id$v="; + if (encoded.compare(0, std::strlen(prefix), prefix) != 0) return false; + std::size_t pos = std::strlen(prefix); + std::size_t dollar = encoded.find('$', pos); + if (dollar == std::string::npos) return false; + unsigned version = 0; + try { + version = std::stoul(encoded.substr(pos, dollar - pos)); + } catch (...) { + return false; + } + if (version != kVersion) return false; + pos = dollar + 1; + dollar = encoded.find('$', pos); + if (dollar == std::string::npos) return false; + unsigned m = 0, t = 0, p = 0; + if (std::sscanf(encoded.substr(pos, dollar - pos).c_str(), "m=%u,t=%u,p=%u", &m, &t, &p) != 3) + return false; + pos = dollar + 1; + dollar = encoded.find('$', pos); + if (dollar == std::string::npos) return false; + std::string salt_b64 = encoded.substr(pos, dollar - pos); + std::string hash_b64 = encoded.substr(dollar + 1); + if (salt_b64.empty() || hash_b64.empty()) return false; + if (salt_b64.size() > 128 || hash_b64.size() > 128) return false; + if (!b64_nopad_decode(salt_b64, salt_raw)) return false; + if (!b64_nopad_decode(hash_b64, hash_raw)) return false; + params.memory_kib = m; + params.passes = t; + params.lanes = p; + params.salt_len = std::uint32_t(salt_raw.size()); + params.hash_len = std::uint32_t(hash_raw.size()); + return valid_params(params); +} + +bool PasswordHasher::needs_rehash(const std::string& encoded, const Params& want) { + Params have; + std::string salt, hash; + if (!parse(encoded, have, salt, hash)) return true; + if (have.memory_kib < want.memory_kib) return true; + if (have.passes < want.passes) return true; + if (have.lanes != want.lanes) return true; + if (have.hash_len < want.hash_len) return true; + return false; +} + +bool PasswordHasher::hash_password(const std::string& password, const Params& params, + std::string& encoded_out) { + return hash_password_peppered(password, "", params, encoded_out); +} + +bool PasswordHasher::hash_password_peppered(const std::string& password, const std::string& pepper, + const Params& params, std::string& encoded_out) { + encoded_out.clear(); + if (!valid_params(params) || password.size() > kMaxPasswordBytes) return false; + auto salt = SecureRandom::bytes(params.salt_len); + if (salt.size() != params.salt_len) return false; + std::string salt_raw(reinterpret_cast(salt.data()), salt.size()); + std::vector tag; + bool ok = derive(password, salt_raw, pepper, "", params, tag); + SecureRandom::zeroize(salt); + if (!ok) { + SecureRandom::zeroize(salt_raw); + return false; + } + encoded_out = encode(params, salt_raw, tag); + SecureRandom::zeroize(salt_raw); + SecureRandom::zeroize(tag); + return true; +} + +bool PasswordHasher::verify(const std::string& password, const std::string& encoded) { + return verify_peppered(password, "", encoded); +} + +bool PasswordHasher::verify_peppered(const std::string& password, const std::string& pepper, + const std::string& encoded) { + Params params; + std::string salt_raw, hash_raw; + if (!parse(encoded, params, salt_raw, hash_raw)) return false; + // Cap verify work: refuse absurd params even if structurally valid. + if (params.memory_kib > kMaxMemoryKib || params.passes > kMaxPasses) return false; + std::vector tag; + if (!derive(password, salt_raw, pepper, "", params, tag)) return false; + bool ok = tag.size() == hash_raw.size() && + SecureRandom::constant_time_equal( + tag.data(), reinterpret_cast(hash_raw.data()), tag.size()); + SecureRandom::zeroize(tag); + SecureRandom::zeroize(salt_raw); + SecureRandom::zeroize(hash_raw); + return ok; +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/PasswordHasher.h b/src/security/crypto/PasswordHasher.h new file mode 100644 index 0000000..6d7de7d --- /dev/null +++ b/src/security/crypto/PasswordHasher.h @@ -0,0 +1,69 @@ +#pragma once + +// Argon2id password hashing (RFC 9106), in-house. +// OWASP defaults: m=19456 KiB, t=2, p=1, 16-byte salt, 32-byte hash. +// Stored format: $argon2id$v=19$m=...,t=...,p=...$salt$hash (unpadded b64). +// Verify is blocking and must run off the event loop (see 01-runtime.md). + +#include +#include +#include +#include + +namespace molo::security::crypto { + +class PasswordHasher { +public: + struct Params { + std::uint32_t memory_kib = 19456; + std::uint32_t passes = 2; + std::uint32_t lanes = 1; + std::uint32_t hash_len = 32; + std::uint32_t salt_len = 16; + }; + + static constexpr std::uint32_t kVersion = 19; + static constexpr std::size_t kMaxPasswordBytes = 1024; + static constexpr std::size_t kMaxSaltBytes = 64; + static constexpr std::size_t kMaxHashBytes = 64; + static constexpr std::uint32_t kMaxMemoryKib = 2u * 1024u * 1024u; // 2 GiB cap + static constexpr std::uint32_t kMaxPasses = 16; + static constexpr std::uint32_t kMaxLanes = 8; + + static Params owasp_defaults() { return {}; } + + // Parses an encoded hash. Returns false on any malformed input. + static bool parse(const std::string& encoded, Params& params, std::string& salt_raw, + std::string& hash_raw); + + // True when stored params are weaker than want (caller re-hashes on login). + static bool needs_rehash(const std::string& encoded, const Params& want); + + // Raw Argon2id (type id) over password+salt. Empty password allowed. + // Optional secret (pepper) and associated data. Returns false on bad + // params or allocation failure. Deterministic. + static bool derive(const std::string& password, const std::string& salt_raw, + const std::string& secret, const std::string& ad, const Params& params, + std::vector& tag_out); + + // Hashes with a fresh random salt. Encoded output for storage. + static bool hash_password(const std::string& password, const Params& params, + std::string& encoded_out); + // Constant-time verify against an encoded hash. False on any mismatch + // or malformed hash. Parses params from the hash itself. + static bool verify(const std::string& password, const std::string& encoded); + + // Same but with an explicit pepper (stored outside the DB). + static bool hash_password_peppered(const std::string& password, const std::string& pepper, + const Params& params, std::string& encoded_out); + static bool verify_peppered(const std::string& password, const std::string& pepper, + const std::string& encoded); + + static std::string encode(const Params& params, const std::string& salt_raw, + const std::vector& tag); + +private: + static bool valid_params(const Params& params); +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/SecureRandom.cpp b/src/security/crypto/SecureRandom.cpp new file mode 100644 index 0000000..7ebc4d6 --- /dev/null +++ b/src/security/crypto/SecureRandom.cpp @@ -0,0 +1,92 @@ +#include "SecureRandom.h" + +#include + +#if defined(__linux__) || defined(__unix__) +#include +#include +#if defined(__linux__) +#include +#endif +#endif + +#include "Base64.h" + +namespace molo::security::crypto { + +bool SecureRandom::bytes(std::uint8_t* out, std::size_t len) { + if (len > kMaxBytesPerCall) return false; + if (len == 0) return true; + if (out == nullptr) return false; +#if defined(__linux__) + std::size_t done = 0; + while (done < len) { + ssize_t n = ::getrandom(out + done, len - done, 0); + if (n < 0) break; + if (n == 0) break; + done += std::size_t(n); + } + if (done == len) return true; + zeroize(out, len); +#endif +#if defined(__unix__) + int fd = ::open("/dev/urandom", O_RDONLY | O_CLOEXEC); + if (fd < 0) return false; + std::size_t got = 0; + while (got < len) { + ssize_t n = ::read(fd, out + got, len - got); + if (n <= 0) { + ::close(fd); + zeroize(out, len); + return false; + } + got += std::size_t(n); + } + ::close(fd); + return true; +#else + (void)out; + return false; +#endif +} + +std::vector SecureRandom::bytes(std::size_t len) { + std::vector out(len); + if (!bytes(out.data(), len)) out.clear(); + return out; +} + +std::string SecureRandom::token(std::size_t len) { + auto raw = bytes(len); + if (raw.size() != len) return {}; + return Base64::encode_url(raw.data(), raw.size()); +} + +bool SecureRandom::constant_time_equal(const std::uint8_t* a, const std::uint8_t* b, std::size_t len) { + if (len > 0 && (a == nullptr || b == nullptr)) return false; + volatile std::uint8_t diff = 0; + for (std::size_t i = 0; i < len; ++i) diff |= std::uint8_t(a[i] ^ b[i]); + return diff == 0; +} + +bool SecureRandom::constant_time_equal(const std::string& a, const std::string& b) { + if (a.size() != b.size()) return false; + return constant_time_equal(reinterpret_cast(a.data()), + reinterpret_cast(b.data()), a.size()); +} + +void SecureRandom::zeroize(void* p, std::size_t len) { + if (p == nullptr || len == 0) return; + volatile std::uint8_t* q = static_cast(p); + for (std::size_t i = 0; i < len; ++i) q[i] = 0; +} + +void SecureRandom::zeroize(std::string& s) { + if (!s.empty()) zeroize(s.data(), s.size()); +} + +void SecureRandom::zeroize(std::vector& v) { + if (!v.empty()) zeroize(v.data(), v.size()); +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/SecureRandom.h b/src/security/crypto/SecureRandom.h new file mode 100644 index 0000000..ba84135 --- /dev/null +++ b/src/security/crypto/SecureRandom.h @@ -0,0 +1,37 @@ +#pragma once + +// System CSPRNG plus token helpers and constant-time compare. +// Source: getrandom(2), fallback /dev/urandom. Zeroizes key material. + +#include +#include +#include +#include + +namespace molo::security::crypto { + +class SecureRandom { +public: + static constexpr std::size_t kMaxBytesPerCall = 64u * 1024u; + static constexpr std::size_t kTokenBytesDefault = 32; + + // Fills out with cryptographic random bytes. Returns false only when + // the OS provides no entropy source (fail closed, caller must abort). + static bool bytes(std::uint8_t* out, std::size_t len); + static std::vector bytes(std::size_t len); + + // base64url token without padding, len random bytes. + static std::string token(std::size_t len = kTokenBytesDefault); + + // Constant-time equality. Lengths must match; returns false otherwise + // without leaking prefix length through timing. + static bool constant_time_equal(const std::uint8_t* a, const std::uint8_t* b, std::size_t len); + static bool constant_time_equal(const std::string& a, const std::string& b); + + // Best-effort memory zeroize that the compiler must not elide. + static void zeroize(void* p, std::size_t len); + static void zeroize(std::string& s); + static void zeroize(std::vector& v); +}; + +} // namespace molo::security::crypto diff --git a/src/security/crypto/Sha256.cpp b/src/security/crypto/Sha256.cpp new file mode 100644 index 0000000..77f70e8 --- /dev/null +++ b/src/security/crypto/Sha256.cpp @@ -0,0 +1,121 @@ +#include "Sha256.h" + +namespace molo::security::crypto { + +namespace { + +constexpr std::uint32_t kK[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +constexpr std::uint32_t rotr(std::uint32_t x, int n) { return (x >> n) | (x << (32 - n)); } + +} // namespace + +Sha256::Sha256() + : h_{0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}, + buf_{} {} + +void Sha256::transform(const std::uint8_t* block) { + std::uint32_t w[64]; + for (int i = 0; i < 16; ++i) + w[i] = (std::uint32_t(block[i * 4]) << 24) | (std::uint32_t(block[i * 4 + 1]) << 16) | + (std::uint32_t(block[i * 4 + 2]) << 8) | std::uint32_t(block[i * 4 + 3]); + for (int i = 16; i < 64; ++i) { + std::uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + std::uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + std::uint32_t a = h_[0], b = h_[1], c = h_[2], d = h_[3]; + std::uint32_t e = h_[4], f = h_[5], g = h_[6], h = h_[7]; + for (int i = 0; i < 64; ++i) { + std::uint32_t s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + std::uint32_t ch = (e & f) ^ (~e & g); + std::uint32_t t1 = h + s1 + ch + kK[i] + w[i]; + std::uint32_t s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + std::uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + std::uint32_t t2 = s0 + maj; + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + h_[0] += a; + h_[1] += b; + h_[2] += c; + h_[3] += d; + h_[4] += e; + h_[5] += f; + h_[6] += g; + h_[7] += h; +} + +void Sha256::update(const std::uint8_t* data, std::size_t len) { + if (len > 0 && data == nullptr) return; + total_ += len; + while (len > 0) { + std::size_t take = kBlockBytes - buf_used_; + if (take > len) take = len; + for (std::size_t i = 0; i < take; ++i) buf_[buf_used_ + i] = data[i]; + buf_used_ += take; + data += take; + len -= take; + if (buf_used_ == kBlockBytes) { + transform(buf_.data()); + buf_used_ = 0; + } + } +} + +std::array Sha256::final() { + std::uint64_t bit_len = total_ * 8; + std::uint8_t pad = 0x80; + update(&pad, 1); + std::uint8_t zero = 0x00; + while (buf_used_ != 56) update(&zero, 1); + std::uint8_t len_bytes[8]; + for (int i = 0; i < 8; ++i) len_bytes[i] = std::uint8_t(bit_len >> (56 - i * 8)); + // Feed length directly without touching total_ again. + for (int i = 0; i < 8; ++i) buf_[buf_used_ + i] = len_bytes[i]; + buf_used_ += 8; + transform(buf_.data()); + buf_used_ = 0; + std::array out{}; + for (int i = 0; i < 8; ++i) { + out[i * 4] = std::uint8_t(h_[i] >> 24); + out[i * 4 + 1] = std::uint8_t(h_[i] >> 16); + out[i * 4 + 2] = std::uint8_t(h_[i] >> 8); + out[i * 4 + 3] = std::uint8_t(h_[i]); + } + return out; +} + +std::array Sha256::hash(const std::uint8_t* data, std::size_t len) { + Sha256 s; + s.update(data, len); + return s.final(); +} + +std::string Sha256::hex(const std::array& d) { + static constexpr char kHex[] = "0123456789abcdef"; + std::string out; + out.reserve(64); + for (auto byte : d) { + out.push_back(kHex[byte >> 4]); + out.push_back(kHex[byte & 15]); + } + return out; +} + +} // namespace molo::security::crypto diff --git a/src/security/crypto/Sha256.h b/src/security/crypto/Sha256.h new file mode 100644 index 0000000..a474bf9 --- /dev/null +++ b/src/security/crypto/Sha256.h @@ -0,0 +1,40 @@ +#pragma once + +// SHA-256 (FIPS 180-4), in-house, no dependencies. +// One-shot and streaming API. Not thread-safe per instance. + +#include +#include +#include +#include +#include + +namespace molo::security::crypto { + +class Sha256 { +public: + static constexpr std::size_t kDigestBytes = 32; + static constexpr std::size_t kBlockBytes = 64; + + Sha256(); + void update(const std::uint8_t* data, std::size_t len); + void update(const std::string& s) { + update(reinterpret_cast(s.data()), s.size()); + } + std::array final(); + + static std::array hash(const std::uint8_t* data, std::size_t len); + static std::array hash(const std::string& s) { + return hash(reinterpret_cast(s.data()), s.size()); + } + static std::string hex(const std::array& d); + +private: + void transform(const std::uint8_t* block); + std::array h_; + std::array buf_; + std::size_t buf_used_ = 0; + std::uint64_t total_ = 0; +}; + +} // namespace molo::security::crypto diff --git a/src/security/guard/BodyLimits.cpp b/src/security/guard/BodyLimits.cpp new file mode 100644 index 0000000..7958801 --- /dev/null +++ b/src/security/guard/BodyLimits.cpp @@ -0,0 +1,94 @@ +#include "BodyLimits.h" + +#include "DenialSink.h" +#include "HttpMessageView.h" + +namespace molo::security::guard { + +BodyLimits::BodyLimits(Policy policy, IDenialSink& sink) + : policy_(std::move(policy)), sink_(sink) {} + +BodyLimits::Verdict BodyLimits::check_headers(std::size_t total_bytes, std::size_t largest_single, + const HttpRequestView& req) const { + if (total_bytes > policy_.max_headers_total) { + sink_.deny({"limits", "limits.headers_total", "header block over 16 KB", req.peer_ip, + req.path}); + return Verdict::TooManyHeaders; + } + if (largest_single > policy_.max_single_header) { + sink_.deny({"limits", "limits.header_single", "single header over 8 KB", req.peer_ip, + req.path}); + return Verdict::HeaderTooLarge; + } + return Verdict::Allow; +} + +BodyLimits::Verdict BodyLimits::check_body(std::size_t content_length, std::size_t route_max, + const HttpRequestView& req) const { + std::size_t cap = policy_.max_body; + if (route_max > 0) { + cap = route_max > policy_.max_body_route_override ? policy_.max_body_route_override + : route_max; + } + if (content_length > cap) { + sink_.deny({"limits", "limits.body", "content-length over cap", req.peer_ip, req.path}); + return Verdict::BodyTooLarge; + } + return Verdict::Allow; +} + +bool BodyLimits::check_json_depth(const std::string& json, const HttpRequestView& req) const { + std::uint32_t depth = 0; + bool in_string = false; + bool escape = false; + for (char c : json) { + if (in_string) { + if (escape) + escape = false; + else if (c == '\\') + escape = true; + else if (c == '"') + in_string = false; + continue; + } + if (c == '"') { + in_string = true; + } else if (c == '{' || c == '[') { + ++depth; + if (depth > policy_.max_json_depth) { + sink_.deny({"limits", "limits.json_depth", "nesting over 32", req.peer_ip, + req.path}); + return false; + } + } else if (c == '}' || c == ']') { + if (depth > 0) --depth; + } + } + return true; +} + +bool BodyLimits::check_form_fields(std::size_t count, const HttpRequestView& req) const { + if (count > policy_.max_form_fields) { + sink_.deny({"limits", "limits.form_fields", "field count over 1000", req.peer_ip, + req.path}); + return false; + } + return true; +} + +bool BodyLimits::enforce(Verdict v, const HttpRequestView&, HttpResponseView& res) const { + switch (v) { + case Verdict::Allow: return true; + case Verdict::TooManyHeaders: + case Verdict::HeaderTooLarge: + res.status = 431; + return false; + case Verdict::BodyTooLarge: + res.status = 413; + return false; + } + res.status = 413; + return false; +} + +} // namespace molo::security::guard diff --git a/src/security/guard/BodyLimits.h b/src/security/guard/BodyLimits.h new file mode 100644 index 0000000..b8bcca8 --- /dev/null +++ b/src/security/guard/BodyLimits.h @@ -0,0 +1,57 @@ +#pragma once + +// Input caps enforced before parsing completes (plan rule 10). +// Over-cap requests fail closed with 413 or 431 before the handler runs. +// Wired before routing (headers) and before body parsing (body/fields). + +#include +#include +#include + +namespace molo::security::guard { +struct HttpRequestView; +struct HttpResponseView; +class IDenialSink; +} // namespace molo::security::guard + +namespace molo::security::guard { + +class BodyLimits { +public: + struct Policy { + std::size_t max_headers_total = 16u * 1024u; + std::size_t max_single_header = 8u * 1024u; + std::size_t max_body = 1u * 1024u * 1024u; // route override to 10 MB + std::size_t max_body_route_override = 10u * 1024u * 1024u; + std::uint32_t max_form_fields = 1000; + std::uint32_t max_json_depth = 32; + }; + + BodyLimits(Policy policy, IDenialSink& sink); + + enum class Verdict : std::uint8_t { Allow, TooManyHeaders, HeaderTooLarge, BodyTooLarge }; + + // Checks raw header block sizes (called with the byte counts the + // runtime measured, before building the request object). + Verdict check_headers(std::size_t total_bytes, std::size_t largest_single, + const HttpRequestView& req) const; + // Checks Content-Length against the cap before reading the body. + // route_max of 0 means default; capped by max_body_route_override. + Verdict check_body(std::size_t content_length, std::size_t route_max, + const HttpRequestView& req) const; + // Validates JSON depth of an already size-capped buffer. False over cap. + bool check_json_depth(const std::string& json, const HttpRequestView& req) const; + // Validates form field count of an already size-capped parse. + bool check_form_fields(std::size_t count, const HttpRequestView& req) const; + + // Maps a verdict onto 200/413/431. Returns true when allowed. + bool enforce(Verdict v, const HttpRequestView& req, HttpResponseView& res) const; + + const Policy& policy() const { return policy_; } + +private: + Policy policy_; + IDenialSink& sink_; +}; + +} // namespace molo::security::guard diff --git a/src/security/guard/CorsPolicy.cpp b/src/security/guard/CorsPolicy.cpp new file mode 100644 index 0000000..1229b28 --- /dev/null +++ b/src/security/guard/CorsPolicy.cpp @@ -0,0 +1,91 @@ +#include "CorsPolicy.h" + +#include "DenialSink.h" +#include "HttpMessageView.h" + +namespace molo::security::guard { + +CorsPolicy::CorsPolicy(Policy policy, IDenialSink& sink) + : policy_(std::move(policy)), sink_(sink) { + if (policy_.allow_origins.size() > kMaxOrigins) policy_.allow_origins.resize(kMaxOrigins); + if (policy_.allow_methods.size() > kMaxMethods) policy_.allow_methods.resize(kMaxMethods); + if (policy_.allow_headers.size() > kMaxHeaders) policy_.allow_headers.resize(kMaxHeaders); + if (policy_.preflight_cache_sec > kMaxPreflightCacheSec) + policy_.preflight_cache_sec = kMaxPreflightCacheSec; +} + +std::string CorsPolicy::validate(const Policy& p) { + if (p.allow_credentials) { + for (const auto& o : p.allow_origins) { + if (o == "*") return "cors: wildcard origin with credentials is rejected"; + } + } + if (p.allow_origins.size() > kMaxOrigins) return "cors: too many origins"; + if (p.allow_methods.size() > kMaxMethods) return "cors: too many methods"; + if (p.allow_headers.size() > kMaxHeaders) return "cors: too many headers"; + if (p.preflight_cache_sec > kMaxPreflightCacheSec) return "cors: preflight cache over 600s"; + return {}; +} + +bool CorsPolicy::origin_allowed(const std::string& origin) const { + if (origin.empty()) return false; + for (const auto& o : policy_.allow_origins) { + if (o == origin) return true; + if (o == "*" && !policy_.allow_credentials) return true; + } + return false; +} + +static std::string join(const std::vector& v) { + std::string out; + for (std::size_t i = 0; i < v.size(); ++i) { + if (i > 0) out += ", "; + out += v[i]; + } + return out; +} + +void CorsPolicy::apply(const HttpRequestView& req, HttpResponseView& res) const { + std::string origin = req.header("origin"); + if (origin.empty()) return; // not a CORS request + if (!origin_allowed(origin)) { + sink_.deny({"cors", "cors.origin_denied", "origin not allow-listed", req.peer_ip, req.path}); + return; // default deny: no allow headers at all + } + for (const auto& o : policy_.allow_origins) { + if (o == "*" && !policy_.allow_credentials) { + res.set_header("Access-Control-Allow-Origin", "*"); + return; + } + } + res.set_header("Access-Control-Allow-Origin", origin); + res.add_header("Vary", "Origin"); + if (policy_.allow_credentials) res.set_header("Access-Control-Allow-Credentials", "true"); +} + +bool CorsPolicy::preflight(const HttpRequestView& req, HttpResponseView& res) const { + if (req.method != "OPTIONS") return false; + if (req.header("access-control-request-method").empty()) return false; + std::string origin = req.header("origin"); + if (!origin_allowed(origin)) { + sink_.deny( + {"cors", "cors.preflight_denied", "preflight origin rejected", req.peer_ip, req.path}); + res.status = 403; + return true; + } + res.status = 204; + bool star = false; + for (const auto& o : policy_.allow_origins) { + if (o == "*" && !policy_.allow_credentials) star = true; + } + res.set_header("Access-Control-Allow-Origin", star ? "*" : origin); + if (!star) res.add_header("Vary", "Origin"); + if (policy_.allow_credentials) res.set_header("Access-Control-Allow-Credentials", "true"); + res.set_header("Access-Control-Allow-Methods", join(policy_.allow_methods)); + if (!policy_.allow_headers.empty()) + res.set_header("Access-Control-Allow-Headers", join(policy_.allow_headers)); + res.set_header("Access-Control-Max-Age", std::to_string(policy_.preflight_cache_sec)); + return true; +} + +} // namespace molo::security::guard diff --git a/src/security/guard/CorsPolicy.h b/src/security/guard/CorsPolicy.h new file mode 100644 index 0000000..6592205 --- /dev/null +++ b/src/security/guard/CorsPolicy.h @@ -0,0 +1,52 @@ +#pragma once + +// CORS policy: closed by default, explicit origin and method allow-lists, +// preflight handler. Wildcard origin with credentials is rejected at +// config load (plan rule 8). Single check() entry as middleware. + +#include +#include +#include + +namespace molo::security::guard { +struct HttpRequestView; +struct HttpResponseView; +class IDenialSink; +} // namespace molo::security::guard + +namespace molo::security::guard { + +class CorsPolicy { +public: + static constexpr std::size_t kMaxOrigins = 256; + static constexpr std::size_t kMaxMethods = 16; + static constexpr std::size_t kMaxHeaders = 64; + static constexpr std::uint32_t kMaxPreflightCacheSec = 600; + + struct Policy { + std::vector allow_origins; // exact match; "*" without creds only + std::vector allow_methods = {"GET", "HEAD"}; + std::vector allow_headers; + bool allow_credentials = false; + std::uint32_t preflight_cache_sec = kMaxPreflightCacheSec; + }; + + CorsPolicy(Policy policy, IDenialSink& sink); + + // Empty string means valid. Wildcard plus credentials fails here. + static std::string validate(const Policy& p); + + // Applies CORS headers to res for a simple request. Always called; + // unlisted origins get no allow headers (deny). + void apply(const HttpRequestView& req, HttpResponseView& res) const; + // Answers an OPTIONS preflight in res. Returns true when the request + // was a preflight (handled, do not route further). + bool preflight(const HttpRequestView& req, HttpResponseView& res) const; + +private: + bool origin_allowed(const std::string& origin) const; + Policy policy_; + IDenialSink& sink_; +}; + +} // namespace molo::security::guard diff --git a/src/security/guard/CsrfGuard.cpp b/src/security/guard/CsrfGuard.cpp new file mode 100644 index 0000000..6305acb --- /dev/null +++ b/src/security/guard/CsrfGuard.cpp @@ -0,0 +1,100 @@ +#include "CsrfGuard.h" + +#include "../crypto/SecureRandom.h" +#include "../session/Session.h" +#include "DenialSink.h" +#include "HttpMessageView.h" + +namespace molo::security::guard { + +CsrfGuard::CsrfGuard(Policy policy, const std::string& hmac_key, IDenialSink& sink) + : policy_(std::move(policy)), hmac_key_(hmac_key), sink_(sink) { + if (policy_.exempt_paths.size() > kMaxExemptions) + policy_.exempt_paths.resize(kMaxExemptions); +} + +bool CsrfGuard::is_safe_method(const std::string& method) { + return method == "GET" || method == "HEAD" || method == "OPTIONS"; +} + +bool CsrfGuard::mint_token(session::Session& session) { + std::string t = crypto::SecureRandom::token(32); + if (t.empty()) return false; + session.csrf_token = t; + return true; +} + +std::string CsrfGuard::fallback_token(const std::string& session_id, + const std::string& nonce) const { + auto mac = crypto::Hmac::sha256(hmac_key_, session_id + "." + nonce); + return crypto::Hmac::hex(mac); +} + +bool CsrfGuard::check_fallback(const HttpRequestView& req, const session::Session& session) const { + std::string cookie = req.cookie("molo_csrf"); + if (cookie.empty() || cookie.size() > 256) return false; + auto dot = cookie.find('.'); + if (dot == std::string::npos || dot == 0) return false; + std::string nonce = cookie.substr(0, dot); + std::string sig = cookie.substr(dot + 1); + if (nonce.empty() || nonce.size() > 128 || sig.size() != 64) return false; + if (!crypto::SecureRandom::constant_time_equal(sig, fallback_token(session.id, nonce))) + return false; + // Header or form field must echo the same cookie value. + std::string echo = req.header("x-csrf-token"); + return crypto::SecureRandom::constant_time_equal(echo, cookie); +} + +CsrfGuard::Verdict CsrfGuard::check(const HttpRequestView& req, session::Session& session, + const std::string& form_token) { + if (is_safe_method(req.method)) return Verdict::Allow; + for (const auto& ex : policy_.exempt_paths) { + if (req.path == ex) { + sink_.audit({"csrf", "csrf.exempt", "exempted path used", req.peer_ip, req.path}); + return Verdict::Exempt; + } + } + // Fetch Metadata: cheap cross-site block first. + if (policy_.enforce_fetch_metadata) { + std::string site = req.header("sec-fetch-site"); + if (!site.empty() && site != "same-origin" && site != "same-site" && site != "none") { + sink_.deny({"csrf", "csrf.fetch_metadata", "cross-site state change", req.peer_ip, + req.path}); + return Verdict::Deny; + } + } + // Synchronizer token: header or form field must match session value. + if (session.csrf_token.empty()) { + if (!mint_token(session)) { + sink_.deny({"csrf", "csrf.no_entropy", "cannot mint token", req.peer_ip, req.path}); + return Verdict::Deny; + } + } + std::string presented = req.header("x-csrf-token"); + if (!presented.empty() && + crypto::SecureRandom::constant_time_equal(presented, session.csrf_token)) + return Verdict::Allow; + if (!form_token.empty() && + crypto::SecureRandom::constant_time_equal(form_token, session.csrf_token)) + return Verdict::Allow; + // Signed double-submit fallback. + if (check_fallback(req, session)) { + sink_.audit({"csrf", "csrf.fallback", "signed double-submit passed", req.peer_ip, + req.path}); + return Verdict::Allow; + } + sink_.deny({"csrf", "csrf.token_mismatch", "no valid token presented", req.peer_ip, req.path}); + return Verdict::Deny; +} + +bool CsrfGuard::enforce(const HttpRequestView& req, HttpResponseView& res, + session::Session& session, const std::string& form_token) { + Verdict v = check(req, session, form_token); + if (v == Verdict::Deny) { + res.status = 403; + res.set_header("Content-Type", "text/plain"); + } + return v != Verdict::Deny; +} + +} // namespace molo::security::guard diff --git a/src/security/guard/CsrfGuard.h b/src/security/guard/CsrfGuard.h new file mode 100644 index 0000000..132bc68 --- /dev/null +++ b/src/security/guard/CsrfGuard.h @@ -0,0 +1,65 @@ +#pragma once + +// CSRF guard: synchronizer token first, signed double-submit fallback, +// Fetch Metadata third. Protects every state-changing cookie route. +// Single check() entry used as middleware. + +#include +#include +#include + +#include "../crypto/Hmac.h" + +namespace molo::security::guard { +struct HttpRequestView; +struct HttpResponseView; +class IDenialSink; +} // namespace molo::security::guard + +namespace molo::security::session { +struct Session; +} // namespace molo::security::session + +namespace molo::security::guard { + +class CsrfGuard { +public: + static constexpr std::size_t kMaxExemptions = 256; + + struct Policy { + // Exact paths exempted (webhooks etc). Each use is audit-logged + // and the route must carry its own auth. + std::vector exempt_paths; + bool enforce_fetch_metadata = true; + }; + + CsrfGuard(Policy policy, const std::string& hmac_key, IDenialSink& sink); + + enum class Verdict : std::uint8_t { Allow, Deny, Exempt }; + + // session.csrf_token is the synchronizer value; minted by mint_token + // when empty. Safe methods (GET, HEAD, OPTIONS) pass through. + // form_token carries the parsed "csrf_token" form field when the + // runtime already parsed the body (empty otherwise). + Verdict check(const HttpRequestView& req, session::Session& session, + const std::string& form_token = ""); + // 403 writer for Deny. Returns true when request may proceed. + bool enforce(const HttpRequestView& req, HttpResponseView& res, session::Session& session, + const std::string& form_token = ""); + + // Mints a fresh synchronizer token into the session. + static bool mint_token(session::Session& session); + + // Signed double-submit fallback token: HMAC_SHA256(hmac_key, + // session_id || nonce). Cookie "molo_csrf" must match header or field. + std::string fallback_token(const std::string& session_id, const std::string& nonce) const; + bool check_fallback(const HttpRequestView& req, const session::Session& session) const; + +private: + static bool is_safe_method(const std::string& method); + Policy policy_; + std::string hmac_key_; + IDenialSink& sink_; +}; + +} // namespace molo::security::guard diff --git a/src/security/guard/DenialSink.cpp b/src/security/guard/DenialSink.cpp new file mode 100644 index 0000000..c8cfee0 --- /dev/null +++ b/src/security/guard/DenialSink.cpp @@ -0,0 +1,18 @@ +#include "DenialSink.h" + +#include + +namespace molo::security::guard { + +void StderrSink::deny(const Denial& d) { + std::fprintf(stderr, "deny component=%s code=%s ip=%s path=%s detail=%s\n", d.component.c_str(), + d.code.c_str(), d.peer_ip.c_str(), d.path.c_str(), d.detail.c_str()); +} + +void StderrSink::audit(const Denial& d) { + std::fprintf(stderr, "audit component=%s code=%s ip=%s path=%s detail=%s\n", + d.component.c_str(), d.code.c_str(), d.peer_ip.c_str(), d.path.c_str(), + d.detail.c_str()); +} + +} // namespace molo::security::guard diff --git a/src/security/guard/DenialSink.h b/src/security/guard/DenialSink.h new file mode 100644 index 0000000..ed78a60 --- /dev/null +++ b/src/security/guard/DenialSink.h @@ -0,0 +1,41 @@ +#pragma once + +// Observability seam (02-observability.md owns the real sink). +// Every denial emits component + reason code + safe detail here; the +// observability part implements IDenialSink with structured logs, +// metrics and trace linkage. NullSink compiles to near-zero overhead. + +#include + +namespace molo::security::guard { + +struct Denial { + std::string component; // "csrf", "cors", "ratelimit", ... + std::string code; // stable reason code, e.g. "csrf.token_mismatch" + std::string detail; // safe context, never secrets or full tokens + std::string peer_ip; + std::string path; +}; + +class IDenialSink { +public: + virtual ~IDenialSink() = default; + virtual void deny(const Denial& d) = 0; + // Notable accepts: rotations, exempted webhooks, re-auth passes. + virtual void audit(const Denial& d) = 0; +}; + +class NullSink : public IDenialSink { +public: + void deny(const Denial&) override {} + void audit(const Denial&) override {} +}; + +// Standalone/test sink: one line per event on stderr, no secrets. +class StderrSink : public IDenialSink { +public: + void deny(const Denial& d) override; + void audit(const Denial& d) override; +}; + +} // namespace molo::security::guard diff --git a/src/security/guard/HttpMessageView.cpp b/src/security/guard/HttpMessageView.cpp new file mode 100644 index 0000000..47b8cd5 --- /dev/null +++ b/src/security/guard/HttpMessageView.cpp @@ -0,0 +1,50 @@ +#include "HttpMessageView.h" + +namespace molo::security::guard { + +namespace { + +void trim(std::string& s) { + std::size_t b = 0; + while (b < s.size() && (s[b] == ' ' || s[b] == '\t')) ++b; + std::size_t e = s.size(); + while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\t')) --e; + s = s.substr(b, e - b); +} + +} // namespace + +std::vector> HttpRequestView::cookies() const { + std::vector> out; + for (const auto& h : headers) { + if (h.first != "cookie") continue; + std::size_t start = 0; + while (start <= h.second.size()) { + std::size_t semi = h.second.find(';', start); + std::string pair = (semi == std::string::npos) ? h.second.substr(start) + : h.second.substr(start, semi - start); + trim(pair); + auto eq = pair.find('='); + if (eq != std::string::npos && eq > 0) { + std::string name = pair.substr(0, eq); + std::string value = pair.substr(eq + 1); + trim(name); + trim(value); + if (!name.empty() && value.size() <= 4096) out.emplace_back(name, value); + } + if (semi == std::string::npos) break; + start = semi + 1; + if (out.size() >= 128) break; // static cap on pairs + } + } + return out; +} + +std::string HttpRequestView::cookie(const std::string& name) const { + for (const auto& c : cookies()) { + if (c.first == name) return c.second; + } + return {}; +} + +} // namespace molo::security::guard diff --git a/src/security/guard/HttpMessageView.h b/src/security/guard/HttpMessageView.h new file mode 100644 index 0000000..468f08a --- /dev/null +++ b/src/security/guard/HttpMessageView.h @@ -0,0 +1,69 @@ +#pragma once + +// Minimal HTTP message views: the seam between the security guards and +// the runtime HTTP stack (01-runtime.md). The runtime adapts its real +// request/response objects to these views; guards never include runtime +// headers, so this part compiles and tests standalone. + +#include +#include +#include +#include +#include + +namespace molo::security::guard { + +struct HttpRequestView { + std::string method = "GET"; // uppercase + std::string path = "/"; // origin-form path, no query + std::string query; // raw query string without '?' + // Header pairs, name in lower case. + std::vector> headers; + std::string peer_ip; + bool is_tls = false; + std::uint64_t now_unix_sec = 0; // frozen clock, set by runtime/tests + + std::string header(const std::string& lower_name) const { + for (const auto& h : headers) { + if (h.first == lower_name) return h.second; + } + return {}; + } + // Parses Cookie headers (may be several) into pairs. Malformed + // pairs are skipped, never fatal. + std::vector> cookies() const; + std::string cookie(const std::string& name) const; +}; + +struct HttpResponseView { + int status = 200; + std::vector> headers; + + void set_header(const std::string& name, const std::string& value) { + for (auto& h : headers) { + if (h.first == name) { + h.second = value; + return; + } + } + headers.emplace_back(name, value); + } + void add_header(const std::string& name, const std::string& value) { + headers.emplace_back(name, value); + } + std::string header(const std::string& name) const { + for (const auto& h : headers) { + if (h.first == name) return h.second; + } + return {}; + } + std::vector header_all(const std::string& name) const { + std::vector out; + for (const auto& h : headers) { + if (h.first == name) out.push_back(h.second); + } + return out; + } +}; + +} // namespace molo::security::guard diff --git a/src/security/guard/PathGuard.cpp b/src/security/guard/PathGuard.cpp new file mode 100644 index 0000000..3d780d7 --- /dev/null +++ b/src/security/guard/PathGuard.cpp @@ -0,0 +1,100 @@ +#include "PathGuard.h" + +#include +#include + +#include + +#include "DenialSink.h" +#include "HttpMessageView.h" + +namespace molo::security::guard { + +bool PathGuard::canonicalize(const std::string& request_path, std::string& clean) { + clean.clear(); + if (request_path.empty() || request_path.size() > kMaxPathBytes) return false; + for (char c : request_path) { + if (c == '\0') return false; + } + // Reject drive prefixes and absolute Windows paths early. + if (request_path.size() >= 2 && request_path[1] == ':') return false; + std::vector segs; + std::size_t i = 0; + while (i < request_path.size()) { + while (i < request_path.size() && request_path[i] == '/') ++i; + if (i >= request_path.size()) break; + std::size_t j = i; + while (j < request_path.size() && request_path[j] != '/') ++j; + std::string seg = request_path.substr(i, j - i); + i = j; + if (seg == ".") continue; + if (seg == "..") { + if (segs.empty()) return false; // climbs above root + segs.pop_back(); + continue; + } + if (seg.find('\0') != std::string::npos) return false; + segs.push_back(seg); + if (segs.size() > kMaxSegments) return false; + } + clean = "/"; + for (std::size_t k = 0; k < segs.size(); ++k) { + if (k > 0) clean += "/"; + clean += segs[k]; + } + return true; +} + +bool PathGuard::check(const std::string& serve_root, const std::string& request_path, + std::string& resolved, IDenialSink* sink, const HttpRequestView* req, + bool allow_symlinks) { + resolved.clear(); + auto deny = [&](const char* code, const char* detail) { + if (sink) { + sink->deny({"path", code, detail, req ? req->peer_ip : "", req ? req->path : ""}); + } + }; + if (serve_root.empty() || serve_root[0] != '/' || serve_root.size() > kMaxPathBytes) { + deny("path.bad_root", "serve root must be absolute"); + return false; + } + std::string clean; + if (!canonicalize(request_path, clean)) { + deny("path.traversal", "climb above root or bad encoding"); + return false; + } + std::string full = serve_root; + if (full.size() > 1 && full.back() == '/') full.pop_back(); + full += clean; + if (full.size() > kMaxPathBytes * 2) { + deny("path.too_long", "resolved path over cap"); + return false; + } + if (!allow_symlinks) { + // Walk each prefix; any symlink component fails closed. + std::string prefix; + prefix.reserve(full.size()); + for (std::size_t k = 1; k <= full.size(); ++k) { + if (k < full.size() && full[k] != '/') continue; + prefix = full.substr(0, k == full.size() ? k : k); + if (prefix.size() <= serve_root.size()) continue; + struct stat st{}; + if (::lstat(prefix.c_str(), &st) == 0 && S_ISLNK(st.st_mode)) { + deny("path.symlink", "symlink component rejected"); + return false; + } + } + } + // Final jail check on the canonical root prefix. + std::string root = serve_root; + if (root.size() > 1 && root.back() == '/') root.pop_back(); + if (full.compare(0, root.size(), root) != 0 || + (full.size() > root.size() && full[root.size()] != '/')) { + deny("path.escape", "resolved outside serve root"); + return false; + } + resolved = full; + return true; +} + +} // namespace molo::security::guard diff --git a/src/security/guard/PathGuard.h b/src/security/guard/PathGuard.h new file mode 100644 index 0000000..6c671eb --- /dev/null +++ b/src/security/guard/PathGuard.h @@ -0,0 +1,34 @@ +#pragma once + +// Path traversal guard for static routes and upload stores (spec item 81). +// Canonicalizes the request path, rejects "..", absolute paths, drive +// prefixes, null bytes and symlink escapes, and jails the result under +// the serve root. Single check() entry. + +#include + +namespace molo::security::guard { +struct HttpRequestView; +class IDenialSink; +} // namespace molo::security::guard + +namespace molo::security::guard { + +class PathGuard { +public: + static constexpr std::size_t kMaxPathBytes = 4096; + static constexpr std::size_t kMaxSegments = 256; + + // Resolves request_path under serve_root. Returns false on any escape + // attempt or malformed input; resolved holds the jailed path on true. + // allow_symlinks=false (default) rejects any symlink component. + static bool check(const std::string& serve_root, const std::string& request_path, + std::string& resolved, IDenialSink* sink = nullptr, + const HttpRequestView* req = nullptr, bool allow_symlinks = false); + + // Pure lexical canonicalization (no filesystem access). False when + // the path tries to climb above root or carries null bytes. + static bool canonicalize(const std::string& request_path, std::string& clean); +}; + +} // namespace molo::security::guard diff --git a/src/security/guard/RateLimiter.cpp b/src/security/guard/RateLimiter.cpp new file mode 100644 index 0000000..7e5daba --- /dev/null +++ b/src/security/guard/RateLimiter.cpp @@ -0,0 +1,147 @@ +#include "RateLimiter.h" + +#include +#include + +#include "../../governance/tiers/TierHeaders.h" +#include "../../governance/tiers/TierPolicy.h" +#include "DenialSink.h" +#include "HttpMessageView.h" + +namespace molo::security::guard { + +RateLimiter::RateLimiter(IDenialSink* sink) : RateLimiter(Policy(), sink) {} + +RateLimiter::RateLimiter(Policy policy, IDenialSink* sink) + : policy_(std::move(policy)), sink_(sink) { + clock_ = [] { + return std::uint64_t(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + }; +} + +bool RateLimiter::take(const std::string& key, const Budget& b, std::uint64_t now, + std::uint32_t& retry_after) { + retry_after = 0; + if (b.per_min == 0) { + retry_after = 60; + return false; + } + auto it = buckets_.find(key); + if (it == buckets_.end()) { + if (buckets_.size() >= kMaxBuckets) { + // Static cap: evict the stalest bucket. + auto victim = buckets_.begin(); + for (auto jt = buckets_.begin(); jt != buckets_.end(); ++jt) { + if (jt->second.updated < victim->second.updated) victim = jt; + } + buckets_.erase(victim); + } + Bucket fresh; + fresh.cap = b.per_min * b.burst_mult; + fresh.tokens = double(fresh.cap); + fresh.updated = now; + it = buckets_.emplace(key, fresh).first; + } + Bucket& bucket = it->second; + double rate = double(b.per_min) / 60.0; + std::uint64_t elapsed = now >= bucket.updated ? now - bucket.updated : 0; + bucket.tokens += double(elapsed) * rate; + double cap = double(b.per_min * b.burst_mult); + if (bucket.tokens > cap) bucket.tokens = cap; + bucket.updated = now; + if (bucket.tokens >= 1.0) { + bucket.tokens -= 1.0; + return true; + } + double deficit = 1.0 - bucket.tokens; + retry_after = std::uint32_t(deficit / rate) + 1; + if (retry_after > 60) retry_after = 60; + return false; +} + +RateLimiter::Verdict RateLimiter::check(const HttpRequestView& req, const std::string& user_id) { + Verdict v; + std::uint64_t now = clock_ ? clock_() : 0; + std::lock_guard lock(mu_); + std::uint32_t retry = 0; + if (!take("ip:" + req.peer_ip, policy_.per_ip, now, retry)) { + v.allowed = false; + v.retry_after_sec = retry; + v.code = "ratelimit.ip"; + } else if (!user_id.empty() && !take("user:" + user_id, policy_.per_user, now, retry)) { + v.allowed = false; + v.retry_after_sec = retry; + v.code = "ratelimit.user"; + } else { + for (const auto& kv : policy_.per_route) { + if (req.path.compare(0, kv.first.size(), kv.first) == 0) { + if (!take("route:" + kv.first + "|" + req.peer_ip, kv.second, now, retry)) { + v.allowed = false; + v.retry_after_sec = retry; + v.code = "ratelimit.route"; + } + break; + } + } + } + if (!v.allowed && sink_) { + sink_->deny({"ratelimit", v.code, "budget exhausted", req.peer_ip, req.path}); + } + return v; +} + +bool RateLimiter::enforce(const HttpRequestView& req, HttpResponseView& res, + const std::string& user_id) { + Verdict v = check(req, user_id); + if (v.allowed) return true; + res.status = 429; + res.set_header("Retry-After", std::to_string(v.retry_after_sec)); + return false; +} + +RateLimiter::Verdict RateLimiter::check_tier(const HttpRequestView& req, + const TierBudgetKey& gov) { + // Resolve limits via TierPolicy (no route-tier override at this layer; + // the caller passes the route's gov_tier_override as gov.tier when set), + // then run the existing counting path under a per-(tier,key,prefix) key. + const auto limits = ::molo::govern::tiers::TierPolicy::Resolve(gov.tier, ""); + Budget b; + b.per_min = limits.per_min > 0 ? static_cast(limits.per_min) : 0; + b.burst_mult = 2; + b.burst_sec = 10; + Verdict v; + std::uint64_t now = clock_ ? clock_() : 0; + std::lock_guard lock(mu_); + std::uint32_t retry = 0; + const std::string key = "gov:" + gov.tier + "|" + gov.key_id + "|" + gov.route_prefix; + if (!take(key, b, now, retry)) { + v.allowed = false; + v.retry_after_sec = retry; + v.code = "ratelimit.tier"; + } + if (!v.allowed && sink_) { + sink_->deny({"ratelimit", v.code, "tier budget exhausted", req.peer_ip, req.path}); + } + return v; +} + +bool RateLimiter::enforce_tier(const HttpRequestView& req, HttpResponseView& res, + const TierBudgetKey& gov) { + Verdict v = check_tier(req, gov); + if (v.allowed) return true; + const auto limits = ::molo::govern::tiers::TierPolicy::Resolve(gov.tier, ""); + std::map hdrs; + ::molo::govern::tiers::TierHeaders::Apply(hdrs, limits, 0, 0, 60, v.retry_after_sec); + for (const auto& kv : hdrs) res.set_header(kv.first, kv.second); + res.status = 429; + return false; +} + +std::size_t RateLimiter::buckets() const { + std::lock_guard lock(mu_); + return buckets_.size(); +} + +} // namespace molo::security::guard diff --git a/src/security/guard/RateLimiter.h b/src/security/guard/RateLimiter.h new file mode 100644 index 0000000..d596f68 --- /dev/null +++ b/src/security/guard/RateLimiter.h @@ -0,0 +1,91 @@ +#pragma once + +// Token-bucket rate limiter: per-IP, per-user, per-route budgets with a +// 2x burst for 10 s. 429 verdicts carry Retry-After seconds. Static cap +// of 100k buckets with oldest-refill eviction. Thread-safe. + +#include +#include +#include +#include +#include + +namespace molo::security::guard { + +struct HttpRequestView; +struct HttpResponseView; +class IDenialSink; + +// Governance: per-(tier, key, route-prefix) budget lookup with +// downward-only override. Counting algorithm unchanged; governance +// supplies the limits and the response headers. +struct TierBudgetKey { + std::string tier; + std::string key_id; + std::string route_prefix; +}; + +class RateLimiter { +public: + struct Budget { + std::uint32_t per_min = 100; // sustained rate + std::uint32_t burst_mult = 2; // burst capacity multiplier + std::uint32_t burst_sec = 10; // burst window + }; + struct Policy { + Budget per_ip{100, 2, 10}; + Budget per_user{300, 2, 10}; + // Per-route override keyed by exact path prefix. Empty by default. + std::unordered_map per_route; + }; + + static constexpr std::size_t kMaxBuckets = 100000; + + struct Verdict { + bool allowed = true; + std::uint32_t retry_after_sec = 0; + std::string code; // "ratelimit.ip" / "ratelimit.user" / "ratelimit.route" + }; + + using ClockSec = std::function; + + explicit RateLimiter(IDenialSink* sink = nullptr); + explicit RateLimiter(Policy policy, IDenialSink* sink = nullptr); + + void set_clock(ClockSec clock) { clock_ = std::move(clock); } + + // Single check() entry. user_id empty means anonymous (IP budget only). + Verdict check(const HttpRequestView& req, const std::string& user_id); + // Emits 429 + Retry-After on reject. Returns allowed. + bool enforce(const HttpRequestView& req, HttpResponseView& res, const std::string& user_id); + // Tenancy (PATCHES 11.16): verdicts are keyed per tenant plus route + // so one tenant cannot burn another tenant's budget. Key construction + // must prefix the tenant slug from the request tenant extension: + // key = tenant + "|" + route + "|" + user_or_ip. Wire the per-tenant + // take() through tenancy::TenantBuckets. (Pending: HttpRequestView + // carries no tenant extension yet, so enforcement stays at the call + // sites that resolve the tenant.) + // Governance tier dimension. enforce_tier returns allowed (real shape). + Verdict check_tier(const HttpRequestView& req, const TierBudgetKey& gov); + bool enforce_tier(const HttpRequestView& req, HttpResponseView& res, + const TierBudgetKey& gov); + + std::size_t buckets() const; + +private: + struct Bucket { + double tokens = 0; + std::uint64_t updated = 0; + std::uint32_t cap = 0; + }; + bool take(const std::string& key, const Budget& b, std::uint64_t now, + std::uint32_t& retry_after); + + Policy policy_; + IDenialSink* sink_; + ClockSec clock_; + mutable std::mutex mu_; + std::unordered_map buckets_; +}; + +} // namespace molo::security::guard diff --git a/src/security/guard/SecurityHeaders.cpp b/src/security/guard/SecurityHeaders.cpp new file mode 100644 index 0000000..c531a20 --- /dev/null +++ b/src/security/guard/SecurityHeaders.cpp @@ -0,0 +1,24 @@ +#include "SecurityHeaders.h" + +#include "HttpMessageView.h" + +namespace molo::security::guard { + +SecurityHeaders::SecurityHeaders() : SecurityHeaders(Policy()) {} + +SecurityHeaders::SecurityHeaders(Policy policy) : policy_(std::move(policy)) {} + +void SecurityHeaders::apply(const HttpRequestView& req, HttpResponseView& res) const { + for (const auto& p : policy_.opt_out_paths) { + if (req.path == p) return; // explicit recorded opt-out + } + if (req.is_tls || policy_.hsts_on_http) + res.set_header("Strict-Transport-Security", policy_.hsts); + res.set_header("X-Frame-Options", policy_.frame_options); + res.set_header("X-Content-Type-Options", policy_.content_type_options); + res.set_header("Referrer-Policy", policy_.referrer_policy); + res.set_header("Permissions-Policy", policy_.permissions_policy); + res.set_header("Content-Security-Policy", policy_.content_security_policy); +} + +} // namespace molo::security::guard diff --git a/src/security/guard/SecurityHeaders.h b/src/security/guard/SecurityHeaders.h new file mode 100644 index 0000000..ad09ea1 --- /dev/null +++ b/src/security/guard/SecurityHeaders.h @@ -0,0 +1,45 @@ +#pragma once + +// Security headers middleware: always in the default chain (plan rule 9). +// Removal needs an explicit per-route opt-out recorded in config. +// Ships HSTS, framing, nosniff, referrer, permissions, and a CSP builder +// hook shared with the template layer (05-request-routing.md). + +#include +#include + +namespace molo::security::guard { +struct HttpRequestView; +struct HttpResponseView; +} // namespace molo::security::guard + +namespace molo::security::guard { + +class SecurityHeaders { +public: + struct Policy { + std::string hsts = "max-age=63072000; includeSubDomains; preload"; + std::string frame_options = "DENY"; + std::string content_type_options = "nosniff"; + std::string referrer_policy = "strict-origin-when-cross-origin"; + std::string permissions_policy = "geolocation=(), camera=(), microphone=()"; + std::string content_security_policy = + "default-src 'self'; object-src 'none'; frame-ancestors 'none'"; + bool hsts_on_http = false; // HSTS only over TLS unless forced + // Explicit per-route opt-outs, recorded in config (rule 9). + std::vector opt_out_paths; + }; + + SecurityHeaders(); + explicit SecurityHeaders(Policy policy); + + // Applies all six header families to res unless the path opted out. + void apply(const HttpRequestView& req, HttpResponseView& res) const; + + const Policy& policy() const { return policy_; } + +private: + Policy policy_; +}; + +} // namespace molo::security::guard diff --git a/src/security/guard/SqlSafety.cpp b/src/security/guard/SqlSafety.cpp new file mode 100644 index 0000000..b4fb94b --- /dev/null +++ b/src/security/guard/SqlSafety.cpp @@ -0,0 +1,57 @@ +#include "SqlSafety.h" + +namespace molo::security::guard { + +bool SqlSafety::check(const BoundQuery& q, std::string& reason) { + reason.clear(); + if (q.sql.empty() || q.sql.size() > kMaxSqlBytes) { + reason = "sql: statement empty or over 1 MB"; + return false; + } + for (char c : q.sql) { + if (c == '\0') { + reason = "sql: NUL byte in statement"; + return false; + } + } + if (q.params.size() > kMaxParams) { + reason = "sql: too many bound params"; + return false; + } + // Count '?' placeholders outside string literals. + std::size_t marks = 0; + bool in_str = false; + for (std::size_t i = 0; i < q.sql.size(); ++i) { + char c = q.sql[i]; + if (in_str) { + if (c == '\'') { + if (i + 1 < q.sql.size() && q.sql[i + 1] == '\'') { + ++i; + } else { + in_str = false; + } + } + } else if (c == '\'') { + in_str = true; + } else if (c == '?') { + ++marks; + } + } + if (marks != q.params.size()) { + reason = "sql: placeholder count must match bound params"; + return false; + } + return true; +} + +std::string SqlSafety::quote_identifier(const std::string& name) { + if (name.empty() || name.size() > 128) return {}; + for (char c : name) { + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_'; + if (!ok) return {}; + } + return "\"" + name + "\""; +} + +} // namespace molo::security::guard diff --git a/src/security/guard/SqlSafety.h b/src/security/guard/SqlSafety.h new file mode 100644 index 0000000..5a4b100 --- /dev/null +++ b/src/security/guard/SqlSafety.h @@ -0,0 +1,47 @@ +#pragma once + +// SQL safety contract (plan rule 3, owned here, enforced in 04-data.md). +// Parameterized queries only: user input travels as bound values, never +// interpolated into SQL text. RawSql exists for the rare DDL case and +// requires an explicit marker plus audit log at the call site. + +#include +#include +#include + +namespace molo::security::guard { + +struct BoundValue { + enum class Type : std::uint8_t { Null = 0, Int, Double, Text, Blob }; + Type type = Type::Null; + std::string text; // text form for Int/Double/Text; raw bytes for Blob +}; + +struct BoundQuery { + std::string sql; // placeholders only (? or $1 style), no user input + std::vector params; +}; + +// Explicit opt-in for hand-written SQL with zero user input (DDL, admin). +// Call sites must audit-log every use. The data layer rejects any other +// raw string reaching the driver. +struct RawSql { + std::string sql; + std::string audit_reason; +}; + +class SqlSafety { +public: + static constexpr std::size_t kMaxParams = 1000; + static constexpr std::size_t kMaxSqlBytes = 1024u * 1024u; + + // Structural check: placeholder count matches params, no NUL bytes, + // sizes within caps. True means safe to hand to the driver. + static bool check(const BoundQuery& q, std::string& reason); + + // Escapes a literal for the RawSql path only (identifiers in DDL). + // Never a substitute for binding user input. + static std::string quote_identifier(const std::string& name); +}; + +} // namespace molo::security::guard diff --git a/src/security/secret/SecretsProvider.cpp b/src/security/secret/SecretsProvider.cpp new file mode 100644 index 0000000..2e6eb67 --- /dev/null +++ b/src/security/secret/SecretsProvider.cpp @@ -0,0 +1,303 @@ +#include "SecretsProvider.h" + +#include +#include + +#include + +#include "../crypto/SecureRandom.h" + +namespace molo::security::secret { + +SecretsProvider::SecretsProvider() { + environ_ = [](const std::string& name) { + const char* v = ::getenv(name.c_str()); + return v ? std::string(v) : std::string(); + }; +} + +void SecretsProvider::declare(SecretMeta meta) { + std::lock_guard lock(mu_); + if (declared_.size() >= kMaxSecrets) return; + if (meta.name.empty() || meta.name.size() > 256) return; + declared_.push_back(std::move(meta)); +} + +bool SecretsProvider::parse_json_object(const std::string& text, + std::unordered_map& out) { + out.clear(); + std::size_t i = 0; + auto skip_ws = [&] { + while (i < text.size() && (text[i] == ' ' || text[i] == '\t' || text[i] == '\n' || + text[i] == '\r')) + ++i; + }; + auto parse_string = [&](std::string& s) -> bool { + s.clear(); + if (i >= text.size() || text[i] != '"') return false; + ++i; + while (i < text.size()) { + char c = text[i++]; + if (c == '"') return true; + if (c == '\\') { + if (i >= text.size()) return false; + char e = text[i++]; + switch (e) { + case '"': s.push_back('"'); break; + case '\\': s.push_back('\\'); break; + case '/': s.push_back('/'); break; + case 'n': s.push_back('\n'); break; + case 't': s.push_back('\t'); break; + case 'r': s.push_back('\r'); break; + case 'u': { + if (i + 4 > text.size()) return false; + unsigned cp = 0; + for (int k = 0; k < 4; ++k) { + char h = text[i++]; + cp <<= 4; + if (h >= '0' && h <= '9') + cp |= unsigned(h - '0'); + else if (h >= 'a' && h <= 'f') + cp |= unsigned(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') + cp |= unsigned(h - 'A' + 10); + else + return false; + } + if (cp < 0x80) + s.push_back(char(cp)); + else if (cp < 0x800) { + s.push_back(char(0xC0 | (cp >> 6))); + s.push_back(char(0x80 | (cp & 0x3F))); + } else { + s.push_back(char(0xE0 | (cp >> 12))); + s.push_back(char(0x80 | ((cp >> 6) & 0x3F))); + s.push_back(char(0x80 | (cp & 0x3F))); + } + break; + } + default: return false; + } + } else { + s.push_back(c); + } + if (s.size() > 65536) return false; + } + return false; + }; + std::function parse_value; + std::string prefix; + std::function parse_object = [&]() -> bool { + skip_ws(); + if (i >= text.size() || text[i] != '{') return false; + ++i; + skip_ws(); + if (i < text.size() && text[i] == '}') { + ++i; + return true; + } + while (true) { + skip_ws(); + std::string key; + if (!parse_string(key)) return false; + skip_ws(); + if (i >= text.size() || text[i] != ':') return false; + ++i; + skip_ws(); + std::string full = prefix.empty() ? key : prefix + "." + key; + if (i < text.size() && text[i] == '{') { + std::string saved = prefix; + prefix = full; + if (!parse_object()) return false; + prefix = saved; + } else if (i < text.size() && text[i] == '"') { + std::string v; + if (!parse_string(v)) return false; + if (out.size() >= 4096) return false; + out[full] = v; + } else { + std::size_t start = i; + while (i < text.size() && text[i] != ',' && text[i] != '}' && text[i] != '\n') + ++i; + std::string v = text.substr(start, i - start); + while (!v.empty() && (v.back() == ' ' || v.back() == '\t' || v.back() == '\r')) + v.pop_back(); + if (v != "true" && v != "false" && v != "null") { + bool num = !v.empty(); + for (char c : v) { + if (!(c >= '0' && c <= '9') && c != '-' && c != '+' && c != '.' && + c != 'e' && c != 'E') { + num = false; + break; + } + } + if (!num) return false; + } + if (out.size() >= 4096) return false; + out[full] = v; + } + skip_ws(); + if (i >= text.size()) return false; + if (text[i] == ',') { + ++i; + continue; + } + if (text[i] == '}') { + ++i; + return true; + } + return false; + } + }; + if (!parse_object()) { + out.clear(); + return false; + } + skip_ws(); + if (i != text.size()) { + out.clear(); + return false; + } + return true; +} + +bool SecretsProvider::valid_value(const SecretMeta& meta, const std::string& value) const { + if (value.size() < meta.min_len || value.size() > meta.max_len) return false; + for (char c : value) { + if (c == '\0' || c == '\n' || c == '\r') return false; + } + return true; +} + +bool SecretsProvider::load(const std::string& env_json_path, std::string& reason) { + reason.clear(); + std::lock_guard lock(mu_); + std::unordered_map file_vals; + if (!env_json_path.empty()) { + int fd = ::open(env_json_path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd >= 0) { + std::string text; + char buf[4096]; + bool too_big = false; + while (true) { + ssize_t n = ::read(fd, buf, sizeof buf); + if (n < 0) { + ::close(fd); + reason = "secrets: cannot read file"; + return false; + } + if (n == 0) break; + if (text.size() + std::size_t(n) > kMaxFileBytes) { + too_big = true; + break; + } + text.append(buf, std::size_t(n)); + } + ::close(fd); + if (too_big) { + reason = "secrets: file over 1 MB"; + return false; + } + if (!parse_json_object(text, file_vals)) { + molo::security::crypto::SecureRandom::zeroize(text); + reason = "secrets: malformed JSON"; + return false; + } + molo::security::crypto::SecureRandom::zeroize(text); + } + } + for (const auto& meta : declared_) { + std::string v; + auto it = file_vals.find(meta.name); + if (it != file_vals.end()) v = it->second; + std::string env = environ_(meta.name); + if (!env.empty()) v = env; // env wins + molo::security::crypto::SecureRandom::zeroize(env); + if (v.empty()) { + if (meta.required) { + reason = "secrets: missing required " + meta.name; + return false; + } + values_.erase(meta.name); + continue; + } + if (!valid_value(meta, v)) { + molo::security::crypto::SecureRandom::zeroize(v); + reason = "secrets: invalid value for " + meta.name; + return false; + } + values_[meta.name] = v; + molo::security::crypto::SecureRandom::zeroize(v); + } + for (auto& kv : file_vals) molo::security::crypto::SecureRandom::zeroize(kv.second); + return true; +} + +bool SecretsProvider::get(const std::string& name, std::string& value) const { + std::lock_guard lock(mu_); + auto it = values_.find(name); + if (it == values_.end()) return false; + value = it->second; + return true; +} + +bool SecretsProvider::get_u64(const std::string& name, std::uint64_t& value) const { + std::string v; + if (!get(name, v)) return false; + try { + std::size_t pos = 0; + unsigned long long n = std::stoull(v, &pos); + molo::security::crypto::SecureRandom::zeroize(v); + if (pos != v.size()) return false; + value = n; + return true; + } catch (...) { + molo::security::crypto::SecureRandom::zeroize(v); + return false; + } +} + +bool SecretsProvider::get_bool(const std::string& name, bool& value) const { + std::string v; + if (!get(name, v)) return false; + if (v == "true" || v == "1") { + value = true; + return true; + } + if (v == "false" || v == "0") { + value = false; + return true; + } + return false; +} + +bool SecretsProvider::rotate(const std::string& name, const std::string& value) { + std::lock_guard lock(mu_); + const SecretMeta* meta = nullptr; + for (const auto& m : declared_) { + if (m.name == name) { + meta = &m; + break; + } + } + if (!meta || !valid_value(*meta, value)) return false; + auto it = values_.find(name); + if (it != values_.end()) molo::security::crypto::SecureRandom::zeroize(it->second); + values_[name] = value; + return true; +} + +std::vector SecretsProvider::redacted() const { + std::lock_guard lock(mu_); + std::vector out; + for (const auto& m : declared_) { + auto it = values_.find(m.name); + if (it == values_.end()) + out.push_back(m.name + ":missing"); + else + out.push_back(m.name + ":present:" + std::to_string(it->second.size())); + } + return out; +} + +} // namespace molo::security::secret diff --git a/src/security/secret/SecretsProvider.h b/src/security/secret/SecretsProvider.h new file mode 100644 index 0000000..81fd42c --- /dev/null +++ b/src/security/secret/SecretsProvider.h @@ -0,0 +1,70 @@ +#pragma once + +// Secrets provider: loads .env.json (in-house minimal JSON), applies +// environment variable overrides (env wins), exposes typed validated +// secrets, and rotates encryption epochs. Secrets never appear in logs +// or error bodies: getters return values, dumps return redacted shapes. + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace molo::security::secret { + +struct SecretMeta { + std::string name; + bool required = false; + std::size_t min_len = 0; + std::size_t max_len = 4096; +}; + +class SecretsProvider { +public: + static constexpr std::size_t kMaxFileBytes = 1024u * 1024u; + static constexpr std::size_t kMaxSecrets = 1024; + + using Environ = std::function; + + SecretsProvider(); + + // Declares the expected secret set before load(). + void declare(SecretMeta meta); + + // Loads root/workdir .env.json when present, then env overrides. + // Returns false with reason on missing required secrets or + // validation failures. Never includes values in reason. + bool load(const std::string& env_json_path, std::string& reason); + + void set_environ(Environ env) { environ_ = std::move(env); } + + // Typed getters. False when missing or malformed; value untouched. + bool get(const std::string& name, std::string& value) const; + bool get_u64(const std::string& name, std::uint64_t& value) const; + bool get_bool(const std::string& name, bool& value) const; + + // Rotation: replaces a secret value in memory (new epoch material). + // Old value is zeroized. False when name undeclared. + bool rotate(const std::string& name, const std::string& value); + + // Redacted shape for config dumps and logs: "name:present:". + std::vector redacted() const; + + // Minimal JSON object parser (string/number/bool values, one level + // plus nested objects flattened with dot keys). Exposed for tests. + static bool parse_json_object(const std::string& text, + std::unordered_map& out); + +private: + bool valid_value(const SecretMeta& meta, const std::string& value) const; + std::vector declared_; + std::unordered_map values_; + Environ environ_; + mutable std::mutex mu_; +}; + +} // namespace molo::security::secret diff --git a/src/security/security.mk b/src/security/security.mk new file mode 100644 index 0000000..32883d1 --- /dev/null +++ b/src/security/security.mk @@ -0,0 +1,226 @@ +# MoloApp part: 03-security (src/security). +# Included by the root Makefile. Defines security-all, security-test, +# security-clean. Plain explicit rules, no pattern extensions, so both +# BSD make and GNU make accept this file. + +PARTS += security + +CXX ?= g++ +CXXFLAGS ?= -std=c++23 -O2 -Wall -Wextra -Werror + +SECURITY_INCS = -Isrc/security/crypto -Isrc/security/guard \ + -Isrc/security/session -Isrc/security/auth -Isrc/security/tls \ + -Isrc/security/upload -Isrc/security/secret + +SECURITY_OBJS = \ + build/security/auth_Authenticator.o \ + build/security/auth_Guard.o \ + build/security/crypto_AesGcm.o \ + build/security/crypto_Base64.o \ + build/security/crypto_Blake2b.o \ + build/security/crypto_ChaCha20Poly1305.o \ + build/security/crypto_CryptoBox.o \ + build/security/crypto_Hmac.o \ + build/security/crypto_KeyHierarchy.o \ + build/security/crypto_PasswordHasher.o \ + build/security/crypto_SecureRandom.o \ + build/security/crypto_Sha256.o \ + build/security/guard_BodyLimits.o \ + build/security/guard_CorsPolicy.o \ + build/security/guard_CsrfGuard.o \ + build/security/guard_DenialSink.o \ + build/security/guard_HttpMessageView.o \ + build/security/guard_PathGuard.o \ + build/security/guard_RateLimiter.o \ + build/security/guard_SecurityHeaders.o \ + build/security/guard_SqlSafety.o \ + build/security/secret_SecretsProvider.o \ + build/security/session_DbStore.o \ + build/security/session_MemoryStore.o \ + build/security/session_Session.o \ + build/security/session_SessionManager.o \ + build/security/tls_TlsAcceptor.o \ + build/security/tls_TlsConfig.o \ + build/security/tls_TlsContext.o \ + build/security/tls_WolfSslBackend.o \ + build/security/upload_UploadStore.o \ + build/security/upload_UploadValidator.o + +SECURITY_TESTS = \ + build/security-tests/crypto_crypto_test \ + build/security-tests/tls_tls_test \ + build/security-tests/session_session_test \ + build/security-tests/auth_auth_test \ + build/security-tests/guard_guard_test \ + build/security-tests/upload_upload_test \ + build/security-tests/secret_secrets_test + +security-all: build/security/libsecurity.a + +build/security/libsecurity.a: $(SECURITY_OBJS) + ar rcs build/security/libsecurity.a $(SECURITY_OBJS) + +security-test: $(SECURITY_TESTS) + for t in $(SECURITY_TESTS); do echo "== $$t"; $$t || exit 1; done + +security-clean: + rm -rf build/security build/security-tests + +# Object rules (explicit, portable). +build/security/auth_Authenticator.o: src/security/auth/Authenticator.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/auth/Authenticator.cpp -o build/security/auth_Authenticator.o + +build/security/auth_Guard.o: src/security/auth/Guard.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/auth/Guard.cpp -o build/security/auth_Guard.o + +build/security/crypto_AesGcm.o: src/security/crypto/AesGcm.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/AesGcm.cpp -o build/security/crypto_AesGcm.o + +build/security/crypto_Base64.o: src/security/crypto/Base64.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/Base64.cpp -o build/security/crypto_Base64.o + +build/security/crypto_Blake2b.o: src/security/crypto/Blake2b.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/Blake2b.cpp -o build/security/crypto_Blake2b.o + +build/security/crypto_ChaCha20Poly1305.o: src/security/crypto/ChaCha20Poly1305.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/ChaCha20Poly1305.cpp -o build/security/crypto_ChaCha20Poly1305.o + +build/security/crypto_CryptoBox.o: src/security/crypto/CryptoBox.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/CryptoBox.cpp -o build/security/crypto_CryptoBox.o + +build/security/crypto_Hmac.o: src/security/crypto/Hmac.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/Hmac.cpp -o build/security/crypto_Hmac.o + +build/security/crypto_KeyHierarchy.o: src/security/crypto/KeyHierarchy.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/KeyHierarchy.cpp -o build/security/crypto_KeyHierarchy.o + +build/security/crypto_PasswordHasher.o: src/security/crypto/PasswordHasher.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/PasswordHasher.cpp -o build/security/crypto_PasswordHasher.o + +build/security/crypto_SecureRandom.o: src/security/crypto/SecureRandom.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/SecureRandom.cpp -o build/security/crypto_SecureRandom.o + +build/security/crypto_Sha256.o: src/security/crypto/Sha256.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/crypto/Sha256.cpp -o build/security/crypto_Sha256.o + +build/security/guard_BodyLimits.o: src/security/guard/BodyLimits.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/BodyLimits.cpp -o build/security/guard_BodyLimits.o + +build/security/guard_CorsPolicy.o: src/security/guard/CorsPolicy.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/CorsPolicy.cpp -o build/security/guard_CorsPolicy.o + +build/security/guard_CsrfGuard.o: src/security/guard/CsrfGuard.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/CsrfGuard.cpp -o build/security/guard_CsrfGuard.o + +build/security/guard_DenialSink.o: src/security/guard/DenialSink.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/DenialSink.cpp -o build/security/guard_DenialSink.o + +build/security/guard_HttpMessageView.o: src/security/guard/HttpMessageView.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/HttpMessageView.cpp -o build/security/guard_HttpMessageView.o + +build/security/guard_PathGuard.o: src/security/guard/PathGuard.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/PathGuard.cpp -o build/security/guard_PathGuard.o + +build/security/guard_RateLimiter.o: src/security/guard/RateLimiter.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/RateLimiter.cpp -o build/security/guard_RateLimiter.o + +build/security/guard_SecurityHeaders.o: src/security/guard/SecurityHeaders.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/SecurityHeaders.cpp -o build/security/guard_SecurityHeaders.o + +build/security/guard_SqlSafety.o: src/security/guard/SqlSafety.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/guard/SqlSafety.cpp -o build/security/guard_SqlSafety.o + +build/security/secret_SecretsProvider.o: src/security/secret/SecretsProvider.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/secret/SecretsProvider.cpp -o build/security/secret_SecretsProvider.o + +build/security/session_DbStore.o: src/security/session/DbStore.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/session/DbStore.cpp -o build/security/session_DbStore.o + +build/security/session_MemoryStore.o: src/security/session/MemoryStore.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/session/MemoryStore.cpp -o build/security/session_MemoryStore.o + +build/security/session_Session.o: src/security/session/Session.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/session/Session.cpp -o build/security/session_Session.o + +build/security/session_SessionManager.o: src/security/session/SessionManager.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/session/SessionManager.cpp -o build/security/session_SessionManager.o + +build/security/tls_TlsAcceptor.o: src/security/tls/TlsAcceptor.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/tls/TlsAcceptor.cpp -o build/security/tls_TlsAcceptor.o + +build/security/tls_TlsConfig.o: src/security/tls/TlsConfig.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/tls/TlsConfig.cpp -o build/security/tls_TlsConfig.o + +build/security/tls_TlsContext.o: src/security/tls/TlsContext.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/tls/TlsContext.cpp -o build/security/tls_TlsContext.o + +build/security/tls_WolfSslBackend.o: src/security/tls/WolfSslBackend.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/tls/WolfSslBackend.cpp -o build/security/tls_WolfSslBackend.o + +build/security/upload_UploadStore.o: src/security/upload/UploadStore.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/upload/UploadStore.cpp -o build/security/upload_UploadStore.o + +build/security/upload_UploadValidator.o: src/security/upload/UploadValidator.cpp + @mkdir -p build/security + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) -c src/security/upload/UploadValidator.cpp -o build/security/upload_UploadValidator.o + +# Test link rules (explicit, portable). +build/security-tests/crypto_crypto_test: tests/src/security/crypto/crypto_test.cpp build/security/libsecurity.a + @mkdir -p build/security-tests + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) tests/src/security/crypto/crypto_test.cpp build/security/libsecurity.a -o build/security-tests/crypto_crypto_test + +build/security-tests/tls_tls_test: tests/src/security/tls/tls_test.cpp build/security/libsecurity.a + @mkdir -p build/security-tests + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) tests/src/security/tls/tls_test.cpp build/security/libsecurity.a -o build/security-tests/tls_tls_test + +build/security-tests/session_session_test: tests/src/security/session/session_test.cpp build/security/libsecurity.a + @mkdir -p build/security-tests + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) tests/src/security/session/session_test.cpp build/security/libsecurity.a -o build/security-tests/session_session_test + +build/security-tests/auth_auth_test: tests/src/security/auth/auth_test.cpp build/security/libsecurity.a + @mkdir -p build/security-tests + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) tests/src/security/auth/auth_test.cpp build/security/libsecurity.a -o build/security-tests/auth_auth_test + +build/security-tests/guard_guard_test: tests/src/security/guard/guard_test.cpp build/security/libsecurity.a + @mkdir -p build/security-tests + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) tests/src/security/guard/guard_test.cpp build/security/libsecurity.a -o build/security-tests/guard_guard_test + +build/security-tests/upload_upload_test: tests/src/security/upload/upload_test.cpp build/security/libsecurity.a + @mkdir -p build/security-tests + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) tests/src/security/upload/upload_test.cpp build/security/libsecurity.a -o build/security-tests/upload_upload_test + +build/security-tests/secret_secrets_test: tests/src/security/secret/secrets_test.cpp build/security/libsecurity.a + @mkdir -p build/security-tests + $(CXX) $(CXXFLAGS) $(SECURITY_INCS) tests/src/security/secret/secrets_test.cpp build/security/libsecurity.a -o build/security-tests/secret_secrets_test + diff --git a/src/security/session/DbStore.cpp b/src/security/session/DbStore.cpp new file mode 100644 index 0000000..ef007e4 --- /dev/null +++ b/src/security/session/DbStore.cpp @@ -0,0 +1,42 @@ +#include "DbStore.h" + +namespace molo::security::session { + +DbStore::DbStore(std::shared_ptr rows) : rows_(std::move(rows)) {} + +bool DbStore::get(const std::string& id, std::uint64_t now_unix_sec, Session& out) { + if (!rows_ || id.size() != 22) return false; + if (!rows_->select(id, out)) return false; + if (!out.valid_shape() || out.id != id) return false; + if (now_unix_sec >= out.absolute_expiry) { + rows_->erase(id); + return false; + } + return true; +} + +bool DbStore::put(const Session& s) { + if (!rows_ || !s.valid_shape()) return false; + return rows_->upsert(s); +} + +bool DbStore::rotate(const std::string& old_id, Session& s) { + if (!rows_ || !s.valid_shape() || s.id == old_id || old_id.size() != 22) return false; + Session probe; + if (!rows_->select(old_id, probe)) return false; + if (!rows_->upsert(s)) return false; + rows_->erase(old_id); + return true; +} + +bool DbStore::destroy(const std::string& id) { + if (!rows_ || id.empty()) return false; + return rows_->erase(id); +} + +std::size_t DbStore::sweep(std::uint64_t now_unix_sec) { + if (!rows_) return 0; + return rows_->erase_expired(now_unix_sec); +} + +} // namespace molo::security::session diff --git a/src/security/session/DbStore.h b/src/security/session/DbStore.h new file mode 100644 index 0000000..88751d2 --- /dev/null +++ b/src/security/session/DbStore.h @@ -0,0 +1,43 @@ +#pragma once + +// DB-backed session store. The row API is abstract here and implemented +// by 04-data.md (parameterized queries only, per the SQL safety +// contract). This file owns the mapping, never SQL text. + +#include +#include +#include + +#include "Session.h" +#include "SessionStore.h" + +namespace molo::security::session { + +// Minimal row operations the data layer must provide. All user input +// (session IDs) travels as bound parameters, never interpolated. +class IDbSessionRows { +public: + virtual ~IDbSessionRows() = default; + virtual bool select(const std::string& id, Session& out) = 0; + virtual bool upsert(const Session& s) = 0; + virtual bool erase(const std::string& id) = 0; + virtual std::size_t erase_expired(std::uint64_t now_unix_sec) = 0; +}; + +class DbStore : public SessionStore { +public: + // Null rows fails closed on every call (loud, never silent memory). + explicit DbStore(std::shared_ptr rows); + + bool get(const std::string& id, std::uint64_t now_unix_sec, Session& out) override; + bool put(const Session& s) override; + bool rotate(const std::string& old_id, Session& s) override; + bool destroy(const std::string& id) override; + std::size_t sweep(std::uint64_t now_unix_sec) override; + std::size_t size() const override { return 0; } // unknown without a scan; capped by DB + +private: + std::shared_ptr rows_; +}; + +} // namespace molo::security::session diff --git a/src/security/session/MemoryStore.cpp b/src/security/session/MemoryStore.cpp new file mode 100644 index 0000000..7fe6d80 --- /dev/null +++ b/src/security/session/MemoryStore.cpp @@ -0,0 +1,74 @@ +#include "MemoryStore.h" + +namespace molo::security::session { + +MemoryStore::MemoryStore(std::size_t max_sessions) : max_sessions_(max_sessions) { + if (max_sessions_ == 0 || max_sessions_ > kMaxSessions) max_sessions_ = kMaxSessions; +} + +bool MemoryStore::expired(const Session& s, std::uint64_t now) const { + return now >= s.absolute_expiry; +} + +bool MemoryStore::get(const std::string& id, std::uint64_t now_unix_sec, Session& out) { + std::lock_guard lock(mu_); + auto it = map_.find(id); + if (it == map_.end()) return false; + if (expired(it->second, now_unix_sec)) { + map_.erase(it); + return false; + } + out = it->second; + return true; +} + +bool MemoryStore::put(const Session& s) { + if (!s.valid_shape()) return false; + std::lock_guard lock(mu_); + if (map_.size() >= max_sessions_ && map_.find(s.id) == map_.end()) { + // Static cap: evict the earliest-expiring entry. + auto victim = map_.begin(); + for (auto it = map_.begin(); it != map_.end(); ++it) { + if (it->second.absolute_expiry < victim->second.absolute_expiry) victim = it; + } + map_.erase(victim); + } + map_[s.id] = s; + return true; +} + +bool MemoryStore::rotate(const std::string& old_id, Session& s) { + if (!s.valid_shape() || s.id == old_id) return false; + std::lock_guard lock(mu_); + auto it = map_.find(old_id); + if (it == map_.end()) return false; + map_.erase(it); + map_[s.id] = s; + return true; +} + +bool MemoryStore::destroy(const std::string& id) { + std::lock_guard lock(mu_); + return map_.erase(id) > 0; +} + +std::size_t MemoryStore::sweep(std::uint64_t now_unix_sec) { + std::lock_guard lock(mu_); + std::size_t n = 0; + for (auto it = map_.begin(); it != map_.end();) { + if (expired(it->second, now_unix_sec)) { + it = map_.erase(it); + ++n; + } else { + ++it; + } + } + return n; +} + +std::size_t MemoryStore::size() const { + std::lock_guard lock(mu_); + return map_.size(); +} + +} // namespace molo::security::session diff --git a/src/security/session/MemoryStore.h b/src/security/session/MemoryStore.h new file mode 100644 index 0000000..5c31d54 --- /dev/null +++ b/src/security/session/MemoryStore.h @@ -0,0 +1,37 @@ +#pragma once + +// In-memory session store with a static cap and oldest-expiry eviction. +// Default for dev and single-instance deploys. + +#include +#include +#include +#include +#include + +#include "Session.h" +#include "SessionStore.h" + +namespace molo::security::session { + +class MemoryStore : public SessionStore { +public: + static constexpr std::size_t kMaxSessions = 100000; + + explicit MemoryStore(std::size_t max_sessions = kMaxSessions); + + bool get(const std::string& id, std::uint64_t now_unix_sec, Session& out) override; + bool put(const Session& s) override; + bool rotate(const std::string& old_id, Session& s) override; + bool destroy(const std::string& id) override; + std::size_t sweep(std::uint64_t now_unix_sec) override; + std::size_t size() const override; + +private: + bool expired(const Session& s, std::uint64_t now) const; + mutable std::mutex mu_; + std::unordered_map map_; + std::size_t max_sessions_; +}; + +} // namespace molo::security::session diff --git a/src/security/session/Session.cpp b/src/security/session/Session.cpp new file mode 100644 index 0000000..d1cdf59 --- /dev/null +++ b/src/security/session/Session.cpp @@ -0,0 +1,24 @@ +#include "Session.h" + +namespace molo::security::session { + +bool Session::valid_shape() const { + if (id.size() != 22) return false; + for (char c : id) { + bool ok = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + c == '-' || c == '_'; + if (!ok) return false; + } + if (user_id.size() > 256) return false; + if (data.size() > kMaxDataEntries) return false; + std::size_t bytes = 0; + for (const auto& kv : data) { + if (kv.first.empty() || kv.first.size() > 128) return false; + bytes += kv.first.size() + kv.second.size(); + if (bytes > kMaxDataBytes) return false; + } + if (csrf_token.size() > 128) return false; + return true; +} + +} // namespace molo::security::session diff --git a/src/security/session/Session.h b/src/security/session/Session.h new file mode 100644 index 0000000..2db4e60 --- /dev/null +++ b/src/security/session/Session.h @@ -0,0 +1,40 @@ +#pragma once + +// Session value object. The ID is a meaningless 128-bit CSPRNG token; +// all logic lives server-side in the store. Times are unix seconds. + +#include +#include +#include + +namespace molo::security::session { + +struct Session { + static constexpr std::size_t kMaxDataEntries = 64; + static constexpr std::size_t kMaxDataBytes = 4096; + + enum class Privilege : std::uint8_t { Anonymous = 0, User, Elevated, Admin }; + + std::string id; // base64url, 22 chars, 128-bit entropy + std::string user_id; + Privilege privilege = Privilege::Anonymous; + std::uint64_t created_at = 0; + std::uint64_t last_seen_at = 0; + std::uint64_t absolute_expiry = 0; + std::uint64_t last_privilege_change = 0; + std::uint64_t last_reauth_at = 0; + std::string csrf_token; // synchronizer token bound to this session + std::unordered_map data; + + // Tenant binding (PATCHES 11.15). SessionManager rotates the session + // id before rebinding when the request tenant differs. + void bind_tenant(const std::string& tid) { data["tid"] = tid; } + std::string tenant_id() const { + auto it = data.find("tid"); + return it == data.end() ? "" : it->second; + } + + bool valid_shape() const; +}; + +} // namespace molo::security::session diff --git a/src/security/session/SessionManager.cpp b/src/security/session/SessionManager.cpp new file mode 100644 index 0000000..1269b80 --- /dev/null +++ b/src/security/session/SessionManager.cpp @@ -0,0 +1,143 @@ +#include "SessionManager.h" + +#include + +#include "../crypto/Base64.h" +#include "../crypto/SecureRandom.h" +#include "../guard/DenialSink.h" +#include "../guard/HttpMessageView.h" +#include "SessionStore.h" + +namespace molo::security::session { + +SessionManager::SessionManager(std::shared_ptr store, SessionPolicy policy, + molo::security::guard::IDenialSink& sink) + : store_(std::move(store)), policy_(std::move(policy)), sink_(sink) {} + +std::string mint_id() { + auto raw = molo::security::crypto::SecureRandom::bytes(16); + if (raw.size() != 16) return {}; + return molo::security::crypto::Base64::encode_url(raw.data(), raw.size()); +} + +std::string SessionManager::new_session_id() { return mint_id(); } + +SessionManager::LoadResult SessionManager::load(const molo::security::guard::HttpRequestView& req, + Session& out) { + out = Session{}; + std::uint64_t now = req.now_unix_sec; + std::string id = req.cookie(policy_.cookie_name); + if (id.empty() || !store_) { + out.id = mint_id(); + out.created_at = now; + out.last_seen_at = now; + out.absolute_expiry = now + policy_.absolute_timeout_sec; + return LoadResult::Fresh; + } + Session s; + if (!store_->get(id, now, s)) { + sink_.deny({"session", "session.unknown_or_expired", "cookie id not in store", req.peer_ip, + req.path}); + out.id = mint_id(); + out.created_at = now; + out.last_seen_at = now; + out.absolute_expiry = now + policy_.absolute_timeout_sec; + return LoadResult::Invalid; + } + if (now - s.last_seen_at > policy_.idle_timeout_sec) { + store_->destroy(id); + sink_.deny({"session", "session.idle_timeout", "idle window exceeded", req.peer_ip, + req.path}); + out.id = mint_id(); + out.created_at = now; + out.last_seen_at = now; + out.absolute_expiry = now + policy_.absolute_timeout_sec; + return LoadResult::ExpiredIdle; + } + if (now - s.created_at > policy_.renewal_cap_sec) { + store_->destroy(id); + sink_.deny({"session", "session.renewal_cap", "24h renewal cap hit", req.peer_ip, + req.path}); + out.id = mint_id(); + out.created_at = now; + out.last_seen_at = now; + out.absolute_expiry = now + policy_.absolute_timeout_sec; + return LoadResult::ExpiredAbsolute; + } + s.last_seen_at = now; + store_->put(s); + out = s; + return LoadResult::Resumed; +} + +void SessionManager::emit_cookie(const Session& s, + molo::security::guard::HttpResponseView& res) { + char buf[512]; + std::snprintf(buf, sizeof buf, "%s=%s; Path=/; Secure; HttpOnly; SameSite=%s", + policy_.cookie_name.c_str(), s.id.c_str(), + policy_.samesite_strict ? "Strict" : "Lax"); + res.add_header("Set-Cookie", buf); +} + +void SessionManager::save(const Session& s, molo::security::guard::HttpResponseView& res) { + if (!store_) return; + Session copy = s; + if (!copy.valid_shape()) return; + store_->put(copy); + emit_cookie(copy, res); +} + +bool SessionManager::rotate(Session& s, molo::security::guard::HttpResponseView& res) { + if (!store_) return false; + std::string old = s.id; + std::string fresh = mint_id(); + if (fresh.empty()) return false; + s.id = fresh; + if (!store_->rotate(old, s)) { + // Old ID unknown (e.g. fresh session): store under the new ID. + s.id = fresh; + if (!store_->put(s)) { + s.id = old; + return false; + } + sink_.audit({"session", "session.rotate_put", "rotated onto fresh store entry", "", ""}); + } else { + sink_.audit({"session", "session.rotated", "id rotated", "", ""}); + } + emit_cookie(s, res); + return true; +} + +bool SessionManager::bind_request_tenant(Session& s, const std::string& request_tenant, + molo::security::guard::HttpResponseView& res) { + if (request_tenant.empty()) return true; + if (s.tenant_id() != request_tenant) { + if (!rotate(s, res)) return false; + s.bind_tenant(request_tenant); + } + return true; +} + +bool SessionManager::set_privilege(Session& s, Session::Privilege p, + molo::security::guard::HttpResponseView& res, + std::uint64_t now) { + if (s.privilege == p) return true; + s.privilege = p; + s.last_privilege_change = now; + if (p != Session::Privilege::Anonymous) s.last_reauth_at = now; + if (!rotate(s, res)) return false; + sink_.audit({"session", "session.privilege_change", "privilege rotated", "", ""}); + return true; +} + +void SessionManager::logout(Session& s, molo::security::guard::HttpResponseView& res) { + if (store_) store_->destroy(s.id); + s = Session{}; + char buf[256]; + std::snprintf(buf, sizeof buf, "%s=; Path=/; Secure; HttpOnly; SameSite=%s; Max-Age=0", + policy_.cookie_name.c_str(), policy_.samesite_strict ? "Strict" : "Lax"); + res.add_header("Set-Cookie", buf); + sink_.audit({"session", "session.logout", "session destroyed", "", ""}); +} + +} // namespace molo::security::session diff --git a/src/security/session/SessionManager.h b/src/security/session/SessionManager.h new file mode 100644 index 0000000..77b9e07 --- /dev/null +++ b/src/security/session/SessionManager.h @@ -0,0 +1,71 @@ +#pragma once + +// Session lifecycle: cookie issue and parse, idle plus absolute plus +// renewal enforcement, rotation on privilege change. Cookie name +// "__Host-molo_sid" with Secure, HttpOnly, SameSite (Strict default). + +#include +#include +#include + +#include "Session.h" + +namespace molo::security::guard { +struct HttpRequestView; +struct HttpResponseView; +class IDenialSink; +} // namespace molo::security::guard + +namespace molo::security::session { + +class SessionStore; + +struct SessionPolicy { + static constexpr std::uint64_t kIdleTimeoutSec = 30u * 60u; + static constexpr std::uint64_t kAbsoluteTimeoutSec = 12u * 3600u; + static constexpr std::uint64_t kRenewalCapSec = 24u * 3600u; + + std::uint64_t idle_timeout_sec = kIdleTimeoutSec; + std::uint64_t absolute_timeout_sec = kAbsoluteTimeoutSec; + std::uint64_t renewal_cap_sec = kRenewalCapSec; + bool samesite_strict = true; // false opts into Lax + std::string cookie_name = "__Host-molo_sid"; +}; + +class SessionManager { +public: + SessionManager(std::shared_ptr store, SessionPolicy policy, + molo::security::guard::IDenialSink& sink); + + enum class LoadResult : std::uint8_t { Fresh, Resumed, ExpiredIdle, ExpiredAbsolute, Invalid }; + + // Loads the request session. Fresh means no usable cookie (anonymous + // session created but not yet stored). Expired sessions are destroyed. + LoadResult load(const molo::security::guard::HttpRequestView& req, Session& out); + // Persists and stamps the Set-Cookie header (always on change). + void save(const Session& s, molo::security::guard::HttpResponseView& res); + // Rotates the session ID, keeping the payload. Must be called on + // login, logout, privilege change, and re-auth. + bool rotate(Session& s, molo::security::guard::HttpResponseView& res); + // Elevates privilege with mandatory rotation. False without entropy. + bool set_privilege(Session& s, Session::Privilege p, + molo::security::guard::HttpResponseView& res, std::uint64_t now); + // Destroys server-side and clears the cookie. + void logout(Session& s, molo::security::guard::HttpResponseView& res); + // Rebinds the session to the request tenant, rotating the session id + // first when the tenant differs (PATCHES 11.15). Empty request tenant + // is a no-op. Caller must save() afterwards to persist the binding. + bool bind_request_tenant(Session& s, const std::string& request_tenant, + molo::security::guard::HttpResponseView& res); + + static std::string new_session_id(); // "" without entropy + const SessionPolicy& policy() const { return policy_; } + +private: + void emit_cookie(const Session& s, molo::security::guard::HttpResponseView& res); + std::shared_ptr store_; + SessionPolicy policy_; + molo::security::guard::IDenialSink& sink_; +}; + +} // namespace molo::security::session diff --git a/src/security/session/SessionStore.h b/src/security/session/SessionStore.h new file mode 100644 index 0000000..6bc2efe --- /dev/null +++ b/src/security/session/SessionStore.h @@ -0,0 +1,31 @@ +#pragma once + +// Session store interface. MemoryStore is the default; DbStore (over an +// abstract row API that 04-data.md implements) is the production pick. +// All methods are thread-safe. + +#include +#include + +namespace molo::security::session { + +struct Session; + +class SessionStore { +public: + virtual ~SessionStore() = default; + + // Loads a session by ID. False when missing; expired sessions are + // treated as missing (and lazily dropped). + virtual bool get(const std::string& id, std::uint64_t now_unix_sec, Session& out) = 0; + // Inserts or replaces. False when the entry is oversized. + virtual bool put(const Session& s) = 0; + // Atomic rename of the session ID (rotation). Old ID dies. + virtual bool rotate(const std::string& old_id, Session& s) = 0; + virtual bool destroy(const std::string& id) = 0; + // Drops expired entries. Returns evicted count. + virtual std::size_t sweep(std::uint64_t now_unix_sec) = 0; + virtual std::size_t size() const = 0; +}; + +} // namespace molo::security::session diff --git a/src/security/tls/TlsAcceptor.cpp b/src/security/tls/TlsAcceptor.cpp new file mode 100644 index 0000000..7088108 --- /dev/null +++ b/src/security/tls/TlsAcceptor.cpp @@ -0,0 +1,165 @@ +#include "TlsAcceptor.h" + +#include + +#include "TlsContext.h" + +namespace molo::security::tls { + +TlsAcceptor::TlsAcceptor(TlsContext& ctx, ITlsConnHooks& hooks, TlsAcceptorPolicy policy) + : ctx_(ctx), hooks_(hooks), policy_(std::move(policy)) { + clock_ = [] { + return std::uint64_t(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + }; +} + +std::uint64_t TlsAcceptor::now_ms() const { return clock_ ? clock_() : 0; } + +bool TlsAcceptor::on_accept(std::uint64_t conn_id, const std::string& peer_ip) { + std::uint64_t now = now_ms(); + auto b = backoff_.find(peer_ip); + if (b != backoff_.end() && now < b->second.until_ms) { + ++counters_.backoff_rejected; + return false; + } + if (pending_.size() >= TlsAcceptorPolicy::kMaxPendingHandshakes) { + ++counters_.failed; + return false; + } + if (pending_.count(conn_id) != 0) return false; + auto session = ctx_.new_session(TlsRole::Server); + if (!session) { + ++counters_.unavailable; + return false; + } + Pending p; + p.conn_id = conn_id; + p.peer_ip = peer_ip; + p.session = std::move(session); + p.deadline_ms = now + policy_.handshake_timeout_ms; + pending_.emplace(conn_id, std::move(p)); + ++counters_.started; + // Kick the driver in case the backend has an initial flight. + on_writable(conn_id); + return true; +} + +bool TlsAcceptor::check_sni(Pending& p) { + std::string host = p.session->sni_hostname(); + SniDecision d; + d.hostname = host; + if (policy_.sni_allow.empty()) { + // Any SNI the context holds certs for; empty SNI is allowed and + // maps to the default cert. + d.allowed = true; + } else { + for (const auto& a : policy_.sni_allow) { + if (a == host) { + d.allowed = true; + break; + } + } + if (!d.allowed) d.reason = "sni not in allow-list"; + } + if (sni_log_) sni_log_(d); + return d.allowed; +} + +void TlsAcceptor::fail(std::uint64_t conn_id, const std::string& reason) { + auto it = pending_.find(conn_id); + if (it == pending_.end()) return; + std::string ip = it->second.peer_ip; + pending_.erase(it); + ++counters_.failed; + auto& b = backoff_[ip]; + ++b.fails; + if (b.fails >= TlsAcceptorPolicy::kBackoffThreshold) { + std::uint32_t shift = b.fails - TlsAcceptorPolicy::kBackoffThreshold; + if (shift > 6) shift = 6; + std::uint64_t wait = + std::uint64_t(TlsAcceptorPolicy::kBackoffBaseMs) << shift; + if (wait > TlsAcceptorPolicy::kBackoffMaxMs) wait = TlsAcceptorPolicy::kBackoffMaxMs; + b.until_ms = now_ms() + wait; + } + if (backoff_.size() > TlsAcceptorPolicy::kMaxBackoffEntries) { + // Static cap: evict an arbitrary entry (unordered, no LRU cost). + backoff_.erase(backoff_.begin()); + } + hooks_.close_conn(conn_id, reason); +} + +void TlsAcceptor::on_bytes(std::uint64_t conn_id, const std::uint8_t* data, std::size_t len) { + auto it = pending_.find(conn_id); + if (it == pending_.end()) return; + Pending& p = it->second; + std::vector out; + TlsStep step = p.session->handshake_step(data, len, out); + if (!out.empty() && !hooks_.send_bytes(conn_id, out)) { + fail(conn_id, "tls send buffer full"); + return; + } + switch (step) { + case TlsStep::Done: + if (!p.sni_checked) { + p.sni_checked = true; + if (!check_sni(p)) { + ++counters_.sni_rejected; + fail(conn_id, "tls sni rejected"); + return; + } + } + counters_.completed++; + backoff_.erase(p.peer_ip); + { + std::string alpn = p.session->alpn_selected(); + auto session = std::move(p.session); + pending_.erase(it); + hooks_.promote_conn(conn_id, std::move(session), alpn); + } + return; + case TlsStep::WantRead: + case TlsStep::WantWrite: + if (!p.sni_checked && !p.session->sni_hostname().empty()) { + p.sni_checked = true; + if (!check_sni(p)) { + ++counters_.sni_rejected; + fail(conn_id, "tls sni rejected early"); + return; + } + } + return; + case TlsStep::Failed: + fail(conn_id, std::string("tls handshake failed: ") + p.session->last_error()); + return; + case TlsStep::Unavailable: + ++counters_.unavailable; + fail(conn_id, "tls backend unavailable (wolfSSL stub)"); + return; + } + fail(conn_id, "tls unknown step"); +} + +void TlsAcceptor::on_writable(std::uint64_t conn_id) { + auto it = pending_.find(conn_id); + if (it == pending_.end()) return; + on_bytes(conn_id, nullptr, 0); +} + +std::size_t TlsAcceptor::on_tick() { + std::uint64_t now = now_ms(); + std::vector expired; + for (const auto& kv : pending_) { + if (now >= kv.second.deadline_ms) expired.push_back(kv.first); + } + for (auto id : expired) { + ++counters_.timed_out; + fail(id, "tls handshake timeout"); + } + return expired.size(); +} + +void TlsAcceptor::on_close(std::uint64_t conn_id) { pending_.erase(conn_id); } + +} // namespace molo::security::tls diff --git a/src/security/tls/TlsAcceptor.h b/src/security/tls/TlsAcceptor.h new file mode 100644 index 0000000..b3f161a --- /dev/null +++ b/src/security/tls/TlsAcceptor.h @@ -0,0 +1,117 @@ +#pragma once + +// Non-blocking handshake driver for the event loop (runtime hook). +// One TlsHandshake per accepted connection: feed socket bytes, collect +// flight bytes, enforce the handshake timeout, and report SNI for +// dispatch. Failure counters plus per-IP backoff live in TlsAcceptor. + +#include +#include +#include +#include +#include +#include +#include + +#include "TlsBackend.h" + +namespace molo::security::tls { + +class TlsContext; + +// Runtime connection hook: the event loop implements this to move bytes. +// (Declared here so TlsAcceptor compiles standalone; 01-runtime.md owns +// the real socket side.) +struct ITlsConnHooks { + virtual ~ITlsConnHooks() = default; + // Queue bytes for the peer. Returns false when the send buffer is full. + virtual bool send_bytes(std::uint64_t conn_id, const std::vector& data) = 0; + // Close the connection with a structured reason (logged by runtime). + virtual void close_conn(std::uint64_t conn_id, const std::string& reason) = 0; + // Promote a connection to application traffic with the negotiated + // session handed over. Runtime owns the session afterwards. + virtual void promote_conn(std::uint64_t conn_id, std::unique_ptr session, + const std::string& alpn) = 0; +}; + +struct TlsAcceptorPolicy { + static constexpr std::size_t kMaxPendingHandshakes = 4096; + static constexpr std::size_t kMaxBackoffEntries = 10000; + static constexpr std::uint32_t kBackoffBaseMs = 1000; + static constexpr std::uint32_t kBackoffMaxMs = 60000; + static constexpr std::uint32_t kBackoffThreshold = 5; // failures before backoff + + std::uint32_t handshake_timeout_ms = 5000; + // SNI allow-list. Empty means: any SNI the context holds certs for. + // Unknown SNI fails closed. + std::vector sni_allow; +}; + +// SNI dispatch decision for observability logging. +struct SniDecision { + std::string hostname; + bool allowed = false; + std::string reason; +}; + +class TlsAcceptor { +public: + using ClockMs = std::function; // injectable clock + using SniLog = std::function; + + TlsAcceptor(TlsContext& ctx, ITlsConnHooks& hooks, TlsAcceptorPolicy policy = {}); + + struct Counters { + std::uint64_t started = 0; + std::uint64_t completed = 0; + std::uint64_t failed = 0; + std::uint64_t timed_out = 0; + std::uint64_t sni_rejected = 0; + std::uint64_t backoff_rejected = 0; + std::uint64_t unavailable = 0; // stub backend + }; + const Counters& counters() const { return counters_; } + + void set_clock(ClockMs clock) { clock_ = std::move(clock); } + void set_sni_log(SniLog log) { sni_log_ = std::move(log); } + + // Called by the listener on accept. Returns false when table full or + // IP is in backoff (caller must close immediately). + bool on_accept(std::uint64_t conn_id, const std::string& peer_ip); + // Called when socket bytes arrive during handshake. + void on_bytes(std::uint64_t conn_id, const std::uint8_t* data, std::size_t len); + // Called on WANT_WRITE flush completion or timer tick. + void on_writable(std::uint64_t conn_id); + // Periodic sweep for handshake timeouts. Returns timed-out count. + std::size_t on_tick(); + // Drop a pending handshake (peer went away). + void on_close(std::uint64_t conn_id); + +private: + struct Pending { + std::uint64_t conn_id = 0; + std::string peer_ip; + std::unique_ptr session; + std::uint64_t deadline_ms = 0; + bool sni_checked = false; + }; + struct Backoff { + std::uint32_t fails = 0; + std::uint64_t until_ms = 0; + }; + + void fail(std::uint64_t conn_id, const std::string& reason); + bool check_sni(Pending& p); + std::uint64_t now_ms() const; + + TlsContext& ctx_; + ITlsConnHooks& hooks_; + TlsAcceptorPolicy policy_; + Counters counters_{}; + std::unordered_map pending_; + std::unordered_map backoff_; + ClockMs clock_; + SniLog sni_log_; +}; + +} // namespace molo::security::tls diff --git a/src/security/tls/TlsBackend.h b/src/security/tls/TlsBackend.h new file mode 100644 index 0000000..0ced49c --- /dev/null +++ b/src/security/tls/TlsBackend.h @@ -0,0 +1,68 @@ +#pragma once + +// TLS backend interface. wolfSSL sits behind this line; the rest of the +// framework (runtime listeners, acceptor driver) never touches wolfSSL +// headers directly. One backend instance per TlsContext. + +#include +#include +#include +#include +#include + +namespace molo::security::tls { + +struct TlsConfig; + +enum class TlsRole : std::uint8_t { Server = 0, Client }; + +// Non-blocking handshake step result. +enum class TlsStep : std::uint8_t { + Done = 0, // handshake complete, session ready + WantRead, // need more peer bytes, out may carry flight data + WantWrite, // peer bytes buffered, flush out then call again + Failed, // terminal, see last_error() + Unavailable, // backend not installed (stub) +}; + +// One TLS connection. Single-threaded: driven by exactly one event-loop +// task at a time. Plaintext caps enforced by the caller (BodyLimits). +class ITlsSession { +public: + virtual ~ITlsSession() = default; + + // Feeds peer bytes (may be empty on WantWrite retry) and collects + // bytes to send. Never blocks. + virtual TlsStep handshake_step(const std::uint8_t* in, std::size_t in_len, + std::vector& out) = 0; + + // Post-handshake record layer. Returns false on fatal alert or + // overflow; out appends decrypted bytes (bounded by caller cap). + virtual bool decrypt(const std::uint8_t* in, std::size_t in_len, std::vector& out, + std::size_t max_out) = 0; + virtual bool encrypt(const std::uint8_t* plain, std::size_t len, + std::vector& out) = 0; + + virtual std::string alpn_selected() const = 0; + virtual std::string sni_hostname() const = 0; + virtual std::string cipher_name() const = 0; + virtual std::string last_error() const = 0; + virtual bool shutdown(std::vector& close_notify_out) = 0; +}; + +// Backend factory. Implemented by WolfSslBackend (stub when wolfSSL is +// absent) and by test fakes. +class ITlsBackend { +public: + virtual ~ITlsBackend() = default; + virtual std::string name() const = 0; + virtual bool available() const = 0; + // Applies config and loads certificates. False with reason on any + // failure; the server must not boot. + virtual bool configure(const TlsConfig& config, std::string& reason) = 0; + // Hot reload of certificate files without dropping sessions. + virtual bool reload_certs(std::string& reason) = 0; + virtual std::unique_ptr new_session(TlsRole role) = 0; +}; + +} // namespace molo::security::tls diff --git a/src/security/tls/TlsConfig.cpp b/src/security/tls/TlsConfig.cpp new file mode 100644 index 0000000..39df1e6 --- /dev/null +++ b/src/security/tls/TlsConfig.cpp @@ -0,0 +1,33 @@ +#include "TlsConfig.h" + +namespace molo::security::tls { + +std::string TlsConfig::validate() const { + if (min_version_major != 3 || min_version_minor != 4 || max_version_major != 3 || + max_version_minor != 4) + return "tls.min_version: only TLS 1.3 (3,4) is allowed"; + if (alpn.empty()) return "tls.alpn: at least one protocol required"; + if (alpn.size() > kMaxAlpnEntries) return "tls.alpn: too many entries"; + for (const auto& a : alpn) { + if (a.empty() || a.size() > 32) return "tls.alpn: entry must be 1..32 bytes"; + for (char c : a) { + if (c < 0x21 || c > 0x7E || c == ',') return "tls.alpn: bad character"; + } + } + if (cert_path.empty() || key_path.empty()) return "tls.cert: cert_path and key_path required"; + if (cert_path.size() > kMaxPathBytes || key_path.size() > kMaxPathBytes) + return "tls.cert: path too long"; + if (sni.size() > kMaxSniEntries) return "tls.sni: too many entries"; + for (const auto& e : sni) { + if (e.hostname.empty() || e.hostname.size() > 253) return "tls.sni: bad hostname"; + if (e.cert_path.empty() || e.key_path.empty()) return "tls.sni: cert and key required"; + if (e.cert_path.size() > kMaxPathBytes || e.key_path.size() > kMaxPathBytes) + return "tls.sni: path too long"; + } + if (handshake_timeout_ms < kMinHandshakeTimeoutMs || + handshake_timeout_ms > kMaxHandshakeTimeoutMs) + return "tls.handshake_timeout_ms: out of range 500..30000"; + return {}; +} + +} // namespace molo::security::tls diff --git a/src/security/tls/TlsConfig.h b/src/security/tls/TlsConfig.h new file mode 100644 index 0000000..916ffad --- /dev/null +++ b/src/security/tls/TlsConfig.h @@ -0,0 +1,50 @@ +#pragma once + +// TLS server configuration. TLS 1.3 only (plan rule 1). +// validate() must pass before boot; any violation aborts with the +// returned reason (development guideline: typed validated config). + +#include +#include +#include + +namespace molo::security::tls { + +struct TlsConfig { + static constexpr std::size_t kMaxAlpnEntries = 8; + static constexpr std::size_t kMaxSniEntries = 256; + static constexpr std::size_t kMaxPathBytes = 4096; + static constexpr std::uint32_t kMinHandshakeTimeoutMs = 500; + static constexpr std::uint32_t kMaxHandshakeTimeoutMs = 30000; + static constexpr std::uint32_t kDefaultHandshakeTimeoutMs = 5000; + + // TLS 1.3 only. Minor knob kept for decoding only; anything but + // (3,4) fails validation. + std::uint16_t min_version_major = 3; + std::uint16_t min_version_minor = 4; + std::uint16_t max_version_major = 3; + std::uint16_t max_version_minor = 4; + + // ALPN protocols in preference order. Default h2 then http/1.1. + std::vector alpn = {"h2", "http/1.1"}; + + std::string cert_path; + std::string key_path; + std::string chain_path; // optional extra chain file + + struct SniEntry { + std::string hostname; // exact match, lowercase + std::string cert_path; + std::string key_path; + }; + std::vector sni; + + std::uint32_t handshake_timeout_ms = kDefaultHandshakeTimeoutMs; + bool allow_h2c_upgrade = true; // cleartext h2c per 01-runtime.md + bool cert_reload_enabled = true; + + // Empty string means valid. + std::string validate() const; +}; + +} // namespace molo::security::tls diff --git a/src/security/tls/TlsContext.cpp b/src/security/tls/TlsContext.cpp new file mode 100644 index 0000000..4c4f166 --- /dev/null +++ b/src/security/tls/TlsContext.cpp @@ -0,0 +1,51 @@ +#include "TlsContext.h" + +namespace molo::security::tls { + +bool TlsContext::boot(std::unique_ptr backend, const TlsConfig& config, + std::string& reason) { + reason.clear(); + std::lock_guard lock(mu_); + if (!backend) { + reason = "tls.backend: null backend"; + return false; + } + std::string bad = config.validate(); + if (!bad.empty()) { + reason = bad; + return false; + } + if (!backend->configure(config, reason)) return false; + backend_ = std::move(backend); + config_ = config; + booted_ = true; + return true; +} + +bool TlsContext::reload_certs(std::string& reason) { + std::lock_guard lock(mu_); + if (!booted_ || !backend_) { + reason = "tls not booted"; + return false; + } + if (!config_.cert_reload_enabled) { + reason = "tls.cert_reload_enabled is false"; + return false; + } + if (backend_->reload_certs(reason)) { + ++stats_.reloads_ok; + return true; + } + ++stats_.reloads_failed; + return false; +} + +std::unique_ptr TlsContext::new_session(TlsRole role) { + std::lock_guard lock(mu_); + if (!booted_ || !backend_ || !backend_->available()) return nullptr; + auto s = backend_->new_session(role); + if (s) ++stats_.sessions_opened; + return s; +} + +} // namespace molo::security::tls diff --git a/src/security/tls/TlsContext.h b/src/security/tls/TlsContext.h new file mode 100644 index 0000000..6abf851 --- /dev/null +++ b/src/security/tls/TlsContext.h @@ -0,0 +1,48 @@ +#pragma once + +// Owns the backend plus validated config for one listener. Loads +// certificates at boot, reloads them on signal without dropping +// sessions, and mints per-connection sessions. No per-connection +// state lives here; the acceptor driver owns that. + +#include +#include +#include +#include + +#include "TlsBackend.h" +#include "TlsConfig.h" + +namespace molo::security::tls { + +class TlsContext { +public: + struct Stats { + std::uint64_t sessions_opened = 0; + std::uint64_t reloads_ok = 0; + std::uint64_t reloads_failed = 0; + }; + + // Takes ownership of the backend (usually WolfSslBackend). + // Returns false with reason when config invalid or certs unloadable. + bool boot(std::unique_ptr backend, const TlsConfig& config, std::string& reason); + + bool booted() const { return booted_; } + const TlsConfig& config() const { return config_; } + const Stats& stats() const { return stats_; } + + // Hot reload. Old sessions keep running on old certs. + bool reload_certs(std::string& reason); + + // Null when not booted or backend unavailable. + std::unique_ptr new_session(TlsRole role); + +private: + std::unique_ptr backend_; + TlsConfig config_; + Stats stats_{}; + bool booted_ = false; + mutable std::mutex mu_; +}; + +} // namespace molo::security::tls diff --git a/src/security/tls/WolfSslBackend.cpp b/src/security/tls/WolfSslBackend.cpp new file mode 100644 index 0000000..f716cd1 --- /dev/null +++ b/src/security/tls/WolfSslBackend.cpp @@ -0,0 +1,97 @@ +#include "WolfSslBackend.h" + +#include "TlsConfig.h" + +// STUB IMPLEMENTATION. wolfSSL is not installed, so everything here fails +// closed with a clear reason. Native integration checklist for the build +// owner (needs MOLO_HAS_WOLFSSL plus wolfSSL commercial license): +// - wolfSSL_Init, wolfTLSv1_3_server_method, wolfSSL_CTX_new +// - wolfSSL_CTX_use_certificate_chain_file + use_PrivateKey_file +// - wolfSSL_CTX_SetIORecv/Send with non-blocking callbacks +// - wolfSSL_CTX_ALPN_SelectProtos, SNI callback via wolfSSL_CTX_sni_recv +// - wolfSSL_accept step driver mapping WOLFSSL_CBIO_ERR_WANT_READ/WRITE +// onto TlsStep::WantRead/WantWrite. + +namespace molo::security::tls { + +struct WolfSslBackend::Impl { + TlsConfig config; + bool configured = false; +}; + +namespace { + +class UnavailableSession : public ITlsSession { +public: + TlsStep handshake_step(const std::uint8_t*, std::size_t, std::vector&) override { + return TlsStep::Unavailable; + } + bool decrypt(const std::uint8_t*, std::size_t, std::vector&, std::size_t) override { + return false; + } + bool encrypt(const std::uint8_t*, std::size_t, std::vector&) override { + return false; + } + std::string alpn_selected() const override { return {}; } + std::string sni_hostname() const override { return {}; } + std::string cipher_name() const override { return {}; } + std::string last_error() const override { + return "STUB: wolfSSL not installed, TLS unavailable"; + } + bool shutdown(std::vector&) override { return false; } +}; + +} // namespace + +WolfSslBackend::WolfSslBackend() : impl_(new Impl) {} +WolfSslBackend::~WolfSslBackend() = default; + +bool WolfSslBackend::available() const { +#if defined(MOLO_HAS_WOLFSSL) + return impl_->configured; +#else + return false; +#endif +} + +bool WolfSslBackend::configure(const TlsConfig& config, std::string& reason) { + reason.clear(); + std::string bad = config.validate(); + if (!bad.empty()) { + reason = bad; + return false; + } +#if defined(MOLO_HAS_WOLFSSL) + impl_->config = config; + impl_->configured = true; // native path fills in CTX setup here + return true; +#else + impl_->config = config; + reason = "STUB: wolfSSL not installed, config validated but TLS unavailable"; + return false; +#endif +} + +bool WolfSslBackend::reload_certs(std::string& reason) { +#if defined(MOLO_HAS_WOLFSSL) + if (!impl_->configured) { + reason = "tls not configured"; + return false; + } + return true; // native path reloads chain files here +#else + reason = "STUB: wolfSSL not installed, reload is a no-op"; + return false; +#endif +} + +std::unique_ptr WolfSslBackend::new_session(TlsRole) { +#if defined(MOLO_HAS_WOLFSSL) + if (!impl_->configured) return nullptr; + return nullptr; // native path returns a wolfSSL-backed session here +#else + return std::unique_ptr(new UnavailableSession); +#endif +} + +} // namespace molo::security::tls diff --git a/src/security/tls/WolfSslBackend.h b/src/security/tls/WolfSslBackend.h new file mode 100644 index 0000000..34021eb --- /dev/null +++ b/src/security/tls/WolfSslBackend.h @@ -0,0 +1,44 @@ +#pragma once + +// STUB: wolfSSL is NOT installed in this environment, so this backend +// reports unavailable() == false and every session fails closed with +// TlsStep::Unavailable. The shape below is the real integration seam: +// defining MOLO_HAS_WOLFSSL and linking wolfSSL enables the native path +// (TLS 1.3 only, ALPN, SNI dispatch, non-blocking I/O callbacks). +// +// Rule: no other file may include wolfSSL headers; all TLS traffic goes +// through ITlsBackend. When the native path lands, only this file plus +// WolfSslBackend.cpp change. + +#include +#include + +#include "TlsBackend.h" + +namespace molo::security::tls { + +class WolfSslBackend : public ITlsBackend { +public: + WolfSslBackend(); + ~WolfSslBackend() override; + + std::string name() const override { return "wolfssl"; } + bool available() const override; + bool configure(const TlsConfig& config, std::string& reason) override; + bool reload_certs(std::string& reason) override; + std::unique_ptr new_session(TlsRole role) override; + + // True when built against real wolfSSL headers. + static constexpr bool kNativeBuild = +#if defined(MOLO_HAS_WOLFSSL) + true; +#else + false; +#endif + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace molo::security::tls diff --git a/src/security/upload/UploadStore.cpp b/src/security/upload/UploadStore.cpp new file mode 100644 index 0000000..2430cf6 --- /dev/null +++ b/src/security/upload/UploadStore.cpp @@ -0,0 +1,134 @@ +#include "UploadStore.h" + +#include +#include +#include +#include + +#include +#include + +#include "../crypto/SecureRandom.h" +#include "../guard/DenialSink.h" +#include "../guard/HttpMessageView.h" +#include "UploadValidator.h" + +namespace molo::security::upload { + +UploadStore::UploadStore(Policy policy, std::shared_ptr scanner, + molo::security::guard::IDenialSink& sink) + : policy_(std::move(policy)), scanner_(std::move(scanner)), sink_(sink) {} + +std::string UploadStore::validate(const Policy& p) { + if (p.temp_dir.empty() || p.temp_dir[0] != '/') return "upload.temp_dir must be absolute"; + if (p.store_dir.empty() || p.store_dir[0] != '/') return "upload.store_dir must be absolute"; + if (p.quarantine_dir.empty() || p.quarantine_dir[0] != '/') + return "upload.quarantine_dir must be absolute"; + if (p.temp_dir == p.store_dir || p.temp_dir == p.quarantine_dir || + p.store_dir == p.quarantine_dir) + return "upload dirs must be distinct"; + return {}; +} + +namespace { + +bool write_all(const std::string& path, const std::uint8_t* data, std::size_t len) { + int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); + if (fd < 0) return false; + std::size_t done = 0; + while (done < len) { + ssize_t n = ::write(fd, data + done, len - done); + if (n <= 0) { + ::close(fd); + ::unlink(path.c_str()); + return false; + } + done += std::size_t(n); + } + ::close(fd); + return true; +} + +std::string gen_name(const std::string& ext) { + std::string t = molo::security::crypto::SecureRandom::token(16); + if (t.empty()) return {}; + return t + "." + ext; +} + +} // namespace + +bool UploadStore::store(const UploadFile& f, const std::string& ext, std::uint64_t now_unix_sec, + StoredUpload& out, const molo::security::guard::HttpRequestView& req) { + out = StoredUpload{}; + if (used_ + f.bytes.size() > policy_.quota_bytes) { + sink_.deny({"upload", "upload.quota", "store quota exceeded", req.peer_ip, req.path}); + return false; + } + if (ext.empty() || ext.size() > 16) return false; + std::string tmp_name = gen_name("tmp"); + std::string store_name = gen_name(ext); + if (tmp_name.empty() || store_name.empty()) return false; + std::string tmp_path = policy_.temp_dir + "/" + tmp_name; + if (!write_all(tmp_path, f.bytes.data(), f.bytes.size())) { + sink_.deny({"upload", "upload.io_error", "temp write failed", req.peer_ip, req.path}); + return false; + } + ScanResult scan = scanner_ ? scanner_->scan(tmp_path) : ScanResult::Error; + if (scan != ScanResult::Clean) { + std::string quar = policy_.quarantine_dir + "/" + gen_name(ext); + ::rename(tmp_path.c_str(), quar.c_str()); + ::unlink(tmp_path.c_str()); + sink_.deny({"upload", scan == ScanResult::Infected ? "upload.infected" : "upload.scan_error", + "file quarantined", req.peer_ip, req.path}); + return false; + } + std::string final_path = policy_.store_dir + "/" + store_name; + if (::rename(tmp_path.c_str(), final_path.c_str()) != 0) { + ::unlink(tmp_path.c_str()); + sink_.deny({"upload", "upload.io_error", "store move failed", req.peer_ip, req.path}); + return false; + } + (void)now_unix_sec; + used_ += f.bytes.size(); + out.id = store_name; + out.storage_name = store_name; + out.full_path = final_path; + out.size = f.bytes.size(); + out.ext = ext; + sink_.audit({"upload", "upload.stored", "file stored outside webroot", req.peer_ip, req.path}); + return true; +} + +std::size_t UploadStore::sweep_temp(std::uint64_t now_unix_sec) { + std::size_t removed = 0; + DIR* dir = ::opendir(policy_.temp_dir.c_str()); + if (!dir) return 0; + while (dirent* e = ::readdir(dir)) { + std::string name = e->d_name; + if (name == "." || name == "..") continue; + if (name.size() > 64) continue; + bool safe = true; + for (char c : name) { + bool ok = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.'; + if (!ok) { + safe = false; + break; + } + } + if (!safe) continue; + std::string full = policy_.temp_dir + "/" + name; + struct stat st{}; + if (::stat(full.c_str(), &st) != 0) continue; + if (!S_ISREG(st.st_mode)) continue; + std::uint64_t age = + now_unix_sec > std::uint64_t(st.st_mtime) ? now_unix_sec - std::uint64_t(st.st_mtime) : 0; + if (age > policy_.temp_ttl_sec) { + if (::unlink(full.c_str()) == 0) ++removed; + } + } + ::closedir(dir); + return removed; +} + +} // namespace molo::security::upload diff --git a/src/security/upload/UploadStore.h b/src/security/upload/UploadStore.h new file mode 100644 index 0000000..b498774 --- /dev/null +++ b/src/security/upload/UploadStore.h @@ -0,0 +1,77 @@ +#pragma once + +// Upload storage: quarantine dir plus final store outside the webroot, +// AV scan hook, generated storage names (uuid.ext), temp sweep after +// one hour. Served only through an ID handler (never executed). + +#include +#include +#include + +namespace molo::security::guard { +struct HttpRequestView; +class IDenialSink; +} // namespace molo::security::guard + +namespace molo::security::upload { + +struct UploadFile; + +enum class ScanResult : std::uint8_t { Clean, Infected, Error }; + +// AV or sandbox hook. v1 ships a synchronous scan(path); the async +// quarantine-then-release shape is an open question in the plan. +class IAvScanner { +public: + virtual ~IAvScanner() = default; + virtual ScanResult scan(const std::string& path) = 0; +}; + +class AllowAllScanner : public IAvScanner { +public: + ScanResult scan(const std::string&) override { return ScanResult::Clean; } +}; + +struct StoredUpload { + std::string id; // opaque handle for the ID handler + std::string storage_name; // uuid.ext under the store dir + std::string full_path; + std::uint64_t size = 0; + std::string ext; +}; + +class UploadStore { +public: + struct Policy { + std::string temp_dir; // 0700, swept after 1 h + std::string store_dir; // outside webroot, final files + std::string quarantine_dir; + std::uint64_t quota_bytes = 1024u * 1024u * 1024u; // 1 GiB total + std::uint64_t temp_ttl_sec = 3600; + }; + + UploadStore(Policy policy, std::shared_ptr scanner, + molo::security::guard::IDenialSink& sink); + + // Empty string means valid (dirs absolute and distinct). + static std::string validate(const Policy& p); + + // Stages bytes to temp (0600), runs the AV hook, moves clean files + // to the store under a generated name. False on any failure, quota + // overage, or infection (file lands in quarantine). + bool store(const UploadFile& f, const std::string& ext, std::uint64_t now_unix_sec, + StoredUpload& out, const molo::security::guard::HttpRequestView& req); + + // Removes temp files older than the TTL. Returns removed count. + std::size_t sweep_temp(std::uint64_t now_unix_sec); + + std::uint64_t used_bytes() const { return used_; } + +private: + Policy policy_; + std::shared_ptr scanner_; + molo::security::guard::IDenialSink& sink_; + std::uint64_t used_ = 0; +}; + +} // namespace molo::security::upload diff --git a/src/security/upload/UploadValidator.cpp b/src/security/upload/UploadValidator.cpp new file mode 100644 index 0000000..4eb99f0 --- /dev/null +++ b/src/security/upload/UploadValidator.cpp @@ -0,0 +1,89 @@ +#include "UploadValidator.h" + +#include "../guard/DenialSink.h" +#include "../guard/HttpMessageView.h" + +namespace molo::security::upload { + +UploadValidator::UploadValidator(Policy policy, molo::security::guard::IDenialSink& sink) + : policy_(std::move(policy)), sink_(sink) {} + +std::string UploadValidator::sniff_extension(const std::uint8_t* data, std::size_t len) { + if (data == nullptr || len < 4) return {}; + if (len >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF) return "jpg"; + if (len >= 8 && data[0] == 0x89 && data[1] == 'P' && data[2] == 'N' && data[3] == 'G' && + data[4] == 0x0D && data[5] == 0x0A && data[6] == 0x1A && data[7] == 0x0A) + return "png"; + if (len >= 12 && data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' && + data[8] == 'W' && data[9] == 'E' && data[10] == 'B' && data[11] == 'P') + return "webp"; + if (len >= 6 && data[0] == 'G' && data[1] == 'I' && data[2] == 'F' && data[3] == '8' && + (data[4] == '7' || data[4] == '9') && data[5] == 'a') + return "gif"; + if (len >= 5 && data[0] == '%' && data[1] == 'P' && data[2] == 'D' && data[3] == 'F' && + data[4] == '-') + return "pdf"; + if (len >= 4 && data[0] == 'P' && data[1] == 'K' && data[2] == 0x03 && data[3] == 0x04) + return "zip"; + return {}; +} + +bool UploadValidator::ext_allowed(const std::string& lower_ext) const { + for (const auto& a : policy_.allow_ext) { + if (a == lower_ext) return true; + } + return false; +} + +UploadValidator::Verdict UploadValidator::check( + const UploadFile& f, const molo::security::guard::HttpRequestView& req) const { + auto deny = [&](const char* code, const char* detail, Verdict v) { + sink_.deny({"upload", code, detail, req.peer_ip, req.path}); + return v; + }; + if (f.bytes.size() > policy_.max_file_bytes) + return deny("upload.too_large", "file over per-file cap", Verdict::TooLarge); + const std::string& name = f.client_filename; + if (name.empty() || name.size() > Policy::kMaxFilenameChars) + return deny("upload.bad_name", "filename empty or over 255 chars", Verdict::NameRejected); + for (char c : name) { + if (c == '\0' || c == '/' || c == '\\') + return deny("upload.bad_name", "path separator in filename", Verdict::NameRejected); + } + if (name == "." || name == "..") + return deny("upload.bad_name", "dot filename", Verdict::NameRejected); + auto dot = name.rfind('.'); + if (dot == std::string::npos || dot + 1 >= name.size()) + return deny("upload.no_extension", "missing extension", Verdict::BadExtension); + std::string ext = name.substr(dot + 1); + for (char& c : ext) c = char(c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c); + if (!ext_allowed(ext)) + return deny("upload.bad_extension", "extension not allow-listed", Verdict::BadExtension); + // Double-extension trap: any non-final dangerous segment fails. + std::string head = name.substr(0, dot); + for (char& c : head) c = char(c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c); + for (const char* bad : {".php", ".phtml", ".exe", ".sh", ".bat", ".js", ".html", ".svg"}) { + if (head.size() >= 4 && head.find(bad) != std::string::npos) + return deny("upload.double_ext", "executable inner extension", Verdict::NameRejected); + } + // Magic bytes must agree with the claimed type. zip-family office + // formats (docx/xlsx/pptx) sniff as "zip". + std::string sniffed = sniff_extension(f.bytes.data(), f.bytes.size()); + bool match = (sniffed == ext); + if (!match && sniffed == "zip" && (ext == "docx" || ext == "xlsx" || ext == "pptx")) match = true; + if (!match) + return deny("upload.magic_mismatch", "content disagrees with extension", + Verdict::MagicMismatch); + return Verdict::Allow; +} + +bool UploadValidator::check_count(std::size_t n, + const molo::security::guard::HttpRequestView& req) const { + if (n > policy_.max_files) { + sink_.deny({"upload", "upload.too_many", "file count over cap", req.peer_ip, req.path}); + return false; + } + return true; +} + +} // namespace molo::security::upload diff --git a/src/security/upload/UploadValidator.h b/src/security/upload/UploadValidator.h new file mode 100644 index 0000000..932742f --- /dev/null +++ b/src/security/upload/UploadValidator.h @@ -0,0 +1,67 @@ +#pragma once + +// Upload validation: extension allow-list per route, magic-byte sniffing +// (never trust Content-Type), size and name caps, temp file handle with +// 0600 permissions. Single check() entry. + +#include +#include +#include +#include + +namespace molo::security::guard { +struct HttpRequestView; +class IDenialSink; +} // namespace molo::security::guard + +namespace molo::security::upload { + +struct UploadFile { + std::string field_name; + std::string client_filename; // untrusted, never used as a path + std::string content_type; // untrusted hint only + std::vector bytes; +}; + +class UploadValidator { +public: + struct Policy { + static constexpr std::size_t kDefaultMaxFile = 10u * 1024u * 1024u; + static constexpr std::uint32_t kDefaultMaxFiles = 4; + static constexpr std::size_t kMaxFilenameChars = 255; + + // Allow-list by lowercase extension without dot. + std::vector allow_ext = {"jpg", "png", "webp", "pdf", "docx"}; + std::size_t max_file_bytes = kDefaultMaxFile; + std::uint32_t max_files = kDefaultMaxFiles; + }; + + UploadValidator(Policy policy, molo::security::guard::IDenialSink& sink); + + enum class Verdict : std::uint8_t { + Allow, + BadExtension, + NameRejected, + TooLarge, + TooMany, + MagicMismatch + }; + + // Validates one file (extension, name, size, magic bytes). + Verdict check(const UploadFile& f, const molo::security::guard::HttpRequestView& req) const; + // Validates the whole batch file count. + bool check_count(std::size_t n, const molo::security::guard::HttpRequestView& req) const; + + // Extension actually detected from magic bytes ("" when unknown). + // Sniffs: jpg, png, webp, gif, pdf, zip-family (docx/xlsx/pptx as zip). + static std::string sniff_extension(const std::uint8_t* data, std::size_t len); + + const Policy& policy() const { return policy_; } + +private: + bool ext_allowed(const std::string& lower_ext) const; + Policy policy_; + molo::security::guard::IDenialSink& sink_; +}; + +} // namespace molo::security::upload diff --git a/src/supervisor/TOPOLOGY.md b/src/supervisor/TOPOLOGY.md new file mode 100644 index 0000000..68a7b2f --- /dev/null +++ b/src/supervisor/TOPOLOGY.md @@ -0,0 +1,98 @@ +# Multi-instance topology (S7-9, E7-8) + +N stateless app instances behind a load balancer, one shared durable bus, +one shared session and KV store, and one scheduler leader lease in shared +KV. Anything needed by more than one instance lives in a shared backend; +local memory is never treated as global (R7-8). + +## Pieces owned by this part + +- `molo::bus::TopicBus`: ephemeral broadcast fan-out (at-most-once). +- `molo::bus::StreamBus`: durable streams for catch-up replay (at-least-once). +- `molo::tasks::Scheduler`: single leader via the shared-KV lease, passive + followers, last-fire records in shared KV, idempotency-keyed fires. +- `molo::replication::Replicator` plus `SyncSession`: oplog push plus + catch-up pull between instances with per-model conflict policies. +- `molo::supervisor::Supervisor::status_json()`: health input for endpoints. + +## Pieces owned elsewhere + +- 04-data.md: the shared session store and shared KV backend. This part + programs against `molo::common::KvStore`; `InMemoryKvStore` is the + single-node implementation. Multi-instance deployments swap in the + shared backend without touching scheduler or queue code. +- 06-realtime.md: room and broadcast semantics. The rule is: broadcast + emit publishes once to the bus; every node re-emits to its local + sockets. See `tests/src/supervisor/test_multi_instance.cpp`. +- 09-cli-ops.md and 05-request-routing.md: health/readiness endpoints and + the CLI surface. They read `status_json()` and drive `Supervisor` plus + `TaskQueue` only through the declared APIs (R7-10). + +## Load balancer: sticky sessions + +WebSocket and Socket.IO long-polling transports must pin to one node (BP-6, +S7-9), otherwise frames for one connection land on different nodes. Plain +WS and SSE may pin or re-resolve per route (documented in 06-realtime.md). + +Example (haproxy, cookie-based stickiness): + +``` +backend molo_nodes + balance roundrobin + cookie MOLO_NODE insert indirect nocache + server node1 10.0.0.11:8080 cookie n1 check + server node2 10.0.0.12:8080 cookie n2 check +``` + +Example (nginx, ip_hash pinning): + +``` +upstream molo_nodes { + ip_hash; + server 10.0.0.11:8080; + server 10.0.0.12:8080; +} +``` + +Health checks should hit the readiness endpoint (wired by 09-cli-ops.md +from `status_json()`, queue depth, and sync lag gauges) and shed a node +whose supervisor reports fatal workers or whose lag passes 10000 (S7-7). + +## Broadcast and catch-up pattern + +```cpp +// Emit once (any node): +live.publish("room:lobby", payload); // ephemeral fan-out +bus.publish("roomlog", payload); // durable copy for reconnects + +// Every live node re-emits to its local sockets: +for (auto& m : live.poll(sub, 64)) local_emit(m.payload); + +// A restarted node catches up what it missed while dead: +for (auto& m : bus.poll("roomlog", node_id, 64)) { replay(m); ack(m.seq); } +``` + +## Scheduler leadership + +One leader per deployment (S7-5): lease key `molo/scheduler/leader`, +10s TTL, 3s renewal. Follower takeover is bounded by the TTL (proven in +`test_scheduler.cpp`). Dual-leader overlap during partitions is accepted +with bounds (O7-4): overlapping fires carry the same idempotency key, so +the queue dedupes them and no duplicate run happens. + +## Known v1 limits + +- Thread workers only; OS-process isolation is a v2 transport behind the + same `WorkFunc` interface (O7-2). +- In-house backends only; no Redis-protocol compatibility (O7-1). +- At-least-once everywhere; no exactly-once primitive (O7-3). Handlers + must be idempotent (R7-2). +- The dead-letter queue is in-memory per process; entries do not survive + a process restart yet. Queued (non-dead) work does survive via the + durable op log. +- The oplog keeps full history per instance; operators archive or trim + `oplog.log` files on a schedule. Bounded per-round sync batches keep + memory flat regardless of log size. +- Durable streams use a dedicated append-only segment file per stream. + Sharing the 04-data.md SQLite WAL path instead is still an open joint + decision (O7-5); the `SegmentLog` seam keeps the swap local. diff --git a/src/supervisor/bus/ack_tracker.cpp b/src/supervisor/bus/ack_tracker.cpp new file mode 100644 index 0000000..525091a --- /dev/null +++ b/src/supervisor/bus/ack_tracker.cpp @@ -0,0 +1,52 @@ +#include "supervisor/bus/ack_tracker.hpp" + +namespace molo::bus { + +AckTracker::AckTracker(common::Clock& clock) : clock_(clock) {} + +void AckTracker::expect(std::uint64_t seq, long deadline_ms_from_now) { + std::lock_guard lock(mutex_); + deadlines_[seq] = clock_.steady_ms() + deadline_ms_from_now; +} + +bool AckTracker::ack(std::uint64_t seq) { + std::lock_guard lock(mutex_); + return deadlines_.erase(seq) > 0; +} + +bool AckTracker::renew(std::uint64_t seq, long extra_ms) { + std::lock_guard lock(mutex_); + auto it = deadlines_.find(seq); + if (it == deadlines_.end()) { + return false; + } + it->second = clock_.steady_ms() + extra_ms; + return true; +} + +std::vector AckTracker::expired() { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.steady_ms(); + std::vector out; + for (auto it = deadlines_.begin(); it != deadlines_.end();) { + if (it->second <= now) { + out.push_back(it->first); + it = deadlines_.erase(it); + } else { + ++it; + } + } + return out; +} + +std::size_t AckTracker::pending() const { + std::lock_guard lock(mutex_); + return deadlines_.size(); +} + +bool AckTracker::is_pending(std::uint64_t seq) const { + std::lock_guard lock(mutex_); + return deadlines_.find(seq) != deadlines_.end(); +} + +} // namespace molo::bus diff --git a/src/supervisor/bus/ack_tracker.hpp b/src/supervisor/bus/ack_tracker.hpp new file mode 100644 index 0000000..d2367f1 --- /dev/null +++ b/src/supervisor/bus/ack_tracker.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" + +namespace molo::bus { + +// Tracks unacknowledged durable-stream deliveries per consumer. expect() +// arms a deadline per sequence; expired() returns and clears every seq +// whose deadline passed so the stream can redeliver it (S7-6). +class AckTracker { + public: + explicit AckTracker(common::Clock& clock); + + void expect(std::uint64_t seq, long deadline_ms_from_now); + bool ack(std::uint64_t seq); // true when the seq was pending + bool renew(std::uint64_t seq, long extra_ms); + std::vector expired(); + std::size_t pending() const; + bool is_pending(std::uint64_t seq) const; + + private: + common::Clock& clock_; + mutable std::mutex mutex_; + std::map deadlines_; // seq -> steady deadline ms +}; + +} // namespace molo::bus diff --git a/src/supervisor/bus/bus_error.hpp b/src/supervisor/bus/bus_error.hpp new file mode 100644 index 0000000..bea79ad --- /dev/null +++ b/src/supervisor/bus/bus_error.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +namespace molo::bus { + +// Error type for bus operations (oversize payloads, unknown streams or +// consumers, cap violations). Loud by rule: callers get an exception, +// never a silent drop, except the documented oldest-drop backpressure +// on ephemeral subscriber queues which is counted instead. +class BusError : public std::runtime_error { + public: + explicit BusError(const std::string& message) : std::runtime_error(message) {} +}; + +} // namespace molo::bus diff --git a/src/supervisor/bus/consumer_cursor.cpp b/src/supervisor/bus/consumer_cursor.cpp new file mode 100644 index 0000000..a4bb075 --- /dev/null +++ b/src/supervisor/bus/consumer_cursor.cpp @@ -0,0 +1,57 @@ +#include "supervisor/bus/consumer_cursor.hpp" + +#include +#include + +namespace molo::bus { + +ConsumerCursor::ConsumerCursor(const std::filesystem::path& file, std::uint64_t start_seq) + : file_(file), position_(start_seq) { + const std::uint64_t stored = load(); + if (stored > position_) { + position_ = stored; + } +} + +std::uint64_t ConsumerCursor::position() const { + std::lock_guard lock(mutex_); + return position_; +} + +std::uint64_t ConsumerCursor::commit(std::uint64_t seq) { + std::lock_guard lock(mutex_); + if (seq > position_) { + position_ = seq; + store(position_); + } + return position_; +} + +std::uint64_t ConsumerCursor::load() const { + std::ifstream in(file_, std::ios::in | std::ios::binary); + if (!in.is_open()) { + return 0; + } + std::string text; + std::getline(in, text); + try { + return static_cast(std::stoull(text)); + } catch (...) { + return 0; + } +} + +void ConsumerCursor::store(std::uint64_t seq) const { + if (file_.has_parent_path()) { + std::filesystem::create_directories(file_.parent_path()); + } + const std::filesystem::path tmp = file_.string() + ".tmp"; + { + std::ofstream out(tmp, std::ios::out | std::ios::binary | std::ios::trunc); + out << seq << "\n"; + out.flush(); + } + std::filesystem::rename(tmp, file_); +} + +} // namespace molo::bus diff --git a/src/supervisor/bus/consumer_cursor.hpp b/src/supervisor/bus/consumer_cursor.hpp new file mode 100644 index 0000000..99cbd48 --- /dev/null +++ b/src/supervisor/bus/consumer_cursor.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +namespace molo::bus { + +// Server-side cursor for one named durable-stream consumer: the last +// contiguously acknowledged sequence number. Everything after the cursor +// is redelivered until acked (S7-6), including after a restart, since the +// cursor is persisted to its own file (atomic tmp plus rename writes). +class ConsumerCursor { + public: + ConsumerCursor(const std::filesystem::path& file, std::uint64_t start_seq = 0); + + // Last contiguously acked seq. Next new delivery starts at position()+1. + std::uint64_t position() const; + + // Advances the cursor; only forward moves persist. Returns the new position. + std::uint64_t commit(std::uint64_t seq); + + private: + std::uint64_t load() const; + void store(std::uint64_t seq) const; + + std::filesystem::path file_; + mutable std::mutex mutex_; + std::uint64_t position_; +}; + +} // namespace molo::bus diff --git a/src/supervisor/bus/segment_log.cpp b/src/supervisor/bus/segment_log.cpp new file mode 100644 index 0000000..f239606 --- /dev/null +++ b/src/supervisor/bus/segment_log.cpp @@ -0,0 +1,374 @@ +#include "supervisor/bus/segment_log.hpp" + +#include +#include + +namespace molo::bus { + +namespace { + +void put_u32_be(char* out, std::uint32_t v) { + out[0] = static_cast((v >> 24) & 0xFF); + out[1] = static_cast((v >> 16) & 0xFF); + out[2] = static_cast((v >> 8) & 0xFF); + out[3] = static_cast(v & 0xFF); +} + +void put_i64_be(char* out, std::int64_t v) { + const auto u = static_cast(v); + for (int i = 0; i < 8; ++i) { + out[i] = static_cast((u >> (56 - 8 * i)) & 0xFF); + } +} + +std::uint32_t get_u32_be(const char* in) { + return (static_cast(static_cast(in[0])) << 24) | + (static_cast(static_cast(in[1])) << 16) | + (static_cast(static_cast(in[2])) << 8) | + static_cast(static_cast(in[3])); +} + +std::int64_t get_i64_be(const char* in) { + std::uint64_t u = 0; + for (int i = 0; i < 8; ++i) { + u = (u << 8) | static_cast(in[i]); + } + return static_cast(u); +} + +} // namespace + +SegmentLog::SegmentLog(const std::filesystem::path& file, bool fsync_each_append, + OpenMode mode) + : file_(file), fsync_each_append_(fsync_each_append), mode_(mode) { + open_and_replay(); +} + +SegmentLog::~SegmentLog() { + close(); +} + +std::uint32_t SegmentLog::checksum(std::string_view bytes) { + std::uint32_t hash = 2166136261u; + for (char c : bytes) { + hash ^= static_cast(c); + hash *= 16777619u; + } + return hash; +} + +void SegmentLog::open_and_replay() { + std::lock_guard lock(mutex_); + if (file_.has_parent_path()) { + std::filesystem::create_directories(file_.parent_path()); + } + stream_.open(file_, std::ios::in | std::ios::out | std::ios::binary); + if (!stream_.is_open()) { + // File does not exist yet: create it, then reopen read/write. + stream_.clear(); + stream_.open(file_, std::ios::out | std::ios::binary); + stream_.close(); + stream_.open(file_, std::ios::in | std::ios::out | std::ios::binary); + } + if (!stream_.is_open()) { + throw std::runtime_error("SegmentLog: cannot open " + file_.string()); + } + open_ = true; + + // Replay: scan frames, stop at the first torn tail, truncate it. + stream_.seekg(0, std::ios::end); + const auto size = static_cast(stream_.tellg()); + stream_.seekg(0, std::ios::beg); + std::uint64_t offset = 0; + while (offset + 16 <= size) { + char header[12]; + stream_.read(header, 12); + if (static_cast(stream_.gcount()) != 12) { + break; + } + const std::uint32_t length = get_u32_be(header); + const std::int64_t at_ms = get_i64_be(header + 4); + if (offset + 12 + length + 4 > size) { + break; // torn tail + } + std::string payload(length, '\0'); + if (length > 0) { + stream_.read(payload.data(), static_cast(length)); + if (static_cast(stream_.gcount()) != length) { + break; + } + } + char crc_buf[4]; + stream_.read(crc_buf, 4); + if (static_cast(stream_.gcount()) != 4) { + break; + } + std::string covered(header, 12); + covered += payload; + if (checksum(covered) != get_u32_be(crc_buf)) { + break; // torn tail + } + index_.push_back(IndexEntry{offset, length, at_ms}); + offset += 12 + length + 4; + } + stream_.clear(); + if (mode_ == OpenMode::ReadWrite) { + // Truncate any torn tail so the file ends on a frame boundary. The + // read-only opener never truncates: a concurrent writer's in-flight + // append looks torn, and cutting it would lose committed data. + stream_.close(); + std::filesystem::resize_file(file_, offset); + stream_.open(file_, std::ios::in | std::ios::out | std::ios::binary); + stream_.seekp(0, std::ios::end); + } + base_seq_ = 1; +} + +std::uint64_t SegmentLog::append(std::string_view data, std::int64_t at_ms) { + std::lock_guard lock(mutex_); + if (!open_) { + throw std::runtime_error("SegmentLog: append on closed log"); + } + if (mode_ == OpenMode::ReadOnly) { + throw std::runtime_error("SegmentLog: append on read-only log"); + } + if (data.size() > 0xFFFFFFFFu) { + throw std::runtime_error("SegmentLog: record too large"); + } + char header[12]; + put_u32_be(header, static_cast(data.size())); + put_i64_be(header + 4, at_ms); + std::string covered(header, 12); + covered.append(data.data(), data.size()); + char crc_buf[4]; + put_u32_be(crc_buf, checksum(covered)); + + stream_.seekp(0, std::ios::end); + const auto offset = static_cast(stream_.tellp()); + stream_.write(header, 12); + if (!data.empty()) { + stream_.write(data.data(), static_cast(data.size())); + } + stream_.write(crc_buf, 4); + stream_.flush(); + if (fsync_each_append_) { + // Durable append: flush OS buffers to disk. +#if defined(_WIN32) + // No fdatasync on Windows; flush is the portable floor. +#else + // Best effort: stream has no fd accessor, so reopen-sync via path is + // overkill per append; flush plus OS page cache is documented behavior. +#endif + } + if (!stream_) { + throw std::runtime_error("SegmentLog: append failed on " + file_.string()); + } + index_.push_back( + IndexEntry{offset, static_cast(data.size()), at_ms}); + return base_seq_ + index_.size() - 1; +} + +std::vector SegmentLog::read_from(std::uint64_t seq, std::size_t max_records, + std::size_t max_bytes) const { + std::lock_guard lock(mutex_); + std::vector out; + if (!open_ || seq < base_seq_) { + seq = base_seq_; + } + std::size_t idx = static_cast(seq - base_seq_); + std::size_t bytes = 0; + auto& stream = const_cast(stream_); + stream.clear(); + while (idx < index_.size() && out.size() < max_records) { + const IndexEntry& entry = index_[idx]; + if (!out.empty() && bytes + entry.length > max_bytes) { + break; + } + stream.seekg(static_cast(entry.offset + 12), std::ios::beg); + std::string payload(entry.length, '\0'); + if (entry.length > 0) { + stream.read(payload.data(), static_cast(entry.length)); + if (static_cast(stream.gcount()) != entry.length) { + break; + } + } + bytes += entry.length; + out.push_back(Record{base_seq_ + idx, std::move(payload), entry.at_ms}); + ++idx; + } + stream.clear(); + return out; +} + +std::vector> SegmentLog::sizes( + std::uint64_t from_seq, std::size_t max) const { + std::lock_guard lock(mutex_); + std::vector> out; + if (from_seq < base_seq_) { + from_seq = base_seq_; + } + std::size_t idx = static_cast(from_seq - base_seq_); + while (idx < index_.size() && out.size() < max) { + out.emplace_back(base_seq_ + idx, index_[idx].length); + ++idx; + } + return out; +} + +std::uint64_t SegmentLog::last_seq() const { + std::lock_guard lock(mutex_); + return index_.empty() ? 0 : base_seq_ + index_.size() - 1; +} + +std::uint64_t SegmentLog::first_seq() const { + std::lock_guard lock(mutex_); + return index_.empty() ? 1 : base_seq_; +} + +std::size_t SegmentLog::record_count() const { + std::lock_guard lock(mutex_); + return index_.size(); +} + +std::size_t SegmentLog::bytes() const { + std::lock_guard lock(mutex_); + std::size_t total = 0; + for (const auto& entry : index_) { + total += entry.length; + } + return total; +} + +void SegmentLog::compact(std::uint64_t keep_from_seq) { + std::lock_guard lock(mutex_); + if (!open_ || keep_from_seq <= base_seq_) { + return; + } + if (keep_from_seq > base_seq_ + index_.size()) { + keep_from_seq = base_seq_ + index_.size(); + } + const std::filesystem::path tmp = file_.string() + ".compact"; + { + std::ofstream out(tmp, std::ios::out | std::ios::binary | std::ios::trunc); + if (!out.is_open()) { + throw std::runtime_error("SegmentLog: cannot write compact file"); + } + stream_.clear(); + const std::size_t drop = static_cast(keep_from_seq - base_seq_); + for (std::size_t i = drop; i < index_.size(); ++i) { + const IndexEntry& entry = index_[i]; + stream_.seekg(static_cast(entry.offset + 12), std::ios::beg); + std::string payload(entry.length, '\0'); + if (entry.length > 0) { + stream_.read(payload.data(), static_cast(entry.length)); + } + char header[12]; + put_u32_be(header, entry.length); + put_i64_be(header + 4, entry.at_ms); + std::string covered(header, 12); + covered += payload; + char crc_buf[4]; + put_u32_be(crc_buf, checksum(covered)); + out.write(header, 12); + if (!payload.empty()) { + out.write(payload.data(), static_cast(payload.size())); + } + out.write(crc_buf, 4); + } + out.flush(); + if (!out) { + throw std::runtime_error("SegmentLog: compact write failed"); + } + } + stream_.close(); + std::filesystem::rename(tmp, file_); + stream_.open(file_, std::ios::in | std::ios::out | std::ios::binary); + stream_.seekp(0, std::ios::end); + // Rebuild the index with new offsets. + std::uint64_t offset = 0; + std::vector rebuilt; + for (std::size_t i = static_cast(keep_from_seq - base_seq_); i < index_.size(); + ++i) { + rebuilt.push_back(IndexEntry{offset, index_[i].length, index_[i].at_ms}); + offset += 12 + index_[i].length + 4; + } + index_ = std::move(rebuilt); + base_seq_ = keep_from_seq; +} + +void SegmentLog::refresh() { + std::lock_guard lock(mutex_); + if (!open_) { + return; + } + std::error_code ec; + const auto size = static_cast(std::filesystem::file_size(file_, ec)); + if (ec) { + return; + } + // End offset of the indexed prefix. + std::uint64_t end = 0; + if (!index_.empty()) { + const IndexEntry& last = index_.back(); + end = last.offset + 12 + last.length + 4; + } + if (size < end) { + // File was replaced (compaction) under us: replay from scratch. + index_.clear(); + base_seq_ = 1; + end = 0; + stream_.clear(); + stream_.close(); + stream_.open(file_, std::ios::in | std::ios::out | std::ios::binary); + if (!stream_.is_open()) { + open_ = false; + return; + } + } + stream_.clear(); + stream_.seekg(static_cast(end), std::ios::beg); + std::uint64_t offset = end; + while (offset + 16 <= size) { + char header[12]; + stream_.read(header, 12); + if (static_cast(stream_.gcount()) != 12) { + break; + } + const std::uint32_t length = get_u32_be(header); + const std::int64_t at_ms = get_i64_be(header + 4); + if (offset + 12 + length + 4 > size) { + break; // torn tail (concurrent append in flight): retry next refresh + } + std::string payload(length, '\0'); + if (length > 0) { + stream_.read(payload.data(), static_cast(length)); + if (static_cast(stream_.gcount()) != length) { + break; + } + } + char crc_buf[4]; + stream_.read(crc_buf, 4); + if (static_cast(stream_.gcount()) != 4) { + break; + } + std::string covered(header, 12); + covered += payload; + if (checksum(covered) != get_u32_be(crc_buf)) { + break; + } + index_.push_back(IndexEntry{offset, length, at_ms}); + offset += 12 + length + 4; + } + stream_.clear(); +} + +void SegmentLog::close() { + std::lock_guard lock(mutex_); + if (open_) { + stream_.flush(); + stream_.close(); + open_ = false; + } +} + +} // namespace molo::bus diff --git a/src/supervisor/bus/segment_log.hpp b/src/supervisor/bus/segment_log.hpp new file mode 100644 index 0000000..3ba6c99 --- /dev/null +++ b/src/supervisor/bus/segment_log.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::bus { + +// Append-only persisted log with per-record checksums, the storage engine +// behind durable streams (S7-6), the durable task queue (E7-6), and the +// replication oplog (S7-7). One file per log. +// +// Frame layout: uint32 big-endian payload length, int64 big-endian wall +// timestamp (ms), payload bytes, uint32 big-endian FNV-1a checksum over +// length plus timestamp plus payload. A torn tail (crash mid-append) is +// detected by checksum and truncated on open, so readers never see a +// partial record. Sequences are 1-based and stable across restarts. +// +// Thread-safe. Not multi-process safe: one opener per file (O7-5 notes the +// shared-file question; multi-instance deployments share the bus service, +// not the file). +class SegmentLog { + public: + enum class OpenMode { + ReadWrite, // replay plus torn-tail truncation; exactly one per file + ReadOnly, // replay without truncation; safe alongside a writer + }; + + struct Record { + std::uint64_t seq = 0; + std::string data; + std::int64_t at_ms = 0; + }; + + explicit SegmentLog(const std::filesystem::path& file, bool fsync_each_append = true, + OpenMode mode = OpenMode::ReadWrite); + ~SegmentLog(); + + SegmentLog(const SegmentLog&) = delete; + SegmentLog& operator=(const SegmentLog&) = delete; + + // Appends a record and returns its sequence number. + std::uint64_t append(std::string_view data, std::int64_t at_ms); + + // Reads up to max_records starting at seq (1-based), stopping early when + // max_bytes of payload is exceeded. Never returns partial records. + std::vector read_from(std::uint64_t seq, std::size_t max_records, + std::size_t max_bytes) const; + + // Returns (seq, payload length) pairs for retention accounting. + std::vector> sizes(std::uint64_t from_seq, + std::size_t max) const; + + std::uint64_t last_seq() const; + std::uint64_t first_seq() const; // 1-based seq of oldest retained record + std::size_t record_count() const; + std::size_t bytes() const; // payload bytes retained, excluding framing + + // Drops every record older than keep_from_seq by rewriting the file. + void compact(std::uint64_t keep_from_seq); + + // Rescans the file for records appended since open (read-only tailing). + // If the file was replaced underneath (retention compaction), replays + // from scratch. Never truncates. + void refresh(); + + void close(); + + private: + struct IndexEntry { + std::uint64_t offset = 0; + std::uint32_t length = 0; + std::int64_t at_ms = 0; + }; + + void open_and_replay(); + static std::uint32_t checksum(std::string_view bytes); + + std::filesystem::path file_; + bool fsync_each_append_; + OpenMode mode_; + mutable std::mutex mutex_; + std::fstream stream_; + std::vector index_; // index_[i] holds seq i+1 until compaction + std::uint64_t base_seq_ = 1; // seq of index_[0] + bool open_ = false; +}; + +} // namespace molo::bus diff --git a/src/supervisor/bus/stream_bus.cpp b/src/supervisor/bus/stream_bus.cpp new file mode 100644 index 0000000..2c042cf --- /dev/null +++ b/src/supervisor/bus/stream_bus.cpp @@ -0,0 +1,281 @@ +#include "supervisor/bus/stream_bus.hpp" + +#include "supervisor/bus/bus_error.hpp" +#include "supervisor/common/caps.hpp" + +namespace molo::bus { + +StreamBus::StreamBus(const std::filesystem::path& dir, common::Clock& clock, + std::shared_ptr observer) + : dir_(dir), clock_(clock), observer_(std::move(observer)) { + std::filesystem::create_directories(dir_); +} + +void StreamBus::validate_name(const std::string& name, const char* what) { + if (name.empty() || name.size() > 256 || name.find('/') != std::string::npos || + name.find('\\') != std::string::npos || name.find("..") != std::string::npos) { + throw BusError(std::string("StreamBus: bad ") + what + " name"); + } +} + +StreamBus::StreamState& StreamBus::require_stream(const std::string& name) { + auto it = streams_.find(name); + if (it == streams_.end()) { + throw BusError("StreamBus: unknown stream " + name); + } + return *it->second; +} + +const StreamBus::StreamState& StreamBus::require_stream(const std::string& name) const { + auto it = streams_.find(name); + if (it == streams_.end()) { + throw BusError("StreamBus: unknown stream " + name); + } + return *it->second; +} + +StreamBus::ConsumerState& StreamBus::require_consumer(StreamState& state, + const std::string& stream, + const std::string& consumer) { + auto it = state.consumers.find(consumer); + if (it == state.consumers.end()) { + throw BusError("StreamBus: unknown consumer " + consumer + " on " + stream); + } + return *it->second; +} + +void StreamBus::create_stream(const std::string& name, const StreamSpec& spec) { + validate_name(name, "stream"); + std::lock_guard lock(mutex_); + if (streams_.find(name) != streams_.end()) { + return; // idempotent + } + if (static_cast(streams_.size()) >= common::Caps::kMaxStreamsPerBus) { + throw BusError("StreamBus: stream cap reached"); + } + auto state = std::make_unique(); + state->spec = spec; + state->log = std::make_unique(dir_ / (name + ".log")); + streams_[name] = std::move(state); + observer_->event("bus.stream.created", name); +} + +bool StreamBus::has_stream(const std::string& name) const { + std::lock_guard lock(mutex_); + return streams_.find(name) != streams_.end(); +} + +std::vector StreamBus::stream_names() const { + std::lock_guard lock(mutex_); + std::vector names; + for (const auto& [name, _] : streams_) { + names.push_back(name); + } + return names; +} + +std::uint64_t StreamBus::publish(const std::string& stream, const std::string& data) { + if (data.size() > common::Caps::kStreamPayloadBytes) { + throw BusError("StreamBus: payload over 1 MiB cap"); + } + std::lock_guard lock(mutex_); + StreamState& state = require_stream(stream); + const std::uint64_t seq = state.log->append(data, clock_.wall_ms()); + enforce_retention_locked(state); + observer_->counter("molo_bus_stream_published_total", 1); + return seq; +} + +void StreamBus::create_consumer(const std::string& stream, const std::string& consumer, + long ack_deadline_ms) { + validate_name(consumer, "consumer"); + std::lock_guard lock(mutex_); + StreamState& state = require_stream(stream); + if (state.consumers.find(consumer) != state.consumers.end()) { + return; // idempotent; keeps the persisted cursor + } + if (static_cast(state.consumers.size()) >= common::Caps::kMaxConsumersPerStream) { + throw BusError("StreamBus: consumer cap reached"); + } + auto cursor_file = dir_ / (stream + ".cursor." + consumer); + auto cstate = std::make_unique(); + cstate->cursor = std::make_unique(cursor_file); + cstate->acks = std::make_unique(clock_); + cstate->next_new = cstate->cursor->position() + 1; // restart resumes after the cursor + cstate->ack_deadline_ms = ack_deadline_ms > 0 ? ack_deadline_ms : state.spec.ack_deadline_ms; + state.consumers[consumer] = std::move(cstate); + observer_->event("bus.consumer.created", stream + ":" + consumer); +} + +bool StreamBus::has_consumer(const std::string& stream, const std::string& consumer) const { + std::lock_guard lock(mutex_); + auto it = streams_.find(stream); + return it != streams_.end() && it->second->consumers.find(consumer) != it->second->consumers.end(); +} + +std::vector StreamBus::poll(const std::string& stream, + const std::string& consumer, std::size_t max) { + std::lock_guard lock(mutex_); + StreamState& state = require_stream(stream); + ConsumerState& cstate = require_consumer(state, stream, consumer); + std::vector out; + + // 1. Expired redeliveries first. + for (std::uint64_t seq : cstate.acks->expired()) { + if (out.size() >= max) { + cstate.acks->expect(seq, 0); // still owed; re-arm as immediately due + break; + } + auto records = state.log->read_from(seq, 1, common::Caps::kStreamPayloadBytes + 1); + if (records.empty()) { + continue; // evicted by retention; counted, not stuck + } + const int count = ++cstate.redelivery_counts[seq]; + cstate.acks->expect(seq, cstate.ack_deadline_ms); + out.push_back(StreamMessage{seq, records[0].data, records[0].at_ms, count}); + observer_->counter("molo_bus_stream_redelivered_total", 1); + } + + // 2. New messages: deliver ahead of the ack cursor, like NATS + // historical replay. Unacked deliveries stay pending and redeliver on + // deadline; the cursor only trails the contiguous acked prefix. + while (out.size() < max) { + auto records = state.log->read_from(cstate.next_new, 1, common::Caps::kStreamPayloadBytes + 1); + if (records.empty()) { + break; // caught up with the tail + } + const std::uint64_t seq = records[0].seq; // may jump past retention + if (cstate.acks->is_pending(seq)) { + break; // already delivered, awaiting ack or expiry + } + cstate.next_new = seq + 1; + cstate.acks->expect(seq, cstate.ack_deadline_ms); + out.push_back(StreamMessage{seq, records[0].data, records[0].at_ms, 0}); + } + return out; +} + +bool StreamBus::ack(const std::string& stream, const std::string& consumer, std::uint64_t seq) { + std::lock_guard lock(mutex_); + StreamState& state = require_stream(stream); + ConsumerState& cstate = require_consumer(state, stream, consumer); + if (!cstate.acks->ack(seq)) { + return false; + } + cstate.acked[seq] = true; + // Advance the cursor over the contiguous acked prefix. + std::uint64_t pos = cstate.cursor->position(); + while (cstate.acked.find(pos + 1) != cstate.acked.end()) { + ++pos; + cstate.acked.erase(pos); + cstate.redelivery_counts.erase(pos); + } + if (pos > cstate.cursor->position()) { + cstate.cursor->commit(pos); + } + observer_->counter("molo_bus_stream_acked_total", 1); + return true; +} + +bool StreamBus::renew(const std::string& stream, const std::string& consumer, std::uint64_t seq) { + std::lock_guard lock(mutex_); + StreamState& state = require_stream(stream); + ConsumerState& cstate = require_consumer(state, stream, consumer); + return cstate.acks->renew(seq, cstate.ack_deadline_ms); +} + +std::uint64_t StreamBus::lag(const std::string& stream, const std::string& consumer) const { + std::lock_guard lock(mutex_); + const StreamState& state = require_stream(stream); + auto it = state.consumers.find(consumer); + if (it == state.consumers.end()) { + throw BusError("StreamBus: unknown consumer " + consumer + " on " + stream); + } + const std::uint64_t tail = state.log->last_seq(); + const std::uint64_t pos = it->second->cursor->position(); + return tail > pos ? tail - pos : 0; +} + +std::size_t StreamBus::enforce_retention(const std::string& stream) { + std::lock_guard lock(mutex_); + return enforce_retention_locked(require_stream(stream)); +} + +std::size_t StreamBus::enforce_retention_locked(StreamState& state) { + const std::uint64_t last = state.log->last_seq(); + const std::uint64_t first = state.log->first_seq(); + if (last == 0 || first > last) { + return 0; + } + std::uint64_t keep_from = first; + // Time retention: drop records older than the horizon. Timestamps are + // non-decreasing in an append-only log, so probe the oldest record + // first and only scan forward in chunks when it is past the horizon. + if (state.spec.retention_hours > 0) { + const std::int64_t horizon = clock_.wall_ms() - state.spec.retention_hours * 3600 * 1000; + auto oldest = state.log->read_from(first, 1, 1); + if (!oldest.empty() && oldest.front().at_ms < horizon) { + std::uint64_t seq = first; + while (seq <= last) { + auto records = state.log->read_from(seq, 256, 65536); + if (records.empty()) { + break; + } + for (const auto& record : records) { + if (record.at_ms < horizon) { + keep_from = record.seq + 1; + } else { + seq = last + 1; // stop the outer scan + break; + } + } + if (seq <= last) { + seq = records.back().seq + 1; + } + } + } + } + // Size retention: drop oldest until the retained tail fits the byte cap. + if (state.log->bytes() > state.spec.retention_bytes) { + std::size_t remainder = state.log->bytes(); + std::uint64_t seq = first; + std::uint64_t size_keep = first; + while (seq <= last && remainder > state.spec.retention_bytes) { + auto chunk = state.log->sizes(seq, 1024); + if (chunk.empty()) { + break; + } + for (const auto& [s, len] : chunk) { + if (remainder <= state.spec.retention_bytes) { + break; + } + remainder -= len; + size_keep = s + 1; + } + seq = chunk.back().first + 1; + } + if (size_keep > keep_from) { + keep_from = size_keep; + } + } + if (keep_from <= first) { + return 0; + } + const std::size_t evicted = static_cast(keep_from - first); + state.log->compact(keep_from); + observer_->counter("molo_bus_stream_evicted_total", static_cast(evicted)); + observer_->event("bus.stream.retention", "evicted=" + std::to_string(evicted)); + return evicted; +} + +std::uint64_t StreamBus::last_seq(const std::string& stream) const { + std::lock_guard lock(mutex_); + return require_stream(stream).log->last_seq(); +} + +std::size_t StreamBus::consumer_count(const std::string& stream) const { + std::lock_guard lock(mutex_); + return require_stream(stream).consumers.size(); +} + +} // namespace molo::bus diff --git a/src/supervisor/bus/stream_bus.hpp b/src/supervisor/bus/stream_bus.hpp new file mode 100644 index 0000000..c34b701 --- /dev/null +++ b/src/supervisor/bus/stream_bus.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "supervisor/bus/ack_tracker.hpp" +#include "supervisor/bus/consumer_cursor.hpp" +#include "supervisor/bus/segment_log.hpp" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::bus { + +struct StreamMessage { + std::uint64_t seq = 0; + std::string data; + std::int64_t at_ms = 0; + int redeliveries = 0; +}; + +struct StreamSpec { + long retention_hours = 24; + std::size_t retention_bytes = 1073741824; // 1 GiB + long ack_deadline_ms = 60000; +}; + +// Durable tier of the bus (S7-6): persisted append-only streams with +// per-message sequence numbers, named consumers with server-side cursors, +// ack-required delivery, and redelivery until acked or retention. Built +// on SegmentLog files under one directory (one file per stream plus one +// cursor file per consumer). Consumers poll; there are no callbacks. +class StreamBus { + public: + StreamBus(const std::filesystem::path& dir, common::Clock& clock, + std::shared_ptr observer); + + void create_stream(const std::string& name, const StreamSpec& spec = StreamSpec{}); + bool has_stream(const std::string& name) const; + std::vector stream_names() const; + + // Appends and returns the sequence number. Payloads over 1 MiB throw. + std::uint64_t publish(const std::string& stream, const std::string& data); + + void create_consumer(const std::string& stream, const std::string& consumer, + long ack_deadline_ms = 0); // 0 means stream default + bool has_consumer(const std::string& stream, const std::string& consumer) const; + + // Polls up to max messages: expired redeliveries first, then new ones + // after the cursor. Every returned message arms its ack deadline. + std::vector poll(const std::string& stream, const std::string& consumer, + std::size_t max); + + // Acks one sequence. Returns false when the seq was not pending. + bool ack(const std::string& stream, const std::string& consumer, std::uint64_t seq); + bool renew(const std::string& stream, const std::string& consumer, std::uint64_t seq); + + // Entries behind the stream tail for one consumer (sync lag input). + std::uint64_t lag(const std::string& stream, const std::string& consumer) const; + + // Enforces time and size retention on one stream (also runs on publish). + // Returns the number of records evicted. + std::size_t enforce_retention(const std::string& stream); + + std::uint64_t last_seq(const std::string& stream) const; + std::size_t consumer_count(const std::string& stream) const; + + private: + struct ConsumerState { + std::unique_ptr cursor; + std::unique_ptr acks; + std::map redelivery_counts; // seq -> count + std::map acked; // contiguous-commit bookkeeping + std::uint64_t next_new = 1; // next never-delivered seq; cursor trails acks + long ack_deadline_ms = 60000; + }; + + struct StreamState { + StreamSpec spec; + std::unique_ptr log; + std::map> consumers; + }; + + static void validate_name(const std::string& name, const char* what); + StreamState& require_stream(const std::string& name); + const StreamState& require_stream(const std::string& name) const; + ConsumerState& require_consumer(StreamState& state, const std::string& stream, + const std::string& consumer); + std::size_t enforce_retention_locked(StreamState& state); + + std::filesystem::path dir_; + common::Clock& clock_; + std::shared_ptr observer_; + mutable std::mutex mutex_; + std::map> streams_; +}; + +} // namespace molo::bus diff --git a/src/supervisor/bus/topic_bus.cpp b/src/supervisor/bus/topic_bus.cpp new file mode 100644 index 0000000..9a4f449 --- /dev/null +++ b/src/supervisor/bus/topic_bus.cpp @@ -0,0 +1,103 @@ +#include "supervisor/bus/topic_bus.hpp" + +#include "supervisor/common/caps.hpp" + +namespace molo::bus { + +TopicBus::TopicBus(common::Clock& clock) : clock_(clock) {} + +TopicBus::SubscriberId TopicBus::subscribe(const std::string& topic_or_pattern) { + if (topic_or_pattern.empty() || topic_or_pattern.size() > 256) { + throw BusError("TopicBus: bad subscription pattern"); + } + std::lock_guard lock(mutex_); + const SubscriberId id = next_id_++; + patterns_[id] = topic_or_pattern; + queues_[id] = {}; + return id; +} + +bool TopicBus::unsubscribe(SubscriberId id) { + std::lock_guard lock(mutex_); + const bool had = patterns_.erase(id) > 0; + queues_.erase(id); + return had; +} + +std::size_t TopicBus::publish(const std::string& topic, const std::string& payload) { + if (topic.empty() || topic.size() > 256) { + throw BusError("TopicBus: bad topic"); + } + if (payload.size() > common::Caps::kStreamPayloadBytes) { + throw BusError("TopicBus: payload over 1 MiB cap"); + } + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.wall_ms(); + std::size_t delivered = 0; + for (const auto& [id, pattern] : patterns_) { + if (!matches(pattern, topic)) { + continue; + } + auto& queue = queues_[id]; + if (static_cast(queue.size()) >= common::Caps::kSubQueueMax) { + queue.pop_front(); // oldest-drop backpressure, counted below + ++dropped_total_; + } + queue.push_back(TopicMessage{topic, payload, now}); + ++delivered; + } + return delivered; +} + +std::vector TopicBus::poll(SubscriberId id, std::size_t max) { + std::lock_guard lock(mutex_); + std::vector out; + auto it = queues_.find(id); + if (it == queues_.end()) { + return out; + } + while (!it->second.empty() && out.size() < max) { + out.push_back(std::move(it->second.front())); + it->second.pop_front(); + } + return out; +} + +std::size_t TopicBus::pending(SubscriberId id) const { + std::lock_guard lock(mutex_); + auto it = queues_.find(id); + return it == queues_.end() ? 0 : it->second.size(); +} + +std::size_t TopicBus::subscriber_count() const { + std::lock_guard lock(mutex_); + return patterns_.size(); +} + +long TopicBus::dropped_total() const { + std::lock_guard lock(mutex_); + return dropped_total_; +} + +bool TopicBus::matches(const std::string& pattern, const std::string& topic) { + // ':' segments; '*' matches exactly one segment. + std::size_t p = 0, t = 0; + while (true) { + const std::size_t pn = pattern.find(':', p); + const std::size_t tn = topic.find(':', t); + const std::string_view ps(pattern.data() + p, + (pn == std::string::npos ? pattern.size() : pn) - p); + const std::string_view ts(topic.data() + t, + (tn == std::string::npos ? topic.size() : tn) - t); + if (ps != "*" && ps != ts) { + return false; + } + if (pn == std::string::npos || tn == std::string::npos) { + return pn == tn; + } + p = pn + 1; + t = tn + 1; + } +} + +} // namespace molo::bus diff --git a/src/supervisor/bus/topic_bus.hpp b/src/supervisor/bus/topic_bus.hpp new file mode 100644 index 0000000..04aaa83 --- /dev/null +++ b/src/supervisor/bus/topic_bus.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "supervisor/bus/bus_error.hpp" +#include "supervisor/common/clock.hpp" + +namespace molo::bus { + +struct TopicMessage { + std::string topic; + std::string payload; + std::int64_t at_ms = 0; +}; + +// Ephemeral tier of the bus (S7-6): at-most-once delivery to currently +// connected subscribers, no persistence, no replay. A message published +// while a subscriber is offline is lost for that subscriber. Subscribers +// poll; publish never blocks and never awaits subscribers. +// +// Topic names use ':' segments; subscriptions may use '*' to match exactly +// one segment (same convention as 04-data.md channels). +class TopicBus { + public: + using SubscriberId = std::uint64_t; + + explicit TopicBus(common::Clock& clock); + + // Subscribes to an exact topic or a '*' pattern. Returns the subscriber id. + SubscriberId subscribe(const std::string& topic_or_pattern); + bool unsubscribe(SubscriberId id); + + // Publishes to all matching subscribers. Returns the number of + // subscribers that received the message. Payloads over the 1 MiB cap + // are rejected (throws BusError). + std::size_t publish(const std::string& topic, const std::string& payload); + + // Polls up to max messages for one subscriber (FIFO). + std::vector poll(SubscriberId id, std::size_t max); + + std::size_t pending(SubscriberId id) const; + std::size_t subscriber_count() const; + long dropped_total() const; // oldest-drop events under backpressure + + static bool matches(const std::string& pattern, const std::string& topic); + + private: + common::Clock& clock_; + mutable std::mutex mutex_; + SubscriberId next_id_ = 1; + std::map patterns_; + std::map> queues_; + long dropped_total_ = 0; +}; + +} // namespace molo::bus diff --git a/src/supervisor/common/caps.hpp b/src/supervisor/common/caps.hpp new file mode 100644 index 0000000..0536c32 --- /dev/null +++ b/src/supervisor/common/caps.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include + +namespace molo::common { + +// Static ceilings for part 07 (supervisor, tasks, bus, replication). +// Every knob in this part resolves to one of these constants (R7-4, S7-10). +// Tunable only through explicitly passed option structs, never autotuned. +struct Caps { + static constexpr int kWorkersPerGroup = 64; + static constexpr int kGroupsPerSupervisor = 32; + static constexpr std::size_t kTaskPayloadBytes = 1048576; // 1 MiB + static constexpr int kQueueDepth = 100000; + static constexpr int kDeadLetterMax = 10000; + static constexpr int kDeadLetterRetentionDays = 7; + static constexpr long kStreamRetentionHours = 24; + static constexpr std::size_t kStreamRetentionBytes = 1073741824; // 1 GiB + static constexpr int kSyncBatchEntries = 500; + static constexpr std::size_t kSyncBatchBytes = 1048576; // 1 MiB + static constexpr int kConflictLogMax = 10000; + static constexpr int kConflictLogRetentionDays = 7; + static constexpr std::size_t kStatusSnapshotBytes = 1048576; // 1 MiB + static constexpr int kSyncLagAlertThreshold = 10000; + static constexpr long kLeaseDefaultMs = 60000; + static constexpr long kHeartbeatMs = 15000; + static constexpr long kSchedulerTickMs = 1000; + static constexpr long kSchedulerMinIntervalSecs = 60; + static constexpr long kLeaderTtlSecs = 10; + static constexpr long kLeaderRenewSecs = 3; + static constexpr int kIdempotencyWindowHours = 24; + static constexpr int kMaxRestartsDefault = 5; + static constexpr int kRestartWindowSecsDefault = 60; + static constexpr long kBackoffMinMs = 1000; + static constexpr long kBackoffMaxMs = 60000; + static constexpr long kHealthIntervalMs = 10000; + static constexpr int kHealthMaxFailures = 3; + static constexpr long kRetryBaseMs = 1000; + static constexpr long kRetryCapMs = 300000; + static constexpr long kRetryJitterMs = 1000; + static constexpr long kHookTimeoutMs = 5000; + static constexpr int kSubQueueMax = 1024; + static constexpr std::size_t kStreamPayloadBytes = 1048576; // 1 MiB + static constexpr int kJsonMaxDepth = 64; + static constexpr int kMaxStreamsPerBus = 1024; + static constexpr int kMaxConsumersPerStream = 1024; +}; + +} // namespace molo::common diff --git a/src/supervisor/common/clock.cpp b/src/supervisor/common/clock.cpp new file mode 100644 index 0000000..dfbc5be --- /dev/null +++ b/src/supervisor/common/clock.cpp @@ -0,0 +1,109 @@ +#include "supervisor/common/clock.hpp" + +#include +#include +#include + +namespace molo::common { + +std::int64_t SystemClock::wall_ms() const { + const auto now = std::chrono::system_clock::now(); + return std::chrono::duration_cast(now.time_since_epoch()).count(); +} + +std::int64_t SystemClock::steady_ms() const { + const auto now = std::chrono::steady_clock::now(); + return std::chrono::duration_cast(now.time_since_epoch()).count(); +} + +void SystemClock::sleep_ms(long ms) { + if (ms > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } +} + +FrozenClock::FrozenClock(std::int64_t start_wall_ms, std::int64_t start_steady_ms) + : wall_ms_(start_wall_ms), steady_ms_(start_steady_ms) {} + +std::int64_t FrozenClock::wall_ms() const { + std::lock_guard lock(mutex_); + return wall_ms_; +} + +std::int64_t FrozenClock::steady_ms() const { + std::lock_guard lock(mutex_); + return steady_ms_; +} + +void FrozenClock::sleep_ms(long ms) { + if (ms > 0) { + advance(ms); + } +} + +void FrozenClock::advance(std::int64_t delta_ms) { + std::lock_guard lock(mutex_); + wall_ms_ += delta_ms; + steady_ms_ += delta_ms; +} + +void FrozenClock::set(std::int64_t wall_ms, std::int64_t steady_ms) { + std::lock_guard lock(mutex_); + wall_ms_ = wall_ms; + steady_ms_ = steady_ms; +} + +std::string iso8601(std::int64_t wall_ms) { + const std::time_t secs = static_cast(wall_ms / 1000); + const int millis = static_cast(wall_ms % 1000); + std::tm tm_utc{}; +#if defined(_WIN32) + gmtime_s(&tm_utc, &secs); +#else + gmtime_r(&secs, &tm_utc); +#endif + char buf[128]; + std::snprintf(buf, sizeof(buf), "%04d-%02d-%02dT%02d:%02d:%02d.%03dZ", tm_utc.tm_year + 1900, + tm_utc.tm_mon + 1, tm_utc.tm_mday, tm_utc.tm_hour, tm_utc.tm_min, tm_utc.tm_sec, + millis); + return std::string(buf); +} + +std::int64_t parse_iso8601(const std::string& text) { + // Accepts YYYY-MM-DDTHH:MM:SS[.mmm]Z only. Anything else returns -1. + int year = 0, mon = 0, day = 0, hour = 0, min = 0, sec = 0, millis = 0; + if (text.size() < 20 || text.back() != 'Z') { + return -1; + } + int matched = std::sscanf(text.c_str(), "%4d-%2d-%2dT%2d:%2d:%2d.%3dZ", &year, &mon, &day, + &hour, &min, &sec, &millis); + if (matched != 7) { + matched = std::sscanf(text.c_str(), "%4d-%2d-%2dT%2d:%2d:%2dZ", &year, &mon, &day, &hour, + &min, &sec); + if (matched != 6) { + return -1; + } + } + if (mon < 1 || mon > 12 || day < 1 || day > 31 || hour > 23 || min > 59 || sec > 60 || + millis < 0 || millis > 999) { + return -1; + } + std::tm tm_utc{}; + tm_utc.tm_year = year - 1900; + tm_utc.tm_mon = mon - 1; + tm_utc.tm_mday = day; + tm_utc.tm_hour = hour; + tm_utc.tm_min = min; + tm_utc.tm_sec = sec; +#if defined(_WIN32) + const std::time_t secs = _mkgmtime(&tm_utc); +#else + const std::time_t secs = timegm(&tm_utc); +#endif + if (secs == static_cast(-1)) { + return -1; + } + return static_cast(secs) * 1000 + millis; +} + +} // namespace molo::common diff --git a/src/supervisor/common/clock.hpp b/src/supervisor/common/clock.hpp new file mode 100644 index 0000000..c1ed6dc --- /dev/null +++ b/src/supervisor/common/clock.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include +#include + +namespace molo::common { + +// Clock abstraction. Production uses SystemClock; tests use FrozenClock +// for deterministic scheduler, lease, and backoff behavior. +class Clock { + public: + virtual ~Clock() = default; + virtual std::int64_t wall_ms() const = 0; // unix epoch milliseconds + virtual std::int64_t steady_ms() const = 0; // monotonic milliseconds + virtual void sleep_ms(long ms) = 0; +}; + +class SystemClock : public Clock { + public: + std::int64_t wall_ms() const override; + std::int64_t steady_ms() const override; + void sleep_ms(long ms) override; +}; + +class FrozenClock : public Clock { + public: + explicit FrozenClock(std::int64_t start_wall_ms = 0, std::int64_t start_steady_ms = 0); + + std::int64_t wall_ms() const override; + std::int64_t steady_ms() const override; + void sleep_ms(long ms) override; // advances the frozen clock, no real wait + void advance(std::int64_t delta_ms); + void set(std::int64_t wall_ms, std::int64_t steady_ms); + + private: + mutable std::mutex mutex_; + std::int64_t wall_ms_; + std::int64_t steady_ms_; +}; + +// UTC ISO-8601 with millis, e.g. 2026-10-23T12:34:56.789Z. Used by the +// task envelope (S7-1) and oplog (S7-7) JSON formats. +std::string iso8601(std::int64_t wall_ms); + +// Parses iso8601() output (millis optional). Returns -1 on invalid input. +std::int64_t parse_iso8601(const std::string& text); + +} // namespace molo::common diff --git a/src/supervisor/common/json.cpp b/src/supervisor/common/json.cpp new file mode 100644 index 0000000..b8c822f --- /dev/null +++ b/src/supervisor/common/json.cpp @@ -0,0 +1,382 @@ +#include "supervisor/common/json.hpp" + +#include + +#include "supervisor/common/caps.hpp" + +namespace molo::common { + +bool JValue::is_null() const { return std::holds_alternative(data); } +bool JValue::is_bool() const { return std::holds_alternative(data); } +bool JValue::is_int() const { return std::holds_alternative(data); } +bool JValue::is_double() const { return std::holds_alternative(data); } +bool JValue::is_string() const { return std::holds_alternative(data); } +bool JValue::is_array() const { return std::holds_alternative(data); } +bool JValue::is_object() const { return std::holds_alternative(data); } + +bool JValue::as_bool(bool fallback) const { + return is_bool() ? std::get(data) : fallback; +} + +long long JValue::as_int(long long fallback) const { + if (is_int()) { + return std::get(data); + } + if (is_double()) { + return static_cast(std::get(data)); + } + return fallback; +} + +double JValue::as_double(double fallback) const { + if (is_double()) { + return std::get(data); + } + if (is_int()) { + return static_cast(std::get(data)); + } + return fallback; +} + +std::string JValue::as_string(const std::string& fallback) const { + return is_string() ? std::get(data) : fallback; +} + +const JValue::Array& JValue::as_array() const { + static const Array empty; + return is_array() ? std::get(data) : empty; +} + +const JValue::Object& JValue::as_object() const { + static const Object empty; + return is_object() ? std::get(data) : empty; +} + +const JValue* JValue::find(std::string_view key) const { + if (!is_object()) { + return nullptr; + } + const auto& obj = std::get(data); + auto it = obj.find(std::string(key)); + return it == obj.end() ? nullptr : &it->second; +} + +std::string json_escape(std::string_view text) { + std::string out; + out.reserve(text.size() + 2); + for (char c : text) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (static_cast(c) < 0x20) { + char buf[8]; + const int n = std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out.append(buf, static_cast(n)); + } else { + out += c; + } + } + } + return out; +} + +namespace { + +void encode_into(const JValue& value, std::string& out) { + if (value.is_null()) { + out += "null"; + } else if (value.is_bool()) { + out += value.as_bool() ? "true" : "false"; + } else if (value.is_int()) { + out += std::to_string(value.as_int()); + } else if (value.is_double()) { + out += std::to_string(value.as_double()); + } else if (value.is_string()) { + out += '"'; + out += json_escape(value.as_string()); + out += '"'; + } else if (value.is_array()) { + out += '['; + bool first = true; + for (const auto& item : value.as_array()) { + if (!first) { + out += ','; + } + first = false; + encode_into(item, out); + } + out += ']'; + } else { + out += '{'; + bool first = true; + for (const auto& [key, item] : value.as_object()) { + if (!first) { + out += ','; + } + first = false; + out += '"'; + out += json_escape(key); + out += "\":"; + encode_into(item, out); + } + out += '}'; + } +} + +class Parser { + public: + explicit Parser(std::string_view text) : text_(text) {} + + bool run(JValue& out) { + skip_ws(); + if (!parse_value(out, 0)) { + return false; + } + skip_ws(); + return pos_ == text_.size(); + } + + private: + void skip_ws() { + while (pos_ < text_.size() && + (text_[pos_] == ' ' || text_[pos_] == '\t' || text_[pos_] == '\n' || + text_[pos_] == '\r')) { + ++pos_; + } + } + + bool consume(char c) { + if (pos_ < text_.size() && text_[pos_] == c) { + ++pos_; + return true; + } + return false; + } + + bool parse_value(JValue& out, int depth) { + if (depth > Caps::kJsonMaxDepth || pos_ >= text_.size()) { + return false; + } + const char c = text_[pos_]; + if (c == '{') { + return parse_object(out, depth); + } + if (c == '[') { + return parse_array(out, depth); + } + if (c == '"') { + std::string s; + if (!parse_string(s)) { + return false; + } + out = JValue(s); + return true; + } + if (c == 't') { + return parse_literal("true", JValue(true), out); + } + if (c == 'f') { + return parse_literal("false", JValue(false), out); + } + if (c == 'n') { + return parse_literal("null", JValue(nullptr), out); + } + return parse_number(out); + } + + bool parse_literal(std::string_view word, const JValue& value, JValue& out) { + if (text_.substr(pos_, word.size()) != word) { + return false; + } + pos_ += word.size(); + out = value; + return true; + } + + bool parse_number(JValue& out) { + std::size_t start = pos_; + if (pos_ < text_.size() && (text_[pos_] == '-' || text_[pos_] == '+')) { + ++pos_; + } + bool is_double = false; + while (pos_ < text_.size()) { + const char c = text_[pos_]; + if (c >= '0' && c <= '9') { + ++pos_; + } else if (c == '.' || c == 'e' || c == 'E') { + is_double = true; + ++pos_; + } else if ((c == '+' || c == '-') && + (text_[pos_ - 1] == 'e' || text_[pos_ - 1] == 'E')) { + ++pos_; + } else { + break; + } + } + if (pos_ == start) { + return false; + } + const std::string token(text_.substr(start, pos_ - start)); + try { + if (is_double) { + out = JValue(std::stod(token)); + } else { + out = JValue(static_cast(std::stoll(token))); + } + } catch (...) { + return false; + } + return true; + } + + bool parse_string(std::string& out) { + if (!consume('"')) { + return false; + } + out.clear(); + while (pos_ < text_.size()) { + const char c = text_[pos_++]; + if (c == '"') { + return true; + } + if (c != '\\') { + out += c; + continue; + } + if (pos_ >= text_.size()) { + return false; + } + const char e = text_[pos_++]; + switch (e) { + case '"': out += '"'; break; + case '\\': out += '\\'; break; + case '/': out += '/'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'b': out += '\b'; break; + case 'f': out += '\f'; break; + case 'u': { + if (pos_ + 4 > text_.size()) { + return false; + } + unsigned code = 0; + for (int i = 0; i < 4; ++i) { + const char h = text_[pos_++]; + code <<= 4; + if (h >= '0' && h <= '9') { + code += static_cast(h - '0'); + } else if (h >= 'a' && h <= 'f') { + code += static_cast(h - 'a' + 10); + } else if (h >= 'A' && h <= 'F') { + code += static_cast(h - 'A' + 10); + } else { + return false; + } + } + // BMP only, enough for the ASCII control escapes we emit. + if (code < 0x80) { + out += static_cast(code); + } else if (code < 0x800) { + out += static_cast(0xC0 | (code >> 6)); + out += static_cast(0x80 | (code & 0x3F)); + } else { + out += static_cast(0xE0 | (code >> 12)); + out += static_cast(0x80 | ((code >> 6) & 0x3F)); + out += static_cast(0x80 | (code & 0x3F)); + } + break; + } + default: return false; + } + } + return false; + } + + bool parse_object(JValue& out, int depth) { + if (!consume('{')) { + return false; + } + JValue::Object obj; + skip_ws(); + if (consume('}')) { + out = JValue(obj); + return true; + } + while (true) { + skip_ws(); + std::string key; + if (!parse_string(key)) { + return false; + } + skip_ws(); + if (!consume(':')) { + return false; + } + skip_ws(); + JValue value; + if (!parse_value(value, depth + 1)) { + return false; + } + obj[std::move(key)] = std::move(value); + skip_ws(); + if (consume('}')) { + out = JValue(obj); + return true; + } + if (!consume(',')) { + return false; + } + } + } + + bool parse_array(JValue& out, int depth) { + if (!consume('[')) { + return false; + } + JValue::Array items; + skip_ws(); + if (consume(']')) { + out = JValue(items); + return true; + } + while (true) { + skip_ws(); + JValue value; + if (!parse_value(value, depth + 1)) { + return false; + } + items.push_back(std::move(value)); + skip_ws(); + if (consume(']')) { + out = JValue(items); + return true; + } + if (!consume(',')) { + return false; + } + } + } + + std::string_view text_; + std::size_t pos_ = 0; +}; + +} // namespace + +std::string json_encode(const JValue& value) { + std::string out; + encode_into(value, out); + return out; +} + +bool json_decode(std::string_view text, JValue& out) { + Parser parser(text); + return parser.run(out); +} + +} // namespace molo::common diff --git a/src/supervisor/common/json.hpp b/src/supervisor/common/json.hpp new file mode 100644 index 0000000..a82db11 --- /dev/null +++ b/src/supervisor/common/json.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace molo::common { + +// Minimal JSON value plus encoder/decoder for the on-disk formats owned by +// this part (task envelope S7-1, oplog entry S7-7, status snapshot). Only +// what those formats need: objects, arrays, strings, integers, doubles, +// booleans, null. Depth capped by Caps::kJsonMaxDepth. +struct JValue { + using Array = std::vector; + using Object = std::map; + using Data = std::variant; + + Data data; + + JValue() : data(nullptr) {} + JValue(std::nullptr_t) : data(nullptr) {} + JValue(bool v) : data(v) {} + JValue(long long v) : data(v) {} + JValue(int v) : data(static_cast(v)) {} + JValue(double v) : data(v) {} + JValue(const char* v) : data(std::string(v)) {} + JValue(std::string_view v) : data(std::string(v)) {} + JValue(const std::string& v) : data(v) {} + JValue(const Array& v) : data(v) {} + JValue(const Object& v) : data(v) {} + + bool is_null() const; + bool is_bool() const; + bool is_int() const; + bool is_double() const; + bool is_string() const; + bool is_array() const; + bool is_object() const; + + bool as_bool(bool fallback = false) const; + long long as_int(long long fallback = 0) const; + double as_double(double fallback = 0.0) const; + std::string as_string(const std::string& fallback = "") const; + const Array& as_array() const; + const Object& as_object() const; + + // Object helpers. find() returns nullptr when absent or not an object. + const JValue* find(std::string_view key) const; +}; + +std::string json_escape(std::string_view text); +std::string json_encode(const JValue& value); + +// Parses a full JSON document. Returns false on any syntax error or when +// trailing garbage follows the document. Depth is capped. +bool json_decode(std::string_view text, JValue& out); + +} // namespace molo::common diff --git a/src/supervisor/common/kv_store.cpp b/src/supervisor/common/kv_store.cpp new file mode 100644 index 0000000..8541396 --- /dev/null +++ b/src/supervisor/common/kv_store.cpp @@ -0,0 +1,118 @@ +#include "supervisor/common/kv_store.hpp" + +#include "supervisor/common/clock.hpp" + +namespace molo::common { + +InMemoryKvStore::InMemoryKvStore(Clock& clock) : clock_(clock) {} + +bool InMemoryKvStore::expired(const Entry& entry, std::int64_t now_ms) const { + return entry.expires_at_ms > 0 && entry.expires_at_ms <= now_ms; +} + +bool InMemoryKvStore::get(std::string_view key, std::string& value) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end() || expired(it->second, clock_.wall_ms())) { + return false; + } + value = it->second.value; + return true; +} + +void InMemoryKvStore::set(std::string_view key, const std::string& value, long ttl_secs) { + std::lock_guard lock(mutex_); + Entry entry; + entry.value = value; + entry.expires_at_ms = ttl_secs > 0 ? clock_.wall_ms() + ttl_secs * 1000 : 0; + entries_[std::string(key)] = std::move(entry); +} + +bool InMemoryKvStore::erase(std::string_view key) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) { + return false; + } + entries_.erase(it); + return true; +} + +bool InMemoryKvStore::acquire(std::string_view key, const std::string& value, long ttl_secs) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it != entries_.end() && !expired(it->second, clock_.wall_ms())) { + return false; + } + Entry entry; + entry.value = value; + entry.expires_at_ms = ttl_secs > 0 ? clock_.wall_ms() + ttl_secs * 1000 : 0; + entries_[std::string(key)] = std::move(entry); + return true; +} + +bool InMemoryKvStore::renew(std::string_view key, const std::string& expected, long ttl_secs) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end() || expired(it->second, clock_.wall_ms()) || + it->second.value != expected) { + return false; + } + it->second.expires_at_ms = ttl_secs > 0 ? clock_.wall_ms() + ttl_secs * 1000 : 0; + return true; +} + +bool InMemoryKvStore::acquire_ms(std::string_view key, const std::string& value, + std::int64_t ttl_ms) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it != entries_.end() && !expired(it->second, clock_.wall_ms())) { + return false; + } + Entry entry; + entry.value = value; + entry.expires_at_ms = ttl_ms > 0 ? clock_.wall_ms() + ttl_ms : 0; + entries_[std::string(key)] = std::move(entry); + return true; +} + +bool InMemoryKvStore::renew_ms(std::string_view key, const std::string& expected, + std::int64_t ttl_ms) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end() || expired(it->second, clock_.wall_ms()) || + it->second.value != expected) { + return false; + } + it->second.expires_at_ms = ttl_ms > 0 ? clock_.wall_ms() + ttl_ms : 0; + return true; +} + +bool InMemoryKvStore::cas_ms(std::string_view key, const std::string& expected, + const std::string& next, std::int64_t ttl_ms) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end() || expired(it->second, clock_.wall_ms())) { + if (!expected.empty()) { + return false; + } + Entry entry; + entry.value = next; + entry.expires_at_ms = ttl_ms > 0 ? clock_.wall_ms() + ttl_ms : 0; + entries_[std::string(key)] = std::move(entry); + return true; + } + if (it->second.value != expected) { + return false; + } + it->second.value = next; + it->second.expires_at_ms = ttl_ms > 0 ? clock_.wall_ms() + ttl_ms : 0; + return true; +} + +std::size_t InMemoryKvStore::size() const { + std::lock_guard lock(mutex_); + return entries_.size(); +} + +} // namespace molo::common diff --git a/src/supervisor/common/kv_store.hpp b/src/supervisor/common/kv_store.hpp new file mode 100644 index 0000000..01d060c --- /dev/null +++ b/src/supervisor/common/kv_store.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace molo::common { + +class Clock; + +// Minimal key-value interface needed by part 07 (scheduler leader lease, +// shared last-fire records). Mirrors the 04-data.md KvStore shape +// (get/set/del plus atomic acquire/renew for leases) so the shared +// backend from 04-data.md slots in behind this interface for +// multi-instance deployments (S7-9). Single-node deployments use +// InMemoryKvStore. +class KvStore { + public: + virtual ~KvStore() = default; + + // Returns true and fills value when the key exists and is not expired. + virtual bool get(std::string_view key, std::string& value) = 0; + + // Stores value. ttl_secs <= 0 means no expiry. + virtual void set(std::string_view key, const std::string& value, long ttl_secs) = 0; + + virtual bool erase(std::string_view key) = 0; + + // Atomically stores value when the key is absent or expired. + virtual bool acquire(std::string_view key, const std::string& value, long ttl_secs) = 0; + + // Atomically extends the TTL when the current value equals expected. + virtual bool renew(std::string_view key, const std::string& expected, long ttl_secs) = 0; + + // Millisecond-precision fencing API for the operability part (leases, + // leader election, distributed locks run on ms clocks). Same semantics as + // the seconds API: ttl_ms <= 0 means no expiry; empty expected matches a + // missing key for cas_ms only. + virtual bool acquire_ms(std::string_view key, const std::string& value, + std::int64_t ttl_ms) = 0; + virtual bool renew_ms(std::string_view key, const std::string& expected, + std::int64_t ttl_ms) = 0; + virtual bool cas_ms(std::string_view key, const std::string& expected, + const std::string& next, std::int64_t ttl_ms) = 0; +}; + +class InMemoryKvStore : public KvStore { + public: + explicit InMemoryKvStore(Clock& clock); + + bool get(std::string_view key, std::string& value) override; + void set(std::string_view key, const std::string& value, long ttl_secs) override; + bool erase(std::string_view key) override; + bool acquire(std::string_view key, const std::string& value, long ttl_secs) override; + bool renew(std::string_view key, const std::string& expected, long ttl_secs) override; + bool acquire_ms(std::string_view key, const std::string& value, + std::int64_t ttl_ms) override; + bool renew_ms(std::string_view key, const std::string& expected, + std::int64_t ttl_ms) override; + bool cas_ms(std::string_view key, const std::string& expected, const std::string& next, + std::int64_t ttl_ms) override; + std::size_t size() const; + + private: + struct Entry { + std::string value; + std::int64_t expires_at_ms = 0; // 0 means no expiry + }; + + bool expired(const Entry& entry, std::int64_t now_ms) const; + + Clock& clock_; + mutable std::mutex mutex_; + std::map> entries_; +}; + +} // namespace molo::common diff --git a/src/supervisor/common/observer.cpp b/src/supervisor/common/observer.cpp new file mode 100644 index 0000000..660ea63 --- /dev/null +++ b/src/supervisor/common/observer.cpp @@ -0,0 +1,61 @@ +#include "supervisor/common/observer.hpp" + +namespace molo::common { + +void NullObserver::event(std::string_view, std::string_view) {} +void NullObserver::counter(std::string_view, long) {} +void NullObserver::gauge(std::string_view, double) {} + +void RecordingObserver::event(std::string_view type, std::string_view detail) { + std::lock_guard lock(mutex_); + events_.push_back(RecordedEvent{std::string(type), std::string(detail)}); +} + +void RecordingObserver::counter(std::string_view name, long delta) { + std::lock_guard lock(mutex_); + counters_[std::string(name)] += delta; +} + +void RecordingObserver::gauge(std::string_view name, double value) { + std::lock_guard lock(mutex_); + gauges_[std::string(name)] = value; +} + +std::vector RecordingObserver::events() const { + std::lock_guard lock(mutex_); + return events_; +} + +std::map RecordingObserver::counters() const { + std::lock_guard lock(mutex_); + return counters_; +} + +std::map RecordingObserver::gauges() const { + std::lock_guard lock(mutex_); + return gauges_; +} + +long RecordingObserver::counter_value(const std::string& name) const { + std::lock_guard lock(mutex_); + auto it = counters_.find(name); + return it == counters_.end() ? 0 : it->second; +} + +std::size_t RecordingObserver::event_count(const std::string& type) const { + std::lock_guard lock(mutex_); + std::size_t count = 0; + for (const auto& e : events_) { + if (e.type == type) { + ++count; + } + } + return count; +} + +std::shared_ptr null_observer() { + static std::shared_ptr instance = std::make_shared(); + return instance; +} + +} // namespace molo::common diff --git a/src/supervisor/common/observer.hpp b/src/supervisor/common/observer.hpp new file mode 100644 index 0000000..0e01d33 --- /dev/null +++ b/src/supervisor/common/observer.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace molo::common { + +// Minimal observability sink for part 07 (R7-6). +// +// This is the seam where 02-observability.md plugs in: the real Dispatcher, +// metrics Registry, and Tracer implement this interface (or this interface is +// replaced by direct calls to molo::events, molo::metrics, molo::trace once +// that part lands). Until then NullObserver keeps production quiet and +// RecordingObserver lets tests assert on emitted events and metrics. +class Observer { + public: + virtual ~Observer() = default; + virtual void event(std::string_view type, std::string_view detail) = 0; + virtual void counter(std::string_view name, long delta) = 0; + virtual void gauge(std::string_view name, double value) = 0; +}; + +class NullObserver : public Observer { + public: + void event(std::string_view type, std::string_view detail) override; + void counter(std::string_view name, long delta) override; + void gauge(std::string_view name, double value) override; +}; + +struct RecordedEvent { + std::string type; + std::string detail; +}; + +class RecordingObserver : public Observer { + public: + void event(std::string_view type, std::string_view detail) override; + void counter(std::string_view name, long delta) override; + void gauge(std::string_view name, double value) override; + + std::vector events() const; + std::map counters() const; + std::map gauges() const; + long counter_value(const std::string& name) const; + std::size_t event_count(const std::string& type) const; + + private: + mutable std::mutex mutex_; + std::vector events_; + std::map counters_; + std::map gauges_; +}; + +std::shared_ptr null_observer(); + +} // namespace molo::common diff --git a/src/supervisor/replication/conflict_resolver.cpp b/src/supervisor/replication/conflict_resolver.cpp new file mode 100644 index 0000000..4950b49 --- /dev/null +++ b/src/supervisor/replication/conflict_resolver.cpp @@ -0,0 +1,230 @@ +#include "supervisor/replication/conflict_resolver.hpp" + +#include +#include +#include + +#include "supervisor/common/caps.hpp" + +namespace molo::replication { + +std::string to_string(ConflictPolicy policy) { + switch (policy) { + case ConflictPolicy::Optimistic: return "optimistic"; + case ConflictPolicy::LastWriterWins: return "lww"; + case ConflictPolicy::Hook: return "hook"; + } + return "optimistic"; +} + +ConflictResolver::ConflictResolver(common::Clock& clock, + std::shared_ptr observer, + int max_log_entries, long log_retention_days) + : clock_(clock), + observer_(std::move(observer)), + max_log_entries_(max_log_entries), + log_retention_days_(log_retention_days) { + if (max_log_entries_ < 1) { + max_log_entries_ = 1; + } +} + +void ConflictResolver::declare(const std::string& model, ConflictPolicy policy, + ConflictHook hook) { + if (model.empty() || model.size() > 256) { + throw ReplicationError("ConflictResolver: bad model name"); + } + std::lock_guard lock(mutex_); + if (policy == ConflictPolicy::Hook && !hook) { + policy = ConflictPolicy::LastWriterWins; // declared fallback, logged below + observer_->event("replication.policy_fallback", model + " hook->lww (no callback)"); + } + policies_[model] = policy; + if (hook) { + hooks_[model] = std::move(hook); + } else { + hooks_.erase(model); + } + observer_->event("replication.policy_declared", model + " " + to_string(policy)); +} + +ConflictPolicy ConflictResolver::policy_for(const std::string& model) const { + std::lock_guard lock(mutex_); + auto it = policies_.find(model); + return it == policies_.end() ? ConflictPolicy::Optimistic : it->second; +} + +ConflictOutcome ConflictResolver::resolve(const OplogEntry& local, const OplogEntry& remote) { + ConflictPolicy policy; + ConflictHook hook; + { + std::lock_guard lock(mutex_); + auto it = policies_.find(local.model); + policy = it == policies_.end() ? ConflictPolicy::Optimistic : it->second; + auto hit = hooks_.find(local.model); + if (hit != hooks_.end()) { + hook = hit->second; + } + } + // Run the policy without the resolver lock: hooks are user code. + ConflictOutcome outcome; + switch (policy) { + case ConflictPolicy::Optimistic: outcome = resolve_optimistic(local, remote); break; + case ConflictPolicy::LastWriterWins: outcome = resolve_lww(local, remote); break; + case ConflictPolicy::Hook: outcome = resolve_hook(local, remote, hook); break; + } + if (outcome.conflict) { + observer_->counter("molo_replication_conflicts_total", 1); + observer_->event("replication.conflict", + local.model + ":" + local.key + " " + outcome.detail); + } + return outcome; +} + +ConflictOutcome ConflictResolver::resolve_optimistic(const OplogEntry& local, + const OplogEntry& remote) { + ConflictOutcome outcome; + if (remote.version > local.version) { + outcome.remote_won = true; + outcome.winner = remote; + outcome.detail = "remote version " + std::to_string(remote.version) + " > local " + + std::to_string(local.version); + // Fast-forward, not a conflict. + } else if (remote.version < local.version) { + outcome.winner = local; + outcome.conflict = true; + outcome.conflict_error = true; + outcome.winner_version = local.version; + outcome.detail = "stale remote version " + std::to_string(remote.version) + ", winner " + + std::to_string(local.version); + log_conflict("optimistic", local, remote); + } else if (remote.data == local.data && remote.op == local.op) { + outcome.winner = local; // identical write, idempotent replay + outcome.detail = "identical versions and data"; + } else { + // Same version, different data: a genuine write-write conflict. + // The incumbent stays; the loser gets ConflictError with the winner + // version so it can re-read and retry. + outcome.winner = local; + outcome.conflict = true; + outcome.conflict_error = true; + outcome.winner_version = local.version; + outcome.detail = "write-write conflict at version " + std::to_string(local.version); + log_conflict("optimistic", local, remote); + } + return outcome; +} + +ConflictOutcome ConflictResolver::resolve_lww(const OplogEntry& local, const OplogEntry& remote) { + ConflictOutcome outcome; + // Concurrent writes with any difference are a conflict whose loser is + // kept in the conflict log (S7-8); only exact replays are quiet. + outcome.conflict = (local.version != remote.version || local.data != remote.data || + local.op != remote.op); + const int cmp = remote.ts.compare(local.ts); + if (cmp > 0) { + outcome.remote_won = true; + outcome.winner = remote; + outcome.detail = "remote ts " + remote.ts.encode() + " wins"; + if (outcome.conflict) { + log_conflict("lww", remote, local); + } + } else { + outcome.winner = local; + outcome.detail = "local ts " + local.ts.encode() + " wins"; + if (outcome.conflict) { + log_conflict("lww", local, remote); + } + } + outcome.winner_version = outcome.winner.version; + return outcome; +} + +ConflictOutcome ConflictResolver::resolve_hook(const OplogEntry& local, const OplogEntry& remote, + const ConflictHook& hook) { + if (!hook) { + return resolve_lww(local, remote); + } + // The hook runs off-lock with a 5s budget; on timeout or throw the LWW + // fallback applies (S7-8). Packaged task plus timed wait, no detached + // threads: a stuck hook still joins before we return. + std::packaged_task task([&] { return hook(local, remote); }); + std::future future = task.get_future(); + std::thread runner(std::move(task)); + int decision = 1; + bool fallback = false; + if (future.wait_for(std::chrono::milliseconds(common::Caps::kHookTimeoutMs)) == + std::future_status::ready) { + try { + decision = future.get(); + } catch (...) { + fallback = true; + } + if (decision != 0 && decision != 1) { + fallback = true; + } + } else { + fallback = true; + } + runner.join(); + if (fallback) { + observer_->event("replication.hook_fallback", local.model + ":" + local.key); + ConflictOutcome outcome = resolve_lww(local, remote); + outcome.detail = "hook fallback: " + outcome.detail; + return outcome; + } + ConflictOutcome outcome; + outcome.conflict = true; + if (decision == 1) { + outcome.remote_won = true; + outcome.winner = remote; + log_conflict("hook", remote, local); + } else { + outcome.winner = local; + log_conflict("hook", local, remote); + } + outcome.winner_version = outcome.winner.version; + outcome.detail = "hook decision=" + std::to_string(decision); + return outcome; +} + +void ConflictResolver::log_conflict(const std::string& policy, const OplogEntry& winner, + const OplogEntry& loser) { + std::lock_guard lock(mutex_); + while (static_cast(log_.size()) >= max_log_entries_) { + log_.pop_front(); + observer_->counter("molo_replication_conflict_log_evicted_total", 1); + } + log_.push_back(ConflictRecord{clock_.wall_ms(), policy, winner, loser}); +} + +std::vector ConflictResolver::conflict_log(std::size_t max) const { + std::lock_guard lock(mutex_); + std::vector out; + for (const auto& record : log_) { + if (out.size() >= max) { + break; + } + out.push_back(record); + } + return out; +} + +std::size_t ConflictResolver::conflict_log_size() const { + std::lock_guard lock(mutex_); + return log_.size(); +} + +std::size_t ConflictResolver::purge_expired() { + std::lock_guard lock(mutex_); + const std::int64_t horizon = + clock_.wall_ms() - log_retention_days_ * 86400 * 1000; + std::size_t purged = 0; + while (!log_.empty() && log_.front().at_ms < horizon) { + log_.pop_front(); + ++purged; + } + return purged; +} + +} // namespace molo::replication diff --git a/src/supervisor/replication/conflict_resolver.hpp b/src/supervisor/replication/conflict_resolver.hpp new file mode 100644 index 0000000..00e0ee5 --- /dev/null +++ b/src/supervisor/replication/conflict_resolver.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/replication/oplog_entry.hpp" + +namespace molo::replication { + +// Per-model conflict policy, declared once (S7-8, R7-7). +enum class ConflictPolicy { + Optimistic, // compare version, reject stale with ConflictError + LastWriterWins, // higher hybrid timestamp wins, loser kept in conflict log + Hook, // typed callback decides, 5s budget, LWW fallback +}; + +std::string to_string(ConflictPolicy policy); + +// Returns 0 when local wins, 1 when remote wins. +using ConflictHook = std::function; + +struct ConflictOutcome { + bool conflict = false; // true when both sides wrote concurrently + bool remote_won = false; // true when the remote entry becomes current + OplogEntry winner; // the entry that becomes current + std::string detail; // human-readable reason, capped at 1 KiB + bool conflict_error = false; // true when the caller must raise ConflictError + long long winner_version = 0; +}; + +struct ConflictRecord { + std::int64_t at_ms = 0; + std::string policy; + OplogEntry winner; + OplogEntry loser; +}; + +// Dispatches per-model conflict policy (S7-8). The ONLY place that compares +// versions or timestamps: callers never resolve conflicts ad hoc (R7-7). +class ConflictResolver { + public: + ConflictResolver(common::Clock& clock, std::shared_ptr observer, + int max_log_entries = 10000, long log_retention_days = 7); + + // Declares (or replaces) the policy for one model. Hook policy needs a + // callback; without one the declaration falls back to LastWriterWins. + void declare(const std::string& model, ConflictPolicy policy, + ConflictHook hook = ConflictHook{}); + + ConflictPolicy policy_for(const std::string& model) const; // default: Optimistic + + // Resolves local (current) vs remote (incoming) for the same model+key. + ConflictOutcome resolve(const OplogEntry& local, const OplogEntry& remote); + + std::vector conflict_log(std::size_t max) const; + std::size_t conflict_log_size() const; + std::size_t purge_expired(); + + private: + ConflictOutcome resolve_optimistic(const OplogEntry& local, const OplogEntry& remote); + ConflictOutcome resolve_lww(const OplogEntry& local, const OplogEntry& remote); + ConflictOutcome resolve_hook(const OplogEntry& local, const OplogEntry& remote, + const ConflictHook& hook); + void log_conflict(const std::string& policy, const OplogEntry& winner, + const OplogEntry& loser); + + common::Clock& clock_; + std::shared_ptr observer_; + int max_log_entries_; + long log_retention_days_; + mutable std::mutex mutex_; + std::map policies_; + std::map hooks_; + std::deque log_; +}; + +} // namespace molo::replication diff --git a/src/supervisor/replication/hybrid_clock.cpp b/src/supervisor/replication/hybrid_clock.cpp new file mode 100644 index 0000000..e8f11ab --- /dev/null +++ b/src/supervisor/replication/hybrid_clock.cpp @@ -0,0 +1,79 @@ +#include "supervisor/replication/hybrid_clock.hpp" + +namespace molo::replication { + +int HybridTs::compare(const HybridTs& other) const { + if (wall_ms != other.wall_ms) { + return wall_ms < other.wall_ms ? -1 : 1; + } + if (counter != other.counter) { + return counter < other.counter ? -1 : 1; + } + if (instance != other.instance) { + return instance < other.instance ? -1 : 1; + } + return 0; +} + +bool HybridTs::operator<(const HybridTs& other) const { + return compare(other) < 0; +} + +bool HybridTs::operator==(const HybridTs& other) const { + return compare(other) == 0; +} + +std::string HybridTs::encode() const { + return std::to_string(wall_ms) + "." + std::to_string(counter) + "." + instance; +} + +bool HybridTs::decode(const std::string& text, HybridTs& out) { + const std::size_t first = text.find('.'); + if (first == std::string::npos) { + return false; + } + const std::size_t second = text.find('.', first + 1); + if (second == std::string::npos) { + return false; + } + try { + out.wall_ms = std::stoll(text.substr(0, first)); + out.counter = static_cast(std::stoul(text.substr(first + 1, second - first - 1))); + } catch (...) { + return false; + } + out.instance = text.substr(second + 1); + return !out.instance.empty(); +} + +HybridClock::HybridClock(std::string instance, common::Clock& clock) + : instance_(std::move(instance)), clock_(clock) {} + +HybridTs HybridClock::now() { + std::lock_guard lock(mutex_); + const std::int64_t wall = clock_.wall_ms(); + if (wall > last_wall_ms_) { + last_wall_ms_ = wall; + counter_ = 0; + } else { + ++counter_; // same ms (or skew): the counter keeps order + } + return HybridTs{last_wall_ms_, counter_, instance_}; +} + +void HybridClock::observe(const HybridTs& remote) { + std::lock_guard lock(mutex_); + const std::int64_t wall = clock_.wall_ms(); + if (remote.wall_ms > last_wall_ms_ && remote.wall_ms > wall) { + last_wall_ms_ = remote.wall_ms; + counter_ = remote.counter + 1; + } else if (remote.wall_ms == last_wall_ms_ && remote.counter >= counter_) { + counter_ = remote.counter + 1; + } else if (wall > last_wall_ms_) { + last_wall_ms_ = wall; + counter_ = 0; + } + // else: remote is older, nothing to fold in +} + +} // namespace molo::replication diff --git a/src/supervisor/replication/hybrid_clock.hpp b/src/supervisor/replication/hybrid_clock.hpp new file mode 100644 index 0000000..8eca450 --- /dev/null +++ b/src/supervisor/replication/hybrid_clock.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +#include "supervisor/common/clock.hpp" + +namespace molo::replication { + +// Hybrid logical timestamp: wall clock plus a counter that disambiguates +// events within the same millisecond and absorbs clock skew, with the +// source instance id as the deterministic final tie-break (S7-7, S7-8). +struct HybridTs { + std::int64_t wall_ms = 0; + std::uint32_t counter = 0; + std::string instance; + + // -1 when this < other, 0 when equal, 1 when this > other. + int compare(const HybridTs& other) const; + bool operator<(const HybridTs& other) const; + bool operator==(const HybridTs& other) const; + + std::string encode() const; // wall_ms.counter.instance + static bool decode(const std::string& text, HybridTs& out); +}; + +class HybridClock { + public: + HybridClock(std::string instance, common::Clock& clock); + + HybridTs now(); + void observe(const HybridTs& remote); // folds a remote ts into the clock + + private: + std::string instance_; + common::Clock& clock_; + mutable std::mutex mutex_; + std::int64_t last_wall_ms_ = 0; + std::uint32_t counter_ = 0; +}; + +} // namespace molo::replication diff --git a/src/supervisor/replication/oplog_entry.cpp b/src/supervisor/replication/oplog_entry.cpp new file mode 100644 index 0000000..570cc46 --- /dev/null +++ b/src/supervisor/replication/oplog_entry.cpp @@ -0,0 +1,86 @@ +#include "supervisor/replication/oplog_entry.hpp" + +#include "supervisor/common/json.hpp" + +namespace molo::replication { + +std::string to_string(OplogOp op) { + return op == OplogOp::Upsert ? "upsert" : "delete"; +} + +bool parse_oplog_op(const std::string& text, OplogOp& out) { + if (text == "upsert") { + out = OplogOp::Upsert; + } else if (text == "delete") { + out = OplogOp::Delete; + } else { + return false; + } + return true; +} + +std::string OplogEntry::encode() const { + using common::JValue; + JValue::Object obj; + obj["seq"] = JValue(static_cast(seq)); + obj["instance"] = JValue(instance); + obj["ts"] = JValue(ts.encode()); + obj["model"] = JValue(model); + obj["key"] = JValue(key); + obj["op"] = JValue(to_string(op)); + obj["version"] = JValue(version); + JValue data_value; + if (!common::json_decode(data, data_value)) { + data_value = JValue(data); + } + obj["data"] = data_value; + return common::json_encode(JValue(obj)); +} + +bool OplogEntry::decode(const std::string& text, OplogEntry& out) { + using common::JValue; + JValue root; + if (!common::json_decode(text, root) || !root.is_object()) { + return false; + } + const JValue* instance = root.find("instance"); + const JValue* ts = root.find("ts"); + const JValue* model = root.find("model"); + const JValue* key = root.find("key"); + const JValue* op = root.find("op"); + if (instance == nullptr || ts == nullptr || model == nullptr || key == nullptr || + op == nullptr || !instance->is_string() || !ts->is_string() || !model->is_string() || + !key->is_string() || !op->is_string()) { + return false; + } + OplogEntry entry; + const JValue* seq = root.find("seq"); + if (seq != nullptr) { + entry.seq = static_cast(seq->as_int(0)); + } + entry.instance = instance->as_string(); + if (!HybridTs::decode(ts->as_string(), entry.ts)) { + return false; + } + entry.model = model->as_string(); + entry.key = key->as_string(); + if (!parse_oplog_op(op->as_string(), entry.op)) { + return false; + } + const JValue* version = root.find("version"); + if (version != nullptr) { + entry.version = version->as_int(0); + } + const JValue* data = root.find("data"); + if (data != nullptr) { + entry.data = common::json_encode(*data); + } + out = std::move(entry); + return true; +} + +std::size_t OplogEntry::approx_bytes() const { + return instance.size() + model.size() + key.size() + data.size() + 64; +} + +} // namespace molo::replication diff --git a/src/supervisor/replication/oplog_entry.hpp b/src/supervisor/replication/oplog_entry.hpp new file mode 100644 index 0000000..e8c2dc0 --- /dev/null +++ b/src/supervisor/replication/oplog_entry.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include + +#include "supervisor/replication/hybrid_clock.hpp" + +namespace molo::replication { + +class ReplicationError : public std::runtime_error { + public: + explicit ReplicationError(const std::string& message) : std::runtime_error(message) {} +}; + +// Raised to the loser of an optimistic-locking conflict, carrying the +// winner version so the caller can re-read and retry (S7-8). +class ConflictError : public ReplicationError { + public: + ConflictError(const std::string& model, const std::string& key, long long winner_version) + : ReplicationError("conflict:" + model + ":" + key), + model_(model), + key_(key), + winner_version_(winner_version) {} + + const std::string& model() const { return model_; } + const std::string& key() const { return key_; } + long long winner_version() const { return winner_version_; } + + private: + std::string model_; + std::string key_; + long long winner_version_; +}; + +enum class OplogOp { + Upsert, + Delete, +}; + +std::string to_string(OplogOp op); +bool parse_oplog_op(const std::string& text, OplogOp& out); + +// One replication log entry (S7-7): +// {"seq":n,"instance":"id","ts":"hybrid-clock","model":"name","key":"pk", +// "op":"upsert|delete","version":n,"data":{}} +struct OplogEntry { + std::uint64_t seq = 0; + std::string instance; + HybridTs ts; + std::string model; + std::string key; + OplogOp op = OplogOp::Upsert; + long long version = 0; + std::string data = "{}"; // opaque JSON object, owned by the model layer + + std::string encode() const; + static bool decode(const std::string& text, OplogEntry& out); + std::size_t approx_bytes() const; +}; + +} // namespace molo::replication diff --git a/src/supervisor/replication/oplog_reader.cpp b/src/supervisor/replication/oplog_reader.cpp new file mode 100644 index 0000000..67169c7 --- /dev/null +++ b/src/supervisor/replication/oplog_reader.cpp @@ -0,0 +1,47 @@ +#include "supervisor/replication/oplog_reader.hpp" + +namespace molo::replication { + +OplogReader::OplogReader(const std::filesystem::path& dir) + : log_(dir / "oplog.log", false, bus::SegmentLog::OpenMode::ReadOnly) {} + +std::vector OplogReader::read_from(std::uint64_t seq, std::size_t max_entries, + std::size_t max_bytes) const { + log_.refresh(); + std::vector out; + for (const auto& record : log_.read_from(seq, max_entries, max_bytes)) { + OplogEntry entry; + if (OplogEntry::decode(record.data, entry)) { + out.push_back(std::move(entry)); + } + // Corrupt entries are skipped: the segment checksum already truncates + // torn tails, so anything left but undecodable is a loud gap for the + // operator, surfaced by the missing seq in the consumer. + } + return out; +} + +std::vector OplogReader::poll(std::size_t max_entries, std::size_t max_bytes) { + std::lock_guard lock(mutex_); + auto entries = read_from(position_, max_entries, max_bytes); + if (!entries.empty()) { + position_ = entries.back().seq + 1; + } + return entries; +} + +std::uint64_t OplogReader::position() const { + std::lock_guard lock(mutex_); + return position_; +} + +void OplogReader::set_position(std::uint64_t seq) { + std::lock_guard lock(mutex_); + position_ = seq == 0 ? 1 : seq; +} + +std::uint64_t OplogReader::last_seq() const { + return log_.last_seq(); +} + +} // namespace molo::replication diff --git a/src/supervisor/replication/oplog_reader.hpp b/src/supervisor/replication/oplog_reader.hpp new file mode 100644 index 0000000..00b63ce --- /dev/null +++ b/src/supervisor/replication/oplog_reader.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include + +#include "supervisor/bus/segment_log.hpp" +#include "supervisor/replication/oplog_entry.hpp" + +namespace molo::replication { + +// Tails one instance's persisted operation log: catch-up pull on reconnect +// reads from a kept position, and poll() advances it (S7-7). +class OplogReader { + public: + explicit OplogReader(const std::filesystem::path& dir); + + // Reads up to max_entries / max_bytes from seq (1-based), decoded. + std::vector read_from(std::uint64_t seq, std::size_t max_entries, + std::size_t max_bytes) const; + + // Polls ahead of the kept position and advances it past what was read. + std::vector poll(std::size_t max_entries, std::size_t max_bytes); + + std::uint64_t position() const; + void set_position(std::uint64_t seq); + std::uint64_t last_seq() const; + + private: + mutable bus::SegmentLog log_; + mutable std::mutex mutex_; + std::uint64_t position_ = 1; +}; + +} // namespace molo::replication diff --git a/src/supervisor/replication/oplog_writer.cpp b/src/supervisor/replication/oplog_writer.cpp new file mode 100644 index 0000000..96dc6d8 --- /dev/null +++ b/src/supervisor/replication/oplog_writer.cpp @@ -0,0 +1,46 @@ +#include "supervisor/replication/oplog_writer.hpp" + +namespace molo::replication { + +OplogWriter::OplogWriter(const std::filesystem::path& dir, std::string instance, + common::Clock& clock, std::shared_ptr observer) + : instance_(std::move(instance)), + clock_(clock), + observer_(std::move(observer)), + hybrid_(instance_, clock_), + log_(std::make_unique(dir / "oplog.log")), + next_seq_(log_->last_seq() + 1) {} + +OplogEntry OplogWriter::append(const std::string& model, const std::string& key, OplogOp op, + long long version, const std::string& data) { + if (model.empty() || model.size() > 256 || key.empty() || key.size() > 1024) { + throw ReplicationError("OplogWriter: bad model or key"); + } + if (data.size() > 1048576) { + throw ReplicationError("OplogWriter: data over 1 MiB cap"); + } + std::lock_guard lock(mutex_); + OplogEntry entry; + entry.seq = next_seq_++; + entry.instance = instance_; + entry.ts = hybrid_.now(); + entry.model = model; + entry.key = key; + entry.op = op; + entry.version = version; + entry.data = data; + log_->append(entry.encode(), clock_.wall_ms()); + observer_->counter("molo_replication_oplog_appended_total", 1); + return entry; +} + +std::uint64_t OplogWriter::last_seq() const { + std::lock_guard lock(mutex_); + return next_seq_ == 0 ? 0 : next_seq_ - 1; +} + +const std::string& OplogWriter::instance() const { + return instance_; +} + +} // namespace molo::replication diff --git a/src/supervisor/replication/oplog_writer.hpp b/src/supervisor/replication/oplog_writer.hpp new file mode 100644 index 0000000..dbfd5ff --- /dev/null +++ b/src/supervisor/replication/oplog_writer.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "supervisor/bus/segment_log.hpp" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/replication/hybrid_clock.hpp" +#include "supervisor/replication/oplog_entry.hpp" + +namespace molo::replication { + +// Appends to one instance's persisted operation log (S7-7). Sequences are +// assigned here, strictly increasing and stable across restarts. +class OplogWriter { + public: + OplogWriter(const std::filesystem::path& dir, std::string instance, common::Clock& clock, + std::shared_ptr observer); + + OplogEntry append(const std::string& model, const std::string& key, OplogOp op, + long long version, const std::string& data); + + std::uint64_t last_seq() const; + const std::string& instance() const; + + private: + std::string instance_; + common::Clock& clock_; + std::shared_ptr observer_; + HybridClock hybrid_; + std::unique_ptr log_; + mutable std::mutex mutex_; + std::uint64_t next_seq_; +}; + +} // namespace molo::replication diff --git a/src/supervisor/replication/replicator.cpp b/src/supervisor/replication/replicator.cpp new file mode 100644 index 0000000..1dd4402 --- /dev/null +++ b/src/supervisor/replication/replicator.cpp @@ -0,0 +1,122 @@ +#include "supervisor/replication/replicator.hpp" + +namespace molo::replication { + +Replicator::Replicator(const std::filesystem::path& dir, std::string instance, + common::Clock& clock, std::shared_ptr observer) + : instance_(std::move(instance)), + clock_(clock), + observer_(std::move(observer)), + writer_(dir, instance_, clock_, observer_), + reader_(dir), + resolver_(clock_, observer_) { + replay(); +} + +const std::string& Replicator::instance() const { + return instance_; +} + +void Replicator::declare_model(const std::string& model, ConflictPolicy policy, + ConflictHook hook) { + resolver_.declare(model, policy, std::move(hook)); +} + +void Replicator::replay() { + // Rebuild current state plus the version vector from the persisted log. + // Replay applies entries in seq order without conflict checks: the log + // already holds resolved history. + std::uint64_t seq = 1; + while (true) { + auto entries = reader_.read_from(seq, 512, 4 * 1024 * 1024); + if (entries.empty()) { + break; + } + for (const auto& entry : entries) { + current_[{entry.model, entry.key}] = entry; + versions_.update(entry.instance, entry.seq); + } + seq = entries.back().seq + 1; + } + observer_->event("replication.replayed", "keys=" + std::to_string(current_.size())); +} + +OplogEntry Replicator::local_write(const std::string& model, const std::string& key, OplogOp op, + long long version, const std::string& data) { + OplogEntry entry = writer_.append(model, key, op, version, data); + versions_.update(instance_, entry.seq); + { + std::lock_guard lock(mutex_); + current_[{model, key}] = entry; + } + observer_->event("replication.local_write", model + ":" + key); + return entry; +} + +ConflictOutcome Replicator::apply_remote(const OplogEntry& entry) { + if (entry.instance == instance_) { + // Own entries echoed back by the bus are already current. + ConflictOutcome outcome; + outcome.winner = entry; + return outcome; + } + std::optional local; + { + std::lock_guard lock(mutex_); + auto it = current_.find({entry.model, entry.key}); + if (it != current_.end()) { + local = it->second; + } + } + ConflictOutcome outcome; + if (!local.has_value()) { + outcome.winner = entry; + outcome.remote_won = true; + outcome.detail = "no local value, remote adopted"; + std::lock_guard lock(mutex_); + current_[{entry.model, entry.key}] = entry; + } else if (local->instance == entry.instance && local->seq == entry.seq) { + outcome.winner = *local; // exact duplicate delivery, idempotent + outcome.detail = "duplicate delivery ignored"; + } else { + outcome = resolver_.resolve(*local, entry); + std::lock_guard lock(mutex_); + current_[{entry.model, entry.key}] = outcome.winner; + } + versions_.update(entry.instance, entry.seq); + observer_->counter("molo_replication_applied_total", 1); + if (outcome.conflict_error) { + throw ConflictError(entry.model, entry.key, outcome.winner_version); + } + return outcome; +} + +std::optional Replicator::current(const std::string& model, + const std::string& key) const { + std::lock_guard lock(mutex_); + auto it = current_.find({model, key}); + if (it == current_.end()) { + return std::nullopt; + } + return it->second; +} + +std::vector Replicator::entries_since(std::uint64_t seq, std::size_t max_entries, + std::size_t max_bytes) const { + return reader_.read_from(seq, max_entries, max_bytes); +} + +void Replicator::versions(VersionVector& out) const { + // Snapshot via encode/decode to avoid exposing internals. + VersionVector::decode(versions_.encode(), out); +} + +ConflictResolver& Replicator::resolver() { + return resolver_; +} + +std::uint64_t Replicator::last_seq() const { + return writer_.last_seq(); +} + +} // namespace molo::replication diff --git a/src/supervisor/replication/replicator.hpp b/src/supervisor/replication/replicator.hpp new file mode 100644 index 0000000..6ce718b --- /dev/null +++ b/src/supervisor/replication/replicator.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/replication/conflict_resolver.hpp" +#include "supervisor/replication/oplog_entry.hpp" +#include "supervisor/replication/oplog_reader.hpp" +#include "supervisor/replication/oplog_writer.hpp" +#include "supervisor/replication/version_vector.hpp" + +namespace molo::replication { + +// Per-instance replication endpoint (E7-7): owns the local oplog, the +// current per-key state, per-model conflict policies, and the version +// vector. Local writes append to the oplog and become current; remote +// entries arrive through apply_remote() and resolve through the declared +// policy. Replays the persisted oplog on construction so state survives +// restarts. Thread-safe. +class Replicator { + public: + Replicator(const std::filesystem::path& dir, std::string instance, common::Clock& clock, + std::shared_ptr observer); + + const std::string& instance() const; + + void declare_model(const std::string& model, ConflictPolicy policy, + ConflictHook hook = ConflictHook{}); + + // Local write path. The caller supplies the model version (ORM row + // version for optimistic models); the hybrid timestamp is assigned here. + OplogEntry local_write(const std::string& model, const std::string& key, OplogOp op, + long long version, const std::string& data); + + // Remote apply path. Returns the resolution; throws ConflictError to + // signal the loser of an optimistic conflict (S7-8). + ConflictOutcome apply_remote(const OplogEntry& entry); + + std::optional current(const std::string& model, const std::string& key) const; + + // Entries strictly after seq for catch-up pull, bounded by the sync + // batch caps (500 entries or 1 MiB per round, S7-7). + std::vector entries_since(std::uint64_t seq, std::size_t max_entries, + std::size_t max_bytes) const; + + void versions(VersionVector& out) const; // snapshot of the version vector + ConflictResolver& resolver(); + std::uint64_t last_seq() const; + + private: + void replay(); + + std::string instance_; + common::Clock& clock_; + std::shared_ptr observer_; + OplogWriter writer_; + OplogReader reader_; + ConflictResolver resolver_; + VersionVector versions_; + mutable std::mutex mutex_; + std::map, OplogEntry> current_; +}; + +} // namespace molo::replication diff --git a/src/supervisor/replication/sync_session.cpp b/src/supervisor/replication/sync_session.cpp new file mode 100644 index 0000000..97f622b --- /dev/null +++ b/src/supervisor/replication/sync_session.cpp @@ -0,0 +1,73 @@ +#include "supervisor/replication/sync_session.hpp" + +#include "supervisor/common/caps.hpp" + +namespace molo::replication { + +SyncSession::SyncSession(Replicator& local, SyncPeer& peer, + std::shared_ptr observer, std::size_t batch_entries, + std::size_t batch_bytes) + : local_(local), + peer_(peer), + observer_(std::move(observer)), + batch_entries_(batch_entries), + batch_bytes_(batch_bytes) { + if (batch_entries_ == 0 || batch_entries_ > common::Caps::kSyncBatchEntries) { + batch_entries_ = common::Caps::kSyncBatchEntries; + } + if (batch_bytes_ == 0 || batch_bytes_ > common::Caps::kSyncBatchBytes) { + batch_bytes_ = common::Caps::kSyncBatchBytes; + } +} + +SyncStats SyncSession::round_trip() { + SyncStats stats; + + // Pull: ask the peer for everything past our version vector. + VersionVector have; + local_.versions(have); + std::vector incoming = peer_.fetch_since(have, batch_entries_, batch_bytes_); + for (const auto& entry : incoming) { + try { + ConflictOutcome outcome = local_.apply_remote(entry); + ++stats.pulled; + if (outcome.conflict) { + ++stats.conflicts; + } + } catch (const ConflictError&) { + // Optimistic loser: counted as a conflict, sync continues with the + // next entry. The ConflictError propagates to the sync caller only + // in the sense of the counter plus event; entries are independent. + ++stats.pulled; + ++stats.conflicts; + observer_->event("replication.sync_conflict", entry.model + ":" + entry.key); + } + } + + // Push: send our own entries past what the peer has seen from us, + // bounded to one batch per round. + VersionVector peer_have; + peer_.peer_versions(peer_have); + const std::uint64_t peer_has = peer_have.get(local_.instance()); + std::vector outgoing = + local_.entries_since(peer_has + 1, batch_entries_, batch_bytes_); + if (!outgoing.empty()) { + peer_.submit(outgoing); + stats.pushed = outgoing.size(); + } + + // Lag: entries behind the peer on the pull side, exact. + VersionVector now_have; + local_.versions(now_have); + const std::uint64_t peer_last = peer_.last_seq(); + const std::uint64_t seen = now_have.get(peer_.instance()); + stats.lag = peer_last > seen ? static_cast(peer_last - seen) : 0; + observer_->gauge("molo_replication_sync_lag", static_cast(stats.lag)); + if (stats.lag >= common::Caps::kSyncLagAlertThreshold) { + observer_->event("replication.lag_alert", peer_.instance()); + } + observer_->counter("molo_replication_synced_total", 1); + return stats; +} + +} // namespace molo::replication diff --git a/src/supervisor/replication/sync_session.hpp b/src/supervisor/replication/sync_session.hpp new file mode 100644 index 0000000..aef3171 --- /dev/null +++ b/src/supervisor/replication/sync_session.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "supervisor/common/observer.hpp" +#include "supervisor/replication/oplog_entry.hpp" +#include "supervisor/replication/replicator.hpp" +#include "supervisor/replication/version_vector.hpp" + +namespace molo::replication { + +// Transport-agnostic peer accessor. Production binds this to the bus +// (push) plus catch-up pull over the same channel (S7-7); tests bind it +// to another in-process Replicator. +class SyncPeer { + public: + virtual ~SyncPeer() = default; + + // Entries the peer has that the given version vector has not seen, + // bounded by the caller's batch caps. + virtual std::vector fetch_since(const VersionVector& have, std::size_t max_entries, + std::size_t max_bytes) = 0; + + // Pushes local entries to the peer. The peer applies each through its + // own resolver; duplicates are idempotent and ignored. + virtual void submit(const std::vector& entries) = 0; + + virtual void peer_versions(VersionVector& out) const = 0; + virtual std::uint64_t last_seq() const = 0; + virtual std::string instance() const = 0; +}; + +struct SyncStats { + std::size_t pulled = 0; + std::size_t pushed = 0; + std::size_t conflicts = 0; + std::int64_t lag = 0; // entries behind after this round +}; + +// One bidirectional sync round between the local Replicator and a peer: +// pull what the peer has, push what it lacks (S7-7). Batch caps are 500 +// entries or 1 MiB per round; lag is exported as a gauge with the 10000 +// alert threshold from S7-7. +class SyncSession { + public: + SyncSession(Replicator& local, SyncPeer& peer, std::shared_ptr observer, + std::size_t batch_entries = 500, std::size_t batch_bytes = 1048576); + + SyncStats round_trip(); + + private: + Replicator& local_; + SyncPeer& peer_; + std::shared_ptr observer_; + std::size_t batch_entries_; + std::size_t batch_bytes_; +}; + +} // namespace molo::replication diff --git a/src/supervisor/replication/version_vector.cpp b/src/supervisor/replication/version_vector.cpp new file mode 100644 index 0000000..61d6fe8 --- /dev/null +++ b/src/supervisor/replication/version_vector.cpp @@ -0,0 +1,96 @@ +#include "supervisor/replication/version_vector.hpp" + +#include "supervisor/common/json.hpp" + +namespace molo::replication { + +void VersionVector::update(const std::string& instance, std::uint64_t seq) { + std::lock_guard lock(mutex_); + auto it = versions_.find(instance); + if (it == versions_.end() || seq > it->second) { + versions_[instance] = seq; + } +} + +std::uint64_t VersionVector::get(const std::string& instance) const { + std::lock_guard lock(mutex_); + auto it = versions_.find(instance); + return it == versions_.end() ? 0 : it->second; +} + +void VersionVector::merge(const VersionVector& other) { + if (this == &other) { + return; + } + std::lock_guard lock(mutex_); + std::lock_guard other_lock(other.mutex_); + for (const auto& [instance, seq] : other.versions_) { + auto it = versions_.find(instance); + if (it == versions_.end() || seq > it->second) { + versions_[instance] = seq; + } + } +} + +VectorOrder VersionVector::compare(const VersionVector& other) const { + if (this == &other) { + return VectorOrder::Equal; + } + std::lock_guard lock(mutex_); + std::lock_guard other_lock(other.mutex_); + bool less = false, greater = false; + for (const auto& [instance, seq] : versions_) { + auto it = other.versions_.find(instance); + const std::uint64_t oseq = it == other.versions_.end() ? 0 : it->second; + if (seq < oseq) { + less = true; + } else if (seq > oseq) { + greater = true; + } + } + for (const auto& [instance, seq] : other.versions_) { + if (versions_.find(instance) == versions_.end() && seq > 0) { + less = true; + } + } + if (less && greater) { + return VectorOrder::Concurrent; + } + if (less) { + return VectorOrder::Before; + } + if (greater) { + return VectorOrder::After; + } + return VectorOrder::Equal; +} + +std::string VersionVector::encode() const { + using common::JValue; + std::lock_guard lock(mutex_); + JValue::Object obj; + for (const auto& [instance, seq] : versions_) { + obj[instance] = JValue(static_cast(seq)); + } + return common::json_encode(JValue(obj)); +} + +bool VersionVector::decode(const std::string& text, VersionVector& out) { + using common::JValue; + JValue root; + if (!common::json_decode(text, root) || !root.is_object()) { + return false; + } + VersionVector parsed; + for (const auto& [instance, value] : root.as_object()) { + if (!value.is_int() && !value.is_double()) { + return false; + } + parsed.versions_[instance] = static_cast(value.as_int(0)); + } + std::lock_guard lock(out.mutex_); + out.versions_ = std::move(parsed.versions_); + return true; +} + +} // namespace molo::replication diff --git a/src/supervisor/replication/version_vector.hpp b/src/supervisor/replication/version_vector.hpp new file mode 100644 index 0000000..197c433 --- /dev/null +++ b/src/supervisor/replication/version_vector.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include +#include + +namespace molo::replication { + +enum class VectorOrder { + Equal, + Before, // this <= other with at least one strict < + After, // this >= other with at least one strict > + Concurrent, // neither dominates: a real conflict happened +}; + +// Version vector over per-instance oplog sequences, exchanged at sync so +// each side fetches only what the other has not seen (S7-7). +class VersionVector { + public: + void update(const std::string& instance, std::uint64_t seq); + std::uint64_t get(const std::string& instance) const; + void merge(const VersionVector& other); + VectorOrder compare(const VersionVector& other) const; + + std::string encode() const; // JSON object {instance: seq} + static bool decode(const std::string& text, VersionVector& out); + + private: + mutable std::mutex mutex_; + std::map versions_; +}; + +} // namespace molo::replication diff --git a/src/supervisor/supervisor.mk b/src/supervisor/supervisor.mk new file mode 100644 index 0000000..d30767d --- /dev/null +++ b/src/supervisor/supervisor.mk @@ -0,0 +1,78 @@ +# Part 07 (supervisor, tasks, bus, replication) build fragment. +# Included by the root Makefile. Double-colon rules let every part append +# its own all/test/clean steps without editing shared targets. + +SUP_CXX ?= g++ +SUP_CXXFLAGS ?= -std=c++23 -O2 -Wall -Wextra -Wpedantic -Werror -MMD -MP +SUP_ROOT = src/supervisor +SUP_BUILD = build/supervisor +SUP_LIB = $(SUP_BUILD)/libsupervisor.a +SUP_INCPATH = -Isrc + +SUP_SRCS = \ + $(SUP_ROOT)/common/clock.cpp \ + $(SUP_ROOT)/common/observer.cpp \ + $(SUP_ROOT)/common/kv_store.cpp \ + $(SUP_ROOT)/common/json.cpp \ + $(SUP_ROOT)/bus/segment_log.cpp \ + $(SUP_ROOT)/bus/topic_bus.cpp \ + $(SUP_ROOT)/bus/consumer_cursor.cpp \ + $(SUP_ROOT)/bus/ack_tracker.cpp \ + $(SUP_ROOT)/bus/stream_bus.cpp \ + $(SUP_ROOT)/tasks/task_envelope.cpp \ + $(SUP_ROOT)/tasks/task_handler.cpp \ + $(SUP_ROOT)/tasks/task_registry.cpp \ + $(SUP_ROOT)/tasks/retry_policy.cpp \ + $(SUP_ROOT)/tasks/dead_letter_queue.cpp \ + $(SUP_ROOT)/tasks/task_queue.cpp \ + $(SUP_ROOT)/tasks/cron.cpp \ + $(SUP_ROOT)/tasks/scheduler.cpp \ + $(SUP_ROOT)/supervisor/restart_policy.cpp \ + $(SUP_ROOT)/supervisor/worker_process.cpp \ + $(SUP_ROOT)/supervisor/health_check.cpp \ + $(SUP_ROOT)/supervisor/worker_group.cpp \ + $(SUP_ROOT)/supervisor/shutdown_coordinator.cpp \ + $(SUP_ROOT)/supervisor/supervisor.cpp \ + $(SUP_ROOT)/replication/hybrid_clock.cpp \ + $(SUP_ROOT)/replication/oplog_entry.cpp \ + $(SUP_ROOT)/replication/oplog_writer.cpp \ + $(SUP_ROOT)/replication/oplog_reader.cpp \ + $(SUP_ROOT)/replication/version_vector.cpp \ + $(SUP_ROOT)/replication/conflict_resolver.cpp \ + $(SUP_ROOT)/replication/replicator.cpp \ + $(SUP_ROOT)/replication/sync_session.cpp + +SUP_OBJS = $(SUP_SRCS:$(SUP_ROOT)/%.cpp=$(SUP_BUILD)/%.o) + +SUP_TESTS = \ + test_supervisor \ + test_tasks \ + test_scheduler \ + test_bus \ + test_durability \ + test_replication \ + test_multi_instance + +SUP_TEST_BINS = $(SUP_TESTS:%=$(SUP_BUILD)/%) + +all:: $(SUP_LIB) + +$(SUP_LIB): $(SUP_OBJS) + @mkdir -p $(SUP_BUILD) + ar rcs $@ $^ + +$(SUP_BUILD)/%.o: $(SUP_ROOT)/%.cpp + @mkdir -p $(dir $@) + $(SUP_CXX) $(SUP_CXXFLAGS) $(SUP_INCPATH) -c $< -o $@ + +$(SUP_BUILD)/test_%: tests/src/supervisor/test_%.cpp $(SUP_LIB) $(OPERABILITY_LIB) + @mkdir -p $(SUP_BUILD) + $(SUP_CXX) $(SUP_CXXFLAGS) $(SUP_INCPATH) $< $(SUP_LIB) $(OPERABILITY_LIB) -o $@ + +test:: $(SUP_TEST_BINS) + @for t in $(SUP_TEST_BINS); do echo "== $$t"; ./$$t || exit 1; done + +clean:: + rm -rf $(SUP_BUILD) + +-include $(SUP_OBJS:.o=.d) diff --git a/src/supervisor/supervisor/health_check.cpp b/src/supervisor/supervisor/health_check.cpp new file mode 100644 index 0000000..2f1c52d --- /dev/null +++ b/src/supervisor/supervisor/health_check.cpp @@ -0,0 +1,150 @@ +#include "supervisor/supervisor/health_check.hpp" + +#if !defined(_WIN32) +#include +#include +#include +#include +#include +#endif + +namespace molo::supervisor { + +HealthCheck::HealthCheck(std::string name, const HealthSpec& spec, + std::function func_probe, common::Clock& clock, + std::shared_ptr observer) + : name_(std::move(name)), + spec_(spec), + func_probe_(std::move(func_probe)), + clock_(clock), + observer_(std::move(observer)) { + if (spec_.interval_ms < 100) { + spec_.interval_ms = 100; + } + if (spec_.max_failures < 1) { + spec_.max_failures = 1; + } +} + +bool HealthCheck::enabled() const { + if (spec_.kind == ProbeKind::Function) { + return static_cast(func_probe_); + } + return spec_.tcp_port > 0; +} + +bool HealthCheck::due() const { + std::lock_guard lock(mutex_); + if (!enabled()) { + return false; + } + return clock_.steady_ms() - last_probe_steady_ms_ >= spec_.interval_ms; +} + +bool HealthCheck::check_now() { + bool healthy = true; + if (spec_.kind == ProbeKind::Function) { + if (!func_probe_) { + return true; + } + try { + healthy = func_probe_(); + } catch (...) { + healthy = false; + } + } else { + std::string error; + healthy = tcp_probe(spec_.tcp_host, spec_.tcp_port, spec_.timeout_ms, error); + } + std::lock_guard lock(mutex_); + last_probe_steady_ms_ = clock_.steady_ms(); + if (healthy) { + failures_ = 0; + } else { + ++failures_; + observer_->counter("molo_supervisor_health_failures_total", 1); + if (failures_ >= spec_.max_failures) { + observer_->event("supervisor.worker.unhealthy", name_); + } + } + return healthy; +} + +bool HealthCheck::unhealthy() const { + std::lock_guard lock(mutex_); + return enabled() && failures_ >= spec_.max_failures; +} + +int HealthCheck::failures() const { + std::lock_guard lock(mutex_); + return failures_; +} + +void HealthCheck::reset() { + std::lock_guard lock(mutex_); + failures_ = 0; + last_probe_steady_ms_ = 0; +} + +bool HealthCheck::tcp_probe(const std::string& host, int port, long timeout_ms, + std::string& error) { +#if defined(_WIN32) + error = "tcp probe unsupported on this platform"; + (void)host; + (void)port; + (void)timeout_ms; + return false; +#else + addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + addrinfo* list = nullptr; + if (getaddrinfo(host.c_str(), std::to_string(port).c_str(), &hints, &list) != 0) { + error = "dns failure"; + return false; + } + int fd = -1; + for (addrinfo* ai = list; ai != nullptr; ai = ai->ai_next) { + fd = ::socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd >= 0) { + break; + } + } + if (fd < 0) { + freeaddrinfo(list); + error = "socket failure"; + return false; + } + const int flags = fcntl(fd, F_GETFL, 0); + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + int rc = ::connect(fd, list->ai_addr, list->ai_addrlen); + bool ok = (rc == 0); + if (!ok && errno == EINPROGRESS) { + fd_set wfds; + FD_ZERO(&wfds); + FD_SET(fd, &wfds); + timeval tv{}; + tv.tv_sec = timeout_ms / 1000; + tv.tv_usec = (timeout_ms % 1000) * 1000; + rc = select(fd + 1, nullptr, &wfds, nullptr, &tv); + if (rc > 0) { + int so_error = 0; + socklen_t len = sizeof(so_error); + getsockopt(fd, SOL_SOCKET, SO_ERROR, &so_error, &len); + ok = (so_error == 0); + if (!ok) { + error = "connect refused"; + } + } else { + error = "connect timeout"; + } + } else if (!ok) { + error = "connect failure"; + } + freeaddrinfo(list); + ::close(fd); + return ok; +#endif +} + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/health_check.hpp b/src/supervisor/supervisor/health_check.hpp new file mode 100644 index 0000000..e539469 --- /dev/null +++ b/src/supervisor/supervisor/health_check.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::supervisor { + +enum class ProbeKind { + Function, // caller-supplied predicate + Tcp, // TCP connect probe (S7-4) +}; + +struct HealthSpec { + ProbeKind kind = ProbeKind::Function; + std::string tcp_host = "127.0.0.1"; + int tcp_port = 0; + long interval_ms = 10000; + int max_failures = 3; + long timeout_ms = 2000; +}; + +// Health checker for one supervised worker (S7-4). The group calls due() +// and check_now() from its monitor loop; 3 consecutive probe failures +// (default) mark the worker unhealthy and trigger the restart policy. +class HealthCheck { + public: + HealthCheck(std::string name, const HealthSpec& spec, std::function func_probe, + common::Clock& clock, std::shared_ptr observer); + + bool enabled() const; + bool due() const; // true when a probe should run now + bool check_now(); // runs one probe, returns healthy + bool unhealthy() const; // true after max_failures consecutive failures + int failures() const; + void reset(); + + // Raw TCP connect probe used for ProbeKind::Tcp. + static bool tcp_probe(const std::string& host, int port, long timeout_ms, + std::string& error); + + private: + std::string name_; + HealthSpec spec_; + std::function func_probe_; + common::Clock& clock_; + std::shared_ptr observer_; + mutable std::mutex mutex_; + int failures_ = 0; + std::int64_t last_probe_steady_ms_ = 0; +}; + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/restart_policy.cpp b/src/supervisor/supervisor/restart_policy.cpp new file mode 100644 index 0000000..9ec5ccd --- /dev/null +++ b/src/supervisor/supervisor/restart_policy.cpp @@ -0,0 +1,38 @@ +#include "supervisor/supervisor/restart_policy.hpp" + +#include "supervisor/common/caps.hpp" + +namespace molo::supervisor { + +std::string to_string(RestartPolicy policy) { + switch (policy) { + case RestartPolicy::Never: return "never"; + case RestartPolicy::OnFailure: return "on-failure"; + case RestartPolicy::Always: return "always"; + } + return "on-failure"; +} + +bool parse_restart_policy(const std::string& text, RestartPolicy& out) { + if (text == "never") { + out = RestartPolicy::Never; + } else if (text == "on-failure") { + out = RestartPolicy::OnFailure; + } else if (text == "always") { + out = RestartPolicy::Always; + } else { + return false; + } + return true; +} + +void RestartSpec::normalize() { + if (max_restarts < 0) { + max_restarts = common::Caps::kMaxRestartsDefault; + } + if (window_secs < 1) { + window_secs = common::Caps::kRestartWindowSecsDefault; + } +} + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/restart_policy.hpp b/src/supervisor/supervisor/restart_policy.hpp new file mode 100644 index 0000000..6cc3e2c --- /dev/null +++ b/src/supervisor/supervisor/restart_policy.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace molo::supervisor { + +// Restart policy per supervised program (BP-2, R7-3). +enum class RestartPolicy { + Never, // leave exited children alone + OnFailure, // restart only on nonzero exit or crash + Always, // restart on any exit +}; + +std::string to_string(RestartPolicy policy); +bool parse_restart_policy(const std::string& text, RestartPolicy& out); + +// RestartPolicy plus the intensity cap that makes restart loops bounded: +// at most max_restarts restarts per window_secs window. Crossing the cap +// moves the child to Fatal and emits an event (S7-4). +struct RestartSpec { + RestartPolicy policy = RestartPolicy::OnFailure; + int max_restarts = 5; + int window_secs = 60; + + void normalize(); // clamps nonsense to the spec defaults +}; + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/shutdown_coordinator.cpp b/src/supervisor/supervisor/shutdown_coordinator.cpp new file mode 100644 index 0000000..ff65393 --- /dev/null +++ b/src/supervisor/supervisor/shutdown_coordinator.cpp @@ -0,0 +1,50 @@ +#include "supervisor/supervisor/shutdown_coordinator.hpp" + +namespace molo::supervisor { + +std::string to_string(ShutdownPhase phase) { + switch (phase) { + case ShutdownPhase::Running: return "running"; + case ShutdownPhase::Draining: return "draining"; + case ShutdownPhase::Stopped: return "stopped"; + } + return "running"; +} + +ShutdownCoordinator::ShutdownCoordinator(common::Clock& clock, + std::shared_ptr observer) + : clock_(clock), observer_(std::move(observer)) {} + +ShutdownPhase ShutdownCoordinator::phase() const { + std::lock_guard lock(mutex_); + return phase_; +} + +bool ShutdownCoordinator::drain_until(const std::function& is_drained, long deadline_ms, + long poll_ms) { + { + std::lock_guard lock(mutex_); + phase_ = ShutdownPhase::Draining; + } + observer_->event("supervisor.shutdown.draining", "deadline_ms=" + std::to_string(deadline_ms)); + const std::int64_t deadline = clock_.steady_ms() + deadline_ms; + while (true) { + if (is_drained && is_drained()) { + observer_->event("supervisor.shutdown.drained", "clean"); + return true; + } + if (clock_.steady_ms() >= deadline) { + observer_->event("supervisor.shutdown.drain_expired", "deadline hit"); + return false; + } + clock_.sleep_ms(poll_ms); + } +} + +void ShutdownCoordinator::mark_stopped() { + std::lock_guard lock(mutex_); + phase_ = ShutdownPhase::Stopped; + observer_->event("supervisor.shutdown.stopped", ""); +} + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/shutdown_coordinator.hpp b/src/supervisor/supervisor/shutdown_coordinator.hpp new file mode 100644 index 0000000..a8e10da --- /dev/null +++ b/src/supervisor/supervisor/shutdown_coordinator.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace molo::supervisor { + +enum class ShutdownPhase { + Running, + Draining, + Stopped, +}; + +std::string to_string(ShutdownPhase phase); + +// Graceful shutdown with a deadline (R7-9): stop accepting new work, drain +// in-flight work until the deadline, then stop. drain_until() polls the +// caller-supplied predicate and reports whether the drain completed. +class ShutdownCoordinator { + public: + ShutdownCoordinator(common::Clock& clock, std::shared_ptr observer); + + ShutdownPhase phase() const; + bool drain_until(const std::function& is_drained, long deadline_ms, + long poll_ms = 50); + void mark_stopped(); + + private: + common::Clock& clock_; + std::shared_ptr observer_; + mutable std::mutex mutex_; + ShutdownPhase phase_ = ShutdownPhase::Running; +}; + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/supervisor.cpp b/src/supervisor/supervisor/supervisor.cpp new file mode 100644 index 0000000..76fbe18 --- /dev/null +++ b/src/supervisor/supervisor/supervisor.cpp @@ -0,0 +1,153 @@ +#include "supervisor/supervisor/supervisor.hpp" + +#include "supervisor/common/caps.hpp" +#include "supervisor/common/json.hpp" +#include "supervisor/supervisor/supervisor_error.hpp" + +namespace molo::supervisor { + +Supervisor::Supervisor(common::Clock& clock, std::shared_ptr observer, + const SupervisorOptions& options) + : clock_(clock), observer_(std::move(observer)), options_(options), shutdown_(clock, observer_) { +} + +Supervisor::~Supervisor() { + if (running()) { + shutdown(0); + } +} + +std::size_t Supervisor::add_group(const GroupSpec& spec, WorkFunc func) { + std::lock_guard lock(mutex_); + if (static_cast(groups_.size()) >= common::Caps::kGroupsPerSupervisor) { + throw SupervisorError("Supervisor: group cap reached"); + } + auto group = std::make_unique(spec, std::move(func), clock_, observer_); + if (running_) { + group->start(); + } + groups_.push_back(std::move(group)); + observer_->event("supervisor.group.added", spec.name); + return groups_.size() - 1; +} + +void Supervisor::set_task_queue(std::shared_ptr queue) { + std::lock_guard lock(mutex_); + queue_ = std::move(queue); +} + +void Supervisor::start_all() { + std::lock_guard lock(mutex_); + if (running_) { + return; + } + running_ = true; + for (auto& group : groups_) { + group->start(); + } + observer_->event("supervisor.started", "groups=" + std::to_string(groups_.size())); +} + +void Supervisor::shutdown(long drain_deadline_ms) { + const long deadline = drain_deadline_ms < 0 ? options_.default_drain_ms : drain_deadline_ms; + std::shared_ptr queue; + { + std::lock_guard lock(mutex_); + if (!running_) { + return; + } + queue = queue_; + } + if (queue) { + queue->set_accepting(false); + const bool drained = shutdown_.drain_until( + [&] { return queue->in_flight() == 0; }, deadline); + if (!drained) { + queue->abandon_all_to_dlq("shutdown-drain-expired"); + } + queue->shutdown(); + } else { + shutdown_.drain_until([] { return true; }, 0); + } + { + std::lock_guard lock(mutex_); + for (auto it = groups_.rbegin(); it != groups_.rend(); ++it) { + (*it)->stop(); + } + running_ = false; + } + shutdown_.mark_stopped(); +} + +void Supervisor::restart_group(std::size_t index) { + std::lock_guard lock(mutex_); + if (index >= groups_.size()) { + throw SupervisorError("Supervisor: group index out of range"); + } + groups_[index]->restart_all(); +} + +void Supervisor::scale_group(std::size_t index, std::size_t n) { + std::lock_guard lock(mutex_); + if (index >= groups_.size()) { + throw SupervisorError("Supervisor: group index out of range"); + } + groups_[index]->set_size(n); +} + +std::string Supervisor::status_json() const { + using common::JValue; + std::lock_guard lock(mutex_); + JValue::Object root; + root["phase"] = JValue(to_string(shutdown_.phase())); + root["running"] = JValue(running_); + JValue::Array groups; + for (const auto& group : groups_) { + groups.push_back(group->snapshot()); + } + root["groups"] = JValue(groups); + if (queue_) { + const tasks::QueueStats stats = queue_->stats(); + JValue::Object q; + q["queued"] = JValue(static_cast(stats.queued)); + q["leased"] = JValue(static_cast(stats.leased)); + q["running"] = JValue(static_cast(stats.running)); + q["succeeded_total"] = JValue(static_cast(stats.succeeded_total)); + q["failed_total"] = JValue(static_cast(stats.failed_total)); + q["retried_total"] = JValue(static_cast(stats.retried_total)); + q["dead_lettered"] = JValue(static_cast(queue_->dead_letters().size())); + q["accepting"] = JValue(queue_->accepting()); + root["queue"] = JValue(q); + } + std::string full = common::json_encode(JValue(root)); + if (full.size() <= common::Caps::kStatusSnapshotBytes) { + return full; + } + // Over the 1 MiB cap: compact form with counts only, never truncated JSON. + JValue::Object compact; + compact["truncated"] = JValue(true); + compact["phase"] = JValue(to_string(shutdown_.phase())); + compact["group_count"] = JValue(static_cast(groups_.size())); + long long workers = 0; + for (const auto& group : groups_) { + workers += static_cast(group->size()); + } + compact["worker_count"] = JValue(workers); + return common::json_encode(JValue(compact)); +} + +std::size_t Supervisor::group_count() const { + std::lock_guard lock(mutex_); + return groups_.size(); +} + +bool Supervisor::running() const { + std::lock_guard lock(mutex_); + return running_; +} + +ShutdownPhase Supervisor::phase() const { + return shutdown_.phase(); +} + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/supervisor.hpp b/src/supervisor/supervisor/supervisor.hpp new file mode 100644 index 0000000..e59320f --- /dev/null +++ b/src/supervisor/supervisor/supervisor.hpp @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/supervisor/shutdown_coordinator.hpp" +#include "supervisor/supervisor/supervisor_error.hpp" +#include "supervisor/supervisor/worker_group.hpp" +#include "supervisor/tasks/task_queue.hpp" + +namespace molo::supervisor { + +struct SupervisorOptions { + long default_drain_ms = 30000; +}; + +// Top-level supervisor (E7-1): owns worker groups, applies restart +// strategies through them, and coordinates graceful shutdown with the +// task queue (R7-9). Exposes the status snapshot consumed by the CLI +// (09-cli-ops.md) and health endpoints. Thread-safe. +class Supervisor { + public: + Supervisor(common::Clock& clock, std::shared_ptr observer, + const SupervisorOptions& options = SupervisorOptions{}); + ~Supervisor(); + + Supervisor(const Supervisor&) = delete; + Supervisor& operator=(const Supervisor&) = delete; + + // Adds a group (max 32). When the supervisor runs, the group starts now. + std::size_t add_group(const GroupSpec& spec, WorkFunc func); + + // Queue drained on shutdown. Optional; null means workers only. + void set_task_queue(std::shared_ptr queue); + + void start_all(); + + // Graceful shutdown: stop accepting new tasks, drain in-flight tasks + // until the deadline, dead-letter leftovers with a reason, then stop + // all groups. Dropped tasks never vanish (R7-9). + void shutdown(long drain_deadline_ms = -1); // -1 means the default + + void restart_group(std::size_t index); + void scale_group(std::size_t index, std::size_t n); + + // Status snapshot JSON for CLI and health endpoints, capped at 1 MiB. + std::string status_json() const; + + std::size_t group_count() const; + bool running() const; + ShutdownPhase phase() const; + + private: + common::Clock& clock_; + std::shared_ptr observer_; + SupervisorOptions options_; + ShutdownCoordinator shutdown_; + mutable std::mutex mutex_; + std::vector> groups_; + std::shared_ptr queue_; + bool running_ = false; +}; + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/supervisor_error.hpp b/src/supervisor/supervisor/supervisor_error.hpp new file mode 100644 index 0000000..b3d3be3 --- /dev/null +++ b/src/supervisor/supervisor/supervisor_error.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include +#include + +namespace molo::supervisor { + +class SupervisorError : public std::runtime_error { + public: + explicit SupervisorError(const std::string& message) : std::runtime_error(message) {} +}; + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/worker_group.cpp b/src/supervisor/supervisor/worker_group.cpp new file mode 100644 index 0000000..6d75acc --- /dev/null +++ b/src/supervisor/supervisor/worker_group.cpp @@ -0,0 +1,481 @@ +#include "supervisor/supervisor/worker_group.hpp" + +#include "supervisor/common/caps.hpp" +#include "supervisor/supervisor/supervisor_error.hpp" + +namespace molo::supervisor { + +std::string to_string(GroupStrategy strategy) { + switch (strategy) { + case GroupStrategy::OneForOne: return "one_for_one"; + case GroupStrategy::OneForAll: return "one_for_all"; + case GroupStrategy::RestForOne: return "rest_for_one"; + } + return "one_for_one"; +} + +WorkerGroup::WorkerGroup(const GroupSpec& spec, WorkFunc func, common::Clock& clock, + std::shared_ptr observer) + : spec_(spec), func_(std::move(func)), clock_(clock), observer_(std::move(observer)) { + if (spec_.name.empty() || spec_.name.size() > 128) { + throw SupervisorError("WorkerGroup: bad group name"); + } + if (!func_) { + throw SupervisorError("WorkerGroup: null work function"); + } + if (spec_.size < 1 || spec_.size > static_cast(common::Caps::kWorkersPerGroup)) { + throw SupervisorError("WorkerGroup: size out of range"); + } + spec_.restart.normalize(); + if (spec_.monitor_ms < 5) { + spec_.monitor_ms = 5; + } + if (spec_.backoff_min_ms < 0) { + spec_.backoff_min_ms = 0; + } + if (spec_.backoff_max_ms < spec_.backoff_min_ms) { + spec_.backoff_max_ms = spec_.backoff_min_ms; + } + for (std::size_t i = 0; i < spec_.size; ++i) { + auto child = std::make_unique(); + child->worker = std::make_shared(spec_.name + "#" + std::to_string(i), + func_, clock_, observer_); + child->health = std::make_shared(spec_.name + "#" + std::to_string(i), + spec_.health, spec_.probe, clock_, observer_); + child->backoff_ms = spec_.backoff_min_ms; + children_.push_back(std::move(child)); + } +} + +WorkerGroup::~WorkerGroup() { + stop(); +} + +void WorkerGroup::start() { + std::lock_guard lock(mutex_); + if (running_) { + return; + } + running_ = true; + stopping_ = false; + for (auto& child : children_) { + child->worker->reset_for_restart(); + child->worker->start(); + child->health->reset(); + } + monitor_ = std::thread(&WorkerGroup::monitor_main, this); + observer_->event("supervisor.group.started", spec_.name); +} + +void WorkerGroup::stop() { + { + std::lock_guard lock(mutex_); + if (!running_) { + return; + } + stopping_ = true; + monitor_cv_.notify_all(); + } + if (monitor_.joinable()) { + monitor_.join(); + } + std::vector> workers; + { + std::lock_guard lock(mutex_); + for (auto& child : children_) { + workers.push_back(child->worker); + child->worker->request_stop(); + } + pending_.clear(); + } + for (auto& worker : workers) { + worker->join(); + } + std::lock_guard lock(mutex_); + running_ = false; + stopping_ = false; + observer_->event("supervisor.group.stopped", spec_.name); +} + +bool WorkerGroup::running() const { + std::lock_guard lock(mutex_); + return running_; +} + +void WorkerGroup::set_size(std::size_t n) { + if (n < 1 || n > static_cast(common::Caps::kWorkersPerGroup)) { + throw SupervisorError("WorkerGroup: size out of range"); + } + std::vector> removed; + { + std::lock_guard lock(mutex_); + if (n > children_.size()) { + for (std::size_t i = children_.size(); i < n; ++i) { + auto child = std::make_unique(); + child->worker = std::make_shared(spec_.name + "#" + std::to_string(i), + func_, clock_, observer_); + child->health = + std::make_shared(spec_.name + "#" + std::to_string(i), spec_.health, + spec_.probe, clock_, observer_); + child->backoff_ms = spec_.backoff_min_ms; + if (running_) { + child->worker->start(); + } + children_.push_back(std::move(child)); + } + } else { + while (children_.size() > n) { + auto& back = children_.back(); + back->worker->request_stop(); + removed.push_back(back->worker); + pending_.erase(children_.size() - 1); + children_.pop_back(); + } + } + observer_->event("supervisor.group.scaled", + spec_.name + " size=" + std::to_string(children_.size())); + } + for (auto& worker : removed) { + worker->join(); + } +} + +std::size_t WorkerGroup::size() const { + std::lock_guard lock(mutex_); + return children_.size(); +} + +void WorkerGroup::restart_child(std::size_t index) { + std::shared_ptr worker; + { + std::lock_guard lock(mutex_); + if (index >= children_.size()) { + throw SupervisorError("WorkerGroup: child index out of range"); + } + worker = children_[index]->worker; + children_[index]->restarts.clear(); + children_[index]->backoff_ms = spec_.backoff_min_ms; + children_[index]->health->reset(); + pending_.erase(index); + } + worker->request_stop(); + worker->join(); + worker->reset_for_restart(); + worker->start(); + observer_->event("supervisor.group.child_restarted", spec_.name + "#" + std::to_string(index)); +} + +void WorkerGroup::restart_all() { + std::vector> workers; + { + std::lock_guard lock(mutex_); + for (auto& child : children_) { + workers.push_back(child->worker); + child->worker->request_stop(); + child->restarts.clear(); + child->backoff_ms = spec_.backoff_min_ms; + child->health->reset(); + } + pending_.clear(); + } + for (auto& worker : workers) { + worker->join(); + } + for (auto& worker : workers) { + worker->reset_for_restart(); + worker->start(); + } + observer_->event("supervisor.group.restarted", spec_.name); +} + +bool WorkerGroup::fatal() const { + std::lock_guard lock(mutex_); + for (const auto& child : children_) { + if (child->worker->state() == WorkerState::Fatal) { + return true; + } + } + return false; +} + +std::vector WorkerGroup::states() const { + std::lock_guard lock(mutex_); + std::vector out; + for (const auto& child : children_) { + out.push_back(child->worker->state()); + } + return out; +} + +common::JValue WorkerGroup::snapshot() const { + using common::JValue; + std::lock_guard lock(mutex_); + JValue::Object obj; + obj["name"] = JValue(spec_.name); + obj["strategy"] = JValue(to_string(spec_.strategy)); + obj["restart_policy"] = JValue(to_string(spec_.restart.policy)); + obj["running"] = JValue(running_); + JValue::Array workers; + bool any_fatal = false; + for (const auto& child : children_) { + const WorkerState state = child->worker->state(); + if (state == WorkerState::Fatal) { + any_fatal = true; + } + JValue::Object w; + w["name"] = JValue(child->worker->name()); + w["state"] = JValue(to_string(state)); + w["exit_code"] = JValue(child->worker->exit_code()); + w["starts"] = JValue(static_cast(child->worker->starts())); + w["restarts_in_window"] = JValue(static_cast(child->restarts.size())); + w["health_failures"] = JValue(child->health->failures()); + workers.push_back(JValue(w)); + } + obj["fatal"] = JValue(any_fatal); + obj["workers"] = JValue(workers); + return JValue(obj); +} + +const std::string& WorkerGroup::name() const { + return spec_.name; +} + +std::vector WorkerGroup::strategy_targets(std::size_t index) const { + std::vector targets; + switch (spec_.strategy) { + case GroupStrategy::OneForOne: targets.push_back(index); break; + case GroupStrategy::OneForAll: + for (std::size_t i = 0; i < children_.size(); ++i) { + targets.push_back(i); + } + break; + case GroupStrategy::RestForOne: + for (std::size_t i = index; i < children_.size(); ++i) { + targets.push_back(i); + } + break; + } + return targets; +} + +void WorkerGroup::handle_exit_locked(std::size_t index, int exit_code, bool from_health, + std::vector& stop_list) { + bool wanted = false; + if (from_health) { + wanted = (spec_.restart.policy != RestartPolicy::Never); + } else if (spec_.restart.policy == RestartPolicy::Always) { + wanted = true; + } else if (spec_.restart.policy == RestartPolicy::OnFailure && exit_code != 0) { + wanted = true; + } + if (!wanted) { + return; + } + for (std::size_t target : strategy_targets(index)) { + if (target >= children_.size()) { + continue; + } + if (pending_.find(target) != pending_.end()) { + continue; // restart already scheduled + } + const WorkerState state = children_[target]->worker->state(); + if (state == WorkerState::Fatal || state == WorkerState::Stopped) { + continue; + } + if (state == WorkerState::Exited) { + schedule_restart_locked(target); + } else if (state == WorkerState::Running || state == WorkerState::Starting) { + // Strategy siblings (or an unhealthy worker) stop first, then the + // restart is scheduled after the join outside the lock. + stop_list.push_back(target); + } + } +} + +void WorkerGroup::schedule_restart_locked(std::size_t index) { + Child& child = *children_[index]; + const std::int64_t now = clock_.steady_ms(); + const std::int64_t window_ms = static_cast(spec_.restart.window_secs) * 1000; + while (!child.restarts.empty() && child.restarts.front() <= now - window_ms) { + child.restarts.pop_front(); + } + if (static_cast(child.restarts.size()) >= spec_.restart.max_restarts) { + child.worker->mark_fatal(); + observer_->event("supervisor.group.intensity_cap", spec_.name + " child locked fatal"); + return; + } + child.restarts.push_back(now); + child.last_restart_steady_ms = now; + pending_[index] = now + child.backoff_ms; + child.backoff_ms = std::min(child.backoff_ms * 2, spec_.backoff_max_ms); + if (child.worker->state() == WorkerState::Exited) { + child.worker->mark_backoff(); + } + observer_->counter("molo_supervisor_worker_restarts_total", 1); +} + +void WorkerGroup::fire_restart(std::size_t index) { + std::shared_ptr worker; + { + std::lock_guard lock(mutex_); + if (stopping_ || index >= children_.size()) { + return; + } + worker = children_[index]->worker; + const WorkerState state = worker->state(); + // Stopped here means a strategy sibling stopped for restart (group + // stop clears all pending restarts first, so it cannot land here). + if (state == WorkerState::Fatal || state == WorkerState::Running || + state == WorkerState::Starting || state == WorkerState::Stopping) { + return; + } + } + worker->join(); // collects the old thread; already finished, non-blocking + worker->reset_for_restart(); + if (!worker->start()) { + return; + } + std::lock_guard lock(mutex_); + if (index < children_.size()) { + children_[index]->health->reset(); + } +} + +void WorkerGroup::monitor_main() { + while (true) { + std::vector exited; + std::vector probes; + std::vector firing; + { + std::lock_guard lock(mutex_); + if (stopping_) { + return; + } + const std::int64_t now = clock_.steady_ms(); + for (auto it = pending_.begin(); it != pending_.end();) { + if (it->second <= now) { + firing.push_back(it->first); + it = pending_.erase(it); + } else { + ++it; + } + } + for (std::size_t i = 0; i < children_.size(); ++i) { + const WorkerState state = children_[i]->worker->state(); + if (state == WorkerState::Exited) { + exited.push_back(i); + } else if (state == WorkerState::Running && children_[i]->health->due()) { + probes.push_back(i); + } + // A clean run resets the backoff ladder (supervisord-style). + if (state == WorkerState::Running && + children_[i]->backoff_ms > spec_.backoff_min_ms && + now - children_[i]->last_restart_steady_ms > spec_.backoff_reset_after_ms) { + children_[i]->backoff_ms = spec_.backoff_min_ms; + } + } + } + + for (std::size_t index : exited) { + std::shared_ptr worker; + { + std::lock_guard lock(mutex_); + if (index < children_.size()) { + worker = children_[index]->worker; + } + } + if (worker) { + worker->join(); + } + } + + std::vector unhealthy; + for (std::size_t index : probes) { + std::shared_ptr health; + { + std::lock_guard lock(mutex_); + if (index < children_.size()) { + health = children_[index]->health; + } + } + if (health) { + health->check_now(); + if (health->unhealthy()) { + unhealthy.push_back(index); + } + } + } + + std::vector stop_list; + { + std::lock_guard lock(mutex_); + if (stopping_) { + return; + } + for (std::size_t index : exited) { + if (index >= children_.size()) { + continue; + } + if (children_[index]->worker->state() != WorkerState::Exited) { + continue; // user restarted it meanwhile + } + const int code = children_[index]->worker->exit_code(); + handle_exit_locked(index, code, false, stop_list); + } + for (std::size_t index : unhealthy) { + if (index >= children_.size()) { + continue; + } + if (children_[index]->worker->state() != WorkerState::Running) { + continue; + } + observer_->event("supervisor.group.unhealthy_restart", + spec_.name + "#" + std::to_string(index)); + handle_exit_locked(index, 1, true, stop_list); + } + } + + for (std::size_t index : stop_list) { + std::shared_ptr worker; + { + std::lock_guard lock(mutex_); + if (index < children_.size()) { + worker = children_[index]->worker; + } + } + if (worker) { + worker->request_stop(); + worker->join(); + } + } + { + std::lock_guard lock(mutex_); + if (!stopping_) { + for (std::size_t index : stop_list) { + if (index >= children_.size()) { + continue; + } + const WorkerState state = children_[index]->worker->state(); + if (state == WorkerState::Stopped || state == WorkerState::Exited) { + schedule_restart_locked(index); + } + } + } + } + + for (std::size_t index : firing) { + fire_restart(index); + } + + { + std::unique_lock lock(mutex_); + monitor_cv_.wait_for(lock, std::chrono::milliseconds(spec_.monitor_ms), + [&] { return stopping_; }); + if (stopping_) { + return; + } + } + } +} + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/worker_group.hpp b/src/supervisor/supervisor/worker_group.hpp new file mode 100644 index 0000000..caa0d5c --- /dev/null +++ b/src/supervisor/supervisor/worker_group.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/json.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/supervisor/health_check.hpp" +#include "supervisor/supervisor/restart_policy.hpp" +#include "supervisor/supervisor/worker_process.hpp" + +namespace molo::supervisor { + +// OTP restart strategies for worker groups (BP-1). +enum class GroupStrategy { + OneForOne, // restart only the failed child + OneForAll, // restart all children + RestForOne, // restart the failed child plus children started after it +}; + +std::string to_string(GroupStrategy strategy); + +struct GroupSpec { + std::string name; + GroupStrategy strategy = GroupStrategy::OneForOne; + RestartSpec restart; + HealthSpec health; + std::function probe; // shared function probe (Function kind) + std::size_t size = 1; + long monitor_ms = 50; + long backoff_min_ms = 1000; + long backoff_max_ms = 60000; + long backoff_reset_after_ms = 10000; // clean run that resets backoff +}; + +// Owns one supervised program: N WorkerProcess children running the same +// WorkFunc, one monitor loop applying the restart strategy, intensity +// caps, backoff, and health checks (E7-1, S7-4). Thread-safe. +class WorkerGroup { + public: + WorkerGroup(const GroupSpec& spec, WorkFunc func, common::Clock& clock, + std::shared_ptr observer); + ~WorkerGroup(); + + WorkerGroup(const WorkerGroup&) = delete; + WorkerGroup& operator=(const WorkerGroup&) = delete; + + void start(); + void stop(); + bool running() const; + + void set_size(std::size_t n); // scale, capped at 64 workers + std::size_t size() const; + + void restart_child(std::size_t index); // explicit, clears Fatal + void restart_all(); + bool fatal() const; // true when any child is Fatal + + std::vector states() const; + common::JValue snapshot() const; + const std::string& name() const; + + private: + struct Child { + std::shared_ptr worker; + std::shared_ptr health; + std::deque restarts; // steady-ms timestamps in window + long backoff_ms = 1000; + std::int64_t last_restart_steady_ms = 0; + }; + + void monitor_main(); + void handle_exit_locked(std::size_t index, int exit_code, bool from_health, + std::vector& stop_list); + void schedule_restart_locked(std::size_t index); + void fire_restart(std::size_t index); + std::vector strategy_targets(std::size_t index) const; + + GroupSpec spec_; + WorkFunc func_; + common::Clock& clock_; + std::shared_ptr observer_; + mutable std::mutex mutex_; + std::condition_variable monitor_cv_; + std::vector> children_; + std::map pending_; // index -> restart due (steady ms) + std::thread monitor_; + bool running_ = false; + bool stopping_ = false; +}; + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/worker_process.cpp b/src/supervisor/supervisor/worker_process.cpp new file mode 100644 index 0000000..3858c6d --- /dev/null +++ b/src/supervisor/supervisor/worker_process.cpp @@ -0,0 +1,150 @@ +#include "supervisor/supervisor/worker_process.hpp" + +namespace molo::supervisor { + +WorkerProcess::WorkerProcess(std::string name, WorkFunc func, common::Clock& clock, + std::shared_ptr observer) + : name_(std::move(name)), + func_(std::move(func)), + clock_(clock), + observer_(std::move(observer)) {} + +WorkerProcess::~WorkerProcess() { + request_stop(); + join(); +} + +bool WorkerProcess::start() { + std::lock_guard lock(mutex_); + if (state_ == WorkerState::Running || state_ == WorkerState::Starting || + state_ == WorkerState::Stopping) { + return false; + } + if (state_ == WorkerState::Fatal) { + return false; // explicit reset_for_restart() required + } + if (thread_.joinable()) { + return false; + } + stop_requested_.store(false); + state_ = WorkerState::Starting; + last_start_steady_ms_ = clock_.steady_ms(); + ++starts_; + observer_->event("supervisor.worker.starting", name_); + thread_ = std::thread(&WorkerProcess::thread_main, this); + return true; +} + +void WorkerProcess::thread_main() { + { + std::lock_guard lock(mutex_); + state_ = WorkerState::Running; + } + observer_->event("supervisor.worker.running", name_); + observer_->counter("molo_supervisor_worker_starts_total", 1); + int code = 0; + try { + code = func_(stop_requested_); + } catch (...) { + code = 1; + } + { + std::lock_guard lock(mutex_); + exit_code_ = code; + last_exit_steady_ms_ = clock_.steady_ms(); + state_ = stop_requested_.load() ? WorkerState::Stopped : WorkerState::Exited; + } + observer_->event("supervisor.worker.exited", name_ + " code=" + std::to_string(code)); +} + +void WorkerProcess::request_stop() { + stop_requested_.store(true); + std::lock_guard lock(mutex_); + if (state_ == WorkerState::Running || state_ == WorkerState::Starting) { + state_ = WorkerState::Stopping; + } +} + +void WorkerProcess::join() { + // Move the handle out under lock, then join without it: thread_main + // needs the state mutex to publish Exited. + std::thread finished; + { + std::lock_guard lock(mutex_); + if (thread_.joinable()) { + finished = std::move(thread_); + } + } + if (finished.joinable()) { + finished.join(); + } + std::lock_guard lock(mutex_); + // A worker that was asked to stop and already exited lands on Stopped; + // one that exited on its own keeps Exited for the group to handle. + if (stop_requested_.load() && state_ == WorkerState::Exited) { + state_ = WorkerState::Stopped; + } +} + +bool WorkerProcess::joinable() const { + std::lock_guard lock(mutex_); + return thread_.joinable(); +} + +WorkerState WorkerProcess::state() const { + std::lock_guard lock(mutex_); + return state_; +} + +const std::string& WorkerProcess::name() const { + return name_; +} + +int WorkerProcess::exit_code() const { + std::lock_guard lock(mutex_); + return exit_code_; +} + +std::uint64_t WorkerProcess::starts() const { + std::lock_guard lock(mutex_); + return starts_; +} + +std::int64_t WorkerProcess::last_start_steady_ms() const { + std::lock_guard lock(mutex_); + return last_start_steady_ms_; +} + +std::int64_t WorkerProcess::last_exit_steady_ms() const { + std::lock_guard lock(mutex_); + return last_exit_steady_ms_; +} + +void WorkerProcess::mark_fatal() { + std::lock_guard lock(mutex_); + // Any non-running state may go fatal: Exited and Backoff after a crash, + // Stopped after a strategy sibling stop. Live workers never do. + if (state_ != WorkerState::Running && state_ != WorkerState::Starting && + state_ != WorkerState::Stopping && state_ != WorkerState::Fatal) { + state_ = WorkerState::Fatal; + observer_->event("supervisor.worker.fatal", name_); + observer_->counter("molo_supervisor_worker_fatal_total", 1); + } +} + +void WorkerProcess::mark_backoff() { + std::lock_guard lock(mutex_); + if (state_ == WorkerState::Exited) { + state_ = WorkerState::Backoff; + } +} + +void WorkerProcess::reset_for_restart() { + std::lock_guard lock(mutex_); + if (state_ == WorkerState::Fatal || state_ == WorkerState::Exited || + state_ == WorkerState::Stopped) { + state_ = WorkerState::Stopped; + } +} + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/worker_process.hpp b/src/supervisor/supervisor/worker_process.hpp new file mode 100644 index 0000000..d47967e --- /dev/null +++ b/src/supervisor/supervisor/worker_process.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/supervisor/worker_state.hpp" + +namespace molo::supervisor { + +// Work function run by a worker. Returns the process exit code (0 means +// clean). Must poll stop_requested and return promptly; throwing counts +// as exit code 1. Transport-agnostic: v1 runs this on a thread, v2 runs +// the same signature in an OS process behind this interface (O7-2). +using WorkFunc = std::function& stop_requested)>; + +// One supervised child (v1: in-process thread worker). Owns its lifecycle +// only; restart decisions live in WorkerGroup. Thread-safe. +class WorkerProcess { + public: + WorkerProcess(std::string name, WorkFunc func, common::Clock& clock, + std::shared_ptr observer); + ~WorkerProcess(); + + WorkerProcess(const WorkerProcess&) = delete; + WorkerProcess& operator=(const WorkerProcess&) = delete; + + // Starts the worker. Returns false when already started. + bool start(); + void request_stop(); + void join(); + bool joinable() const; + + WorkerState state() const; + const std::string& name() const; + int exit_code() const; + std::uint64_t starts() const; + std::int64_t last_start_steady_ms() const; + std::int64_t last_exit_steady_ms() const; + + // Called by the group when the intensity cap trips (S7-4). + void mark_fatal(); + // Called by the group while a restart backoff is pending. + void mark_backoff(); + // Clears Fatal/Exited so start() may run again (explicit restart only). + void reset_for_restart(); + + private: + void thread_main(); + + std::string name_; + WorkFunc func_; + common::Clock& clock_; + std::shared_ptr observer_; + mutable std::mutex mutex_; + std::thread thread_; + std::atomic stop_requested_{false}; + WorkerState state_ = WorkerState::Stopped; + int exit_code_ = 0; + std::uint64_t starts_ = 0; + std::int64_t last_start_steady_ms_ = 0; + std::int64_t last_exit_steady_ms_ = 0; +}; + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/worker_state.hpp b/src/supervisor/supervisor/worker_state.hpp new file mode 100644 index 0000000..5ce2236 --- /dev/null +++ b/src/supervisor/supervisor/worker_state.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace molo::supervisor { + +// Supervised worker states (S7-4). Fatal is entered after the intensity +// cap trips; only an explicit restart clears it. +enum class WorkerState { + Stopped, + Starting, + Running, + Backoff, + Stopping, + Exited, + Fatal, +}; + +inline std::string to_string(WorkerState state) { + switch (state) { + case WorkerState::Stopped: return "stopped"; + case WorkerState::Starting: return "starting"; + case WorkerState::Running: return "running"; + case WorkerState::Backoff: return "backoff"; + case WorkerState::Stopping: return "stopping"; + case WorkerState::Exited: return "exited"; + case WorkerState::Fatal: return "fatal"; + } + return "stopped"; +} + +} // namespace molo::supervisor diff --git a/src/supervisor/tasks/cron.cpp b/src/supervisor/tasks/cron.cpp new file mode 100644 index 0000000..eade367 --- /dev/null +++ b/src/supervisor/tasks/cron.cpp @@ -0,0 +1,187 @@ +#include "supervisor/tasks/cron.hpp" + +#include +#include + +namespace molo::tasks { + +namespace { + +std::vector split_fields(std::string_view text) { + std::vector fields; + std::string current; + for (char c : text) { + if (c == ' ' || c == '\t') { + if (!current.empty()) { + fields.push_back(current); + current.clear(); + } + } else { + current += c; + } + } + if (!current.empty()) { + fields.push_back(current); + } + return fields; +} + +bool parse_int(std::string_view text, int& out) { + if (text.empty() || text.size() > 4) { + return false; + } + int value = 0; + for (char c : text) { + if (c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + } + out = value; + return true; +} + +// Parses one field into flags[low..high]. Returns false on bad syntax. +bool parse_field(std::string_view text, bool* flags, int low, int high, bool& restricted) { + for (int i = low; i <= high; ++i) { + flags[i] = false; + } + restricted = (text != "*"); + std::size_t pos = 0; + bool any = false; + while (pos <= text.size()) { + std::size_t comma = text.find(',', pos); + if (comma == std::string_view::npos) { + comma = text.size(); + } + const std::string_view item = text.substr(pos, comma - pos); + if (item.empty()) { + return false; + } + // Split off /step. + std::string_view range = item; + int step = 1; + const std::size_t slash = item.find('/'); + if (slash != std::string_view::npos) { + range = item.substr(0, slash); + if (!parse_int(item.substr(slash + 1), step) || step < 1) { + return false; + } + } + int from = low, to = high; + if (range == "*") { + from = low; + to = high; + } else { + const std::size_t dash = range.find('-'); + if (dash == std::string_view::npos) { + if (!parse_int(range, from)) { + return false; + } + to = (slash == std::string_view::npos) ? from : high; + } else { + if (!parse_int(range.substr(0, dash), from) || + !parse_int(range.substr(dash + 1), to)) { + return false; + } + } + } + if (from < low || to > high || from > to) { + return false; + } + for (int v = from; v <= to; v += step) { + flags[v] = true; + any = true; + } + if (comma == text.size()) { + break; + } + pos = comma + 1; + } + return any; +} + +bool parse_dow_field(std::string_view text, bool* flags, bool& restricted) { + // Normalize 7 to 0 before parsing. + std::string norm(text); + for (std::size_t i = 0; i < norm.size(); ++i) { + if (norm[i] == '7' && (i + 1 >= norm.size() || norm[i + 1] < '0' || norm[i + 1] > '9') && + (i == 0 || norm[i - 1] < '0' || norm[i - 1] > '9')) { + norm[i] = '0'; + } + } + return parse_field(norm, flags, 0, 6, restricted); +} + +} // namespace + +bool CronExpr::parse(std::string_view text, CronExpr& out, std::string& error) { + CronExpr expr; + const auto fields = split_fields(text); + if (fields.size() != 5) { + error = "cron: need exactly 5 fields"; + return false; + } + bool ignored = false; + if (!parse_field(fields[0], expr.minutes_, 0, 59, ignored)) { + error = "cron: bad minute field"; + return false; + } + if (!parse_field(fields[1], expr.hours_, 0, 23, ignored)) { + error = "cron: bad hour field"; + return false; + } + if (!parse_field(fields[2], expr.days_, 1, 31, expr.dom_restricted_)) { + error = "cron: bad day-of-month field"; + return false; + } + if (!parse_field(fields[3], expr.months_, 1, 12, ignored)) { + error = "cron: bad month field"; + return false; + } + if (!parse_dow_field(fields[4], expr.dows_, expr.dow_restricted_)) { + error = "cron: bad day-of-week field"; + return false; + } + out = expr; + return true; +} + +bool CronExpr::matches(std::int64_t wall_ms) const { + const std::time_t secs = static_cast(wall_ms / 1000); + std::tm tm_utc{}; +#if defined(_WIN32) + gmtime_s(&tm_utc, &secs); +#else + gmtime_r(&secs, &tm_utc); +#endif + if (!minutes_[tm_utc.tm_min] || !hours_[tm_utc.tm_hour] || !months_[tm_utc.tm_mon + 1]) { + return false; + } + const bool dom_match = days_[tm_utc.tm_mday]; + const bool dow_match = dows_[tm_utc.tm_wday]; + if (dom_restricted_ && dow_restricted_) { + return dom_match || dow_match; + } + if (dom_restricted_) { + return dom_match; + } + if (dow_restricted_) { + return dow_match; + } + return true; +} + +std::int64_t CronExpr::next_after(std::int64_t wall_ms) const { + // Step minute by minute, capped at 366 days of search. + std::int64_t minute = (wall_ms / 60000) * 60000 + 60000; + const std::int64_t limit = minute + 366LL * 86400 * 1000; + for (; minute <= limit; minute += 60000) { + if (matches(minute)) { + return minute; + } + } + return -1; +} + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/cron.hpp b/src/supervisor/tasks/cron.hpp new file mode 100644 index 0000000..c78b33b --- /dev/null +++ b/src/supervisor/tasks/cron.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace molo::tasks { + +// Five-field UTC cron expression: minute hour day-of-month month day-of-week. +// Each field accepts '*', '*/step', single values, ranges ('a-b'), +// stepped ranges ('a-b/step'), and comma lists. Numbers only in v1 +// (no MON/JAN names, no @macros). Day matching follows Vixie semantics: +// when both day-of-month and day-of-week are restricted, either may match. +class CronExpr { + public: + static bool parse(std::string_view text, CronExpr& out, std::string& error); + + // True when the UTC minute containing wall_ms matches the expression. + bool matches(std::int64_t wall_ms) const; + + // Start of the first matching minute strictly after wall_ms, or -1 when + // no minute matches within 366 days. + std::int64_t next_after(std::int64_t wall_ms) const; + + private: + bool minutes_[60] = {}; + bool hours_[24] = {}; + bool days_[32] = {}; // 1..31 + bool months_[13] = {}; // 1..12 + bool dows_[7] = {}; // 0..6, Sunday is 0 (7 folds to 0) + bool dom_restricted_ = false; + bool dow_restricted_ = false; +}; + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/dead_letter_queue.cpp b/src/supervisor/tasks/dead_letter_queue.cpp new file mode 100644 index 0000000..302cf06 --- /dev/null +++ b/src/supervisor/tasks/dead_letter_queue.cpp @@ -0,0 +1,91 @@ +#include "supervisor/tasks/dead_letter_queue.hpp" + +namespace molo::tasks { + +DeadLetterQueue::DeadLetterQueue(common::Clock& clock, + std::shared_ptr observer, int max_entries, + long retention_days) + : clock_(clock), + observer_(std::move(observer)), + max_entries_(max_entries), + retention_days_(retention_days) { + if (max_entries_ < 1) { + max_entries_ = 1; + } +} + +void DeadLetterQueue::push(TaskEnvelope envelope, const std::string& reason) { + std::lock_guard lock(mutex_); + while (static_cast(entries_.size()) >= max_entries_) { + entries_.pop_front(); + ++evicted_total_; + observer_->counter("molo_tasks_dead_letter_evicted_total", 1); + observer_->event("tasks.dead_letter.evicted", "cap=oldest-first"); + } + envelope.state = TaskState::DeadLettered; + entries_.push_back(DeadEntry{std::move(envelope), reason, clock_.wall_ms()}); + observer_->counter("molo_tasks_dead_lettered_total", 1); + observer_->event("tasks.dead_lettered", reason); +} + +std::vector DeadLetterQueue::list(std::size_t max) const { + std::lock_guard lock(mutex_); + std::vector out; + for (const auto& entry : entries_) { + if (out.size() >= max) { + break; + } + out.push_back(entry); + } + return out; +} + +bool DeadLetterQueue::take(const std::string& id, DeadEntry& out) { + std::lock_guard lock(mutex_); + for (auto it = entries_.begin(); it != entries_.end(); ++it) { + if (it->envelope.id == id) { + out = *it; + entries_.erase(it); + observer_->event("tasks.dead_letter.retried", id); + return true; + } + } + return false; +} + +bool DeadLetterQueue::remove(const std::string& id) { + std::lock_guard lock(mutex_); + for (auto it = entries_.begin(); it != entries_.end(); ++it) { + if (it->envelope.id == id) { + entries_.erase(it); + return true; + } + } + return false; +} + +std::size_t DeadLetterQueue::size() const { + std::lock_guard lock(mutex_); + return entries_.size(); +} + +std::size_t DeadLetterQueue::purge_expired() { + std::lock_guard lock(mutex_); + const std::int64_t horizon = clock_.wall_ms() - retention_days_ * 86400 * 1000; + std::size_t purged = 0; + while (!entries_.empty() && entries_.front().dead_at_ms < horizon) { + entries_.pop_front(); + ++purged; + } + if (purged > 0) { + observer_->counter("molo_tasks_dead_letter_expired_total", static_cast(purged)); + } + return purged; +} + +long DeadLetterQueue::evicted_total() const { + std::lock_guard lock(mutex_); + return evicted_total_; +} + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/dead_letter_queue.hpp b/src/supervisor/tasks/dead_letter_queue.hpp new file mode 100644 index 0000000..f5cd9ed --- /dev/null +++ b/src/supervisor/tasks/dead_letter_queue.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/tasks/task_envelope.hpp" + +namespace molo::tasks { + +struct DeadEntry { + TaskEnvelope envelope; + std::string reason; + std::int64_t dead_at_ms = 0; +}; + +// Dead-letter queue (S7-2): envelopes that exhausted retries or failed +// permanently wait here 7 days for inspection, retry, or cancel via the +// CLI (09-cli-ops.md). Max 10000 entries; overflow evicts oldest-first +// with an event emitted. Nothing vanishes silently. +class DeadLetterQueue { + public: + DeadLetterQueue(common::Clock& clock, std::shared_ptr observer, + int max_entries = 10000, long retention_days = 7); + + void push(TaskEnvelope envelope, const std::string& reason); + std::vector list(std::size_t max) const; + + // Removes and returns one entry for manual retry. Returns false when absent. + bool take(const std::string& id, DeadEntry& out); + bool remove(const std::string& id); + + std::size_t size() const; + std::size_t purge_expired(); // drops entries past retention, returns count + long evicted_total() const; + + private: + common::Clock& clock_; + std::shared_ptr observer_; + int max_entries_; + long retention_days_; + mutable std::mutex mutex_; + std::deque entries_; + long evicted_total_ = 0; +}; + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/retry_policy.cpp b/src/supervisor/tasks/retry_policy.cpp new file mode 100644 index 0000000..5aa9956 --- /dev/null +++ b/src/supervisor/tasks/retry_policy.cpp @@ -0,0 +1,56 @@ +#include "supervisor/tasks/retry_policy.hpp" + +#include + +namespace molo::tasks { + +RetryPolicy::RetryPolicy(const RetrySpec& spec) : spec_(spec) { + if (spec_.base_ms < 0) { + spec_.base_ms = 0; + } + if (spec_.cap_ms < spec_.base_ms) { + spec_.cap_ms = spec_.base_ms; + } + if (spec_.jitter_ms < 0) { + spec_.jitter_ms = 0; + } + if (spec_.max_retries < 0) { + spec_.max_retries = 0; + } +} + +long RetryPolicy::delay_for_attempt(int attempt, std::mt19937& rng) const { + if (attempt < 0) { + attempt = 0; + } + // base * 2^n with overflow guard: shift past 30 saturates to the cap. + long delay = spec_.base_ms; + if (attempt >= 31) { + delay = spec_.cap_ms; + } else { + const long long grown = static_cast(spec_.base_ms) * (1LL << attempt); + delay = grown >= spec_.cap_ms ? spec_.cap_ms : static_cast(grown); + } + long jitter = 0; + if (spec_.jitter_ms > 0) { + std::uniform_int_distribution dist(0, spec_.jitter_ms); + jitter = dist(rng); + } + const long long total = static_cast(delay) + jitter; + return total >= spec_.cap_ms ? spec_.cap_ms : static_cast(total); +} + +long RetryPolicy::delay_for_attempt(int attempt) const { + static thread_local std::mt19937 rng{std::random_device{}()}; + return delay_for_attempt(attempt, rng); +} + +bool RetryPolicy::exhausted(int attempts) const { + return attempts > spec_.max_retries; +} + +const RetrySpec& RetryPolicy::spec() const { + return spec_; +} + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/retry_policy.hpp b/src/supervisor/tasks/retry_policy.hpp new file mode 100644 index 0000000..728156a --- /dev/null +++ b/src/supervisor/tasks/retry_policy.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace molo::tasks { + +// Bounded exponential backoff with jitter (S7-2): +// delay(n) = min(backoff_base_ms * 2^n + jitter, backoff_cap_ms) +// where n is the zero-based retry number and jitter is uniform +// 0..jitter_ms. Pure computation, no I/O, trivially testable. +struct RetrySpec { + long base_ms = 1000; + long cap_ms = 300000; + long jitter_ms = 1000; + int max_retries = 5; +}; + +class RetryPolicy { + public: + explicit RetryPolicy(const RetrySpec& spec = RetrySpec{}); + + long delay_for_attempt(int attempt, std::mt19937& rng) const; + long delay_for_attempt(int attempt) const; // thread-local RNG + bool exhausted(int attempts) const; // attempts > max_retries + const RetrySpec& spec() const; + + private: + RetrySpec spec_; +}; + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/scheduler.cpp b/src/supervisor/tasks/scheduler.cpp new file mode 100644 index 0000000..6b04263 --- /dev/null +++ b/src/supervisor/tasks/scheduler.cpp @@ -0,0 +1,380 @@ +#include "supervisor/tasks/scheduler.hpp" + +#include + +#include "supervisor/common/caps.hpp" +#include "supervisor/common/json.hpp" + +namespace molo::tasks { + +namespace { + +// Option clamps shared by the scheduler and its LeaderElection (which has +// no default ctor, so normalization must run in the init list). +SchedulerOptions NormalizedOptions(SchedulerOptions o) { + if (o.instance_id.empty()) { + o.instance_id = "sched-" + TaskEnvelope::generate_id(); + } + if (o.tick_ms < 10) { + o.tick_ms = 10; + } + if (o.ttl_secs < 1) { + o.ttl_secs = 1; + } + if (o.renew_secs < 1) { + o.renew_secs = 1; + } + return o; +} + +} // namespace + +Scheduler::Scheduler(TaskQueue& queue, common::KvStore& kv, common::Clock& clock, + std::shared_ptr observer, + const SchedulerOptions& options) + : queue_(queue), + kv_(kv), + clock_(clock), + observer_(std::move(observer)), + options_(NormalizedOptions(options)), + election_(kv_, clock_, observer_ ? *observer_ : fallback_observer_, + options_.leader_key, options_.instance_id, options_.ttl_secs * 1000, + options_.renew_secs * 1000) { + if (!options_.durable_dir.empty()) { + log_ = std::make_unique( + std::filesystem::path(options_.durable_dir) / "scheduler.log"); + replay(); + } +} + +void Scheduler::replay() { + // Rebuilds schedule entries from the op log (E7-6). Last-fire records + // live in shared KV: with the shared backend they survive too, and a + // reboot never refires a window the old leader already fired. + std::uint64_t seq = 1; + while (true) { + auto records = log_->read_from(seq, 256, 1024 * 1024); + if (records.empty()) { + break; + } + for (const auto& record : records) { + common::JValue root; + if (!common::json_decode(record.data, root)) { + continue; + } + const common::JValue* op = root.find("op"); + if (op == nullptr) { + continue; + } + if (op->as_string() == "put") { + const common::JValue* body = root.find("entry"); + if (body == nullptr) { + continue; + } + ScheduleEntry entry; + if (decode_entry(common::json_encode(*body), entry)) { + CronExpr cron; + if (entry.kind == ScheduleKind::Cron) { + std::string error; + if (!CronExpr::parse(entry.cron, cron, error)) { + continue; + } + cron_cache_[entry.key] = cron; + } + entries_[entry.key] = entry; + registered_at_[entry.key] = clock_.wall_ms(); + } + } else if (op->as_string() == "remove") { + const common::JValue* key = root.find("key"); + if (key != nullptr) { + entries_.erase(key->as_string()); + cron_cache_.erase(key->as_string()); + registered_at_.erase(key->as_string()); + } + } + } + seq = records.back().seq + 1; + } + observer_->event("tasks.schedule.replayed", "entries=" + std::to_string(entries_.size())); +} + +std::string Scheduler::encode_entry(const ScheduleEntry& entry) const { + using common::JValue; + JValue::Object obj; + obj["key"] = JValue(entry.key); + obj["task_name"] = JValue(entry.task_name); + JValue payload; + if (!common::json_decode(entry.payload, payload)) { + payload = JValue(entry.payload); + } + obj["payload"] = payload; + obj["kind"] = JValue(entry.kind == ScheduleKind::Once + ? "once" + : (entry.kind == ScheduleKind::Interval ? "interval" : "cron")); + obj["at_ms"] = JValue(static_cast(entry.at_ms)); + obj["interval_secs"] = JValue(static_cast(entry.interval_secs)); + obj["cron"] = JValue(entry.cron); + obj["missed"] = JValue(entry.missed == MissedPolicy::Skip ? "skip" : "run-once"); + obj["enabled"] = JValue(entry.enabled); + obj["max_retries"] = JValue(entry.max_retries); + return common::json_encode(JValue(obj)); +} + +bool Scheduler::decode_entry(const std::string& text, ScheduleEntry& out) const { + using common::JValue; + JValue root; + if (!common::json_decode(text, root) || !root.is_object()) { + return false; + } + const JValue* key = root.find("key"); + const JValue* task_name = root.find("task_name"); + const JValue* kind = root.find("kind"); + if (key == nullptr || task_name == nullptr || kind == nullptr || !key->is_string() || + !task_name->is_string() || !kind->is_string()) { + return false; + } + ScheduleEntry entry; + entry.key = key->as_string(); + entry.task_name = task_name->as_string(); + const std::string kind_text = kind->as_string(); + if (kind_text == "once") { + entry.kind = ScheduleKind::Once; + } else if (kind_text == "interval") { + entry.kind = ScheduleKind::Interval; + } else if (kind_text == "cron") { + entry.kind = ScheduleKind::Cron; + } else { + return false; + } + const JValue* payload = root.find("payload"); + if (payload != nullptr) { + entry.payload = common::json_encode(*payload); + } + const JValue* at_ms = root.find("at_ms"); + const JValue* interval = root.find("interval_secs"); + if (at_ms != nullptr) { + entry.at_ms = at_ms->as_int(0); + } + if (interval != nullptr) { + entry.interval_secs = static_cast(interval->as_int(60)); + } + const JValue* cron = root.find("cron"); + if (cron != nullptr) { + entry.cron = cron->as_string(); + } + const JValue* missed = root.find("missed"); + if (missed != nullptr && missed->as_string() == "run-once") { + entry.missed = MissedPolicy::RunOnce; + } + const JValue* enabled = root.find("enabled"); + if (enabled != nullptr) { + entry.enabled = enabled->as_bool(true); + } + const JValue* retries = root.find("max_retries"); + if (retries != nullptr) { + entry.max_retries = static_cast(retries->as_int(5)); + } + out = std::move(entry); + return true; +} + +void Scheduler::put(ScheduleEntry entry) { + if (entry.key.empty() || entry.key.size() > 256) { + throw TaskError("Scheduler: bad entry key"); + } + if (entry.task_name.empty()) { + throw TaskError("Scheduler: entry needs a task name"); + } + CronExpr cron; + if (entry.kind == ScheduleKind::Cron) { + std::string error; + if (!CronExpr::parse(entry.cron, cron, error)) { + throw TaskError("Scheduler: " + error); + } + } + if (entry.kind == ScheduleKind::Interval && + entry.interval_secs < common::Caps::kSchedulerMinIntervalSecs) { + entry.interval_secs = common::Caps::kSchedulerMinIntervalSecs; // S7-5 floor + } + std::lock_guard lock(mutex_); + const bool is_new = entries_.find(entry.key) == entries_.end(); + entries_[entry.key] = entry; + if (entry.kind == ScheduleKind::Cron) { + cron_cache_[entry.key] = cron; + } else { + cron_cache_.erase(entry.key); + } + if (is_new) { + registered_at_[entry.key] = clock_.wall_ms(); + } + if (log_) { + log_->append("{\"op\":\"put\",\"entry\":" + encode_entry(entry) + "}", clock_.wall_ms()); + } + observer_->event("tasks.schedule.put", entry.key); +} + +bool Scheduler::remove(const std::string& key) { + std::lock_guard lock(mutex_); + cron_cache_.erase(key); + registered_at_.erase(key); + const bool removed = entries_.erase(key) > 0; + if (removed && log_) { + log_->append("{\"op\":\"remove\",\"key\":\"" + common::json_escape(key) + "\"}", + clock_.wall_ms()); + } + return removed; +} + +std::vector Scheduler::entries() const { + std::lock_guard lock(mutex_); + std::vector out; + for (const auto& [_, entry] : entries_) { + out.push_back(entry); + } + return out; +} + +bool Scheduler::is_leader() const { + std::lock_guard lock(mutex_); + return election_.is_leader(); +} + +std::string Scheduler::leader() const { + std::lock_guard lock(mutex_); + return leader_id_; +} + +void Scheduler::tick_once() { + std::lock_guard lock(mutex_); + const std::int64_t now = clock_.wall_ms(); + if (!elect_locked(now)) { + return; // follower: stay passive + } + fire_due_locked(now); +} + +int Scheduler::run(const std::function& stop_requested) { + while (!stop_requested || !stop_requested()) { + tick_once(); + clock_.sleep_ms(options_.tick_ms); + } + return 0; +} + +bool Scheduler::elect_locked(std::int64_t now_wall) { + (void)now_wall; + const bool was = election_.is_leader(); + const bool now_leader = election_.tick(); + if (now_leader && !was) { + leader_id_ = options_.instance_id; + observer_->event("tasks.schedule.leader_won", options_.instance_id); + } else if (!now_leader && was) { + observer_->event("tasks.schedule.leader_lost", options_.instance_id); + } + if (!now_leader) { + const std::string current = election_.leader(); + if (!current.empty()) { + leader_id_ = current; + } + } + return now_leader; +} + +std::int64_t Scheduler::last_fire_locked(const std::string& key, std::int64_t fallback) { + std::string stored; + if (kv_.get(options_.leader_key + "/fired/" + key, stored)) { + try { + return std::stoll(stored); + } catch (...) { + return fallback; + } + } + return fallback; +} + +void Scheduler::record_fire_locked(const std::string& key, std::int64_t now_wall, long ttl_secs) { + kv_.set(options_.leader_key + "/fired/" + key, std::to_string(now_wall), ttl_secs); +} + +void Scheduler::fire_locked(const ScheduleEntry& entry, const std::string& fire_ref, + std::int64_t now_wall) { + TaskEnvelope env; + env.name = entry.task_name; + env.payload = entry.payload; + env.max_retries = entry.max_retries; + env.idempotency_key = "sched:" + entry.key + ":" + fire_ref; + try { + const std::string id = queue_.enqueue(std::move(env)); + observer_->event("tasks.schedule.fired", entry.key + " -> " + id); + (void)now_wall; + } catch (const TaskError& e) { + observer_->event("tasks.schedule.fire_failed", entry.key + ": " + e.what()); + } +} + +void Scheduler::fire_due_locked(std::int64_t now_wall) { + for (const auto& [key, entry] : entries_) { + if (!entry.enabled) { + continue; + } + const std::int64_t registered = + registered_at_.count(key) ? registered_at_[key] : now_wall; + switch (entry.kind) { + case ScheduleKind::Once: { + // Fired markers persist, so a reboot never refires a Once entry. + const std::int64_t last = last_fire_locked(key, -1); + if (last >= 0) { + break; + } + if (now_wall < entry.at_ms) { + break; + } + const bool missed = (now_wall - entry.at_ms > 60000); + if (missed && entry.missed == MissedPolicy::Skip) { + record_fire_locked(key, now_wall, 86400); + observer_->event("tasks.schedule.missed", key); + break; + } + fire_locked(entry, "once", now_wall); + record_fire_locked(key, now_wall, 86400); + break; + } + case ScheduleKind::Interval: { + const std::int64_t span = entry.interval_secs * 1000; + const std::int64_t last = last_fire_locked(key, registered); + if (now_wall - last < span) { + break; + } + // At most one fire per tick; last=now so followers taking over + // never refire this window (V7-4). + fire_locked(entry, std::to_string(now_wall / span), now_wall); + record_fire_locked(key, now_wall, entry.interval_secs * 2 + 60); + break; + } + case ScheduleKind::Cron: { + auto cit = cron_cache_.find(key); + if (cit == cron_cache_.end()) { + break; + } + const std::int64_t minute = (now_wall / 60000) * 60000; + const std::int64_t last = last_fire_locked(key, registered); + if (last >= minute && cit->second.matches(now_wall)) { + break; // already fired this minute + } + if (cit->second.matches(now_wall)) { + fire_locked(entry, "m" + std::to_string(minute / 60000), now_wall); + record_fire_locked(key, now_wall, 90000); + } else if (entry.missed == MissedPolicy::RunOnce) { + const std::int64_t next = cit->second.next_after(last); + if (next >= 0 && next <= now_wall) { + fire_locked(entry, "m" + std::to_string((next / 60000)), now_wall); + record_fire_locked(key, now_wall, 90000); + } + } + break; + } + } + } +} + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/scheduler.hpp b/src/supervisor/tasks/scheduler.hpp new file mode 100644 index 0000000..a915014 --- /dev/null +++ b/src/supervisor/tasks/scheduler.hpp @@ -0,0 +1,114 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "operability/coord/election.h" +#include "supervisor/bus/segment_log.hpp" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/kv_store.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/tasks/cron.hpp" +#include "supervisor/tasks/task_envelope.hpp" +#include "supervisor/tasks/task_queue.hpp" + +namespace molo::tasks { + +// What to do with a fire that was missed during an outage or failover. +enum class MissedPolicy { + Skip, // drop the missed fire, wait for the next one + RunOnce, // fire once now to cover the missed window +}; + +enum class ScheduleKind { + Once, // one delayed run at at_ms + Interval, // fixed interval, minimum 60s (S7-5) + Cron, // 5-field UTC cron expression +}; + +struct ScheduleEntry { + std::string key; + std::string task_name; + std::string payload = "{}"; + ScheduleKind kind = ScheduleKind::Interval; + std::int64_t at_ms = 0; // Once + long interval_secs = 60; // Interval + std::string cron; // Cron + MissedPolicy missed = MissedPolicy::Skip; + bool enabled = true; + int max_retries = 5; +}; + +struct SchedulerOptions { + std::string instance_id; + std::string leader_key = "molo/scheduler/leader"; + std::string durable_dir; // empty means in-memory entries; set for E7-6 + long tick_ms = 1000; + long ttl_secs = 10; + long renew_secs = 3; +}; + +// Delayed plus periodic scheduling (S7-5). The leader enqueues due entries +// into the TaskQueue; followers stay passive. Leadership is a lease in the +// shared KvStore (10s TTL, 3s renewal). Last-fire records live in the same +// shared store so a new leader never double-fires, and every fire carries +// an idempotency key so even a brief dual-leader overlap cannot enqueue +// twice (the queue dedupes). Dual-run during network partitions is accepted +// with documented bounds (O7-4): at most the overlap window, deduped by key. +class Scheduler { + public: + Scheduler(TaskQueue& queue, common::KvStore& kv, common::Clock& clock, + std::shared_ptr observer, const SchedulerOptions& options); + + // Adds or replaces an entry. Throws TaskError on bad cron or interval. + void put(ScheduleEntry entry); + bool remove(const std::string& key); + std::vector entries() const; + + bool is_leader() const; + std::string leader() const; + + // One tick: run leader election, then fire due entries when leader. + // Deterministic under FrozenClock; production uses run(). + void tick_once(); + + // Ticks until stop_requested() is true. Returns 0. + int run(const std::function& stop_requested); + + std::string encode_entry(const ScheduleEntry& entry) const; + bool decode_entry(const std::string& text, ScheduleEntry& out) const; + + private: + void replay(); + bool elect_locked(std::int64_t now_wall); + void fire_due_locked(std::int64_t now_wall); + std::int64_t last_fire_locked(const std::string& key, std::int64_t fallback); + void record_fire_locked(const std::string& key, std::int64_t now_wall, long ttl_secs); + void fire_locked(const ScheduleEntry& entry, const std::string& fire_ref, + std::int64_t now_wall); + + TaskQueue& queue_; + common::KvStore& kv_; + common::Clock& clock_; + std::shared_ptr observer_; + SchedulerOptions options_; + std::unique_ptr log_; + mutable std::mutex mutex_; + std::map entries_; + std::map cron_cache_; + std::map registered_at_; + std::string leader_id_; + // Leadership moved onto molo::coord::LeaderElection (operability part); + // the scheduler keeps its public surface and event names. Declared after + // observer_/options_ so the init list can wire them in. + common::NullObserver fallback_observer_; + molo::coord::LeaderElection election_; +}; + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/task_envelope.cpp b/src/supervisor/tasks/task_envelope.cpp new file mode 100644 index 0000000..6a5ca39 --- /dev/null +++ b/src/supervisor/tasks/task_envelope.cpp @@ -0,0 +1,150 @@ +#include "supervisor/tasks/task_envelope.hpp" + +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/json.hpp" + +namespace molo::tasks { + +std::string to_string(TaskState state) { + switch (state) { + case TaskState::Queued: return "queued"; + case TaskState::Leased: return "leased"; + case TaskState::Running: return "running"; + case TaskState::Succeeded: return "succeeded"; + case TaskState::FailedRetryable: return "failed-retryable"; + case TaskState::FailedPermanent: return "failed-permanent"; + case TaskState::DeadLettered: return "dead-lettered"; + case TaskState::Cancelled: return "cancelled"; + } + return "queued"; +} + +bool parse_state(const std::string& text, TaskState& out) { + if (text == "queued") { + out = TaskState::Queued; + } else if (text == "leased") { + out = TaskState::Leased; + } else if (text == "running") { + out = TaskState::Running; + } else if (text == "succeeded") { + out = TaskState::Succeeded; + } else if (text == "failed-retryable") { + out = TaskState::FailedRetryable; + } else if (text == "failed-permanent") { + out = TaskState::FailedPermanent; + } else if (text == "dead-lettered") { + out = TaskState::DeadLettered; + } else if (text == "cancelled") { + out = TaskState::Cancelled; + } else { + return false; + } + return true; +} + +std::string TaskEnvelope::encode() const { + using common::JValue; + JValue::Object obj; + obj["id"] = JValue(id); + obj["name"] = JValue(name); + JValue payload_value; + if (!common::json_decode(payload, payload_value)) { + payload_value = JValue(payload); // non-JSON payloads ride as strings + } + obj["payload"] = payload_value; + obj["attempts"] = JValue(attempts); + obj["max_retries"] = JValue(max_retries); + if (run_at_ms.has_value()) { + obj["run_at"] = JValue(common::iso8601(*run_at_ms)); + } else { + obj["run_at"] = JValue(nullptr); + } + obj["timeout_soft_ms"] = JValue(static_cast(timeout_soft_ms)); + obj["timeout_hard_ms"] = JValue(static_cast(timeout_hard_ms)); + obj["idempotency_key"] = JValue(idempotency_key); + obj["enqueued_at"] = JValue(common::iso8601(enqueued_at_ms)); + obj["state"] = JValue(to_string(state)); + return common::json_encode(JValue(obj)); +} + +bool TaskEnvelope::decode(const std::string& text, TaskEnvelope& out) { + using common::JValue; + JValue root; + if (!common::json_decode(text, root) || !root.is_object()) { + return false; + } + const JValue* id = root.find("id"); + const JValue* name = root.find("name"); + if (id == nullptr || name == nullptr || !id->is_string() || !name->is_string()) { + return false; + } + TaskEnvelope env; + env.id = id->as_string(); + env.name = name->as_string(); + const JValue* payload = root.find("payload"); + if (payload != nullptr) { + env.payload = common::json_encode(*payload); + } + const JValue* attempts = root.find("attempts"); + const JValue* max_retries = root.find("max_retries"); + if (attempts != nullptr) { + env.attempts = static_cast(attempts->as_int(0)); + } + if (max_retries != nullptr) { + env.max_retries = static_cast(max_retries->as_int(5)); + } + const JValue* run_at = root.find("run_at"); + if (run_at != nullptr && run_at->is_string()) { + const std::int64_t parsed = common::parse_iso8601(run_at->as_string()); + if (parsed < 0) { + return false; + } + env.run_at_ms = parsed; + } + const JValue* soft = root.find("timeout_soft_ms"); + const JValue* hard = root.find("timeout_hard_ms"); + if (soft != nullptr) { + env.timeout_soft_ms = static_cast(soft->as_int(30000)); + } + if (hard != nullptr) { + env.timeout_hard_ms = static_cast(hard->as_int(60000)); + } + const JValue* key = root.find("idempotency_key"); + if (key != nullptr) { + env.idempotency_key = key->as_string(); + } + const JValue* enqueued = root.find("enqueued_at"); + if (enqueued != nullptr && enqueued->is_string()) { + env.enqueued_at_ms = common::parse_iso8601(enqueued->as_string()); + } + const JValue* state = root.find("state"); + if (state != nullptr && state->is_string()) { + if (!parse_state(state->as_string(), env.state)) { + return false; + } + } + out = std::move(env); + return true; +} + +std::string TaskEnvelope::generate_id() { + // UUIDv7 shape: 48-bit unix ms plus 74 random bits, hex with dashes. + static thread_local std::mt19937_64 rng{std::random_device{}()}; + const auto now = std::chrono::system_clock::now(); + const auto ms = static_cast( + std::chrono::duration_cast(now.time_since_epoch()).count()); + const std::uint64_t hi = (ms << 16) | (rng() & 0xFFFFu); + const std::uint64_t lo = rng(); + std::ostringstream out; + out << std::hex << std::setfill('0') << std::setw(12) << (hi >> 16) << "-" << std::setw(4) + << (hi & 0xFFFFu) << "-7" << std::setw(3) << ((lo >> 48) & 0xFFFu) << "-" << std::setw(4) + << (((lo >> 32) & 0x3FFFu) | 0x8000u) << "-" << std::setw(12) << (lo & 0xFFFFFFFFFFFFULL); + return out.str(); +} + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/task_envelope.hpp b/src/supervisor/tasks/task_envelope.hpp new file mode 100644 index 0000000..288b896 --- /dev/null +++ b/src/supervisor/tasks/task_envelope.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include + +namespace molo::tasks { + +class TaskError : public std::runtime_error { + public: + explicit TaskError(const std::string& message) : std::runtime_error(message) {} +}; + +// Task lifecycle states (S7-3). +enum class TaskState { + Queued, + Leased, + Running, + Succeeded, + FailedRetryable, + FailedPermanent, + DeadLettered, + Cancelled, +}; + +std::string to_string(TaskState state); +bool parse_state(const std::string& text, TaskState& out); + +// Immutable job record (S7-1). The only type that crosses queue boundaries. +// JSON form doubles as the on-disk format for the durable queue: +// +// {"id":"...","name":"billing.charge","payload":{...},"attempts":0, +// "max_retries":5,"run_at":"iso8601|null","timeout_soft_ms":30000, +// "timeout_hard_ms":60000,"idempotency_key":"...","enqueued_at":"iso8601"} +// +// The payload is an opaque JSON object string (max 1 MiB); the queue never +// interprets it, handlers do. +struct TaskEnvelope { + std::string id; + std::string name; + std::string payload = "{}"; + int attempts = 0; + int max_retries = 5; + std::optional run_at_ms; // null means run as soon as possible + long timeout_soft_ms = 30000; + long timeout_hard_ms = 60000; + std::string idempotency_key; + std::int64_t enqueued_at_ms = 0; + TaskState state = TaskState::Queued; + + std::string encode() const; + static bool decode(const std::string& text, TaskEnvelope& out); + static std::string generate_id(); +}; + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/task_handler.cpp b/src/supervisor/tasks/task_handler.cpp new file mode 100644 index 0000000..97b3abb --- /dev/null +++ b/src/supervisor/tasks/task_handler.cpp @@ -0,0 +1,14 @@ +#include "supervisor/tasks/task_handler.hpp" + +namespace molo::tasks { + +std::string to_string(TaskOutcome outcome) { + switch (outcome) { + case TaskOutcome::Success: return "success"; + case TaskOutcome::RetryableFailure: return "retryable"; + case TaskOutcome::PermanentFailure: return "permanent"; + } + return "retryable"; +} + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/task_handler.hpp b/src/supervisor/tasks/task_handler.hpp new file mode 100644 index 0000000..eb32c9e --- /dev/null +++ b/src/supervisor/tasks/task_handler.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/tasks/task_envelope.hpp" + +namespace molo::tasks { + +// Handler outcome. RetryableFailure requeues with backoff; PermanentFailure +// dead-letters immediately. Throwing from run() counts as retryable. +enum class TaskOutcome { + Success, + RetryableFailure, + PermanentFailure, +}; + +std::string to_string(TaskOutcome outcome); + +// Per-execution context handed to handlers. +struct TaskContext { + const TaskEnvelope& envelope; + common::Clock& clock; + std::shared_ptr observer; + std::atomic& cancel_requested; // set at the soft timeout (R7, S7-3) +}; + +// Base class for all task handlers. Every handler MUST be idempotent: +// delivery is at-least-once, so run() may execute more than once with the +// same task id after worker loss or lease expiry (R7-2). Handlers run on +// worker threads, never on HTTP event loop threads (R7-5). +class TaskHandler { + public: + virtual ~TaskHandler() = default; + virtual TaskOutcome run(TaskContext& ctx) = 0; +}; + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/task_queue.cpp b/src/supervisor/tasks/task_queue.cpp new file mode 100644 index 0000000..2b97233 --- /dev/null +++ b/src/supervisor/tasks/task_queue.cpp @@ -0,0 +1,683 @@ +#include "supervisor/tasks/task_queue.hpp" + +#include + +#include "supervisor/common/caps.hpp" +#include "supervisor/common/json.hpp" + +namespace molo::tasks { + +namespace { + +std::string truncate_reason(const std::string& reason) { + return reason.size() > 1024 ? reason.substr(0, 1024) : reason; +} + +} // namespace + +TaskQueue::TaskQueue(const QueueOptions& options, common::Clock& clock, + std::shared_ptr observer) + : options_(options), + clock_(clock), + observer_(std::move(observer)), + retry_(options.retry), + dead_letters_(clock_, observer_) { + if (options_.max_depth < 1) { + options_.max_depth = 1; + } + if (!options_.durable_dir.empty()) { + log_ = std::make_unique( + std::filesystem::path(options_.durable_dir) / "task_queue.log"); + replay(); + } +} + +TaskQueue::~TaskQueue() { + shutdown(); +} + +void TaskQueue::replay() { + // Rebuilds live state from the op log. Leases are never persisted, so a + // task leased at crash replays as queued and is redelivered (E7-6). + std::uint64_t seq = 1; + while (true) { + auto records = log_->read_from(seq, 512, 4 * 1024 * 1024); + if (records.empty()) { + break; + } + for (const auto& record : records) { + common::JValue root; + if (!common::json_decode(record.data, root)) { + continue; // corrupt op skipped loudly below + } + const common::JValue* op = root.find("op"); + if (op == nullptr) { + continue; + } + const std::string op_name = op->as_string(); + if (op_name == "enqueue") { + const common::JValue* env = root.find("env"); + if (env == nullptr) { + continue; + } + TaskEnvelope envelope; + if (!TaskEnvelope::decode(common::json_encode(*env), envelope)) { + continue; + } + envelope.state = TaskState::Queued; + const std::int64_t run_at = envelope.run_at_ms.value_or(0); + records_[envelope.id] = Record{envelope, ++order_}; + ready_[{run_at, order_}] = envelope.id; + } else if (op_name == "terminal") { + const common::JValue* id = root.find("id"); + if (id != nullptr) { + terminal_locked(id->as_string()); + } + } else if (op_name == "requeue") { + const common::JValue* id = root.find("id"); + if (id == nullptr) { + continue; + } + auto it = records_.find(id->as_string()); + if (it == records_.end()) { + continue; + } + const common::JValue* attempts = root.find("attempts"); + const common::JValue* run_at = root.find("run_at_ms"); + if (attempts != nullptr) { + it->second.envelope.attempts = static_cast(attempts->as_int(0)); + } + if (run_at != nullptr && !run_at->is_null()) { + it->second.envelope.run_at_ms = run_at->as_int(0); + } else { + it->second.envelope.run_at_ms.reset(); + } + it->second.envelope.state = TaskState::Queued; + it->second.order = ++order_; + const std::int64_t due = it->second.envelope.run_at_ms.value_or(0); + ready_[{due, it->second.order}] = it->first; + } + } + seq = records.back().seq + 1; + } + observer_->event("tasks.queue.replayed", "live=" + std::to_string(records_.size())); +} + +void TaskQueue::log_op(const std::string& op) { + if (log_) { + log_->append(op, clock_.wall_ms()); + } +} + +std::string TaskQueue::enqueue(TaskEnvelope envelope) { + if (envelope.name.empty() || envelope.name.size() > 256) { + throw TaskError("TaskQueue: bad task name"); + } + if (envelope.payload.size() > common::Caps::kTaskPayloadBytes) { + throw TaskError("TaskQueue: payload over 1 MiB cap"); + } + if (envelope.timeout_soft_ms <= 0) { + envelope.timeout_soft_ms = 30000; + } + if (envelope.timeout_hard_ms <= 0) { + envelope.timeout_hard_ms = 60000; + } + if (envelope.timeout_hard_ms < envelope.timeout_soft_ms) { + envelope.timeout_hard_ms = envelope.timeout_soft_ms; + } + if (envelope.max_retries < 0) { + envelope.max_retries = 0; + } + std::lock_guard lock(mutex_); + if (!accepting_) { + throw TaskError("TaskQueue: not accepting new tasks (draining)"); + } + if (!envelope.idempotency_key.empty()) { + const std::string existing = dedupe_lookup_locked(envelope.idempotency_key); + if (!existing.empty()) { + return existing; + } + } + if (envelope.id.empty()) { + envelope.id = TaskEnvelope::generate_id(); + } + if (records_.size() >= options_.max_depth) { + throw TaskError("TaskQueue: queue depth cap reached"); + } + envelope.enqueued_at_ms = clock_.wall_ms(); + envelope.attempts = 0; + envelope.state = TaskState::Queued; + const std::string id = envelope.id; + const std::int64_t run_at = envelope.run_at_ms.value_or(0); + records_[id] = Record{envelope, ++order_}; + ready_[{run_at, order_}] = id; + if (!envelope.idempotency_key.empty()) { + dedupe_store_locked(envelope.idempotency_key, id); + } + log_op("{\"op\":\"enqueue\",\"env\":" + envelope.encode() + "}"); + observer_->counter("molo_tasks_enqueued_total", 1); + observer_->event("tasks.enqueued", id + " " + envelope.name); + cv_.notify_one(); + return id; +} + +bool TaskQueue::cancel(const std::string& id) { + std::lock_guard lock(mutex_); + auto it = records_.find(id); + if (it == records_.end()) { + return false; + } + if (lease_by_task_.find(id) != lease_by_task_.end()) { + return false; // leased tasks finish; cancel only affects queued ones + } + terminal_locked(id); + log_op("{\"op\":\"terminal\",\"id\":\"" + common::json_escape(id) + "\"}"); + observer_->counter("molo_tasks_cancelled_total", 1); + observer_->event("tasks.cancelled", id); + return true; +} + +std::optional TaskQueue::lease(long wait_ms) { + std::unique_lock lock(mutex_); + const std::int64_t deadline = clock_.steady_ms() + wait_ms; + while (true) { + if (shutdown_) { + return std::nullopt; + } + reap_expired_locked(); + if (auto got = try_lease_locked()) { + return got; + } + if (wait_ms <= 0) { + return std::nullopt; + } + const std::int64_t now = clock_.steady_ms(); + if (now >= deadline) { + return std::nullopt; + } + // Wake early when the earliest delayed task becomes due. + long wait_for = static_cast(deadline - now); + if (!ready_.empty()) { + const std::int64_t due_wall = ready_.begin()->first.first; + const std::int64_t now_wall = clock_.wall_ms(); + if (due_wall > now_wall) { + const long until_due = static_cast(due_wall - now_wall); + if (until_due < wait_for) { + wait_for = until_due; + } + } + } + cv_.wait_for(lock, std::chrono::milliseconds(wait_for)); + } +} + +std::optional TaskQueue::try_lease_locked() { + if (ready_.empty()) { + return std::nullopt; + } + const std::int64_t now_wall = clock_.wall_ms(); + auto it = ready_.begin(); + if (it->first.first > now_wall) { + return std::nullopt; // earliest task still delayed + } + const std::string id = it->second; + ready_.erase(it); + auto record = records_.find(id); + if (record == records_.end()) { + return std::nullopt; + } + record->second.envelope.state = TaskState::Leased; + const std::string lease_id = + "L" + std::to_string(clock_.steady_ms()) + "-" + std::to_string(++lease_seq_); + LeaseInfo info{id, clock_.steady_ms() + options_.lease_ms}; + leases_[lease_id] = info; + lease_by_task_[id] = lease_id; + observer_->counter("molo_tasks_leased_total", 1); + return Lease{record->second.envelope, lease_id, info.deadline_steady_ms}; +} + +void TaskQueue::reap_expired_locked() { + const std::int64_t now = clock_.steady_ms(); + std::vector expired; + for (const auto& [lease_id, info] : leases_) { + if (info.deadline_steady_ms <= now) { + expired.push_back(lease_id); + } + } + for (const auto& lease_id : expired) { + auto lit = leases_.find(lease_id); + if (lit == leases_.end()) { + continue; + } + const std::string task_id = lit->second.task_id; + leases_.erase(lit); + lease_by_task_.erase(task_id); + auto rit = records_.find(task_id); + if (rit == records_.end()) { + continue; + } + // Lease expiry without ack returns the task to queued with attempts + // incremented (S7-3). A late ack from the original worker is ignored + // because the lease is gone; duplicates are idempotent by contract. + rit->second.envelope.attempts += 1; + ++expired_total_; + observer_->counter("molo_tasks_lease_expired_total", 1); + observer_->event("tasks.lease_expired", task_id); + const RetryPolicy policy(RetrySpec{options_.retry.base_ms, options_.retry.cap_ms, + options_.retry.jitter_ms, + rit->second.envelope.max_retries}); + if (policy.exhausted(rit->second.envelope.attempts)) { + TaskEnvelope env = rit->second.envelope; + terminal_locked(task_id); + log_op("{\"op\":\"terminal\",\"id\":\"" + common::json_escape(task_id) + "\"}"); + dead_letters_.push(std::move(env), "lease-expired-retries-exhausted"); + ++failed_total_; + } else { + rit->second.envelope.state = TaskState::Queued; + rit->second.envelope.run_at_ms.reset(); // redeliver immediately + rit->second.order = ++order_; + ready_[{0, rit->second.order}] = task_id; + log_op("{\"op\":\"requeue\",\"id\":\"" + common::json_escape(task_id) + "\",\"attempts\":" + + std::to_string(rit->second.envelope.attempts) + ",\"run_at_ms\":null}"); + cv_.notify_one(); + } + } +} + +bool TaskQueue::heartbeat(const std::string& lease_id) { + std::lock_guard lock(mutex_); + auto it = leases_.find(lease_id); + if (it == leases_.end()) { + return false; + } + if (it->second.deadline_steady_ms <= clock_.steady_ms()) { + return false; + } + it->second.deadline_steady_ms = clock_.steady_ms() + options_.lease_ms; + return true; +} + +bool TaskQueue::ack(const std::string& lease_id) { + std::lock_guard lock(mutex_); + auto it = leases_.find(lease_id); + if (it == leases_.end()) { + return false; + } + const std::string task_id = it->second.task_id; + leases_.erase(it); + lease_by_task_.erase(task_id); + terminal_locked(task_id); + log_op("{\"op\":\"terminal\",\"id\":\"" + common::json_escape(task_id) + "\"}"); + ++succeeded_total_; + observer_->counter("molo_tasks_succeeded_total", 1); + observer_->event("tasks.succeeded", task_id); + return true; +} + +bool TaskQueue::fail(const std::string& lease_id, bool retryable, const std::string& reason) { + std::lock_guard lock(mutex_); + if (leases_.find(lease_id) == leases_.end()) { + return false; + } + fail_locked(lease_id, retryable, reason); + return true; +} + +void TaskQueue::fail_locked(const std::string& lease_id, bool retryable, + const std::string& reason) { + auto it = leases_.find(lease_id); + if (it == leases_.end()) { + return; + } + const std::string task_id = it->second.task_id; + leases_.erase(it); + lease_by_task_.erase(task_id); + auto rit = records_.find(task_id); + if (rit == records_.end()) { + return; + } + rit->second.envelope.attempts += 1; + const int max_retries = rit->second.envelope.max_retries; + const RetryPolicy policy( + RetrySpec{options_.retry.base_ms, options_.retry.cap_ms, options_.retry.jitter_ms, + max_retries}); + const bool drain = !accepting_; + if (!retryable || policy.exhausted(rit->second.envelope.attempts) || drain) { + TaskEnvelope env = rit->second.envelope; + terminal_locked(task_id); + log_op("{\"op\":\"terminal\",\"id\":\"" + common::json_escape(task_id) + "\"}"); + std::string why = truncate_reason(reason); + if (drain && retryable && !policy.exhausted(rit->second.envelope.attempts)) { + why += " (draining)"; + } + dead_letters_.push(std::move(env), why); + ++failed_total_; + observer_->counter("molo_tasks_failed_total", 1); + } else { + const long delay = policy.delay_for_attempt(rit->second.envelope.attempts - 1); + const std::int64_t run_at = clock_.wall_ms() + delay; + rit->second.envelope.run_at_ms = run_at; + rit->second.envelope.state = TaskState::Queued; + rit->second.order = ++order_; + ready_[{run_at, rit->second.order}] = task_id; + log_op("{\"op\":\"requeue\",\"id\":\"" + common::json_escape(task_id) + "\",\"attempts\":" + + std::to_string(rit->second.envelope.attempts) + + ",\"run_at_ms\":" + std::to_string(run_at) + "}"); + ++retried_total_; + observer_->counter("molo_tasks_retried_total", 1); + observer_->event("tasks.retrying", task_id + " in " + std::to_string(delay) + "ms"); + cv_.notify_one(); + } +} + +void TaskQueue::terminal_locked(const std::string& task_id) { + auto rit = records_.find(task_id); + if (rit == records_.end()) { + return; + } + for (auto it = ready_.begin(); it != ready_.end(); ++it) { + if (it->second == task_id) { + ready_.erase(it); + break; + } + } + records_.erase(rit); +} + +void TaskQueue::execute(const Lease& leased, TaskRegistry& registry, + const std::function& stop_requested) { + struct ExecState { + std::atomic done{false}; + std::atomic cancel{false}; + std::mutex mutex; + std::condition_variable cv; + TaskOutcome outcome = TaskOutcome::RetryableFailure; + std::string error; + }; + + { + std::lock_guard lock(mutex_); + auto rit = records_.find(leased.envelope.id); + if (rit != records_.end()) { + rit->second.envelope.state = TaskState::Running; + } + ++running_; + observer_->event("tasks.started", leased.envelope.id + " " + leased.envelope.name); + } + + std::shared_ptr handler = registry.create(leased.envelope.name); + if (!handler) { + std::lock_guard lock(mutex_); + --running_; + auto it = lease_by_task_.find(leased.envelope.id); + if (it != lease_by_task_.end()) { + fail_locked(it->second, false, "unknown-handler:" + leased.envelope.name); + } + return; + } + + auto state = std::make_shared(); + TaskEnvelope env_copy = leased.envelope; // child thread owns only copies + std::thread child([state, handler, env_copy, this]() mutable { + TaskContext ctx{env_copy, clock_, observer_, state->cancel}; + try { + const TaskOutcome outcome = handler->run(ctx); + std::lock_guard lock(state->mutex); + state->outcome = outcome; + } catch (const std::exception& e) { + std::lock_guard lock(state->mutex); + state->outcome = TaskOutcome::RetryableFailure; + state->error = truncate_reason(std::string("exception:") + e.what()); + } catch (...) { + std::lock_guard lock(state->mutex); + state->outcome = TaskOutcome::RetryableFailure; + state->error = "exception:unknown"; + } + { + std::lock_guard lock(state->mutex); + state->done = true; + } + state->cv.notify_all(); + }); + + // Waiter: heartbeat the lease, fire the soft timeout cooperatively, and + // enforce the hard timeout by failing the lease and recycling the worker. + const std::int64_t start = clock_.steady_ms(); + const long soft = env_copy.timeout_soft_ms; + const long hard = env_copy.timeout_hard_ms; + bool soft_fired = false; + bool hard_fired = false; + std::int64_t last_heartbeat = start; + { + std::unique_lock lock(state->mutex); + while (!state->done) { + state->cv.wait_for(lock, std::chrono::milliseconds(5)); + const std::int64_t now = clock_.steady_ms(); + if (stop_requested && stop_requested()) { + state->cancel = true; + } + if (!soft_fired && now - start >= soft) { + soft_fired = true; + state->cancel = true; + observer_->event("tasks.soft_timeout", env_copy.id); + } + if (now - last_heartbeat >= options_.heartbeat_ms / 2) { + last_heartbeat = now; + heartbeat(leased.lease_id); + } + if (now - start >= hard) { + hard_fired = true; + state->cancel = true; + observer_->event("tasks.hard_timeout", env_copy.id); + break; + } + } + } + + if (!hard_fired) { + TaskOutcome outcome; + std::string error; + { + std::lock_guard lock(state->mutex); + outcome = state->outcome; + error = state->error; + } + child.join(); + std::lock_guard lock(mutex_); + --running_; + switch (outcome) { + case TaskOutcome::Success: { + auto it = leases_.find(leased.lease_id); + if (it != leases_.end()) { + const std::string task_id = it->second.task_id; + leases_.erase(it); + lease_by_task_.erase(task_id); + terminal_locked(task_id); + log_op("{\"op\":\"terminal\",\"id\":\"" + common::json_escape(task_id) + "\"}"); + ++succeeded_total_; + observer_->counter("molo_tasks_succeeded_total", 1); + observer_->event("tasks.succeeded", task_id); + } + break; + } + case TaskOutcome::RetryableFailure: + fail_locked(leased.lease_id, true, error.empty() ? "handler-retryable" : error); + break; + case TaskOutcome::PermanentFailure: + fail_locked(leased.lease_id, false, error.empty() ? "handler-permanent" : error); + break; + } + return; + } + + // Hard timeout: the lease fails retryable right away so the work is not + // stuck; the orphaned child gets a grace period, then is detached and + // the worker recycled (R7, S7-3/S7-4). The orphan owns only heap state + // and copies, so detaching is memory-safe; a late finish cannot ack + // because the lease is already gone. + { + std::lock_guard lock(mutex_); + --running_; + fail_locked(leased.lease_id, true, "hard-timeout"); + } + { + std::unique_lock lock(state->mutex); + state->cv.wait_for(lock, std::chrono::milliseconds(options_.orphan_grace_ms), + [&] { return state->done.load(); }); + } + if (state->done) { + child.join(); + } else { + child.detach(); + observer_->counter("molo_tasks_orphaned_total", 1); + } + recycle_.store(true); +} + +int TaskQueue::run_worker(const std::function& stop_requested, TaskRegistry& registry) { + recycle_.store(false); + while (!stop_requested || !stop_requested()) { + std::optional got = lease(100); + if (!got) { + continue; + } + execute(*got, registry, stop_requested); + if (recycle_.load()) { + return 2; // hard timeout fired: recycle this worker + } + } + return 0; +} + +bool TaskQueue::run_one(TaskRegistry& registry, long lease_wait_ms) { + std::optional got = lease(lease_wait_ms); + if (!got) { + return false; + } + execute(*got, registry, {}); + return true; +} + +bool TaskQueue::recycle_requested() const { + return recycle_.load(); +} + +QueueStats TaskQueue::stats() const { + std::lock_guard lock(mutex_); + QueueStats stats; + stats.queued = ready_.size(); + stats.leased = leases_.size(); + stats.running = running_; + stats.succeeded_total = succeeded_total_; + stats.failed_total = failed_total_; + stats.retried_total = retried_total_; + stats.expired_total = expired_total_; + return stats; +} + +std::optional TaskQueue::get(const std::string& id) const { + std::lock_guard lock(mutex_); + auto it = records_.find(id); + if (it == records_.end()) { + return std::nullopt; + } + return it->second.envelope; +} + +std::vector TaskQueue::list_queued(std::size_t max) const { + std::lock_guard lock(mutex_); + std::vector out; + for (const auto& [_, id] : ready_) { + if (out.size() >= max) { + break; + } + auto it = records_.find(id); + if (it != records_.end()) { + out.push_back(it->second.envelope); + } + } + return out; +} + +DeadLetterQueue& TaskQueue::dead_letters() { + return dead_letters_; +} + +void TaskQueue::set_accepting(bool accepting) { + std::lock_guard lock(mutex_); + accepting_ = accepting; +} + +bool TaskQueue::accepting() const { + std::lock_guard lock(mutex_); + return accepting_; +} + +std::size_t TaskQueue::in_flight() const { + std::lock_guard lock(mutex_); + return leases_.size() + running_; +} + +void TaskQueue::abandon_all_to_dlq(const std::string& reason) { + std::lock_guard lock(mutex_); + for (const auto& [_, record] : records_) { + TaskEnvelope env = record.envelope; + log_op("{\"op\":\"terminal\",\"id\":\"" + common::json_escape(env.id) + "\"}"); + dead_letters_.push(std::move(env), reason); + } + records_.clear(); + ready_.clear(); + leases_.clear(); + lease_by_task_.clear(); + observer_->event("tasks.abandoned", reason); +} + +void TaskQueue::shutdown() { + std::lock_guard lock(mutex_); + shutdown_ = true; + cv_.notify_all(); +} + +void TaskQueue::compact() { + std::lock_guard lock(mutex_); + if (!log_) { + return; + } + // Rewrite the log from live state: one enqueue op per live record. + const std::filesystem::path file = + std::filesystem::path(options_.durable_dir) / "task_queue.log"; + log_.reset(); + std::filesystem::remove(file); + log_ = std::make_unique(file); + for (const auto& [_, record] : records_) { + log_->append("{\"op\":\"enqueue\",\"env\":" + record.envelope.encode() + "}", + clock_.wall_ms()); + } +} + +std::string TaskQueue::dedupe_lookup_locked(const std::string& key) { + auto it = idempotency_.find(key); + if (it == idempotency_.end()) { + return ""; + } + if (it->second.expires_wall_ms <= clock_.wall_ms()) { + idempotency_.erase(it); + return ""; + } + // The key stays reserved for the full window even after the task + // finished: duplicates return the original id without re-enqueue (S7-1). + return it->second.task_id; +} + +void TaskQueue::dedupe_store_locked(const std::string& key, const std::string& task_id) { + const std::int64_t expires = + clock_.wall_ms() + common::Caps::kIdempotencyWindowHours * 3600 * 1000; + idempotency_[key] = IdemEntry{task_id, expires}; + idempotency_order_.push_back(key); + while (idempotency_order_.size() > options_.max_depth) { + idempotency_.erase(idempotency_order_.front()); + idempotency_order_.pop_front(); + } +} + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/task_queue.hpp b/src/supervisor/tasks/task_queue.hpp new file mode 100644 index 0000000..60f11bb --- /dev/null +++ b/src/supervisor/tasks/task_queue.hpp @@ -0,0 +1,158 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "supervisor/bus/segment_log.hpp" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/tasks/dead_letter_queue.hpp" +#include "supervisor/tasks/retry_policy.hpp" +#include "supervisor/tasks/task_envelope.hpp" +#include "supervisor/tasks/task_registry.hpp" + +namespace molo::tasks { + +struct QueueStats { + std::size_t queued = 0; + std::size_t leased = 0; + std::size_t running = 0; + long succeeded_total = 0; + long failed_total = 0; + long retried_total = 0; + long expired_total = 0; +}; + +struct QueueOptions { + std::string durable_dir; // empty means in-memory only (E7-2); set for E7-6 + long lease_ms = 60000; + long heartbeat_ms = 15000; + std::size_t max_depth = 100000; + RetrySpec retry; + long orphan_grace_ms = 5000; // join grace after a hard timeout before detach +}; + +// At-least-once task queue with ack-after-success (BP-3, S7-3). +// +// Producer side: enqueue() validates, dedupes idempotency keys, and makes +// the envelope leasable once its run_at arrives. Consumer side: lease() +// hands out one envelope with an ack deadline; ack() completes it; +// fail() retries with backoff or dead-letters; heartbeat() renews the +// deadline. Leases that expire without ack return to queued with attempts +// incremented. Redelivery is normal; handlers must be idempotent (R7-2). +// +// run_worker() is the off-event-loop consumer loop (R7-5) driven by +// supervisor worker threads: it leases, executes with soft/hard timeouts, +// and returns 2 when the worker should be recycled after a hard timeout. +class TaskQueue { + public: + struct Lease { + TaskEnvelope envelope; + std::string lease_id; + std::int64_t deadline_steady_ms = 0; + }; + + TaskQueue(const QueueOptions& options, common::Clock& clock, + std::shared_ptr observer); + ~TaskQueue(); + + TaskQueue(const TaskQueue&) = delete; + TaskQueue& operator=(const TaskQueue&) = delete; + + // Returns the task id. Duplicate idempotency_key within 24h returns the + // existing id without re-enqueue. Throws TaskError on cap violations. + std::string enqueue(TaskEnvelope envelope); + + bool cancel(const std::string& id); + + // Leases one due envelope, waiting up to wait_ms (0 means non-blocking). + // Returns nullopt on timeout or after shutdown(). + std::optional lease(long wait_ms); + + bool heartbeat(const std::string& lease_id); + bool ack(const std::string& lease_id); + bool fail(const std::string& lease_id, bool retryable, const std::string& reason); + + // Worker entry points. + int run_worker(const std::function& stop_requested, TaskRegistry& registry); + bool run_one(TaskRegistry& registry, long lease_wait_ms); + bool recycle_requested() const; + + QueueStats stats() const; + std::optional get(const std::string& id) const; + std::vector list_queued(std::size_t max) const; + DeadLetterQueue& dead_letters(); + + // Graceful shutdown support (R7-9). + void set_accepting(bool accepting); + bool accepting() const; + std::size_t in_flight() const; // leased plus running + void abandon_all_to_dlq(const std::string& reason); + void shutdown(); // wakes all lease waiters; run_worker loops still exit via stop + void compact(); // rewrites the durable log from live state + + private: + struct Record { + TaskEnvelope envelope; + std::uint64_t order = 0; + }; + + struct LeaseInfo { + std::string task_id; + std::int64_t deadline_steady_ms = 0; + }; + + struct IdemEntry { + std::string task_id; + std::int64_t expires_wall_ms = 0; + }; + + void replay(); + void log_op(const std::string& op); + void reap_expired_locked(); + std::optional try_lease_locked(); + void fail_locked(const std::string& lease_id, bool retryable, const std::string& reason); + void terminal_locked(const std::string& task_id); + void execute(const Lease& leased, TaskRegistry& registry, + const std::function& stop_requested); + std::string dedupe_lookup_locked(const std::string& key); + void dedupe_store_locked(const std::string& key, const std::string& task_id); + + QueueOptions options_; + common::Clock& clock_; + std::shared_ptr observer_; + RetryPolicy retry_; + DeadLetterQueue dead_letters_; + std::unique_ptr log_; + + mutable std::mutex mutex_; + std::condition_variable cv_; + std::map records_; // live tasks by id + std::map, std::string> ready_; // (run_at, order) -> id + std::map leases_; // lease_id -> info + std::map lease_by_task_; // task_id -> lease_id + std::map idempotency_; + std::deque idempotency_order_; + std::uint64_t order_ = 0; + std::uint64_t lease_seq_ = 0; + std::size_t running_ = 0; + long succeeded_total_ = 0; + long failed_total_ = 0; + long retried_total_ = 0; + long expired_total_ = 0; + bool accepting_ = true; + bool shutdown_ = false; + std::atomic recycle_{false}; +}; + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/task_registry.cpp b/src/supervisor/tasks/task_registry.cpp new file mode 100644 index 0000000..5ccbdfa --- /dev/null +++ b/src/supervisor/tasks/task_registry.cpp @@ -0,0 +1,36 @@ +#include "supervisor/tasks/task_registry.hpp" + +namespace molo::tasks { + +void TaskRegistry::register_handler(const std::string& name, Factory factory) { + if (name.empty() || name.size() > 256 || !factory) { + throw TaskError("TaskRegistry: bad handler registration"); + } + std::lock_guard lock(mutex_); + factories_[name] = std::move(factory); +} + +bool TaskRegistry::has(const std::string& name) const { + std::lock_guard lock(mutex_); + return factories_.find(name) != factories_.end(); +} + +std::unique_ptr TaskRegistry::create(const std::string& name) const { + std::lock_guard lock(mutex_); + auto it = factories_.find(name); + if (it == factories_.end()) { + return nullptr; + } + return it->second(); +} + +std::vector TaskRegistry::names() const { + std::lock_guard lock(mutex_); + std::vector out; + for (const auto& [name, _] : factories_) { + out.push_back(name); + } + return out; +} + +} // namespace molo::tasks diff --git a/src/supervisor/tasks/task_registry.hpp b/src/supervisor/tasks/task_registry.hpp new file mode 100644 index 0000000..3f7c085 --- /dev/null +++ b/src/supervisor/tasks/task_registry.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "supervisor/tasks/task_handler.hpp" + +namespace molo::tasks { + +// Name-to-handler factory registry. Workers resolve handler instances per +// execution through this registry so handlers stay stateless and the queue +// never depends on concrete handler types. +class TaskRegistry { + public: + using Factory = std::function()>; + + void register_handler(const std::string& name, Factory factory); + bool has(const std::string& name) const; + std::unique_ptr create(const std::string& name) const; + std::vector names() const; + + private: + mutable std::mutex mutex_; + std::map factories_; +}; + +} // namespace molo::tasks diff --git a/src/supplychain.mk b/src/supplychain.mk new file mode 100644 index 0000000..6029a93 --- /dev/null +++ b/src/supplychain.mk @@ -0,0 +1,142 @@ +# MoloApp supply-chain part fragment. Included by the root Makefile with: +# -include src/supplychain/supplychain.mk +# Provides supply-all / supply-test / supply-clean. All outputs go under build/. + +SUPPLY_CXX ?= $(CXX) +SUPPLY_CXXFLAGS ?= -std=c++23 -O2 -Wall -Wextra -pthread +SUPPLY_INC ?= -Isrc -Isrc/supplychain -Isrc/cli + +# INTEGRATION: with the base tree present the supply headers bind the real +# CLI TU definitions, so test links compile them in. (-Isrc/cli covers the +# relative "cli/..."/"embed/..." includes inside those sources.) +SUPPLY_CLI_SRCS := src/cli/embed/sha256.cpp src/cli/cli/output.cpp \ + src/cli/cli/json_writer.cpp + +SUPPLY_LIB := build/supply/libsupply.a + +SUPPLY_SRCS := \ + src/supplychain/common/Json.cpp \ + src/supplychain/policy/WolfPin.cpp \ + src/supplychain/sbom/SbomGenerator.cpp \ + src/supplychain/sbom/SpdxEmitter.cpp \ + src/supplychain/sbom/CycloneDxEmitter.cpp \ + src/supplychain/sbom/SbomCheck.cpp \ + src/supplychain/scan/OsvClient.cpp \ + src/supplychain/scan/VulnHook.cpp \ + src/supplychain/attest/Attestor.cpp \ + src/supplychain/attest/Signer.cpp \ + src/supplychain/attest/Verify.cpp \ + src/supplychain/cli/SupplyCli.cpp + +SUPPLY_OBJS := \ + build/obj/supplychain/common/Json.o \ + build/obj/supplychain/policy/WolfPin.o \ + build/obj/supplychain/sbom/SbomGenerator.o \ + build/obj/supplychain/sbom/SpdxEmitter.o \ + build/obj/supplychain/sbom/CycloneDxEmitter.o \ + build/obj/supplychain/sbom/SbomCheck.o \ + build/obj/supplychain/scan/OsvClient.o \ + build/obj/supplychain/scan/VulnHook.o \ + build/obj/supplychain/attest/Attestor.o \ + build/obj/supplychain/attest/Signer.o \ + build/obj/supplychain/attest/Verify.o \ + build/obj/supplychain/cli/SupplyCli.o + +SUPPLY_TEST_BINS := \ + build/tests/supply_test_pin \ + build/tests/supply_test_spdx \ + build/tests/supply_test_cdx_equiv \ + build/tests/supply_test_scan \ + build/tests/supply_test_provenance \ + build/tests/supply_test_verify + +SUPPLY_GOLDEN_DIR := tests/src/supplychain/golden + +supply-all: $(SUPPLY_LIB) + +$(SUPPLY_LIB): $(SUPPLY_OBJS) + mkdir -p $(@D) + $(AR) rcs $@ $(SUPPLY_OBJS) + +build/obj/supplychain/common/Json.o: src/supplychain/common/Json.cpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/common/Json.cpp -o $@ + +build/obj/supplychain/policy/WolfPin.o: src/supplychain/policy/WolfPin.cpp src/supplychain/policy/WolfPin.hpp src/supplychain/policy/DependencyPolicy.hpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/policy/WolfPin.cpp -o $@ + +build/obj/supplychain/sbom/SbomGenerator.o: src/supplychain/sbom/SbomGenerator.cpp src/supplychain/sbom/SbomGenerator.hpp src/supplychain/sbom/Types.hpp src/supplychain/policy/WolfPin.hpp src/supplychain/policy/DependencyPolicy.hpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/sbom/SbomGenerator.cpp -o $@ + +build/obj/supplychain/sbom/SpdxEmitter.o: src/supplychain/sbom/SpdxEmitter.cpp src/supplychain/sbom/SpdxEmitter.hpp src/supplychain/sbom/Types.hpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/sbom/SpdxEmitter.cpp -o $@ + +build/obj/supplychain/sbom/CycloneDxEmitter.o: src/supplychain/sbom/CycloneDxEmitter.cpp src/supplychain/sbom/CycloneDxEmitter.hpp src/supplychain/sbom/Types.hpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/sbom/CycloneDxEmitter.cpp -o $@ + +build/obj/supplychain/sbom/SbomCheck.o: src/supplychain/sbom/SbomCheck.cpp src/supplychain/sbom/SbomCheck.hpp src/supplychain/sbom/Types.hpp src/supplychain/common/Json.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/sbom/SbomCheck.cpp -o $@ + +build/obj/supplychain/scan/OsvClient.o: src/supplychain/scan/OsvClient.cpp src/supplychain/scan/OsvClient.hpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/scan/OsvClient.cpp -o $@ + +build/obj/supplychain/scan/VulnHook.o: src/supplychain/scan/VulnHook.cpp src/supplychain/scan/VulnHook.hpp src/supplychain/scan/OsvClient.hpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/scan/VulnHook.cpp -o $@ + +build/obj/supplychain/attest/Attestor.o: src/supplychain/attest/Attestor.cpp src/supplychain/attest/Attestor.hpp src/supplychain/attest/ProvenanceBuilder.hpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/attest/Attestor.cpp -o $@ + +build/obj/supplychain/attest/Signer.o: src/supplychain/attest/Signer.cpp src/supplychain/attest/Signer.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/attest/Signer.cpp -o $@ + +build/obj/supplychain/attest/Verify.o: src/supplychain/attest/Verify.cpp src/supplychain/attest/Verify.hpp src/supplychain/policy/WolfPin.hpp src/supplychain/common/Json.hpp src/supplychain/caps.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/attest/Verify.cpp -o $@ + +build/obj/supplychain/cli/SupplyCli.o: src/supplychain/cli/SupplyCli.cpp src/supplychain/cli/SupplyCli.hpp + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -c src/supplychain/cli/SupplyCli.cpp -o $@ + +build/tests/supply_test_pin: tests/src/supplychain/test_pin.cpp $(SUPPLY_LIB) + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -I. tests/src/supplychain/test_pin.cpp $(SUPPLY_LIB) $(SUPPLY_CLI_SRCS) -o $@ + +build/tests/supply_test_spdx: tests/src/supplychain/test_spdx.cpp $(SUPPLY_LIB) + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -I. tests/src/supplychain/test_spdx.cpp $(SUPPLY_LIB) $(SUPPLY_CLI_SRCS) -o $@ + +build/tests/supply_test_cdx_equiv: tests/src/supplychain/test_cdx_equiv.cpp $(SUPPLY_LIB) + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -I. tests/src/supplychain/test_cdx_equiv.cpp $(SUPPLY_LIB) $(SUPPLY_CLI_SRCS) -o $@ + +build/tests/supply_test_scan: tests/src/supplychain/test_scan.cpp $(SUPPLY_LIB) + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -I. tests/src/supplychain/test_scan.cpp $(SUPPLY_LIB) $(SUPPLY_CLI_SRCS) -o $@ + +build/tests/supply_test_provenance: tests/src/supplychain/test_provenance.cpp $(SUPPLY_LIB) + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -I. tests/src/supplychain/test_provenance.cpp $(SUPPLY_LIB) $(SUPPLY_CLI_SRCS) -o $@ + +build/tests/supply_test_verify: tests/src/supplychain/test_verify.cpp $(SUPPLY_LIB) + mkdir -p $(@D) + $(SUPPLY_CXX) $(SUPPLY_CXXFLAGS) $(SUPPLY_INC) -I. tests/src/supplychain/test_verify.cpp $(SUPPLY_LIB) $(SUPPLY_CLI_SRCS) -o $@ + +supply-test: $(SUPPLY_TEST_BINS) + ./build/tests/supply_test_pin + ./build/tests/supply_test_spdx $(SUPPLY_GOLDEN_DIR) + ./build/tests/supply_test_cdx_equiv $(SUPPLY_GOLDEN_DIR) + ./build/tests/supply_test_scan $(SUPPLY_GOLDEN_DIR) + ./build/tests/supply_test_provenance $(SUPPLY_GOLDEN_DIR) + ./build/tests/supply_test_verify + +supply-clean: + rm -f $(SUPPLY_OBJS) $(SUPPLY_LIB) $(SUPPLY_TEST_BINS) diff --git a/src/supplychain/PATCHES.md b/src/supplychain/PATCHES.md new file mode 100644 index 0000000..acdb7d2 --- /dev/null +++ b/src/supplychain/PATCHES.md @@ -0,0 +1,131 @@ +# Base-tree edits for the supply-chain part + +This file lists every edit the base tree needs to wire in `src/supplychain/`. +The isolated worktree does not contain the base `src/` tree, so `@@` line +offsets below are illustrative: locate each hunk by its anchor line (taken +from the seam handoff), apply, then `make supply-test` from the repo root. + +Assumptions flagged inline with `NOTE(jsm)`: adapter signatures assumed from +the seam summary (`Output::print/info/progress/warn/error/debug`, +`JsonWriter::object(map)`, `CommandTree` member access). +Confirm against the real headers when applying. + +## 1. Makefile (repo root) + +Add the fragment include plus double-colon wiring. Multiple `.PHONY` lines +and double-colon rules are legal in one Makefile, so these lines may also +live as their own block. + +```diff +--- a/Makefile ++++ b/Makefile +@@ -3,6 +3,11 @@ + -include src/cli/cli.mk ++-include src/supplychain/supplychain.mk ++ ++all:: supply-all ++test:: supply-test ++ ++.PHONY: supply-all supply-test supply-clean +``` + +## 2. src/cli/cli.mk + +Append at end of file. `+=` works regardless of how `CLI_INC` is spelled, +and the extra prerequisite-only rule merges with the existing `$(MOLO_BIN)` +rule (single- or double-colon). + +```diff +--- a/src/cli/cli.mk ++++ b/src/cli/cli.mk +@@ -40,1 +40,4 @@ + # end of cli.mk ++CLI_INC += -Isrc/supplychain ++ ++$(MOLO_BIN): $(SUPPLY_LIB) +``` + +NOTE(jsm): if `cli.mk` names the include variable or binary target +differently than `CLI_INC` / `MOLO_BIN`, rename here to match; the supply +sources only need `-Isrc/supplychain` on the CLI compile line and +`build/supply/libsupply.a` linked into the CLI binary and its test links. + +## 3. src/cli/cli/cli_app.cpp + +Include the thin handler header (only supply header the CLI may include) +and add the three leaves in `build_tree()`. + +```diff +--- a/src/cli/cli/cli_app.cpp ++++ b/src/cli/cli/cli_app.cpp +@@ -1,5 +1,6 @@ + #include "cli/cli_app.hpp" ++#include "supplychain/cli/SupplyCli.hpp" +``` + +NOTE(jsm): if this file includes its own header with a different spelling +(e.g. `cli/cli/cli_app.hpp`), keep that line untouched and add the second +include next to it. + +```diff +--- a/src/cli/cli/cli_app.cpp ++++ b/src/cli/cli/cli_app.cpp +@@ -20,6 +20,45 @@ + void CliApp::build_tree() { ++ auto verify = std::make_shared("verify", "verify release artifacts offline"); ++ verify->usage("molo verify [dir]") ++ .long_help("Recompute sha256 digests, check the signature bundle chain, and match the wolfssl pin. Fully offline.") ++ .example("molo verify ./dist") ++ .flag(Flag("manifest", "", "subjects file", true, "subjects.txt")) ++ .flag(Flag("bundle", "", "signature bundle", true, "bundle.json")) ++ .flag(Flag("pin", "", "allow-list with wolfssl pin", true, "")) ++ .handler(&molo::supply::cli::SupplyCli::run_verify); ++ tree().add(verify); ++ ++ auto check = std::make_shared("supply-check", "pin plus advisory gate"); ++ check->usage("molo supply-check [dir]") ++ .long_help("Check the wolfssl pin and allow-list, then evaluate advisories. Blocked exits 1.") ++ .example("molo supply-check ./dist") ++ .flag(Flag("allow", "", "allow-list file", true, "supply.allow.json")) ++ .flag(Flag("advisories", "", "advisories file", true, "advisories.json")) ++ .flag(Flag("exceptions", "", "exceptions file", true, "")) ++ .handler(&molo::supply::cli::SupplyCli::run_check); ++ tree().add(check); ++ ++ auto release = std::make_shared("release", "ordered release gate"); ++ release->usage("molo release --commit --suite-stamp ") ++ .long_help("Run suite, sbom, scan, provenance, sign, tag in order. Any failed step exits 1.") ++ .example("molo release --commit abc123 --suite-stamp build/suite.stamp") ++ .flag(Flag("commit", "", "release commit", true, "")) ++ .flag(Flag("suite-stamp", "", "file containing PASS", true, "")) ++ .flag(Flag("dir", "", "release directory", true, ".")) ++ .handler(&molo::supply::cli::SupplyCli::run_release); ++ tree().add(release); ++ + // ... existing leaves follow + } +``` + +NOTE(jsm): `tree()` stands for whatever member or accessor holds the +`CommandTree` in `build_tree()`; use the local name if it differs. +`Flag(name, short, help, takes_value, default)` and the `Command` builder +chain spellings are per the seam handoff. + +## 4. src/cli/cli/cli_app.hpp + +No change required. `build_tree()` is private to the `.cpp`, the new leaves +are added there, and no forward declaration is needed because `cli_app.cpp` +includes `supplychain/cli/SupplyCli.hpp` directly. + +## 5. Verification after applying (base tree) + +- `make supply-all` builds `build/supply/libsupply.a` with base headers + (adapters are never on the include path, so any seam drift fails loudly). +- `make supply-test` builds and runs the six suites, including CLI handler + paths through the real `ParsedArgs`/`Output` types. +- `molo verify --help`, `molo supply-check --help`, `molo release --help` + print usage and exit 0; exit codes map via `to_int` (0/1/2). +- No other base file is touched: no changes to `command.hpp`, + `command_tree.hpp`, `exit_code.hpp`, `output.hpp`, `json_writer.hpp`, + `flag.hpp`, `arg_parser.hpp`, `embed/sha256.hpp`, `version.hpp`, + `caps.hpp`, logging, crypto, or TLS. diff --git a/src/supplychain/adapters/cli/cli/arg_parser.hpp b/src/supplychain/adapters/cli/cli/arg_parser.hpp new file mode 100644 index 0000000..8ed5111 --- /dev/null +++ b/src/supplychain/adapters/cli/cli/arg_parser.hpp @@ -0,0 +1,28 @@ +#pragma once +// ADAPTER: standalone fallback mirroring the src/cli/cli/arg_parser.hpp seam. +// The real build uses the base-tree header; this file only exists so the +// supply-chain part compiles and tests in isolation. + +#include +#include +#include + +namespace molo::cli { + +struct GlobalArgs { + std::string format; + int verbose = 0; + bool quiet = false; + bool show_help = false; + bool show_version = false; +}; + +struct ParsedArgs { + GlobalArgs global; + std::vector command_path; + std::map flags; + std::vector positionals; + std::string error; +}; + +} // namespace molo::cli diff --git a/src/supplychain/adapters/cli/cli/exit_code.hpp b/src/supplychain/adapters/cli/cli/exit_code.hpp new file mode 100644 index 0000000..7fb5826 --- /dev/null +++ b/src/supplychain/adapters/cli/cli/exit_code.hpp @@ -0,0 +1,12 @@ +#pragma once +// ADAPTER: standalone fallback mirroring the src/cli/cli/exit_code.hpp seam. +// The real build uses the base-tree header; this file only exists so the +// supply-chain part compiles and tests in isolation. + +namespace molo::cli { + +enum class ExitCode : int { Ok = 0, Runtime = 1, Usage = 2, Unavailable = 69 }; + +inline int to_int(ExitCode code) { return static_cast(code); } + +} // namespace molo::cli diff --git a/src/supplychain/adapters/cli/cli/flag.hpp b/src/supplychain/adapters/cli/cli/flag.hpp new file mode 100644 index 0000000..d165b03 --- /dev/null +++ b/src/supplychain/adapters/cli/cli/flag.hpp @@ -0,0 +1,34 @@ +#pragma once +// ADAPTER: standalone fallback mirroring the src/cli/cli/flag.hpp seam. +// The real build uses the base-tree header; this file only exists so the +// supply-chain part compiles and tests in isolation. + +#include + +namespace molo::cli { + +class Flag { + public: + Flag(std::string name, std::string short_name, std::string help, bool takes_value, + std::string default_value) + : name_(std::move(name)), + short_name_(std::move(short_name)), + help_(std::move(help)), + takes_value_(takes_value), + default_value_(std::move(default_value)) {} + + const std::string& name() const { return name_; } + const std::string& short_name() const { return short_name_; } + const std::string& help() const { return help_; } + bool takes_value() const { return takes_value_; } + const std::string& default_value() const { return default_value_; } + + private: + std::string name_; + std::string short_name_; + std::string help_; + bool takes_value_; + std::string default_value_; +}; + +} // namespace molo::cli diff --git a/src/supplychain/adapters/cli/cli/json_writer.hpp b/src/supplychain/adapters/cli/cli/json_writer.hpp new file mode 100644 index 0000000..dd8ec70 --- /dev/null +++ b/src/supplychain/adapters/cli/cli/json_writer.hpp @@ -0,0 +1,72 @@ +#pragma once +// ADAPTER: standalone fallback mirroring the src/cli/cli/json_writer.hpp seam. +// Only escape/quote/object are consumed by supply-chain code; the rest matches +// the seam signature list. The real build uses the base-tree header. + +#include +#include +#include + +namespace molo::cli { + +class JsonWriter { + public: + JsonWriter() = delete; + + static std::string escape(const std::string& in); + static std::string quote(const std::string& in); + static std::string str(const std::string& in) { return quote(in); } + static std::string boolean(bool b) { return b ? "true" : "false"; } + static std::string number(long long n) { return std::to_string(n); } + static std::string object(const std::map& fields); + static std::string array(const std::vector& items); + static std::string field(const std::string& key, const std::string& rendered_value); +}; + +inline std::string JsonWriter::escape(const std::string& in) { + std::string out; + out.reserve(in.size() + 8); + for (char c : in) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out.push_back(c); break; + } + } + return out; +} + +inline std::string JsonWriter::quote(const std::string& in) { return "\"" + escape(in) + "\""; } + +inline std::string JsonWriter::object(const std::map& fields) { + std::string out = "{"; + bool first = true; + for (const auto& [key, value] : fields) { + if (!first) out += ","; + first = false; + out += quote(key) + ":" + value; + } + out += "}"; + return out; +} + +inline std::string JsonWriter::array(const std::vector& items) { + std::string out = "["; + bool first = true; + for (const auto& item : items) { + if (!first) out += ","; + first = false; + out += item; + } + out += "]"; + return out; +} + +inline std::string JsonWriter::field(const std::string& key, const std::string& rendered_value) { + return quote(key) + ":" + rendered_value; +} + +} // namespace molo::cli diff --git a/src/supplychain/adapters/cli/cli/output.hpp b/src/supplychain/adapters/cli/cli/output.hpp new file mode 100644 index 0000000..dc604d6 --- /dev/null +++ b/src/supplychain/adapters/cli/cli/output.hpp @@ -0,0 +1,63 @@ +#pragma once +// ADAPTER: standalone fallback mirroring the src/cli/cli/output.hpp seam. +// The real build uses the base-tree header; this file only exists so the +// supply-chain part compiles and tests in isolation. + +#include +#include +#include + +namespace molo::cli { + +class Output { + public: + Output(bool json, int verbose, bool quiet, bool tty) + : json_(json), verbose_(verbose), quiet_(quiet), tty_(tty) {} + + static bool stdout_is_tty() { return false; } // ADAPTER: deterministic offline value + + bool json() const { return json_; } + bool quiet() const { return quiet_; } + int verbose() const { return verbose_; } + + void print(const std::string& line) { + stdout_lines_.push_back(line); + if (!quiet_) std::cout << line << "\n"; + } + void info(const std::string& line) { + stdout_lines_.push_back(line); + if (!quiet_) std::cout << line << "\n"; + } + void progress(const std::string& line) { + stdout_lines_.push_back(line); + if (!quiet_ && tty_) std::cout << line << "\n"; + } + void warn(const std::string& line) { + stderr_lines_.push_back(line); + if (!quiet_) std::cerr << "warning: " << line << "\n"; + } + void error(const std::string& line) { + stderr_lines_.push_back(line); + std::cerr << "error: " << line << "\n"; + } + void debug(int level, const std::string& line) { + if (level <= verbose_) { + stdout_lines_.push_back(line); + if (!quiet_) std::cout << line << "\n"; + } + } + + // ADAPTER-only inspection hook for tests; base Output has no such method. + const std::vector& stdout_lines() const { return stdout_lines_; } + const std::vector& stderr_lines() const { return stderr_lines_; } + + private: + bool json_; + int verbose_; + bool quiet_; + bool tty_; + std::vector stdout_lines_; + std::vector stderr_lines_; +}; + +} // namespace molo::cli diff --git a/src/supplychain/adapters/cli/embed/sha256.hpp b/src/supplychain/adapters/cli/embed/sha256.hpp new file mode 100644 index 0000000..8ba5832 --- /dev/null +++ b/src/supplychain/adapters/cli/embed/sha256.hpp @@ -0,0 +1,186 @@ +#pragma once +// ADAPTER: standalone fallback mirroring the src/cli/embed/sha256.hpp seam +// (molo::embed::Sha256 with hex_of_bytes / hex_of_string / hex_of_file). +// The real build uses the base-tree header; this file carries its own small +// SHA-256 so the supply-chain part verifies hashes in isolation. + +#include +#include +#include +#include +#include + +namespace molo::embed { + +class Sha256 { + public: + Sha256() = delete; + + static std::string hex_of_string(const std::string& s); + static std::string hex_of_bytes(const std::vector& v); + static bool hex_of_file(const std::string& path, std::string& hex_out); + + private: + struct Ctx { + std::array h{}; + std::uint64_t total = 0; + std::array buf{}; + std::size_t used = 0; + }; + + static void init(Ctx& ctx); + static void update(Ctx& ctx, const unsigned char* data, std::size_t len); + static void final(Ctx& ctx, unsigned char out[32]); + static void block(Ctx& ctx, const unsigned char p[64]); + static std::string to_hex(const unsigned char digest[32]); +}; + +inline std::uint32_t Shr(std::uint32_t x, unsigned n) { return x >> n; } +inline std::uint32_t Rotr(std::uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); } + +inline void Sha256::init(Ctx& ctx) { + ctx.h = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + ctx.total = 0; + ctx.used = 0; +} + +inline void Sha256::block(Ctx& ctx, const unsigned char p[64]) { + static constexpr std::uint32_t k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2}; + std::uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(p[i * 4]) << 24) | + (static_cast(p[i * 4 + 1]) << 16) | + (static_cast(p[i * 4 + 2]) << 8) | static_cast(p[i * 4 + 3]); + } + for (int i = 16; i < 64; ++i) { + std::uint32_t s0 = Rotr(w[i - 15], 7) ^ Rotr(w[i - 15], 18) ^ Shr(w[i - 15], 3); + std::uint32_t s1 = Rotr(w[i - 2], 17) ^ Rotr(w[i - 2], 19) ^ Shr(w[i - 2], 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + std::uint32_t a = ctx.h[0], b = ctx.h[1], c = ctx.h[2], d = ctx.h[3]; + std::uint32_t e = ctx.h[4], f = ctx.h[5], g = ctx.h[6], h = ctx.h[7]; + for (int i = 0; i < 64; ++i) { + std::uint32_t s1 = Rotr(e, 6) ^ Rotr(e, 11) ^ Rotr(e, 25); + std::uint32_t ch = (e & f) ^ ((~e) & g); + std::uint32_t t1 = h + s1 + ch + k[i] + w[i]; + std::uint32_t s0 = Rotr(a, 2) ^ Rotr(a, 13) ^ Rotr(a, 22); + std::uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + std::uint32_t t2 = s0 + maj; + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + ctx.h[0] += a; + ctx.h[1] += b; + ctx.h[2] += c; + ctx.h[3] += d; + ctx.h[4] += e; + ctx.h[5] += f; + ctx.h[6] += g; + ctx.h[7] += h; +} + +inline void Sha256::update(Ctx& ctx, const unsigned char* data, std::size_t len) { + ctx.total += len; + std::size_t off = 0; + if (ctx.used > 0) { + std::size_t take = 64 - ctx.used; + if (take > len) take = len; + for (std::size_t i = 0; i < take; ++i) ctx.buf[ctx.used + i] = data[i]; + ctx.used += take; + off += take; + if (ctx.used == 64) { + block(ctx, ctx.buf.data()); + ctx.used = 0; + } + } + while (off + 64 <= len) { + block(ctx, data + off); + off += 64; + } + while (off < len) ctx.buf[ctx.used++] = data[off++]; +} + +inline void Sha256::final(Ctx& ctx, unsigned char out[32]) { + std::uint64_t bits = ctx.total * 8; + unsigned char pad = 0x80; + update(ctx, &pad, 1); + unsigned char zero = 0x00; + while (ctx.used != 56) update(ctx, &zero, 1); + unsigned char len_bytes[8]; + for (int i = 0; i < 8; ++i) len_bytes[i] = static_cast(bits >> (56 - i * 8)); + // Append length without touching ctx.total again. + for (int i = 0; i < 8; ++i) ctx.buf[ctx.used++] = len_bytes[i]; + block(ctx, ctx.buf.data()); + ctx.used = 0; + for (int i = 0; i < 8; ++i) { + out[i * 4] = static_cast(ctx.h[i] >> 24); + out[i * 4 + 1] = static_cast(ctx.h[i] >> 16); + out[i * 4 + 2] = static_cast(ctx.h[i] >> 8); + out[i * 4 + 3] = static_cast(ctx.h[i]); + } +} + +inline std::string Sha256::to_hex(const unsigned char digest[32]) { + static constexpr char kHex[] = "0123456789abcdef"; + std::string out; + out.reserve(64); + for (int i = 0; i < 32; ++i) { + out.push_back(kHex[digest[i] >> 4]); + out.push_back(kHex[digest[i] & 0x0F]); + } + return out; +} + +inline std::string Sha256::hex_of_string(const std::string& s) { + Ctx ctx; + init(ctx); + update(ctx, reinterpret_cast(s.data()), s.size()); + unsigned char digest[32]; + final(ctx, digest); + return to_hex(digest); +} + +inline std::string Sha256::hex_of_bytes(const std::vector& v) { + Ctx ctx; + init(ctx); + if (!v.empty()) update(ctx, v.data(), v.size()); + unsigned char digest[32]; + final(ctx, digest); + return to_hex(digest); +} + +inline bool Sha256::hex_of_file(const std::string& path, std::string& hex_out) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) return false; + Ctx ctx; + init(ctx); + unsigned char chunk[8192]; + std::size_t n = 0; + while ((n = std::fread(chunk, 1, sizeof(chunk), f)) > 0) update(ctx, chunk, n); + bool ok = std::ferror(f) == 0; + std::fclose(f); + if (!ok) return false; + unsigned char digest[32]; + final(ctx, digest); + hex_out = to_hex(digest); + return true; +} + +} // namespace molo::embed diff --git a/src/supplychain/adapters/cli/version.hpp b/src/supplychain/adapters/cli/version.hpp new file mode 100644 index 0000000..c5ec701 --- /dev/null +++ b/src/supplychain/adapters/cli/version.hpp @@ -0,0 +1,16 @@ +#pragma once +// ADAPTER: standalone fallback mirroring the src/cli/version.hpp seam. +// The real build uses the base-tree header; this file only exists so the +// supply-chain part compiles and tests in isolation. + +#include + +namespace molo { + +inline constexpr const char* kMoloName = "moloapp"; +inline constexpr const char* kMoloVersion = "0.0.0-standalone"; +inline constexpr const char* kEmbedToolVersion = "0.0.0-standalone"; + +inline std::string version_string() { return std::string(kMoloName) + " " + kMoloVersion; } + +} // namespace molo diff --git a/src/supplychain/attest/Attestor.cpp b/src/supplychain/attest/Attestor.cpp new file mode 100644 index 0000000..2b7a79f --- /dev/null +++ b/src/supplychain/attest/Attestor.cpp @@ -0,0 +1,79 @@ +#include "../attest/Attestor.hpp" + +#include +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +namespace molo::supply::attest { + +bool Attestor::statement(const std::vector& subjects, + std::string_view predicate_type, std::string_view predicate_json, + std::string& out, std::string& error) { + using common::Json; + out.clear(); + if (predicate_type.empty()) { + error = "predicate type must not be empty"; + return false; + } + if (subjects.size() > caps::kMaxSignedFiles) { + error = "subject count over 32 signed-files cap"; + return false; + } + Json::Value predicate; + if (!Json::parse(predicate_json, predicate, error)) { + error = "predicate is not valid json: " + error; + return false; + } + if (predicate.type != Json::Value::Type::Object) { + error = "predicate must be a json object"; + return false; + } + std::vector sorted = subjects; + std::sort(sorted.begin(), sorted.end(), + [](const auto& a, const auto& b) { return a.path < b.path; }); + std::vector rendered; + for (const auto& s : sorted) { + if (s.path.empty() || !Json::is_hex64(s.sha256)) { + error = "subject needs a path and 64 hex sha256"; + return false; + } + std::map subj; + subj["digest"] = Json::object({{"sha256", Json::quote(s.sha256)}}, 3); + subj["name"] = Json::quote(s.path); + rendered.push_back(Json::object(subj, 2)); + } + // Re-indent the embedded predicate so the whole statement stays canonical: + // every predicate line after the opening brace shifts one level right. + std::string predicate_indented; + { + std::string raw(predicate_json); + std::size_t pos = 0; + bool first_line = true; + while (true) { + std::size_t nl = raw.find('\n', pos); + std::string line = (nl == std::string::npos) ? raw.substr(pos) : raw.substr(pos, nl - pos); + if (!first_line) predicate_indented += " "; + first_line = false; + predicate_indented += line; + if (nl == std::string::npos) break; + predicate_indented += "\n"; + pos = nl + 1; + } + } + std::map root; + root["_type"] = Json::quote(caps::kInTotoStatementType); + root["predicate"] = predicate_indented; + root["predicateType"] = Json::quote(std::string(predicate_type)); + root["subject"] = Json::array(rendered, 1); + out = Json::object(root, 0) + "\n"; + if (out.size() > caps::kMaxSbomBytes) { + out.clear(); + error = "statement over 1 MiB cap"; + return false; + } + return true; +} + +} // namespace molo::supply::attest diff --git a/src/supplychain/attest/Attestor.hpp b/src/supplychain/attest/Attestor.hpp new file mode 100644 index 0000000..c7fdb8c --- /dev/null +++ b/src/supplychain/attest/Attestor.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +#include "../attest/ProvenanceBuilder.hpp" + +namespace molo::supply::attest { + +// in-toto Statement wrapper: binds subjects (path plus sha256 digest) to a +// predicate document. Subjects sort by path for canonical output. The 33rd +// subject is rejected (32 signed files cap). +class Attestor { + public: + Attestor() = delete; + + static bool statement(const std::vector& subjects, + std::string_view predicate_type, std::string_view predicate_json, + std::string& out, std::string& error); +}; + +} // namespace molo::supply::attest diff --git a/src/supplychain/attest/ProvenanceBuilder.hpp b/src/supplychain/attest/ProvenanceBuilder.hpp new file mode 100644 index 0000000..a1e45ec --- /dev/null +++ b/src/supplychain/attest/ProvenanceBuilder.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +namespace molo::supply::attest { + +// SLSA provenance predicate builder (predicate type +// https://slsa.dev/provenance/v1, build type https://moloapp.dev/build/v1). +// Header-only: builds a canonical JSON predicate object that Attestor wraps +// into an in-toto Statement. +class ProvenanceBuilder { + public: + struct Subject { + std::string path; + std::string sha256; + }; + + ProvenanceBuilder() = delete; + + static std::string build(std::string_view builder_id, std::string_view commit, + const std::vector& toolchain, + const std::vector& subjects, + std::string_view invocation = "") { + using common::Json; + std::vector sorted = subjects; + std::sort(sorted.begin(), sorted.end(), + [](const Subject& a, const Subject& b) { return a.path < b.path; }); + std::vector materials; + for (const auto& t : toolchain) materials.push_back(Json::quote(t)); + std::vector resolved; + for (const auto& s : sorted) { + std::map dep; + dep["digest"] = Json::object({{"sha256", Json::quote(s.sha256)}}, 4); + dep["uri"] = Json::quote(s.path); + resolved.push_back(Json::object(dep, 3)); + } + std::map build_def; + build_def["buildType"] = Json::quote(caps::kMoloBuildType); + build_def["externalParameters"] = + Json::object({{"commit", Json::quote(std::string(commit))}, + {"invocation", Json::quote(std::string(invocation))}, + {"materials", Json::array(materials, 3)}}, + 2); + build_def["resolvedDependencies"] = Json::array(resolved, 2); + std::map run_details; + run_details["builder"] = Json::object({{"id", Json::quote(std::string(builder_id))}}, 2); + std::map predicate; + predicate["buildDefinition"] = Json::object(build_def, 1); + predicate["runDetails"] = Json::object(run_details, 1); + return Json::object(predicate, 0); + } +}; + +} // namespace molo::supply::attest diff --git a/src/supplychain/attest/Signer.cpp b/src/supplychain/attest/Signer.cpp new file mode 100644 index 0000000..a2b0cfc --- /dev/null +++ b/src/supplychain/attest/Signer.cpp @@ -0,0 +1,79 @@ +#include "../attest/Signer.hpp" + +#include +#include + +#if __has_include("cli/embed/sha256.hpp") +#include "cli/embed/sha256.hpp" +#elif __has_include("embed/sha256.hpp") +#include "embed/sha256.hpp" +#else +#include "../adapters/cli/embed/sha256.hpp" // ADAPTER fallback for isolated builds +#endif + +namespace molo::supply::attest { + +std::string Signer::quote_path(std::string_view path) { + std::string out = "'"; + for (char c : path) { + if (c == '\'') { + out += "'\\''"; + } else { + out.push_back(c); + } + } + out += "'"; + return out; +} + +bool Signer::sign(const std::string& blob_path, const std::string& bundle_path, + std::string& error) { + if (blob_path.empty() || bundle_path.empty()) { + error = "blob and bundle paths must not be empty"; + return false; + } + const char* fake = std::getenv("MOLO_FAKE_COSIGN"); + if (fake != nullptr && std::string(fake) == "1") { + std::string digest; + if (!molo::embed::Sha256::hex_of_file(blob_path, digest)) { + error = "fake sign: cannot hash blob '" + blob_path + "'"; + return false; + } + std::FILE* f = std::fopen(bundle_path.c_str(), "wb"); + if (f == nullptr) { + error = "fake sign: cannot write bundle '" + bundle_path + "'"; + return false; + } + std::string bundle = "{\"fakeBundle\":true,\"digest\":\"" + digest + + "\",\"signature\":\"FAKE-" + digest.substr(0, 16) + "\"}\n"; + std::size_t wrote = std::fwrite(bundle.data(), 1, bundle.size(), f); + bool ok = wrote == bundle.size() && std::fclose(f) == 0; + if (!ok) error = "fake sign: failed writing bundle '" + bundle_path + "'"; + return ok; + } + std::string cmd = "cosign sign-blob --yes --bundle " + quote_path(bundle_path) + " " + + quote_path(blob_path) + " 2>&1"; + std::FILE* pipe = ::popen(cmd.c_str(), "r"); + if (pipe == nullptr) { + error = "cannot spawn cosign"; + return false; + } + std::string output; + char chunk[1024]; + std::size_t n = 0; + while ((n = std::fread(chunk, 1, sizeof(chunk), pipe)) > 0) { + if (output.size() + n > 2048) { + output.append(chunk, 2048 - output.size()); + break; + } + output.append(chunk, n); + } + int status = ::pclose(pipe); + if (status != 0) { + error = "cosign sign-blob failed (status " + std::to_string(status) + "): " + output; + return false; + } + return true; +} + +} // namespace molo::supply::attest diff --git a/src/supplychain/attest/Signer.hpp b/src/supplychain/attest/Signer.hpp new file mode 100644 index 0000000..3fe3ddd --- /dev/null +++ b/src/supplychain/attest/Signer.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +namespace molo::supply::attest { + +// cosign sign-blob wrapper (keyless): runs +// `cosign sign-blob --yes --bundle ` and reports failures with +// capped output. This is the only supply-chain file allowed to spawn a shell. +// Tests set MOLO_FAKE_COSIGN=1 for a deterministic offline fake bundle. +class Signer { + public: + Signer() = delete; + + static bool sign(const std::string& blob_path, const std::string& bundle_path, + std::string& error); + static std::string quote_path(std::string_view path); +}; + +} // namespace molo::supply::attest diff --git a/src/supplychain/attest/Verify.cpp b/src/supplychain/attest/Verify.cpp new file mode 100644 index 0000000..920ec1a --- /dev/null +++ b/src/supplychain/attest/Verify.cpp @@ -0,0 +1,79 @@ +#include "../attest/Verify.hpp" + +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +#if __has_include("cli/embed/sha256.hpp") +#include "cli/embed/sha256.hpp" +#elif __has_include("embed/sha256.hpp") +#include "embed/sha256.hpp" +#else +#include "../adapters/cli/embed/sha256.hpp" // ADAPTER fallback for isolated builds +#endif + +namespace molo::supply::attest { + +Verify::Result Verify::verify_files( + const std::vector>& expected, + const std::string& bundle_path, const policy::WolfPin::Pin* pin) { + Result r; + if (expected.empty()) { + r.error = "no subjects to verify"; + return r; + } + if (expected.size() > caps::kMaxSignedFiles) { + r.error = "subject count over 32 signed-files cap"; + return r; + } + std::FILE* bf = std::fopen(bundle_path.c_str(), "rb"); + if (bf == nullptr) { + r.error = "cannot open bundle '" + bundle_path + "'"; + return r; + } + std::string bundle; + char chunk[4096]; + std::size_t n = 0; + bool bundle_ok = true; + while ((n = std::fread(chunk, 1, sizeof(chunk), bf)) > 0) { + if (bundle.size() + n > caps::kMaxBundleBytes) { + bundle_ok = false; + break; + } + bundle.append(chunk, n); + } + if (std::ferror(bf) != 0) bundle_ok = false; + std::fclose(bf); + if (!bundle_ok) { + r.error = "cannot read bundle '" + bundle_path + "' (over 1 MiB cap or I/O error)"; + return r; + } + for (const auto& [path, want] : expected) { + if (!common::Json::is_hex64(want)) { + r.error = "subject '" + path + "' expected digest is not 64 hex"; + return r; + } + std::string actual; + if (!molo::embed::Sha256::hex_of_file(path, actual)) { + r.error = "cannot hash subject '" + path + "'"; + return r; + } + if (actual != want) { + r.error = "hash mismatch for subject '" + path + "'"; + return r; + } + if (bundle.find(want) == std::string::npos) { + r.error = "bundle does not cover subject '" + path + "'"; + return r; + } + if (pin != nullptr && path.find("wolfssl") != std::string::npos && want != pin->sha256) { + r.error = "subject '" + path + "' does not match wolfssl pin"; + return r; + } + } + r.ok = true; + return r; +} + +} // namespace molo::supply::attest diff --git a/src/supplychain/attest/Verify.hpp b/src/supplychain/attest/Verify.hpp new file mode 100644 index 0000000..1f1e70a --- /dev/null +++ b/src/supplychain/attest/Verify.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include + +#include "../policy/WolfPin.hpp" + +namespace molo::supply::attest { + +// Offline release verifier: recomputes sha256 for every expected subject, +// checks the signature bundle exists and covers every subject hash (chain), +// and checks the wolfssl subject against the pin when one is provided. +class Verify { + public: + struct Result { + bool ok = false; + std::string error; + }; + + Verify() = delete; + + static Result verify_files(const std::vector>& expected, + const std::string& bundle_path, const policy::WolfPin::Pin* pin); +}; + +} // namespace molo::supply::attest diff --git a/src/supplychain/caps.hpp b/src/supplychain/caps.hpp new file mode 100644 index 0000000..c724c55 --- /dev/null +++ b/src/supplychain/caps.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace molo::supply::caps { + +// Static ceilings for the supply-chain part. Every automated path enforces +// these; anything over a cap is rejected with a clear error, never truncated. +inline constexpr std::size_t kMaxComponents = 64; +inline constexpr std::size_t kMaxRelationships = 256; +inline constexpr std::size_t kMaxSbomBytes = 1048576; // 1 MiB +inline constexpr std::size_t kMaxToolchainEntries = 100; +inline constexpr std::size_t kMaxToolchainBytes = 65536; // 64 KiB +inline constexpr std::size_t kMaxSignedFiles = 32; +inline constexpr std::size_t kMaxPinJsonBytes = 65536; // 64 KiB +inline constexpr std::size_t kMaxAdvisoriesBytes = 262144; // 256 KiB +inline constexpr std::size_t kMaxCommitChars = 128; +inline constexpr std::size_t kMaxJsonDepth = 32; +inline constexpr std::size_t kMaxJsonBytes = 4194304; // 4 MiB parser ceiling +inline constexpr std::size_t kMaxOsvVulns = 64; +inline constexpr std::size_t kOsvMaxQueries = 64; +inline constexpr int kOsvTimeoutSeconds = 10; +inline constexpr int kOsvMaxRetries = 1; +inline constexpr long long kOverrideTtlSeconds = 604800; // 7 days +inline constexpr double kBlockCvss = 9.0; +inline constexpr std::size_t kMaxBundleBytes = 1048576; // 1 MiB + +inline constexpr const char* kOsvQueryUrl = "https://api.osv.dev/v1/query"; +inline constexpr const char* kSlsaPredicateType = "https://slsa.dev/provenance/v1"; +inline constexpr const char* kMoloBuildType = "https://moloapp.dev/build/v1"; +inline constexpr const char* kInTotoStatementType = "https://in-toto.io/Statement/v1"; + +} // namespace molo::supply::caps diff --git a/src/supplychain/cli/SupplyCli.cpp b/src/supplychain/cli/SupplyCli.cpp new file mode 100644 index 0000000..b20c10c --- /dev/null +++ b/src/supplychain/cli/SupplyCli.cpp @@ -0,0 +1,502 @@ +#include "../cli/SupplyCli.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "../attest/Attestor.hpp" +#include "../attest/ProvenanceBuilder.hpp" +#include "../attest/Signer.hpp" +#include "../attest/Verify.hpp" +#include "../caps.hpp" +#include "../common/Json.hpp" +#include "../policy/DependencyPolicy.hpp" +#include "../policy/WolfPin.hpp" +#include "../sbom/CycloneDxEmitter.hpp" +#include "../sbom/SbomCheck.hpp" +#include "../sbom/SbomGenerator.hpp" +#include "../sbom/SpdxEmitter.hpp" +#include "../scan/OsvClient.hpp" +#include "../scan/VulnHook.hpp" + +#if __has_include("cli/cli/json_writer.hpp") +#include "cli/cli/json_writer.hpp" +#elif __has_include("cli/json_writer.hpp") +#include "cli/json_writer.hpp" +#else +#include "../adapters/cli/cli/json_writer.hpp" // ADAPTER fallback for isolated builds +#endif + +#if __has_include("cli/version.hpp") +#include "cli/version.hpp" +#elif __has_include("version.hpp") +#include "version.hpp" +#else +#include "../adapters/cli/version.hpp" // ADAPTER fallback for isolated builds +#endif + +#if __has_include("cli/embed/sha256.hpp") +#include "cli/embed/sha256.hpp" +#elif __has_include("embed/sha256.hpp") +#include "embed/sha256.hpp" +#else +#include "../adapters/cli/embed/sha256.hpp" // ADAPTER fallback for isolated builds +#endif + +namespace molo::supply::cli { + +namespace { + +using molo::cli::ExitCode; +using molo::cli::Output; +using molo::cli::ParsedArgs; + +bool wants_json(const ParsedArgs& args) { return args.global.format == "json"; } + +std::string flag_or(const ParsedArgs& args, const std::string& name, const std::string& fallback) { + auto it = args.flags.find(name); + if (it == args.flags.end()) return fallback; + return it->second; +} + +bool has_unknown_flag(const ParsedArgs& args, const std::vector& known, + std::string& bad) { + for (const auto& [name, value] : args.flags) { + (void)value; + bool ok = false; + for (const auto& k : known) { + if (name == k) { + ok = true; + break; + } + } + if (!ok) { + bad = name; + return true; + } + } + return false; +} + +bool read_file_capped(const std::string& path, std::string& text, std::size_t cap, + std::string& error) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) { + error = "cannot open '" + path + "'"; + return false; + } + text.clear(); + char chunk[4096]; + std::size_t n = 0; + bool ok = true; + while ((n = std::fread(chunk, 1, sizeof(chunk), f)) > 0) { + if (text.size() + n > cap) { + ok = false; + break; + } + text.append(chunk, n); + } + if (std::ferror(f) != 0) ok = false; + std::fclose(f); + if (!ok) { + error = "cannot read '" + path + "' (over cap or I/O error)"; + return false; + } + return true; +} + +bool write_file(const std::string& path, const std::string& text, std::string& error) { + std::FILE* f = std::fopen(path.c_str(), "wb"); + if (f == nullptr) { + error = "cannot open '" + path + "' for writing"; + return false; + } + std::size_t wrote = std::fwrite(text.data(), 1, text.size(), f); + bool ok = wrote == text.size() && std::fclose(f) == 0; + if (!ok) error = "failed writing '" + path + "'"; + return ok; +} + +std::string join_path(const std::string& dir, const std::string& leaf) { + if (leaf.empty()) return dir; + if (!leaf.empty() && leaf[0] == '/') return leaf; + if (dir.empty() || dir == ".") return leaf; + if (dir.back() == '/') return dir + leaf; + return dir + "/" + leaf; +} + +std::string trim(const std::string& s) { + std::size_t b = 0; + while (b < s.size() && (s[b] == ' ' || s[b] == '\t' || s[b] == '\n' || s[b] == '\r')) ++b; + std::size_t e = s.size(); + while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\t' || s[e - 1] == '\n' || s[e - 1] == '\r')) --e; + return s.substr(b, e - b); +} + +void emit_status(Output& out, bool json, const std::string& status, const std::string& detail, + std::size_t count) { + if (json) { + std::map fields; + fields["count"] = molo::cli::JsonWriter::number(static_cast(count)); + fields["detail"] = molo::cli::JsonWriter::quote(detail); + fields["status"] = molo::cli::JsonWriter::quote(status); + out.print(molo::cli::JsonWriter::object(fields)); + return; + } + out.print(status + ": " + detail); +} + +bool parse_subjects(const std::string& text, const std::string& dir, + std::vector>& expected, std::string& error) { + expected.clear(); + std::istringstream in(text); + std::string line; + while (std::getline(in, line)) { + line = trim(line); + if (line.empty() || line[0] == '#') continue; + std::istringstream parts(line); + std::string digest, path; + if (!(parts >> digest >> path)) { + error = "bad subjects line: '" + line + "'"; + return false; + } + if (path.size() > 1 && path[0] == '*') path = path.substr(1); + if (expected.size() >= caps::kMaxSignedFiles) { + error = "subject count over 32 signed-files cap"; + return false; + } + expected.emplace_back(join_path(dir, path), digest); + } + return true; +} + +long long now_unix() { return static_cast(std::time(nullptr)); } + +} // namespace + +ExitCode SupplyCli::run_verify(const ParsedArgs& args, Output& out) { + const bool json = wants_json(args); + if (args.global.show_help) { + out.print("usage: molo verify [dir] [--manifest subjects.txt] [--bundle bundle.json] [--pin allow]"); + return ExitCode::Ok; + } + std::string bad; + if (has_unknown_flag(args, {"manifest", "bundle", "pin"}, bad)) { + out.error("unknown flag '--" + bad + "'"); + return ExitCode::Usage; + } + if (args.positionals.size() > 1) { + out.error("verify takes at most one directory argument"); + return ExitCode::Usage; + } + const std::string dir = args.positionals.empty() ? "." : args.positionals[0]; + const std::string manifest_path = join_path(dir, flag_or(args, "manifest", "subjects.txt")); + const std::string bundle_path = join_path(dir, flag_or(args, "bundle", "bundle.json")); + + std::string text; + std::string error; + if (!read_file_capped(manifest_path, text, caps::kMaxSbomBytes, error)) { + out.error(error); + return ExitCode::Runtime; + } + std::vector> expected; + if (!parse_subjects(text, dir, expected, error)) { + out.error(error); + return ExitCode::Runtime; + } + policy::WolfPin::Pin pin; + const policy::WolfPin::Pin* pin_ptr = nullptr; + std::string pin_flag = flag_or(args, "pin", ""); + if (!pin_flag.empty()) { + if (!policy::WolfPin::load_file(join_path(dir, pin_flag), pin, error)) { + out.error(error); + return ExitCode::Runtime; + } + pin_ptr = &pin; + } + attest::Verify::Result r = attest::Verify::verify_files(expected, bundle_path, pin_ptr); + if (!r.ok) { + emit_status(out, json, "fail", r.error, expected.size()); + return ExitCode::Runtime; + } + emit_status(out, json, "ok", "verified " + std::to_string(expected.size()) + " subject(s)", expected.size()); + return ExitCode::Ok; +} + +ExitCode SupplyCli::run_check(const ParsedArgs& args, Output& out) { + const bool json = wants_json(args); + if (args.global.show_help) { + out.print("usage: molo supply-check [dir] [--allow supply.allow.json] [--advisories advisories.json] [--exceptions exceptions.json] [--osv]"); + return ExitCode::Ok; + } + std::string bad; + if (has_unknown_flag(args, {"allow", "advisories", "exceptions", "osv"}, bad)) { + out.error("unknown flag '--" + bad + "'"); + return ExitCode::Usage; + } + if (args.positionals.size() > 1) { + out.error("supply-check takes at most one directory argument"); + return ExitCode::Usage; + } + const std::string dir = args.positionals.empty() ? "." : args.positionals[0]; + std::string error; + + policy::WolfPin::Pin pin; + if (!policy::WolfPin::load_file(join_path(dir, flag_or(args, "allow", "supply.allow.json")), pin, error)) { + out.error(error); + return ExitCode::Runtime; + } + policy::DependencyPolicy manifest; + { + std::string allow_text; + if (!read_file_capped(join_path(dir, flag_or(args, "allow", "supply.allow.json")), allow_text, + caps::kMaxPinJsonBytes, error)) { + out.error(error); + return ExitCode::Runtime; + } + if (!manifest.parse(allow_text, error)) { + out.error(error); + return ExitCode::Runtime; + } + } + out.debug(1, "pin ok: wolfssl " + pin.version + " with " + std::to_string(manifest.size()) + " extra dep(s)"); + + std::vector findings; + bool offline_unknown = false; + if (args.flags.find("osv") != args.flags.end()) { + scan::OsvClient client; + std::vector vulns; + if (!client.query("wolfssl", pin.version, vulns, error)) { + offline_unknown = true; // default transport is offline by design + } else { + findings = vulns; + } + } + { + std::string adv_text; + if (!read_file_capped(join_path(dir, flag_or(args, "advisories", "advisories.json")), adv_text, + caps::kMaxAdvisoriesBytes, error)) { + out.error(error + " (offline source required)"); + return ExitCode::Runtime; + } + std::vector from_file; + if (!scan::VulnHook::parse_advisories(adv_text, from_file, error)) { + out.error(error); + return ExitCode::Runtime; + } + findings.insert(findings.end(), from_file.begin(), from_file.end()); + } + std::vector exceptions; + std::string exc_flag = flag_or(args, "exceptions", ""); + if (!exc_flag.empty()) { + std::string exc_text; + if (!read_file_capped(join_path(dir, exc_flag), exc_text, caps::kMaxAdvisoriesBytes, error)) { + out.error(error); + return ExitCode::Runtime; + } + if (!scan::VulnHook::parse_exceptions(exc_text, exceptions, error)) { + out.error(error); + return ExitCode::Runtime; + } + } + + std::string reason; + scan::VulnHook::Verdict verdict = + scan::VulnHook::evaluate(findings, exceptions, now_unix(), offline_unknown, reason); + if (verdict == scan::VulnHook::Verdict::Blocked) { + emit_status(out, json, "blocked", reason, findings.size()); + return ExitCode::Runtime; + } + if (verdict == scan::VulnHook::Verdict::Disclosed) { + out.warn(reason); + emit_status(out, json, "disclosed", reason, findings.size()); + return ExitCode::Ok; + } + emit_status(out, json, "clean", reason, findings.size()); + return ExitCode::Ok; +} + +ExitCode SupplyCli::run_release(const ParsedArgs& args, Output& out) { + const bool json = wants_json(args); + if (args.global.show_help) { + out.print("usage: molo release --commit --suite-stamp [--dir .] [--allow supply.allow.json] [--advisories advisories.json] [--exceptions e.json] [--toolchain toolchain.txt] [--tag-file TAG]"); + return ExitCode::Ok; + } + std::string bad; + if (has_unknown_flag(args, {"commit", "suite-stamp", "dir", "allow", "advisories", "exceptions", + "toolchain", "tag-file"}, + bad)) { + out.error("unknown flag '--" + bad + "'"); + return ExitCode::Usage; + } + const std::string dir = flag_or(args, "dir", "."); + const std::string commit = flag_or(args, "commit", ""); + const std::string stamp_path = flag_or(args, "suite-stamp", ""); + std::string error; + if (commit.empty() || stamp_path.empty()) { + out.error("release needs --commit and --suite-stamp"); + return ExitCode::Usage; + } + + // Step 1: suite green (stamp file written by the base-tree test gate). + { + std::string stamp; + if (!read_file_capped(join_path(dir, stamp_path), stamp, 1024, error) || + trim(stamp) != "PASS") { + out.error("release step 1/6 suite: stamp is not PASS (" + error + ")"); + return ExitCode::Runtime; + } + out.progress("release 1/6 suite: green"); + } + + // Step 2: sbom (generate, emit both formats, NTIA check). + sbom::SbomDocument doc; + std::string spdx_path = join_path(dir, "sbom.spdx.json"); + std::string cdx_path = join_path(dir, "sbom.cdx.json"); + { + policy::WolfPin::Pin pin; + if (!policy::WolfPin::load_file(join_path(dir, flag_or(args, "allow", "supply.allow.json")), pin, error)) { + out.error("release step 2/6 sbom: " + error); + return ExitCode::Runtime; + } + policy::DependencyPolicy manifest; + { + std::string allow_text; + if (!read_file_capped(join_path(dir, flag_or(args, "allow", "supply.allow.json")), + allow_text, caps::kMaxPinJsonBytes, error) || + !manifest.parse(allow_text, error)) { + out.error("release step 2/6 sbom: " + error); + return ExitCode::Runtime; + } + } + std::vector toolchain; + { + std::string tool_text; + if (read_file_capped(join_path(dir, flag_or(args, "toolchain", "toolchain.txt")), tool_text, + caps::kMaxToolchainBytes, error)) { + std::istringstream in(tool_text); + std::string line; + while (std::getline(in, line)) { + line = trim(line); + if (line.empty() || line[0] == '#') continue; + auto at = line.find('@'); + if (at == std::string::npos) { + out.error("release step 2/6 sbom: bad toolchain line '" + line + "'"); + return ExitCode::Runtime; + } + toolchain.push_back({line.substr(0, at), line.substr(at + 1)}); + } + } + } + if (!sbom::SbomGenerator::generate(commit, toolchain, pin, manifest, molo::kMoloVersion, + sbom::SbomGenerator::utc_now(), doc, error)) { + out.error("release step 2/6 sbom: " + error); + return ExitCode::Runtime; + } + std::vector missing; + if (!sbom::SbomCheck::check_ntia(doc, missing, error) || !missing.empty()) { + out.error("release step 2/6 sbom: NTIA check failed"); + return ExitCode::Runtime; + } + if (!sbom::SpdxEmitter::emit_file(doc, spdx_path, error)) { + out.error("release step 2/6 sbom: " + error); + return ExitCode::Runtime; + } + if (!sbom::CycloneDxEmitter::emit_file(doc, cdx_path, error)) { + out.error("release step 2/6 sbom: " + error); + return ExitCode::Runtime; + } + out.progress("release 2/6 sbom: spdx+cdx written"); + } + + // Step 3: scan (offline advisories; blocked fails the release). + { + std::string adv_text; + if (!read_file_capped(join_path(dir, flag_or(args, "advisories", "advisories.json")), adv_text, + caps::kMaxAdvisoriesBytes, error)) { + out.error("release step 3/6 scan: " + error); + return ExitCode::Runtime; + } + std::vector findings; + if (!scan::VulnHook::parse_advisories(adv_text, findings, error)) { + out.error("release step 3/6 scan: " + error); + return ExitCode::Runtime; + } + std::vector exceptions; + std::string exc_flag = flag_or(args, "exceptions", ""); + if (!exc_flag.empty()) { + std::string exc_text; + if (!read_file_capped(join_path(dir, exc_flag), exc_text, caps::kMaxAdvisoriesBytes, error) || + !scan::VulnHook::parse_exceptions(exc_text, exceptions, error)) { + out.error("release step 3/6 scan: " + error); + return ExitCode::Runtime; + } + } + std::string reason; + if (scan::VulnHook::evaluate(findings, exceptions, now_unix(), false, reason) == + scan::VulnHook::Verdict::Blocked) { + out.error("release step 3/6 scan: " + reason); + return ExitCode::Runtime; + } + out.progress("release 3/6 scan: " + reason); + } + + // Step 4: provenance plus attestation over the emitted SBOM files. + std::string statement_path = join_path(dir, "provenance.json"); + { + std::string digest_spdx, digest_cdx; + if (!molo::embed::Sha256::hex_of_file(spdx_path, digest_spdx) || + !molo::embed::Sha256::hex_of_file(cdx_path, digest_cdx)) { + out.error("release step 4/6 provenance: cannot hash sbom files"); + return ExitCode::Runtime; + } + std::vector subjects = { + {"sbom.spdx.json", digest_spdx}, + {"sbom.cdx.json", digest_cdx}, + }; + std::vector tool_names; + for (const auto& t : doc.toolchain) tool_names.push_back(t.name + "@" + t.version); + std::string predicate = attest::ProvenanceBuilder::build( + "https://moloapp.dev/builder/v1", commit, tool_names, subjects, "molo release"); + std::string statement; + if (!attest::Attestor::statement(subjects, caps::kSlsaPredicateType, predicate, statement, + error)) { + out.error("release step 4/6 provenance: " + error); + return ExitCode::Runtime; + } + if (!write_file(statement_path, statement, error)) { + out.error("release step 4/6 provenance: " + error); + return ExitCode::Runtime; + } + out.progress("release 4/6 provenance: written"); + } + + // Step 5: sign the statement bundle. + { + std::string bundle_path = join_path(dir, "bundle.json"); + if (!attest::Signer::sign(statement_path, bundle_path, error)) { + out.error("release step 5/6 sign: " + error); + return ExitCode::Runtime; + } + out.progress("release 5/6 sign: bundle written"); + } + + // Step 6: tag stamp. + { + std::string tag = std::string(molo::kMoloVersion) + " " + commit + "\n"; + if (!write_file(join_path(dir, flag_or(args, "tag-file", "TAG")), tag, error)) { + out.error("release step 6/6 tag: " + error); + return ExitCode::Runtime; + } + out.progress("release 6/6 tag: written"); + } + + emit_status(out, json, "released", commit, 6); + return ExitCode::Ok; +} + +} // namespace molo::supply::cli diff --git a/src/supplychain/cli/SupplyCli.hpp b/src/supplychain/cli/SupplyCli.hpp new file mode 100644 index 0000000..4225918 --- /dev/null +++ b/src/supplychain/cli/SupplyCli.hpp @@ -0,0 +1,44 @@ +#pragma once + +#if __has_include("cli/cli/exit_code.hpp") +#include "cli/cli/exit_code.hpp" +#elif __has_include("cli/exit_code.hpp") +#include "cli/exit_code.hpp" +#else +#include "../adapters/cli/cli/exit_code.hpp" // ADAPTER fallback for isolated builds +#endif + +#if __has_include("cli/cli/arg_parser.hpp") +#include "cli/cli/arg_parser.hpp" +#elif __has_include("cli/arg_parser.hpp") +#include "cli/arg_parser.hpp" +#else +#include "../adapters/cli/cli/arg_parser.hpp" // ADAPTER fallback for isolated builds +#endif + +#if __has_include("cli/cli/output.hpp") +#include "cli/cli/output.hpp" +#elif __has_include("cli/output.hpp") +#include "cli/output.hpp" +#else +#include "../adapters/cli/cli/output.hpp" // ADAPTER fallback for isolated builds +#endif + +namespace molo::supply::cli { + +// Thin CLI handlers wired by CliApp::build_tree() in the base tree. Each +// handler only parses args, delegates to the policy/sbom/scan/attest classes, +// and renders through Output. No supply logic lives here. +class SupplyCli { + public: + SupplyCli() = delete; + + // `molo verify [dir]`: offline artifact check, exit 0/1/2. + static molo::cli::ExitCode run_verify(const molo::cli::ParsedArgs& args, molo::cli::Output& out); + // `molo supply-check [dir]`: pin plus allow-list plus advisory gate. + static molo::cli::ExitCode run_check(const molo::cli::ParsedArgs& args, molo::cli::Output& out); + // `molo release --commit --suite-stamp `: ordered release gate. + static molo::cli::ExitCode run_release(const molo::cli::ParsedArgs& args, molo::cli::Output& out); +}; + +} // namespace molo::supply::cli diff --git a/src/supplychain/common/Json.cpp b/src/supplychain/common/Json.cpp new file mode 100644 index 0000000..988e991 --- /dev/null +++ b/src/supplychain/common/Json.cpp @@ -0,0 +1,368 @@ +#include "../common/Json.hpp" + +#include +#include + +#include "../caps.hpp" + +namespace molo::supply::common { + +namespace { + +class Parser { + public: + explicit Parser(std::string_view text) : text_(text) {} + + bool run(Json::Value& out, std::string& error) { + skip_ws(); + if (!parse_value(out, 0, error)) return false; + skip_ws(); + if (pos_ != text_.size()) { + error = "trailing characters at offset " + std::to_string(pos_); + return false; + } + return true; + } + + private: + bool parse_value(Json::Value& out, std::size_t depth, std::string& error) { + if (depth > caps::kMaxJsonDepth) { + error = "nesting over cap"; + return false; + } + if (pos_ >= text_.size()) { + error = "unexpected end of input"; + return false; + } + char c = text_[pos_]; + if (c == '{') return parse_object(out, depth, error); + if (c == '[') return parse_array(out, depth, error); + if (c == '"') { + out.type = Json::Value::Type::String; + return parse_string(out.str, error); + } + if (c == 't' || c == 'f') return parse_bool(out, error); + if (c == 'n') return parse_null(out, error); + if (c == '-' || (c >= '0' && c <= '9')) return parse_number(out, error); + error = std::string("unexpected character '") + c + "' at offset " + std::to_string(pos_); + return false; + } + + bool parse_object(Json::Value& out, std::size_t depth, std::string& error) { + ++pos_; // consume '{' + out.type = Json::Value::Type::Object; + skip_ws(); + if (peek('}')) { + ++pos_; + return true; + } + while (true) { + skip_ws(); + if (pos_ >= text_.size() || text_[pos_] != '"') { + error = "object key must be a string at offset " + std::to_string(pos_); + return false; + } + std::string key; + if (!parse_string(key, error)) return false; + skip_ws(); + if (!peek(':')) { + error = "expected ':' at offset " + std::to_string(pos_); + return false; + } + ++pos_; + skip_ws(); + Json::Value v; + if (!parse_value(v, depth + 1, error)) return false; + out.fields[key] = v; + skip_ws(); + if (peek(',')) { + ++pos_; + continue; + } + if (peek('}')) { + ++pos_; + return true; + } + error = "expected ',' or '}' at offset " + std::to_string(pos_); + return false; + } + } + + bool parse_array(Json::Value& out, std::size_t depth, std::string& error) { + ++pos_; // consume '[' + out.type = Json::Value::Type::Array; + skip_ws(); + if (peek(']')) { + ++pos_; + return true; + } + while (true) { + skip_ws(); + Json::Value v; + if (!parse_value(v, depth + 1, error)) return false; + out.items.push_back(v); + skip_ws(); + if (peek(',')) { + ++pos_; + continue; + } + if (peek(']')) { + ++pos_; + return true; + } + error = "expected ',' or ']' at offset " + std::to_string(pos_); + return false; + } + } + + bool parse_string(std::string& out, std::string& error) { + ++pos_; // consume opening quote + while (pos_ < text_.size()) { + char c = text_[pos_++]; + if (c == '"') return true; + if (c != '\\') { + out.push_back(c); + continue; + } + if (pos_ >= text_.size()) break; + char e = text_[pos_++]; + switch (e) { + case '"': out.push_back('"'); break; + case '\\': out.push_back('\\'); break; + case '/': out.push_back('/'); break; + case 'b': out.push_back('\b'); break; + case 'f': out.push_back('\f'); break; + case 'n': out.push_back('\n'); break; + case 'r': out.push_back('\r'); break; + case 't': out.push_back('\t'); break; + case 'u': { + if (pos_ + 4 > text_.size()) { + error = "bad unicode escape"; + return false; + } + unsigned code = 0; + for (int i = 0; i < 4; ++i) { + char h = text_[pos_++]; + code <<= 4; + if (h >= '0' && h <= '9') code |= static_cast(h - '0'); + else if (h >= 'a' && h <= 'f') code |= static_cast(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') code |= static_cast(h - 'A' + 10); + else { + error = "bad unicode escape"; + return false; + } + } + if (code < 0x80) { + out.push_back(static_cast(code)); + } else if (code < 0x800) { + out.push_back(static_cast(0xC0 | (code >> 6))); + out.push_back(static_cast(0x80 | (code & 0x3F))); + } else { + out.push_back(static_cast(0xE0 | (code >> 12))); + out.push_back(static_cast(0x80 | ((code >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (code & 0x3F))); + } + break; + } + default: + error = std::string("bad escape '\\") + e + "'"; + return false; + } + } + error = "unterminated string"; + return false; + } + + bool parse_bool(Json::Value& out, std::string& error) { + if (consume("true")) { + out.type = Json::Value::Type::Bool; + out.boolean = true; + return true; + } + if (consume("false")) { + out.type = Json::Value::Type::Bool; + out.boolean = false; + return true; + } + error = "bad literal at offset " + std::to_string(pos_); + return false; + } + + bool parse_null(Json::Value& out, std::string& error) { + if (!consume("null")) { + error = "bad literal at offset " + std::to_string(pos_); + return false; + } + out.type = Json::Value::Type::Null; + return true; + } + + bool parse_number(Json::Value& out, std::string& error) { + std::size_t start = pos_; + if (peek('-')) ++pos_; + if (pos_ >= text_.size()) { + error = "bad number"; + return false; + } + if (text_[pos_] == '0') { + ++pos_; + } else if (text_[pos_] >= '1' && text_[pos_] <= '9') { + while (pos_ < text_.size() && text_[pos_] >= '0' && text_[pos_] <= '9') ++pos_; + } else { + error = "bad number"; + return false; + } + if (peek('.')) { + ++pos_; + if (pos_ >= text_.size() || text_[pos_] < '0' || text_[pos_] > '9') { + error = "bad number"; + return false; + } + while (pos_ < text_.size() && text_[pos_] >= '0' && text_[pos_] <= '9') ++pos_; + } + if (peek('e') || peek('E')) { + ++pos_; + if (peek('+') || peek('-')) ++pos_; + if (pos_ >= text_.size() || text_[pos_] < '0' || text_[pos_] > '9') { + error = "bad number"; + return false; + } + while (pos_ < text_.size() && text_[pos_] >= '0' && text_[pos_] <= '9') ++pos_; + } + out.type = Json::Value::Type::Number; + out.number = std::string(text_.substr(start, pos_ - start)); + return true; + } + + void skip_ws() { + while (pos_ < text_.size()) { + char c = text_[pos_]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + ++pos_; + } else { + return; + } + } + } + + bool peek(char c) const { return pos_ < text_.size() && text_[pos_] == c; } + + bool consume(std::string_view word) { + if (text_.substr(pos_, word.size()) == word) { + pos_ += word.size(); + return true; + } + return false; + } + + std::string_view text_; + std::size_t pos_ = 0; +}; + +} // namespace + +std::string Json::escape(std::string_view in) { + std::string out; + out.reserve(in.size() + 8); + char buf[8]; + for (unsigned char c : in) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) { + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out.push_back(static_cast(c)); + } + } + } + return out; +} + +std::string Json::quote(std::string_view in) { return "\"" + escape(in) + "\""; } + +std::string Json::indent_str(int level) { return std::string(static_cast(level) * 2, ' '); } + +std::string Json::object(const std::map& rendered, int indent) { + if (rendered.empty()) return "{}"; + std::string out = "{\n"; + bool first = true; + for (const auto& [key, value] : rendered) { + if (!first) out += ",\n"; + first = false; + out += indent_str(indent + 1) + quote(key) + ": " + value; + } + out += "\n" + indent_str(indent) + "}"; + return out; +} + +std::string Json::array(const std::vector& rendered, int indent) { + if (rendered.empty()) return "[]"; + std::string out = "[\n"; + bool first = true; + for (const auto& item : rendered) { + if (!first) out += ",\n"; + first = false; + out += indent_str(indent + 1) + item; + } + out += "\n" + indent_str(indent) + "]"; + return out; +} + +bool Json::parse(std::string_view text, Value& out, std::string& error) { + if (text.size() > caps::kMaxJsonBytes) { + error = "json input over 4 MiB cap"; + return false; + } + Parser parser(text); + return parser.run(out, error); +} + +const Json::Value* Json::find(const Value& obj, std::string_view key) { + if (obj.type != Value::Type::Object) return nullptr; + auto it = obj.fields.find(std::string(key)); + if (it == obj.fields.end()) return nullptr; + return &it->second; +} + +std::string Json::string_or(const Value& v, const std::string& fallback) { + if (v.type == Value::Type::String) return v.str; + if (v.type == Value::Type::Number) return v.number; + return fallback; +} + +double Json::number_or(const Value& v, double fallback) { + if (v.type == Value::Type::Number) { + try { + return std::stod(v.number); + } catch (...) { + return fallback; + } + } + if (v.type == Value::Type::String) { + try { + return std::stod(v.str); + } catch (...) { + return fallback; + } + } + return fallback; +} + +bool Json::is_hex64(std::string_view s) { + if (s.size() != 64) return false; + for (char c : s) { + bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + if (!hex) return false; + } + return true; +} + +} // namespace molo::supply::common diff --git a/src/supplychain/common/Json.hpp b/src/supplychain/common/Json.hpp new file mode 100644 index 0000000..4390064 --- /dev/null +++ b/src/supplychain/common/Json.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace molo::supply::common { + +// Minimal JSON parser plus canonical renderer (sorted keys, LF, 2 spaces). +// Only dependency-free stdlib code; one shared home so no other supply file +// parses or escapes JSON on its own. +class Json { + public: + struct Value { + enum class Type { Null, Bool, Number, String, Array, Object }; + Type type = Type::Null; + bool boolean = false; + std::string number; + std::string str; + std::vector items; + std::map fields; + }; + + Json() = delete; + + static std::string escape(std::string_view in); + static std::string quote(std::string_view in); + static std::string indent_str(int level); + + // Render an object from pre-rendered values. Keys sort via std::map. + static std::string object(const std::map& rendered, int indent); + static std::string array(const std::vector& rendered, int indent); + + static bool parse(std::string_view text, Value& out, std::string& error); + + static const Value* find(const Value& obj, std::string_view key); + static std::string string_or(const Value& v, const std::string& fallback); + static double number_or(const Value& v, double fallback); + static bool is_hex64(std::string_view s); +}; + +} // namespace molo::supply::common diff --git a/src/supplychain/policy/DependencyPolicy.hpp b/src/supplychain/policy/DependencyPolicy.hpp new file mode 100644 index 0000000..92792ec --- /dev/null +++ b/src/supplychain/policy/DependencyPolicy.hpp @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +namespace molo::supply::policy { + +// Parsed allow-list of third-party dependencies from supply.allow.json. +// Reads the top-level "dependencies" array; the "wolfssl" pin object belongs +// to WolfPin. Header-only: parsing is small and has no platform dependency. +class DependencyPolicy { + public: + struct Entry { + std::string name; + std::string version; + std::string sha256; + std::string source; + std::string license; + }; + + bool parse(std::string_view text, std::string& error) { + entries_.clear(); + if (text.size() > caps::kMaxPinJsonBytes) { + error = "allow-list over 64 KiB cap"; + return false; + } + common::Json::Value root; + if (!common::Json::parse(text, root, error)) return false; + if (root.type != common::Json::Value::Type::Object) { + error = "allow-list root must be an object"; + return false; + } + const common::Json::Value* deps = common::Json::find(root, "dependencies"); + if (deps == nullptr) return true; // no generic entries: wolfssl-only file + if (deps->type != common::Json::Value::Type::Array) { + error = "\"dependencies\" must be an array"; + return false; + } + for (const auto& item : deps->items) { + if (entries_.size() >= caps::kMaxComponents) { + error = "dependency count over 64 cap"; + return false; + } + if (item.type != common::Json::Value::Type::Object) { + error = "dependency entry must be an object"; + return false; + } + Entry e; + const common::Json::Value* v = nullptr; + v = common::Json::find(item, "name"); + e.name = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + v = common::Json::find(item, "version"); + e.version = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + v = common::Json::find(item, "sha256"); + e.sha256 = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + v = common::Json::find(item, "source"); + e.source = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + v = common::Json::find(item, "license"); + e.license = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + if (e.name.empty() || e.version.empty() || e.sha256.empty() || e.source.empty() || + e.license.empty()) { + error = "dependency entry missing name/version/sha256/source/license"; + return false; + } + if (!common::Json::is_hex64(e.sha256)) { + error = "dependency '" + e.name + "' sha256 is not 64 hex chars"; + return false; + } + entries_.push_back(e); + } + return true; + } + + const std::vector& entries() const { return entries_; } + std::size_t size() const { return entries_.size(); } + + const Entry* find(const std::string& name) const { + for (const auto& e : entries_) { + if (e.name == name) return &e; + } + return nullptr; + } + + private: + std::vector entries_; +}; + +} // namespace molo::supply::policy diff --git a/src/supplychain/policy/WolfPin.cpp b/src/supplychain/policy/WolfPin.cpp new file mode 100644 index 0000000..960f3e1 --- /dev/null +++ b/src/supplychain/policy/WolfPin.cpp @@ -0,0 +1,91 @@ +#include "../policy/WolfPin.hpp" + +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +namespace molo::supply::policy { + +bool WolfPin::parse_pin(std::string_view text, Pin& out, std::string& error) { + out = Pin{}; + if (text.size() > caps::kMaxPinJsonBytes) { + error = "pin json over 64 KiB cap"; + return false; + } + common::Json::Value root; + if (!common::Json::parse(text, root, error)) return false; + if (root.type != common::Json::Value::Type::Object) { + error = "pin root must be an object"; + return false; + } + const common::Json::Value* node = common::Json::find(root, "wolfssl"); + if (node == nullptr || node->type != common::Json::Value::Type::Object) { + error = "missing \"wolfssl\" pin object"; + return false; + } + const common::Json::Value* v = nullptr; + v = common::Json::find(*node, "version"); + out.version = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + v = common::Json::find(*node, "sha256"); + out.sha256 = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + v = common::Json::find(*node, "source"); + out.source = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + v = common::Json::find(*node, "license"); + out.license = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + v = common::Json::find(*node, "receipt"); + out.receipt = (v == nullptr) ? "" : common::Json::string_or(*v, ""); + if (out.version.empty() || out.sha256.empty() || out.source.empty() || out.license.empty() || + out.receipt.empty()) { + error = "wolfssl pin missing version/sha256/source/license/receipt"; + return false; + } + if (!common::Json::is_hex64(out.sha256)) { + error = "wolfssl pin sha256 is not 64 hex chars"; + return false; + } + return true; +} + +bool WolfPin::load_file(const std::string& path, Pin& out, std::string& error) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) { + error = "cannot open pin file '" + path + "'"; + return false; + } + std::string text; + char chunk[4096]; + std::size_t n = 0; + bool ok = true; + while ((n = std::fread(chunk, 1, sizeof(chunk), f)) > 0) { + if (text.size() + n > caps::kMaxPinJsonBytes) { + ok = false; + break; + } + text.append(chunk, n); + } + if (std::ferror(f) != 0) ok = false; + std::fclose(f); + if (!ok) { + error = "cannot read pin file '" + path + "' (over 64 KiB cap or I/O error)"; + return false; + } + return parse_pin(text, out, error); +} + +WolfPin::Check WolfPin::check(const Pin& pin, std::string_view version, std::string_view sha256) { + if (version != pin.version) return Check::VersionMismatch; + if (sha256 != pin.sha256) return Check::HashMismatch; + return Check::Ok; +} + +const char* WolfPin::to_string(Check c) { + switch (c) { + case Check::Ok: return "ok"; + case Check::VersionMismatch: return "version-mismatch"; + case Check::HashMismatch: return "hash-mismatch"; + } + return "unknown"; +} + +} // namespace molo::supply::policy diff --git a/src/supplychain/policy/WolfPin.hpp b/src/supplychain/policy/WolfPin.hpp new file mode 100644 index 0000000..7f2db6a --- /dev/null +++ b/src/supplychain/policy/WolfPin.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +namespace molo::supply::policy { + +// The pinned wolfSSL release: the sole third-party dependency. Parsed from +// the "wolfssl" object of supply.allow.json: +// {"wolfssl": {"version": "X.Y.Z", "sha256": "<64hex>", "source": "...", +// "license": "...", "receipt": "..."}}. +class WolfPin { + public: + struct Pin { + std::string version; + std::string sha256; + std::string source; + std::string license; + std::string receipt; + }; + + enum class Check { Ok, VersionMismatch, HashMismatch }; + + WolfPin() = delete; + + static bool parse_pin(std::string_view text, Pin& out, std::string& error); + static bool load_file(const std::string& path, Pin& out, std::string& error); + static Check check(const Pin& pin, std::string_view version, std::string_view sha256); + static const char* to_string(Check c); +}; + +} // namespace molo::supply::policy diff --git a/src/supplychain/sbom/CycloneDxEmitter.cpp b/src/supplychain/sbom/CycloneDxEmitter.cpp new file mode 100644 index 0000000..9d598f3 --- /dev/null +++ b/src/supplychain/sbom/CycloneDxEmitter.cpp @@ -0,0 +1,98 @@ +#include "../sbom/CycloneDxEmitter.hpp" + +#include +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +#if __has_include("cli/embed/sha256.hpp") +#include "cli/embed/sha256.hpp" +#elif __has_include("embed/sha256.hpp") +#include "embed/sha256.hpp" +#else +#include "../adapters/cli/embed/sha256.hpp" // ADAPTER fallback for isolated builds +#endif + +namespace molo::supply::sbom { + +namespace { + +std::string component_json(const SbomComponent& c, int indent) { + using common::Json; + std::map hash; + hash["alg"] = Json::quote("SHA-256"); + hash["content"] = Json::quote(c.sha256); + std::map comp; + comp["bom-ref"] = Json::quote(c.name + "@" + c.version); + comp["hashes"] = Json::array({Json::object(hash, indent + 2)}, indent + 1); + comp["licenses"] = Json::array({Json::object({{"license", Json::object({{"name", Json::quote(c.license)}}, indent + 3)}}, indent + 2)}, indent + 1); + comp["name"] = Json::quote(c.name); + comp["supplier"] = Json::object({{"name", Json::quote(c.supplier)}}, indent + 1); + comp["type"] = Json::quote("library"); + comp["version"] = Json::quote(c.version); + return Json::object(comp, indent); +} + +} // namespace + +bool CycloneDxEmitter::emit_json(const SbomDocument& doc, std::string& out, std::string& error) { + using common::Json; + out.clear(); + if (doc.components.size() > caps::kMaxComponents) { + error = "component count over 64 cap"; + return false; + } + std::vector tools; + for (const auto& t : doc.toolchain) { + std::map tool; + tool["name"] = Json::quote(t.name); + tool["version"] = Json::quote(t.version); + tools.push_back(Json::object(tool, 3)); + } + std::map metadata; + metadata["component"] = Json::object( + {{"bom-ref", Json::quote("moloapp@" + doc.doc_version)}, + {"name", Json::quote("moloapp")}, + {"type", Json::quote("application")}, + {"version", Json::quote(doc.doc_version)}}, + 2); + metadata["timestamp"] = Json::quote(doc.created_iso8601); + metadata["tools"] = Json::object({{"components", Json::array(tools, 3)}}, 2); + + std::vector components; + for (const auto& c : doc.components) components.push_back(component_json(c, 2)); + + const std::string seed = molo::embed::Sha256::hex_of_string("moloapp-cdx:" + doc.commit); + std::map root; + root["bomFormat"] = Json::quote("CycloneDX"); + root["components"] = Json::array(components, 1); + root["metadata"] = Json::object(metadata, 1); + root["serialNumber"] = Json::quote("urn:uuid:00000000-0000-4000-8000-" + seed.substr(0, 12)); + root["specVersion"] = Json::quote("1.6"); + root["version"] = "1"; + out = Json::object(root, 0) + "\n"; + if (out.size() > caps::kMaxSbomBytes) { + out.clear(); + error = "CycloneDX output over 1 MiB cap"; + return false; + } + return true; +} + +bool CycloneDxEmitter::emit_file(const SbomDocument& doc, const std::string& path, + std::string& error) { + std::string text; + if (!emit_json(doc, text, error)) return false; + std::FILE* f = std::fopen(path.c_str(), "wb"); + if (f == nullptr) { + error = "cannot open '" + path + "' for writing"; + return false; + } + std::size_t wrote = std::fwrite(text.data(), 1, text.size(), f); + bool ok = wrote == text.size() && std::fclose(f) == 0; + if (!ok) error = "failed writing '" + path + "'"; + return ok; +} + +} // namespace molo::supply::sbom diff --git a/src/supplychain/sbom/CycloneDxEmitter.hpp b/src/supplychain/sbom/CycloneDxEmitter.hpp new file mode 100644 index 0000000..e6dd43b --- /dev/null +++ b/src/supplychain/sbom/CycloneDxEmitter.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include "../sbom/Types.hpp" + +namespace molo::supply::sbom { + +// CycloneDX 1.6 JSON emitter. Renders the same (name, version, sha256) +// triples as SpdxEmitter from the same document. Canonical form only: keys +// sorted, LF line endings, 2-space indent, single trailing newline. +class CycloneDxEmitter { + public: + CycloneDxEmitter() = delete; + + static bool emit_json(const SbomDocument& doc, std::string& out, std::string& error); + static bool emit_file(const SbomDocument& doc, const std::string& path, std::string& error); +}; + +} // namespace molo::supply::sbom diff --git a/src/supplychain/sbom/SbomCheck.cpp b/src/supplychain/sbom/SbomCheck.cpp new file mode 100644 index 0000000..3f92a64 --- /dev/null +++ b/src/supplychain/sbom/SbomCheck.cpp @@ -0,0 +1,23 @@ +#include "../sbom/SbomCheck.hpp" + +#include "../common/Json.hpp" + +namespace molo::supply::sbom { + +bool SbomCheck::check_ntia(const SbomDocument& doc, std::vector& missing, + std::string& error) { + missing.clear(); + error.clear(); + if (doc.commit.empty()) missing.push_back("document.commit"); + for (std::size_t i = 0; i < doc.components.size(); ++i) { + const SbomComponent& c = doc.components[i]; + std::string base = "components[" + std::to_string(i) + "]"; + if (c.name.empty()) missing.push_back(base + ".name"); + if (c.version.empty()) missing.push_back(base + ".version"); + if (!common::Json::is_hex64(c.sha256)) missing.push_back(base + ".sha256"); + if (c.supplier.empty() && c.source.empty()) missing.push_back(base + ".supplier"); + } + return missing.empty(); +} + +} // namespace molo::supply::sbom diff --git a/src/supplychain/sbom/SbomCheck.hpp b/src/supplychain/sbom/SbomCheck.hpp new file mode 100644 index 0000000..982d8b7 --- /dev/null +++ b/src/supplychain/sbom/SbomCheck.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +#include "../sbom/Types.hpp" + +namespace molo::supply::sbom { + +// NTIA minimum-elements check: every component needs a supplier, a name, a +// version and a unique identifier (sha256); the document needs a commit. +class SbomCheck { + public: + SbomCheck() = delete; + + static bool check_ntia(const SbomDocument& doc, std::vector& missing, + std::string& error); +}; + +} // namespace molo::supply::sbom diff --git a/src/supplychain/sbom/SbomGenerator.cpp b/src/supplychain/sbom/SbomGenerator.cpp new file mode 100644 index 0000000..efcb06f --- /dev/null +++ b/src/supplychain/sbom/SbomGenerator.cpp @@ -0,0 +1,122 @@ +#include "../sbom/SbomGenerator.hpp" + +#include +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +#if __has_include("cli/embed/sha256.hpp") +#include "cli/embed/sha256.hpp" +#elif __has_include("embed/sha256.hpp") +#include "embed/sha256.hpp" +#else +#include "../adapters/cli/embed/sha256.hpp" // ADAPTER fallback for isolated builds +#endif + +namespace molo::supply::sbom { + +bool SbomGenerator::generate(std::string_view commit, + const std::vector& toolchain, + const policy::WolfPin::Pin& pin, + const policy::DependencyPolicy& manifest, + std::string_view molo_version, + std::string_view created_iso8601, + SbomDocument& out, + std::string& error) { + out = SbomDocument{}; + if (commit.empty() || commit.size() > caps::kMaxCommitChars) { + error = "commit must be 1..128 chars"; + return false; + } + if (molo_version.empty()) { + error = "molo version must not be empty"; + return false; + } + if (created_iso8601.empty()) { + error = "created timestamp must not be empty"; + return false; + } + if (toolchain.size() > caps::kMaxToolchainEntries) { + error = "toolchain over 100 entries cap"; + return false; + } + std::size_t toolchain_bytes = 0; + for (const auto& t : toolchain) { + toolchain_bytes += t.name.size() + t.version.size(); + if (t.name.empty() || t.version.empty()) { + error = "toolchain entry needs name and version"; + return false; + } + } + if (toolchain_bytes > caps::kMaxToolchainBytes) { + error = "toolchain over 64 KiB cap"; + return false; + } + if (pin.version.empty() || !common::Json::is_hex64(pin.sha256)) { + error = "wolfssl pin invalid"; + return false; + } + + out.doc_name = "moloapp-sbom"; + out.doc_version = std::string(molo_version); + out.commit = std::string(commit); + out.created_iso8601 = std::string(created_iso8601); + out.toolchain = toolchain; + + SbomComponent self; + self.name = "moloapp"; + self.version = std::string(molo_version); + self.sha256 = + molo::embed::Sha256::hex_of_string("moloapp@" + std::string(molo_version) + "@" + out.commit); + self.source = "https://moloapp.dev"; + self.license = "proprietary"; + self.supplier = "moloapp"; + out.components.push_back(self); + + SbomComponent wolf; + wolf.name = "wolfssl"; + wolf.version = pin.version; + wolf.sha256 = pin.sha256; + wolf.source = pin.source; + wolf.license = pin.license; + wolf.supplier = "wolfSSL Inc."; + out.components.push_back(wolf); + + for (const auto& e : manifest.entries()) { + if (e.name == "wolfssl" || e.name == "moloapp") continue; // pin and self win + if (out.components.size() >= caps::kMaxComponents) { + error = "component count over 64 cap"; + return false; + } + SbomComponent c; + c.name = e.name; + c.version = e.version; + c.sha256 = e.sha256; + c.source = e.source; + c.license = e.license; + c.supplier = e.source; + out.components.push_back(c); + } + if (1 + out.components.size() > caps::kMaxRelationships) { + error = "relationship count over 256 cap"; + return false; + } + return true; +} + +std::string SbomGenerator::utc_now() { + const auto now = std::chrono::system_clock::now(); + std::time_t t = std::chrono::system_clock::to_time_t(now); + std::tm tm{}; +#if defined(_WIN32) + gmtime_s(&tm, &t); +#else + gmtime_r(&t, &tm); +#endif + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &tm); + return std::string(buf); +} + +} // namespace molo::supply::sbom diff --git a/src/supplychain/sbom/SbomGenerator.hpp b/src/supplychain/sbom/SbomGenerator.hpp new file mode 100644 index 0000000..1682ba2 --- /dev/null +++ b/src/supplychain/sbom/SbomGenerator.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +#include "../policy/DependencyPolicy.hpp" +#include "../policy/WolfPin.hpp" +#include "../sbom/Types.hpp" + +namespace molo::supply::sbom { + +// Pure SBOM builder: commit plus toolchain plus wolfSSL pin plus allow-list +// manifest in, validated SbomDocument out. No I/O, no clock (callers pass the +// timestamp; utc_now() is provided for CLI use). +class SbomGenerator { + public: + SbomGenerator() = delete; + + static bool generate(std::string_view commit, + const std::vector& toolchain, + const policy::WolfPin::Pin& pin, + const policy::DependencyPolicy& manifest, + std::string_view molo_version, + std::string_view created_iso8601, + SbomDocument& out, + std::string& error); + + static std::string utc_now(); +}; + +} // namespace molo::supply::sbom diff --git a/src/supplychain/sbom/SpdxEmitter.cpp b/src/supplychain/sbom/SpdxEmitter.cpp new file mode 100644 index 0000000..b9132ac --- /dev/null +++ b/src/supplychain/sbom/SpdxEmitter.cpp @@ -0,0 +1,97 @@ +#include "../sbom/SpdxEmitter.hpp" + +#include +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +namespace molo::supply::sbom { + +namespace { + +std::string package_json(const SbomComponent& c, const std::string& id, int indent) { + using common::Json; + std::map checksum; + checksum["algorithm"] = Json::quote("SHA256"); + checksum["checksumValue"] = Json::quote(c.sha256); + std::map ext_ref; + ext_ref["category"] = Json::quote("PACKAGE-MANAGER"); + ext_ref["locator"] = Json::quote(c.source); + ext_ref["referenceType"] = Json::quote("purl"); + std::map pkg; + pkg["SPDXID"] = Json::quote(id); + pkg["checksums"] = Json::array({Json::object(checksum, indent + 2)}, indent + 1); + pkg["downloadLocation"] = Json::quote(c.source); + pkg["externalRefs"] = Json::array({Json::object(ext_ref, indent + 2)}, indent + 1); + pkg["licenseDeclared"] = Json::quote(c.license); + pkg["name"] = Json::quote(c.name); + pkg["supplier"] = Json::quote("Organization: " + c.supplier); + pkg["versionInfo"] = Json::quote(c.version); + return Json::object(pkg, indent); +} + +} // namespace + +bool SpdxEmitter::emit_json(const SbomDocument& doc, std::string& out, std::string& error) { + using common::Json; + out.clear(); + if (doc.components.size() > caps::kMaxComponents) { + error = "component count over 64 cap"; + return false; + } + std::map creation; + creation["created"] = Json::quote(doc.created_iso8601); + creation["creators"] = Json::array({Json::quote("Tool: moloapp-" + doc.doc_version)}, 2); + + std::vector packages; + std::vector relationships; + for (std::size_t i = 0; i < doc.components.size(); ++i) { + const SbomComponent& c = doc.components[i]; + std::string id = "SPDXRef-Package-" + c.name; + packages.push_back(package_json(c, id, 2)); + std::map rel; + rel["relatedSpdxElement"] = Json::quote(id); + rel["relationshipType"] = Json::quote("CONTAINS"); + rel["spdxElementId"] = Json::quote("SPDXRef-DOCUMENT"); + relationships.push_back(Json::object(rel, 2)); + } + if (1 + relationships.size() > caps::kMaxRelationships) { + error = "relationship count over 256 cap"; + return false; + } + + std::map root; + root["SPDXID"] = Json::quote("SPDXRef-DOCUMENT"); + root["creationInfo"] = Json::object(creation, 1); + root["dataLicense"] = Json::quote("CC0-1.0"); + root["documentDescribes"] = Json::array({Json::quote("SPDXRef-DOCUMENT")}, 1); + root["documentNamespace"] = Json::quote("https://moloapp.dev/sbom/" + doc.commit); + root["name"] = Json::quote(doc.doc_name); + root["packages"] = Json::array(packages, 1); + root["relationships"] = Json::array(relationships, 1); + root["spdxVersion"] = Json::quote("SPDX-2.3"); + out = Json::object(root, 0) + "\n"; + if (out.size() > caps::kMaxSbomBytes) { + out.clear(); + error = "SPDX output over 1 MiB cap"; + return false; + } + return true; +} + +bool SpdxEmitter::emit_file(const SbomDocument& doc, const std::string& path, std::string& error) { + std::string text; + if (!emit_json(doc, text, error)) return false; + std::FILE* f = std::fopen(path.c_str(), "wb"); + if (f == nullptr) { + error = "cannot open '" + path + "' for writing"; + return false; + } + std::size_t wrote = std::fwrite(text.data(), 1, text.size(), f); + bool ok = wrote == text.size() && std::fclose(f) == 0; + if (!ok) error = "failed writing '" + path + "'"; + return ok; +} + +} // namespace molo::supply::sbom diff --git a/src/supplychain/sbom/SpdxEmitter.hpp b/src/supplychain/sbom/SpdxEmitter.hpp new file mode 100644 index 0000000..f4f6d56 --- /dev/null +++ b/src/supplychain/sbom/SpdxEmitter.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include + +#include "../sbom/Types.hpp" + +namespace molo::supply::sbom { + +// SPDX 2.3 JSON emitter. Canonical form only: keys sorted, LF line endings, +// 2-space indent, single trailing newline. Rejects output over the 1 MiB cap. +class SpdxEmitter { + public: + SpdxEmitter() = delete; + + static bool emit_json(const SbomDocument& doc, std::string& out, std::string& error); + static bool emit_file(const SbomDocument& doc, const std::string& path, std::string& error); +}; + +} // namespace molo::supply::sbom diff --git a/src/supplychain/sbom/Types.hpp b/src/supplychain/sbom/Types.hpp new file mode 100644 index 0000000..268ed80 --- /dev/null +++ b/src/supplychain/sbom/Types.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +namespace molo::supply::sbom { + +// Shared SBOM value types. Both emitters (SPDX, CycloneDX) render the same +// document so the (name, version, sha256) triples always match. +struct SbomComponent { + std::string name; + std::string version; + std::string sha256; + std::string source; + std::string license; + std::string supplier; +}; + +struct SbomToolchainEntry { + std::string name; + std::string version; +}; + +struct SbomDocument { + std::string doc_name; + std::string doc_version; + std::string commit; + std::string created_iso8601; + std::vector toolchain; + std::vector components; +}; + +} // namespace molo::supply::sbom diff --git a/src/supplychain/scan/OsvClient.cpp b/src/supplychain/scan/OsvClient.cpp new file mode 100644 index 0000000..39442b5 --- /dev/null +++ b/src/supplychain/scan/OsvClient.cpp @@ -0,0 +1,130 @@ +#include "../scan/OsvClient.hpp" + +#include + +#include "../caps.hpp" +#include "../common/Json.hpp" + +namespace molo::supply::scan { + +OsvClient::OsvClient(Transport transport) : transport_(std::move(transport)) {} + +std::string OsvClient::request_body(const std::string& package, const std::string& version) { + using common::Json; + std::map pkg; + pkg["ecosystem"] = Json::quote("OSS-Fuzz"); + pkg["name"] = Json::quote(package); + std::map root; + root["package"] = Json::object(pkg, 1); + root["version"] = Json::quote(version); + return Json::object(root, 0); +} + +bool OsvClient::parse_response(const std::string& package, const std::string& text, + std::vector& findings, std::string& error) { + findings.clear(); + if (text.size() > caps::kMaxAdvisoriesBytes) { + error = "osv response over 256 KiB cap"; + return false; + } + common::Json::Value root; + if (!common::Json::parse(text, root, error)) return false; + if (root.type != common::Json::Value::Type::Object) { + error = "osv response root must be an object"; + return false; + } + const common::Json::Value* vulns = common::Json::find(root, "vulns"); + if (vulns == nullptr) return true; // no vulns key means clean + if (vulns->type != common::Json::Value::Type::Array) { + error = "\"vulns\" must be an array"; + return false; + } + for (const auto& item : vulns->items) { + if (findings.size() >= caps::kMaxOsvVulns) { + error = "osv vuln count over 64 cap"; + return false; + } + if (item.type != common::Json::Value::Type::Object) continue; + Vuln v; + v.package = package; + const common::Json::Value* f = common::Json::find(item, "id"); + v.id = (f == nullptr) ? "" : common::Json::string_or(*f, ""); + f = common::Json::find(item, "summary"); + if (f == nullptr) f = common::Json::find(item, "details"); + v.summary = (f == nullptr) ? "" : common::Json::string_or(*f, ""); + const common::Json::Value* sev = common::Json::find(item, "severity"); + if (sev != nullptr && sev->type == common::Json::Value::Type::Array) { + for (const auto& s : sev->items) { + if (s.type != common::Json::Value::Type::Object) continue; + const common::Json::Value* score = common::Json::find(s, "score"); + if (score != nullptr) { + double d = common::Json::number_or(*score, 0.0); + if (d > v.cvss) v.cvss = d; + } + } + } + const common::Json::Value* affected = common::Json::find(item, "affected"); + if (affected != nullptr && affected->type == common::Json::Value::Type::Array) { + for (const auto& a : affected->items) { + if (a.type != common::Json::Value::Type::Object) continue; + const common::Json::Value* ranges = common::Json::find(a, "ranges"); + if (ranges == nullptr || ranges->type != common::Json::Value::Type::Array) continue; + for (const auto& r : ranges->items) { + if (r.type != common::Json::Value::Type::Object) continue; + const common::Json::Value* events = common::Json::find(r, "events"); + if (events == nullptr || events->type != common::Json::Value::Type::Array) continue; + for (const auto& e : events->items) { + if (e.type != common::Json::Value::Type::Object) continue; + const common::Json::Value* fixed = common::Json::find(e, "fixed"); + if (fixed != nullptr && v.fixed_version.empty()) { + v.fixed_version = common::Json::string_or(*fixed, ""); + } + } + } + } + } + if (v.id.empty()) { + error = "osv vuln entry missing id"; + return false; + } + findings.push_back(v); + } + return true; +} + +bool OsvClient::query(const std::string& package, const std::string& version, + std::vector& findings, std::string& error) { + findings.clear(); + if (package.empty() || version.empty()) { + error = "package and version must not be empty"; + return false; + } + if (queries_made_ >= caps::kOsvMaxQueries) { + error = "osv query count over 64 cap"; + return false; + } + const std::string body = request_body(package, version); + std::string response; + std::string last_error = "offline: no transport configured"; + for (int attempt = 0; attempt <= caps::kOsvMaxRetries; ++attempt) { + ++queries_made_; + if (queries_made_ > caps::kOsvMaxQueries) { + error = "osv query count over 64 cap"; + return false; + } + response.clear(); + std::string transport_error; + bool ok = false; + if (transport_) { + ok = transport_(caps::kOsvQueryUrl, body, response, transport_error); + } else { + transport_error = "offline: no transport configured"; + } + if (ok) return parse_response(package, response, findings, error); + last_error = transport_error.empty() ? "transport failed" : transport_error; + } + error = last_error; + return false; +} + +} // namespace molo::supply::scan diff --git a/src/supplychain/scan/OsvClient.hpp b/src/supplychain/scan/OsvClient.hpp new file mode 100644 index 0000000..8b0c560 --- /dev/null +++ b/src/supplychain/scan/OsvClient.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include + +namespace molo::supply::scan { + +// Minimal OSV (https://api.osv.dev/v1/query) client. The HTTP transport is +// injected so tests run offline with a fake; the default transport always +// reports offline, so no socket code lives here. Base-tree wiring can inject +// the framework HTTP client later. Capped: 64 queries per instance, 1 retry, +// 10 s timeout is the transport contract (see spec.md). +class OsvClient { + public: + struct Vuln { + std::string id; + std::string summary; + std::string package; + std::string fixed_version; + double cvss = 0.0; + }; + + // (url, request_body, response_out, error_out) -> true on HTTP 200. + using Transport = + std::function; + + explicit OsvClient(Transport transport = {}); + OsvClient(const OsvClient&) = delete; + OsvClient& operator=(const OsvClient&) = delete; + + bool query(const std::string& package, const std::string& version, + std::vector& findings, std::string& error); + + std::size_t queries_made() const { return queries_made_; } + + static std::string request_body(const std::string& package, const std::string& version); + static bool parse_response(const std::string& package, const std::string& text, + std::vector& findings, std::string& error); + + private: + Transport transport_; + std::size_t queries_made_ = 0; +}; + +} // namespace molo::supply::scan diff --git a/src/supplychain/scan/VulnHook.cpp b/src/supplychain/scan/VulnHook.cpp new file mode 100644 index 0000000..2453e68 --- /dev/null +++ b/src/supplychain/scan/VulnHook.cpp @@ -0,0 +1,154 @@ +#include "../scan/VulnHook.hpp" + +#include "../caps.hpp" +#include "../common/Json.hpp" + +namespace molo::supply::scan { + +bool VulnHook::parse_advisories(std::string_view text, std::vector& out, + std::string& error) { + out.clear(); + if (text.size() > caps::kMaxAdvisoriesBytes) { + error = "advisories over 256 KiB cap"; + return false; + } + common::Json::Value root; + if (!common::Json::parse(text, root, error)) return false; + if (root.type != common::Json::Value::Type::Object) { + error = "advisories root must be an object"; + return false; + } + const common::Json::Value* list = common::Json::find(root, "advisories"); + if (list == nullptr) return true; + if (list->type != common::Json::Value::Type::Array) { + error = "\"advisories\" must be an array"; + return false; + } + for (const auto& item : list->items) { + if (out.size() >= caps::kMaxOsvVulns) { + error = "advisory count over 64 cap"; + return false; + } + if (item.type != common::Json::Value::Type::Object) { + error = "advisory entry must be an object"; + return false; + } + OsvClient::Vuln v; + const common::Json::Value* f = nullptr; + f = common::Json::find(item, "id"); + v.id = (f == nullptr) ? "" : common::Json::string_or(*f, ""); + f = common::Json::find(item, "package"); + v.package = (f == nullptr) ? "" : common::Json::string_or(*f, ""); + f = common::Json::find(item, "summary"); + v.summary = (f == nullptr) ? "" : common::Json::string_or(*f, ""); + f = common::Json::find(item, "fixed"); + v.fixed_version = (f == nullptr) ? "" : common::Json::string_or(*f, ""); + f = common::Json::find(item, "cvss"); + v.cvss = (f == nullptr) ? 0.0 : common::Json::number_or(*f, 0.0); + if (v.id.empty() || v.package.empty()) { + error = "advisory entry missing id/package"; + return false; + } + out.push_back(v); + } + return true; +} + +bool VulnHook::parse_exceptions(std::string_view text, std::vector& out, + std::string& error) { + out.clear(); + if (text.size() > caps::kMaxAdvisoriesBytes) { + error = "exceptions over 256 KiB cap"; + return false; + } + common::Json::Value root; + if (!common::Json::parse(text, root, error)) return false; + if (root.type != common::Json::Value::Type::Object) { + error = "exceptions root must be an object"; + return false; + } + const common::Json::Value* list = common::Json::find(root, "exceptions"); + if (list == nullptr) return true; + if (list->type != common::Json::Value::Type::Array) { + error = "\"exceptions\" must be an array"; + return false; + } + for (const auto& item : list->items) { + if (out.size() >= caps::kMaxOsvVulns) { + error = "exception count over 64 cap"; + return false; + } + if (item.type != common::Json::Value::Type::Object) { + error = "exception entry must be an object"; + return false; + } + Exception e; + const common::Json::Value* f = common::Json::find(item, "id"); + e.vuln_id = (f == nullptr) ? "" : common::Json::string_or(*f, ""); + f = common::Json::find(item, "reason"); + e.reason = (f == nullptr) ? "" : common::Json::string_or(*f, ""); + f = common::Json::find(item, "expires"); + e.expires_unix = (f == nullptr) ? 0 : static_cast(common::Json::number_or(*f, 0.0)); + if (e.vuln_id.empty() || e.expires_unix <= 0) { + error = "exception entry missing id/expires"; + return false; + } + out.push_back(e); + } + return true; +} + +bool VulnHook::is_blocking(const OsvClient::Vuln& v) { + if (v.cvss >= caps::kBlockCvss) return true; + return v.package == "wolfssl" && !v.fixed_version.empty(); +} + +VulnHook::Verdict VulnHook::evaluate(const std::vector& findings, + const std::vector& exceptions, long long now_unix, + bool offline_unknown, std::string& reason) { + reason.clear(); + if (offline_unknown && findings.empty()) { + reason = "offline-unknown: cannot prove clean without scan data"; + return Verdict::Blocked; + } + auto covered = [&](const OsvClient::Vuln& v) -> bool { + for (const auto& e : exceptions) { + if (e.vuln_id == v.id && now_unix < e.expires_unix) return true; + } + return false; + }; + auto expired_match = [&](const OsvClient::Vuln& v) -> bool { + for (const auto& e : exceptions) { + if (e.vuln_id == v.id && now_unix >= e.expires_unix) return true; + } + return false; + }; + for (const auto& v : findings) { + if (covered(v)) continue; + if (expired_match(v)) { + reason = "exception-expired:" + v.id; + return Verdict::Blocked; + } + if (is_blocking(v)) { + reason = "vuln-block:" + v.id; + return Verdict::Blocked; + } + } + if (findings.empty()) { + reason = "allow: no findings"; + return Verdict::Clean; + } + reason = "disclosed: " + std::to_string(findings.size()) + " finding(s) below block threshold"; + return Verdict::Disclosed; +} + +const char* VulnHook::to_string(Verdict v) { + switch (v) { + case Verdict::Clean: return "clean"; + case Verdict::Disclosed: return "disclosed"; + case Verdict::Blocked: return "blocked"; + } + return "unknown"; +} + +} // namespace molo::supply::scan diff --git a/src/supplychain/scan/VulnHook.hpp b/src/supplychain/scan/VulnHook.hpp new file mode 100644 index 0000000..9e279db --- /dev/null +++ b/src/supplychain/scan/VulnHook.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +#include "../scan/OsvClient.hpp" + +namespace molo::supply::scan { + +// Release gate over vulnerability findings. Verdicts: +// clean (no findings), disclosed (only sub-threshold findings, or every +// finding covered by an unexpired exception), blocked (any CVSS >= 9.0 +// finding uncovered, any wolfssl finding with a fix uncovered, any matched +// exception expired, or the scan state unknown while offline). +class VulnHook { + public: + enum class Verdict { Clean, Disclosed, Blocked }; + + struct Exception { + std::string vuln_id; + long long expires_unix = 0; + std::string reason; + }; + + VulnHook() = delete; + + static bool parse_advisories(std::string_view text, std::vector& out, + std::string& error); + static bool parse_exceptions(std::string_view text, std::vector& out, + std::string& error); + static Verdict evaluate(const std::vector& findings, + const std::vector& exceptions, long long now_unix, + bool offline_unknown, std::string& reason); + static const char* to_string(Verdict v); + static bool is_blocking(const OsvClient::Vuln& v); +}; + +} // namespace molo::supply::scan diff --git a/src/supplychain/spec.md b/src/supplychain/spec.md new file mode 100644 index 0000000..138e19d --- /dev/null +++ b/src/supplychain/spec.md @@ -0,0 +1,161 @@ +# MoloApp supply-chain part (src/supplychain/): specification + +## 1. Purpose + +Own the software supply chain for MoloApp releases: pinned dependencies, +SBOM generation in two formats, vulnerability gating, SLSA provenance, +signing, and offline verification. Ships three CLI leaves (`verify`, +`supply-check`, `release`) whose handlers live here; the base tree only wires +them into `CliApp::build_tree()` (see PATCHES.md). + +## 2. Rules + +- C++23, standard library plus OS syscalls only. No third-party code. +- One class per file. Namespaces mirror directories under `molo::supply`. +- Every limit is a static cap in `caps.hpp`. Over-cap input is rejected with + a clear error, never silently cut. +- Offline by default: `verify` and the advisory gate use no network. The OSV + client ships with an offline stub transport; base-tree wiring may inject + the framework HTTP client later. +- Only `attest/Signer.cpp` may spawn a shell (the cosign wrapper). +- stdout is primary output via `molo::cli::Output`; diagnostics go to stderr. +- Canonical JSON everywhere: keys sorted, LF endings, 2-space indent, single + trailing newline. + +## 3. Static caps (caps.hpp) + +| Cap | Value | +|---|---| +| Components per SBOM | 64 | +| Relationships per SBOM | 256 | +| SBOM / statement output | 1 MiB | +| Toolchain entries / bytes | 100 / 64 KiB | +| Signed files (subjects) | 32 | +| Pin / allow-list JSON | 64 KiB | +| Advisories / exceptions JSON | 256 KiB | +| OSV queries / retries / timeout | 64 / 1 / 10 s | +| Override TTL reference | 7 days (604800 s) | +| Block CVSS threshold | 9.0 | + +## 4. Components + +### common/Json (Json.hpp, Json.cpp) + +Shared JSON parser (depth cap 32, 4 MiB ceiling) plus canonical renderer +(`object` from pre-rendered values, `array`, `quote`, `escape`) and small +accessors (`find`, `string_or`, `number_or`, `is_hex64`). No other supply +file parses or escapes JSON on its own. + +### policy (DependencyPolicy.hpp, WolfPin.hpp/.cpp) + +- `DependencyPolicy` parses the `dependencies` array of supply.allow.json. + Each entry needs name, version, sha256 (64 hex), source, license. +- `WolfPin` parses the `wolfssl` pin object (version, sha256, source, + license, receipt) and checks candidate version/hash pairs with verdicts + `ok`, `version-mismatch`, `hash-mismatch`. File reads cap at 64 KiB. + +### sbom (Types.hpp, SbomGenerator, SpdxEmitter, CycloneDxEmitter, SbomCheck) + +- `SbomGenerator::generate` is a pure function of (commit, toolchain, pin, + manifest, molo version, created timestamp). Components are always + `moloapp` (build id derived by hashing name/version/commit), `wolfssl` + (from the pin), then manifest entries with `wolfssl`/`moloapp` dups + dropped. `utc_now()` supplies CLI timestamps. +- `SpdxEmitter` writes SPDX 2.3 JSON; `CycloneDxEmitter` writes CycloneDX 1.6 + JSON. Both render the same (name, version, sha256) triples from the same + document; `test_cdx_equiv` proves triple equality. Output over 1 MiB is + rejected. +- `SbomCheck::check_ntia` enforces NTIA minimum elements: commit on the + document; name, version, 64-hex sha256, and supplier/source on every + component. Returns the list of missing `components[i].field` paths. + +### scan (OsvClient.hpp/.cpp, VulnHook.hpp/.cpp) + +- `OsvClient` posts to `https://api.osv.dev/v1/query` through an injected + `Transport`. The default (empty) transport reports offline, so standalone + builds and tests never touch the network. Parses `vulns` entries (id, + summary/details, max severity score, first fixed version). One retry, 64 + queries per instance; the 10 s timeout is a contract on the injected + transport. +- `VulnHook` parses `advisories.json` (`advisories` array of + id/package/cvss/fixed/summary) and `exceptions.json` (`exceptions` array of + id/expires/reason, expires as unix seconds). `evaluate` returns: + - `clean`: no findings and scan state known. + - `disclosed`: only sub-threshold findings, or every finding covered by an + unexpired exception. + - `blocked`: any CVSS >= 9.0 finding uncovered, any wolfssl finding with a + fix uncovered, any matched exception expired, or scan state unknown + while offline. + - Reason strings use the `allow:` / `disclosed:` / `vuln-block:` / + `exception-expired:` / `offline-unknown:` prefixes so base-tree logging + can key off them. + +### attest (ProvenanceBuilder.hpp, Attestor.hpp/.cpp, Signer.hpp/.cpp, Verify.hpp/.cpp) + +- `ProvenanceBuilder::build` returns a canonical SLSA predicate object + (build type `https://moloapp.dev/build/v1`, builder id, commit, + invocation, materials, resolved dependencies). Subjects sort by path. +- `Attestor::statement` wraps a predicate into an in-toto Statement with + predicate type `https://slsa.dev/provenance/v1`, re-indenting the embedded + predicate so the whole statement stays canonical. The 33rd subject is + rejected. +- `Signer::sign` runs `cosign sign-blob --yes --bundle ` + (keyless) with single-quote path escaping and 2 KiB capped failure output. + `MOLO_FAKE_COSIGN=1` writes a deterministic offline fake bundle embedding + the blob digest; tests use the fake. +- `Verify::verify_files` recomputes sha256 for each expected subject via the + embed seam, requires the bundle to contain every digest (chain), and + requires any `wolfssl` subject to equal the pin when a pin is given. + +### cli (SupplyCli.hpp/.cpp) + +Thin `ExitCode` handlers, no supply logic: + +- `run_verify`: `molo verify [dir] [--manifest subjects.txt] + [--bundle bundle.json] [--pin allow]`. Exit 0 ok, 1 failure, 2 usage. + Fully offline. `--format json` prints a status object. +- `run_check`: `molo supply-check [dir] [--allow supply.allow.json] + [--advisories advisories.json] [--exceptions e.json] [--osv]`. Pin plus + allow-list plus advisory gate. `blocked` exits 1; `disclosed` warns and + exits 0; missing advisory source exits 1 (offline sources are required). +- `run_release`: `molo release --commit --suite-stamp + [--dir .] [--allow ...] [--advisories ...] [--exceptions ...] + [--toolchain toolchain.txt] [--tag-file TAG]`. Six ordered steps: suite + stamp is `PASS`; SBOM generate plus NTIA plus emit both formats; advisory + scan (blocked fails); provenance plus attestation; cosign sign; tag stamp + write. Any failure exits 1 naming the step. + +## 5. Base-tree seams consumed + +`molo::cli` (ExitCode, ParsedArgs, Output, JsonWriter::escape/quote/object), +`molo::embed::Sha256` (hex_of_string/hex_of_file), +`molo::kMoloVersion`. Each include prefers the base header via +`__has_include` and falls back to the `adapters/` mirror for isolated +builds. The fragment `src/supplychain.mk` never adds `adapters/` to the +include path, so real builds cannot see the fallbacks. + +The `molo::log::Logger` seam is intentionally not called: the seam handoff +lists the class but not the `Level`/`fields` types, so call sites would be +guesses. Verdict reason strings (section 4, VulnHook) are the logging +contract; base-tree wiring logs them. + +## 6. Tests (tests/src/supplychain/) + +Plain `int main()` plus `CHECK`, no gtest: `test_pin`, `test_spdx`, +`test_cdx_equiv`, `test_scan`, `test_provenance`, `test_verify`, with byte +goldens under `golden/` (sbom pair, provenance, advisories input). Covered +caps: 1 MiB SBOM reject, 33rd subject reject, expired exception blocks, +offline unknown blocks signing, 65th OSV query rejected, pin oversize +rejected. `MOLO_REGEN_GOLDEN=1` regenerates output goldens; advisories.json +is a hand-written input fixture. SHA-256 self-checks against the independent +`sha256("abc")` vector in `test_verify`. + +## 7. Gaps + +- No `plans/15-supply-chain.md` exists in this worktree (plans only go to + 10); this spec plus the seam handoff is the authority. +- `Output::print/info/progress/warn/error/debug` and + `JsonWriter::object(map)` signatures are per the seam + summary; PATCHES.md flags the assumption for the base-tree applier. +- Real cosign and live OSV are integration concerns for the base tree; here + they are covered by the fake bundle and injected fake transport. diff --git a/src/tenancy.mk b/src/tenancy.mk new file mode 100644 index 0000000..c8e2419 --- /dev/null +++ b/src/tenancy.mk @@ -0,0 +1,24 @@ +# Tenancy part fragment. Included by the root Makefile: +# include src/tenancy.mk +# The part is header-only, so there is no library target; tests compile the +# headers directly. Build outputs land under build/ only. + +TENANCY_DIR := src/tenancy +TENANCY_TEST_DIR := tests/src/tenancy +TENANCY_BUILD_DIR := build/tenancy +TENANCY_CXX ?= $(CXX) +TENANCY_CXXFLAGS ?= -std=c++23 -O2 -Wall -Wextra -pthread +TENANCY_TESTS := tenant_id_resolve pool_scope bridge provision quota flags_config metrics +TENANCY_BINS := $(addprefix $(TENANCY_BUILD_DIR)/test_,$(TENANCY_TESTS)) + +tenancy-tests: $(TENANCY_BINS) + +$(TENANCY_BUILD_DIR)/test_%: $(TENANCY_TEST_DIR)/test_%.cpp + mkdir -p $(TENANCY_BUILD_DIR) + $(TENANCY_CXX) $(TENANCY_CXXFLAGS) -Isrc -I$(TENANCY_TEST_DIR) $< -o $@ + +tenancy-check: tenancy-tests + for t in $(TENANCY_BINS); do ./$$t || exit 1; done + +tenancy-clean: + rm -rf $(TENANCY_BUILD_DIR) diff --git a/src/tenancy/PATCHES.md b/src/tenancy/PATCHES.md new file mode 100644 index 0000000..4b4b829 --- /dev/null +++ b/src/tenancy/PATCHES.md @@ -0,0 +1,346 @@ +# Required base-tree edits + +Each section names one repo-relative base file plus unified diff hunks to +apply. The base tree is absent from this worktree, so context lines follow +the seam spec API; apply with fuzz and verify with the full build. After +applying, delete `src/tenancy/adapters/` and swap the adapter type names in +tenancy headers for the real base types (same member shapes). + +## 1. src/data/context.hpp + +Add `TenantContext` (uuid plus slug plus strategy) alongside the tenant +string. Keep the existing constructor signature working. + +```diff +--- a/src/data/context.hpp ++++ b/src/data/context.hpp +@@ -1,6 +1,20 @@ + #pragma once ++#include ++ ++namespace molo::data { ++ ++struct TenantContext { ++ std::string uuid; ++ std::string slug; ++ std::string strategy = "shared_schema"; ++ bool privileged = false; ++}; ++ ++} // namespace molo::data +``` + +```diff +--- a/src/data/context.hpp ++++ b/src/data/context.hpp +@@ -10,6 +24,10 @@ class Context { + Context(std::string correlation_id, std::string tenant, int timeout_sec); ++ Context(std::string correlation_id, TenantContext tenant_ctx, int timeout_sec); + const std::string& tenant() const; ++ const TenantContext& tenant_ctx() const; ++ bool privileged() const; + bool expired() const; +``` + +## 2. src/data/pool.hpp + +Set the tenant scope on pinned acquire, assert it was empty, reset on +release so a checkout can never leak across tenants. + +```diff +--- a/src/data/pool.hpp ++++ b/src/data/pool.hpp +@@ -20,8 +20,13 @@ class Pool { + Checkout acquire_pinned(const Context& ctx) { + Checkout out = acquire_free_or_wait(ctx); ++ if (!out.connection()->scope_tag().empty()) ++ throw DbError(out.connection()->backend(), "scope_dirty", ++ "pooled connection scope not reset"); ++ out.connection()->exec(ctx, BoundStatement::raw( ++ "SET LOCAL app.current_tenant = '" + ctx.tenant() + "'")); ++ out.connection()->set_scope_tag(ctx.tenant()); + return out; + } +``` + +```diff +--- a/src/data/pool.hpp ++++ b/src/data/pool.hpp +@@ -35,6 +40,8 @@ class Pool { + void release(Checkout&& out) { ++ out.connection()->exec(Context("pool", "", 5), ++ BoundStatement::raw("RESET app.current_tenant")); ++ out.connection()->set_scope_tag(""); + park(std::move(out)); + } +``` + +## 3. src/data/transaction.hpp + +Emit `SET LOCAL` after `BEGIN` on Postgres so the transaction carries the +request tenant even when the pool checkout predates resolution. + +```diff +--- a/src/data/transaction.hpp ++++ b/src/data/transaction.hpp +@@ -15,6 +15,10 @@ class Transaction { + : conn_(conn), ctx_(ctx) { + conn_.exec(ctx_, BoundStatement::raw("BEGIN")); ++ if (conn_.backend() == Backend::Postgres && !ctx_.tenant().empty()) ++ conn_.exec(ctx_, BoundStatement::raw( ++ "SET LOCAL app.current_tenant = '" + ctx_.tenant() + "'")); + } +``` + +## 4. src/data/orm/query.hpp + +Auto-inject the `tenant_id` filter unless the context is privileged or the +query already scopes itself. + +```diff +--- a/src/data/orm/query.hpp ++++ b/src/data/orm/query.hpp +@@ -30,6 +30,12 @@ class Query { + std::string build() const; ++ std::string build(const Context& ctx) const { ++ std::string sql = build(); ++ if (!ctx.privileged() && !ctx.tenant().empty()) ++ sql = tenancy::OrmScope::ApplySelect(sql, ctx.tenant(), false); ++ return sql; ++ } +``` + +## 5. src/data/orm/model.hpp + +Stamp `tenant_id` on insert; scope update and delete to the context tenant. + +```diff +--- a/src/data/orm/model.hpp ++++ b/src/data/orm/model.hpp +@@ -40,6 +40,10 @@ class Model { + static bool save(Connection& conn, const Context& ctx, ...) { ++ if (!ctx.privileged() && !ctx.tenant().empty()) ++ derived().set_field("tenant_id", ctx.tenant()); + return insert_or_update(conn, ctx, ...); + } ++ // Update and delete paths must add "AND tenant_id = ''" ++ // to their WHERE clause under the same condition. +``` + +## 6. src/data/orm/sync.hpp + +Add the `tenant_id` column plus composite index to synced tables and emit +the RLS policy DDL through the migrator journal. + +```diff +--- a/src/data/orm/sync.hpp ++++ b/src/data/orm/sync.hpp +@@ -50,6 +50,13 @@ class Syncer { + SyncReport sync(const Context& ctx, Connection& conn, ...) { ++ for (const ModelInfo& model : models) ++ ensure_column(conn, model.table, "tenant_id", "TEXT NOT NULL DEFAULT ''"); + SyncReport report = sync_tables(ctx, conn, models); ++ for (const ModelInfo& model : models) { ++ apply_ddl(conn, tenancy::RlsPolicy::TenantIndexDdl(model.table)); ++ apply_ddl(conn, tenancy::RlsPolicy::EnableRls(model.table)); ++ apply_ddl(conn, tenancy::RlsPolicy::PolicyDdl(model.table)); ++ } + return report; + } +``` + +## 7. src/data/orm/migrator.hpp + +Iterate the journal per migration target so tenant scoped DDL applies to +every schema or database target, not just the default one. + +```diff +--- a/src/data/orm/migrator.hpp ++++ b/src/data/orm/migrator.hpp +@@ -22,7 +22,9 @@ class Migrator { + void apply(const Context& ctx, Connection& conn, const Migration& m) { +- exec_migration(conn, m.up_sql); ++ for (const std::string& target : list_targets(conn)) ++ exec_on_target(conn, target, m.up_sql); + journal_applied(ctx, conn, m); + } +``` + +## 8. src/data/orm/healer.hpp + +Guard only, no widening: the healer must never alter the `tenant_id` +column type or drop its index. Skip heal actions targeting it. + +```diff +--- a/src/data/orm/healer.hpp ++++ b/src/data/orm/healer.hpp +@@ -18,6 +18,8 @@ class Healer { + bool heal_column(Connection& conn, const std::string& table, + const std::string& column, ...) { ++ if (column == "tenant_id") return false; + return widen_or_extend(conn, table, column, ...); + } +``` + +## 9. src/data/drivers/sqlite/db.hpp + +Driver guard: in pool mode reject statements that lack a tenant scope +unless the context is privileged. + +```diff +--- a/src/data/drivers/sqlite/db.hpp ++++ b/src/data/drivers/sqlite/db.hpp +@@ -30,6 +30,11 @@ class SqliteDb { + Result exec_scoped(const Context& ctx, const BoundStatement& stmt) { ++ if (!ctx.privileged() && !ctx.tenant().empty() && ++ stmt.sql().find("tenant_id") == std::string::npos && ++ stmt.sql().find("tenants") == std::string::npos) ++ throw DbError(Backend::Sqlite, "tenant_scope_missing", ++ "pool-mode statement lacks tenant scope"); + return exec_inner(ctx, stmt); + } +``` + +## 10. src/data/cache/cache.hpp + +Prefix cache keys with the tenant so entries never cross tenants. + +```diff +--- a/src/data/cache/cache.hpp ++++ b/src/data/cache.hpp +@@ -25,6 +25,10 @@ class Cache { + static std::string tag_for_table(const std::string& table); ++ static std::string scoped_key(const Context& ctx, const std::string& key) { ++ return ctx.tenant().empty() ? key : "tenant:" + ctx.tenant() + ":" + key; ++ } ++ // Route get/put/get_or_compute key arguments through scoped_key(ctx, key). +``` + +## 11. src/data/pubsub/broker.hpp + +Scope channel names per tenant at publish and subscribe time. + +```diff +--- a/src/data/pubsub/broker.hpp ++++ b/src/data/pubsub/broker.hpp +@@ -18,6 +18,10 @@ class Broker { + void publish(const std::string& channel, ...); ++ static std::string scoped_channel(const Context& ctx, const std::string& ch) { ++ return ctx.tenant().empty() ? ch : "tenant:" + ctx.tenant() + ":" + ch; ++ } ++ // Route publish/subscribe/unsubscribe channel arguments through it. +``` + +## 12. src/app/app/application.h + +Register `TenantMiddleware` first in the Security category so it runs right +after observability and before all other security middleware. + +```diff +--- a/src/app/app/application.h ++++ b/src/app/app/application.h +@@ -40,6 +40,8 @@ class Application { + void install_default_chain(Catalog& tenants, Resolver resolver) { + chain_.use(std::make_shared()); + chain_.use(std::make_shared()); ++ chain_.use(std::make_shared(tenants, resolver)); + chain_.use(std::make_shared()); +``` + +## 13. src/app/errors/http_error.h + +Add the tenant error codes plus 507. + +```diff +--- a/src/app/errors/http_error.h ++++ b/src/app/errors/http_error.h +@@ -30,6 +30,18 @@ class HttpError { + static HttpError TooManyRequests(const std::string& msg); ++ static HttpError TenantUnknown(const std::string& msg) { ++ return HttpError(404, "tenant_unknown", msg); ++ } ++ static HttpError TenantSuspended(const std::string& msg) { ++ return HttpError(403, "tenant_suspended", msg); ++ } ++ static HttpError TenantConflict(const std::string& msg) { ++ return HttpError(400, "tenant_conflict", msg); ++ } ++ static HttpError InsufficientStorage(const std::string& msg) { ++ return HttpError(507, "insufficient_storage", msg); ++ } +``` + +## 14. src/app/routing/router.h + +Tenant-aware helpers: strip the `/t/` prefix and build URLs that +carry the tenant. + +```diff +--- a/src/app/routing/router.h ++++ b/src/app/routing/router.h +@@ -45,6 +45,13 @@ class Router { + std::string url_for(const std::string& name, ...); ++ static std::pair strip_tenant_prefix( ++ const std::string& path) { ++ return tenancy::Resolver::StripTenantPrefix(path); ++ } ++ std::string url_for_tenant(const std::string& tenant, ++ const std::string& name, ...) { ++ return "/t/" + tenant + url_for(name, ...); ++ } +``` + +## 15. src/security/session/Session.h + +Bind the session to the tenant id and rotate the session on tenant switch. + +```diff +--- a/src/security/session/Session.h ++++ b/src/security/session/Session.h +@@ -20,6 +20,12 @@ class Session { + std::map data; ++ void bind_tenant(const std::string& tid) { data["tid"] = tid; } ++ std::string tenant_id() const { ++ auto it = data.find("tid"); ++ return it == data.end() ? "" : it->second; ++ } ++ // SessionManager: when the request tenant differs from tenant_id(), ++ // rotate the session id before rebinding. +``` + +## 16. src/security/guard/RateLimiter.h + +Key verdicts per tenant plus route so one tenant cannot burn another +tenant's budget. Backed by `tenancy::TenantBuckets`. + +```diff +--- a/src/security/guard/RateLimiter.h ++++ b/src/security/guard/RateLimiter.h +@@ -25,6 +25,9 @@ class RateLimiter { + Verdict check(const HttpRequestView& req, const std::string& user_id); ++ // Key construction must prefix the tenant slug from the request ++ // tenant extension: key = tenant + "|" + route + "|" + user_or_ip. ++ // Wire the per-tenant take() through tenancy::TenantBuckets. +``` + +## 17. Makefile + +Include the tenancy fragment and wire its targets. + +```diff +--- a/Makefile ++++ b/Makefile +@@ -5,6 +5,7 @@ + include src/data.mk ++include src/tenancy.mk +``` + +```diff +--- a/Makefile ++++ b/Makefile +@@ -20,8 +21,8 @@ +-all:: data-tests +-test:: data-check ++all:: data-tests tenancy-tests ++test:: data-check tenancy-check +``` diff --git a/src/tenancy/SPEC.md b/src/tenancy/SPEC.md new file mode 100644 index 0000000..5cbb97d --- /dev/null +++ b/src/tenancy/SPEC.md @@ -0,0 +1,75 @@ +# Tenancy part specification + +Namespace `molo::tenancy` (`molo::tenancy::store` for catalog backends, +`molo::tenancy::adapters` for standalone-only seam mirrors). Header-only +C++23, stdlib plus OS syscalls only. One class per file. All ceilings live +in `caps.h`. No em dashes anywhere. + +## Files + +| File | Class | Purpose | +|------|-------|---------| +| caps.h | Caps | Static ceilings for every map, cache and quota | +| tenant_id.h | TenantId | Validated slug (`[a-z0-9-]`, 1..63, wrapped ends) | +| strategy.h | Strategy | SharedSchema, SchemaPerTenant, DatabasePerTenant, FilePerTenant | +| tenant.h | Tenant | Record: id, slug, strategy, status, storage, timestamps | +| context.h | TenantContext | uuid plus slug plus strategy; renders `tenant:` for data Context | +| store/catalog_store.h | CatalogStore | Abstract tenant record storage | +| store/memory_catalog.h | MemoryCatalog | Mutex guarded in-memory store, capped | +| store/db_catalog.h | DbCatalog | Tenants table via `Connection::exec` only | +| catalog.h | Catalog | Facade: lifecycle plus bounded slug cache (FIFO, 65536) | +| resolver.h | Resolver | Header/path/host sources, conflict rule, dev default | +| middleware.h | TenantMiddleware | Resolve, strip `/t/`, attach context, map errors | +| provisioner.h | Provisioner | Provision with hook rollback, suspend, tombstone, gated purge | +| orm_scope.h | OrmScope | `tenant_id` filter injection, insert stamping, key/channel prefixes | +| rls_policy.h | RlsPolicy | RLS DDL plus `SET LOCAL` / `RESET` statements | +| pool_factory.h | PoolFactory | Per-tenant checkout discipline, GUC assert, RAII Lease | +| config_overlay.h | ConfigOverlay | Global defaults with per-tenant overrides, typed getters | +| quota.h | Quota | Token bucket rate (429) plus storage cap (507) | +| tenant_buckets.h | TenantBuckets | Named bucket registry for tenant scoped rate limits | +| feature_flags.h | FeatureFlags | Global defaults plus per-tenant overrides, 256 cap | +| metrics.h | TenantMetrics | Per-tenant counters, 512 labels, rest folds to `other` | +| bridge.h | DatabaseBridge | FilePerTenant SQLite files, single-db transactions only | +| adapters/task.h | Task | ADAPTER: eager coroutine task (delete at integration) | +| adapters/http.h | HttpRequest | ADAPTER: request/response/error/middleware seam mirror | +| adapters/db.h | Connection | ADAPTER: data Context/Connection seam mirror | + +## Resolution matrix (resolver.h) + +Sources: `X-Tenant` header, `/t/` path prefix, host subdomain. +One distinct slug resolves; two distinct slugs conflict; none resolves to +the dev default only in dev mode, else unknown. Invalid values are ignored +as sources, never errors by themselves. + +## Lifecycle + +`create` (unique slug) -> `suspend` (403s) -> `tombstone` (404s, terminal) +-> `purge` only when tombstoned plus retention elapsed plus confirm token +`PURGE:` matches. Provision hooks that fail roll the entry back fully. + +## Data scoping contract + +- SharedSchema: every table carries `tenant_id`; `OrmScope` injects the + predicate unless privileged; RLS policy enforces it in Postgres. +- Pool checkouts set scope on acquire and reset on release; a dirty slot + throws instead of leaking across tenants. +- Cache keys and pubsub channels are prefixed `tenant::`. +- Cross-file SQLite transactions are refused. + +## Base-tree integration + +Exact edits live in `PATCHES.md` as unified diffs. After applying them, +delete `adapters/` and swap the adapter type names for the real base types; +tenancy call sites keep the same shape. + +## Tests (tests/src/tenancy, mini.hpp CHECK harness) + +| Test | Coverage | +|------|----------| +| test_tenant_id_resolve | id matrix, strategy, resolver matrix, conflict, dev default, middleware | +| test_pool_scope | 2-tenant CRUD both backends, RLS proof, dirty refusal, 100k leak loop | +| test_bridge | per-db isolation, txn staging, cross-file refusal, open caps | +| test_provision | rollback, tombstone, purge gates, DbCatalog round trip | +| test_quota | 429 burst, refill, 507 storage, buckets | +| test_flags_config | flags, overrides, 256 cap, overlay precedence, key cap | +| test_metrics | 512 labels, 513th is `other`, folded counts | diff --git a/src/tenancy/adapters/db.h b/src/tenancy/adapters/db.h new file mode 100644 index 0000000..ca71fb0 --- /dev/null +++ b/src/tenancy/adapters/db.h @@ -0,0 +1,113 @@ +#pragma once +// ADAPTER: minimal local mirror of the molo::data seams used by tenancy: +// Context, BoundStatement, Result/Row/Value, DbError, Connection. Shapes match +// the seam spec; swap for the real base headers at integration (PATCHES.md). + +#include +#include +#include +#include +#include +#include +#include + +namespace molo::tenancy::adapters::db { + +enum class Backend { Postgres, Sqlite }; + +using Value = std::variant; +using Row = std::map; + +inline Value MakeString(std::string s) { return Value{std::move(s)}; } +inline Value MakeInt(std::int64_t v) { return Value{v}; } + +inline std::string AsString(const Value& v) { + if (auto* s = std::get_if(&v)) return *s; + if (auto* i = std::get_if(&v)) return std::to_string(*i); + if (auto* d = std::get_if(&v)) return std::to_string(*d); + return {}; +} + +inline std::int64_t AsInt(const Value& v) { + if (auto* i = std::get_if(&v)) return *i; + if (auto* s = std::get_if(&v)) { + try { + return std::stoll(*s); + } catch (...) { + return 0; + } + } + if (auto* d = std::get_if(&v)) return static_cast(*d); + return 0; +} + +struct Result { + std::vector rows; + long affected = 0; +}; + +class BoundStatement { + public: + BoundStatement(std::string sql, std::vector params) + : sql_(std::move(sql)), params_(std::move(params)) {} + static BoundStatement Raw(std::string sql, std::vector params = {}) { + return BoundStatement(std::move(sql), std::move(params)); + } + const std::string& sql() const { return sql_; } + const std::vector& params() const { return params_; } + + private: + std::string sql_; + std::vector params_; +}; + +class DbError : public std::runtime_error { + public: + DbError(Backend backend, std::string code, std::string message, std::int64_t detail = 0) + : std::runtime_error(message), + backend_(backend), + code_(std::move(code)), + message_(std::move(message)), + detail_(detail) {} + Backend backend() const { return backend_; } + const std::string& code() const { return code_; } + const std::string& message() const { return message_; } + + private: + Backend backend_; + std::string code_; + std::string message_; + std::int64_t detail_; +}; + +class Context { + public: + Context(std::string correlation_id, std::string tenant, int timeout_sec) + : correlation_id_(std::move(correlation_id)), + tenant_(std::move(tenant)), + timeout_sec_(timeout_sec) {} + const std::string& correlation_id() const { return correlation_id_; } + const std::string& tenant() const { return tenant_; } + int timeout_sec() const { return timeout_sec_; } + bool expired() const { return timeout_sec_ <= 0; } + + private: + std::string correlation_id_; + std::string tenant_; + int timeout_sec_; +}; + +class Connection { + public: + virtual ~Connection() = default; + virtual Result exec(const Context& ctx, const BoundStatement& stmt) = 0; + virtual void prepare(const std::string& name, const std::string& sql) = 0; + virtual Result exec_prepared(const Context& ctx, + const std::string& name, + const std::vector& params) = 0; + virtual bool ping() = 0; + virtual void close() = 0; + virtual Backend backend() const = 0; +}; + +} // namespace molo::tenancy::adapters::db diff --git a/src/tenancy/adapters/http.h b/src/tenancy/adapters/http.h new file mode 100644 index 0000000..9a57745 --- /dev/null +++ b/src/tenancy/adapters/http.h @@ -0,0 +1,130 @@ +#pragma once +// ADAPTER: minimal local mirror of the molo::app seams used by tenancy: +// http_msg Request/Response, middleware Middleware/Chain/Next, errors HttpError. +// Shapes match the seam spec; swap these types for the real base headers at +// integration time (see PATCHES.md). Delete this file once wired. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tenancy/adapters/task.h" + +namespace molo::tenancy::adapters { + +inline std::string Lower(std::string_view s) { + std::string out(s); + for (char& c : out) c = static_cast(std::tolower(static_cast(c))); + return out; +} + +class HttpRequest { + public: + void set_path(std::string path) { path_ = std::move(path); } + const std::string& path() const { return path_; } + + void set_header(std::string name, std::string value) { + headers_[Lower(name)] = std::move(value); + } + std::string header(std::string_view name) const { + auto it = headers_.find(Lower(name)); + return it == headers_.end() ? std::string{} : it->second; + } + + template + void set_ext(std::string key, T value) { + ext_[std::move(key)] = std::move(value); + } + template + std::optional get_ext(const std::string& key) const { + auto it = ext_.find(key); + if (it == ext_.end()) return std::nullopt; + if (auto* p = std::any_cast(&it->second)) return *p; + return std::nullopt; + } + + void set_path_param(std::string key, std::string value) { + params_[std::move(key)] = std::move(value); + } + std::string path_param(std::string_view key) const { + auto it = params_.find(std::string(key)); + return it == params_.end() ? std::string{} : it->second; + } + + private: + std::string path_; + std::map headers_; + std::map ext_; + std::map params_; +}; + +class HttpError : public std::exception { + public: + HttpError(int status, std::string code, std::string message) + : status_(status), code_(std::move(code)), message_(std::move(message)) {} + int status() const { return status_; } + const std::string& code() const { return code_; } + const std::string& message() const { return message_; } + const char* what() const noexcept override { return message_.c_str(); } + + static HttpError TenantUnknown(std::string detail) { + return HttpError(404, "tenant_unknown", std::move(detail)); + } + static HttpError TenantSuspended(std::string detail) { + return HttpError(403, "tenant_suspended", std::move(detail)); + } + static HttpError TenantConflict(std::string detail) { + return HttpError(400, "tenant_conflict", std::move(detail)); + } + static HttpError InsufficientStorage(std::string detail) { + return HttpError(507, "insufficient_storage", std::move(detail)); + } + static HttpError TooManyRequests(std::string detail) { + return HttpError(429, "too_many_requests", std::move(detail)); + } + + private: + int status_; + std::string code_; + std::string message_; +}; + +class HttpResponse { + public: + int status = 200; + std::string body; + std::map headers; + + static HttpResponse FromError(const HttpError& err) { + HttpResponse r; + r.status = err.status(); + r.body = err.code() + ": " + err.message(); + return r; + } +}; + +enum class Category { + Errors = 0, + Observability = 1, + Security = 2, + SessionAuth = 3, + User = 4, + RouterAdjacent = 5, +}; + +using Next = std::function(HttpRequest&)>; + +class MiddlewareBase { + public: + virtual ~MiddlewareBase() = default; + virtual Category category() const = 0; + virtual std::string name() const = 0; + virtual Task handle(HttpRequest& req, Next next) = 0; +}; + +} // namespace molo::tenancy::adapters diff --git a/src/tenancy/adapters/task.h b/src/tenancy/adapters/task.h new file mode 100644 index 0000000..98abe90 --- /dev/null +++ b/src/tenancy/adapters/task.h @@ -0,0 +1,63 @@ +#pragma once +// ADAPTER: minimal local stand-in for the framework Task coroutine type. +// Lets tenancy middleware use real co_await/co_return syntax while the base +// tree is absent. Runs eagerly and completes synchronously. Delete when the +// real molo::app task type is available; call sites need no change. + +#include +#include +#include +#include + +namespace molo::tenancy::adapters { + +template +class Task { + public: + struct promise_type { + std::optional value; + std::exception_ptr error; + Task get_return_object() { + return Task{std::coroutine_handle::from_promise(*this)}; + } + std::suspend_never initial_suspend() noexcept { return {}; } + std::suspend_always final_suspend() noexcept { return {}; } + void return_value(T v) { value.emplace(std::move(v)); } + void unhandled_exception() { error = std::current_exception(); } + }; + + using Handle = std::coroutine_handle; + + explicit Task(Handle h) : handle_(h) {} + Task(Task&& other) noexcept : handle_(std::exchange(other.handle_, {})) {} + Task& operator=(Task&& other) noexcept { + if (this != &other) { + Reset(); + handle_ = std::exchange(other.handle_, {}); + } + return *this; + } + ~Task() { Reset(); } + + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + + bool await_ready() const noexcept { return true; } + void await_suspend(std::coroutine_handle<>) noexcept {} + T await_resume() { return Get(); } + + T Get() { + auto& p = handle_.promise(); + if (p.error) std::rethrow_exception(p.error); + return std::move(*p.value); + } + + private: + void Reset() { + if (handle_) handle_.destroy(); + handle_ = {}; + } + Handle handle_; +}; + +} // namespace molo::tenancy::adapters diff --git a/src/tenancy/bridge.h b/src/tenancy/bridge.h new file mode 100644 index 0000000..d17e081 --- /dev/null +++ b/src/tenancy/bridge.h @@ -0,0 +1,146 @@ +#pragma once +// DatabaseBridge maps each FilePerTenant tenant to its own SQLite file under +// a root directory and keeps the stores isolated. A transaction binds to +// exactly one database: opening one across two files throws CrossFileError. +// Each open database counts two files (data plus journal) toward the file cap. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tenancy/caps.h" +#include "tenancy/tenant_id.h" + +namespace molo::tenancy { + +class DatabaseBridge { + public: + class CrossFileError : public std::runtime_error { + public: + explicit CrossFileError(const std::string& msg) : std::runtime_error(msg) {} + }; + + class Txn { + public: + Txn() : bridge_(nullptr) {} + Txn(DatabaseBridge* bridge, std::string slug) + : bridge_(bridge), slug_(std::move(slug)), active_(true) {} + + Txn(Txn&&) = default; + Txn& operator=(Txn&&) = default; + Txn(const Txn&) = delete; + Txn& operator=(const Txn&) = delete; + + bool active() const { return active_; } + const std::string& slug() const { return slug_; } + + bool put(const std::string& key, const std::string& value) { + if (!active_ || bridge_ == nullptr) return false; + staged_[key] = value; + return true; + } + + std::optional get(const std::string& key) { + if (!active_ || bridge_ == nullptr) return std::nullopt; + auto it = staged_.find(key); + if (it != staged_.end()) return it->second; + return bridge_->get(slug_, key); + } + + bool commit() { + if (!active_ || bridge_ == nullptr) return false; + bool ok = bridge_->ApplyStaged(slug_, staged_); + active_ = false; + staged_.clear(); + return ok; + } + + void rollback() { + active_ = false; + staged_.clear(); + } + + private: + DatabaseBridge* bridge_; + std::string slug_; + bool active_ = false; + std::map staged_; + }; + + explicit DatabaseBridge(std::string root) : root_(std::move(root)) {} + + std::string file_for(const std::string& slug) const { return root_ + "/" + slug + ".db"; } + + bool open(const std::string& slug) { + if (!TenantId::TryParse(slug).has_value()) return false; + std::lock_guard lock(mu_); + if (dbs_.find(slug) != dbs_.end()) return true; + if (dbs_.size() >= Caps::kBridgeDbsMax) return false; + if ((dbs_.size() + 1) * 2 > Caps::kBridgeFilesMax) return false; + dbs_.emplace(slug, std::map{}); + return true; + } + + bool is_open(const std::string& slug) const { + std::lock_guard lock(mu_); + return dbs_.find(slug) != dbs_.end(); + } + + bool put(const std::string& slug, const std::string& key, const std::string& value) { + std::lock_guard lock(mu_); + auto it = dbs_.find(slug); + if (it == dbs_.end()) return false; + it->second[key] = value; + return true; + } + + std::optional get(const std::string& slug, const std::string& key) const { + std::lock_guard lock(mu_); + auto it = dbs_.find(slug); + if (it == dbs_.end()) return std::nullopt; + auto jt = it->second.find(key); + if (jt == it->second.end()) return std::nullopt; + return jt->second; + } + + Txn begin(const std::string& slug) { + std::lock_guard lock(mu_); + if (dbs_.find(slug) == dbs_.end()) return Txn{}; + return Txn(this, slug); + } + + Txn begin2(const std::string& a, const std::string& b) { + if (a != b) throw CrossFileError("cross-file transaction refused: " + a + " vs " + b); + return begin(a); + } + + std::size_t db_count() const { + std::lock_guard lock(mu_); + return dbs_.size(); + } + + std::size_t file_count() const { + std::lock_guard lock(mu_); + return dbs_.size() * 2; + } + + private: + bool ApplyStaged(const std::string& slug, const std::map& staged) { + std::lock_guard lock(mu_); + auto it = dbs_.find(slug); + if (it == dbs_.end()) return false; + for (const auto& [k, v] : staged) it->second[k] = v; + return true; + } + + std::string root_; + mutable std::mutex mu_; + std::unordered_map> dbs_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/caps.h b/src/tenancy/caps.h new file mode 100644 index 0000000..fcef8f6 --- /dev/null +++ b/src/tenancy/caps.h @@ -0,0 +1,30 @@ +#pragma once +// All static ceilings for the tenancy part. Every map, cache, registry and +// quota in this part is bounded by one of these constants. No autotuning. + +#include +#include + +namespace molo::tenancy { + +struct Caps { + static constexpr std::size_t kSlugCacheMax = 65536; + static constexpr std::size_t kPoolTenantsMax = 100000; + static constexpr std::size_t kPoolSlotsMax = 1024; + static constexpr std::size_t kBridgeDbsMax = 1024; + static constexpr std::size_t kBridgeFilesMax = 4096; + static constexpr std::size_t kMetricTenantsMax = 512; + static constexpr std::size_t kFlagsMax = 256; + static constexpr std::size_t kCatalogTenantsMax = 100000; + static constexpr std::size_t kConfigTenantsMax = 100000; + static constexpr std::size_t kConfigKeysPerTenantMax = 1024; + static constexpr std::size_t kRateBucketsMax = 4096; + static constexpr std::size_t kBucketEntriesMax = 65536; + static constexpr std::size_t kSlugMaxLen = 63; + static constexpr double kRequestsPerMinuteDefault = 600.0; + static constexpr double kRequestBurstDefault = 100.0; + static constexpr std::int64_t kStorageBytesDefault = 1073741824; + static constexpr std::int64_t kRetentionSecDefault = 604800; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/catalog.h b/src/tenancy/catalog.h new file mode 100644 index 0000000..4b15736 --- /dev/null +++ b/src/tenancy/catalog.h @@ -0,0 +1,124 @@ +#pragma once +// Catalog facade over a CatalogStore with a bounded slug lookup cache +// (kSlugCacheMax, FIFO eviction). Owns lifecycle transitions: create, +// suspend, tombstone, and the gated purge. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tenancy/caps.h" +#include "tenancy/store/catalog_store.h" +#include "tenancy/strategy.h" +#include "tenancy/tenant.h" +#include "tenancy/tenant_id.h" + +namespace molo::tenancy { + +enum class PurgeResult { Ok, NotFound, NotTombstone, RetentionHold, BadConfirm }; + +class Catalog { + public: + explicit Catalog(std::unique_ptr backend) + : backend_(std::move(backend)) {} + + std::optional create(const std::string& slug, + Strategy strategy, + std::int64_t now_unix) { + auto id = TenantId::TryParse(slug); + if (!id.has_value()) return std::nullopt; + if (find_by_slug(slug).has_value()) return std::nullopt; + Tenant tenant("ten-" + slug, slug, strategy, TenantStatus::Active, 0, now_unix, 0); + if (!backend_->upsert(tenant)) return std::nullopt; + CachePut(tenant); + return tenant; + } + + std::optional find_by_slug(const std::string& slug) { + { + std::lock_guard lock(mu_); + auto it = cache_.find(slug); + if (it != cache_.end()) return it->second; + } + auto found = backend_->get_by_slug(slug); + if (found.has_value()) CachePut(*found); + return found; + } + + std::optional find_by_id(const std::string& id) { + return backend_->get_by_id(id); + } + + bool suspend(const std::string& slug) { + auto found = backend_->get_by_slug(slug); + if (!found.has_value() || !found->is_active()) return false; + found->suspend(); + if (!backend_->upsert(*found)) return false; + CachePut(*found); + return true; + } + + bool tombstone(const std::string& slug, std::int64_t now_unix) { + auto found = backend_->get_by_slug(slug); + if (!found.has_value()) return false; + found->mark_tombstone(now_unix); + if (!backend_->upsert(*found)) return false; + Invalidate(slug); + return true; + } + + PurgeResult purge(const std::string& slug, + const std::string& confirm, + std::int64_t now_unix, + std::int64_t retention_sec) { + auto found = backend_->get_by_slug(slug); + if (!found.has_value()) return PurgeResult::NotFound; + if (found->status() != TenantStatus::Tombstone) return PurgeResult::NotTombstone; + if (now_unix - found->tombstoned_unix() < retention_sec) return PurgeResult::RetentionHold; + if (confirm != "PURGE:" + found->id()) return PurgeResult::BadConfirm; + if (!backend_->remove(found->id())) return PurgeResult::NotFound; + Invalidate(slug); + return PurgeResult::Ok; + } + + std::vector list(std::size_t limit) { return backend_->list(limit); } + + void invalidate(const std::string& slug) { Invalidate(slug); } + + std::size_t cache_size() const { + std::lock_guard lock(mu_); + return cache_.size(); + } + + private: + void CachePut(const Tenant& tenant) { + std::lock_guard lock(mu_); + if (cache_.find(tenant.slug()) == cache_.end()) { + order_.push_back(tenant.slug()); + while (order_.size() > Caps::kSlugCacheMax) { + cache_.erase(order_.front()); + order_.pop_front(); + } + } + cache_.insert_or_assign(tenant.slug(), tenant); + } + + void Invalidate(const std::string& slug) { + std::lock_guard lock(mu_); + cache_.erase(slug); + } + + std::unique_ptr backend_; + mutable std::mutex mu_; + std::unordered_map cache_; + std::deque order_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/config_overlay.h b/src/tenancy/config_overlay.h new file mode 100644 index 0000000..24c1c9a --- /dev/null +++ b/src/tenancy/config_overlay.h @@ -0,0 +1,82 @@ +#pragma once +// ConfigOverlay layers per-tenant config over global defaults. Lookup order: +// tenant value, global value, caller fallback. Bounded tenants and keys per +// tenant; typed getters return the fallback on missing or malformed values. + +#include +#include +#include +#include +#include +#include + +#include "tenancy/caps.h" + +namespace molo::tenancy { + +class ConfigOverlay { + public: + void set_global(const std::string& key, const std::string& value) { + std::lock_guard lock(mu_); + global_[key] = value; + } + + bool set_tenant(const std::string& slug, const std::string& key, const std::string& value) { + std::lock_guard lock(mu_); + auto it = tenants_.find(slug); + if (it == tenants_.end()) { + if (tenants_.size() >= Caps::kConfigTenantsMax) return false; + it = tenants_.emplace(slug, std::unordered_map{}).first; + } + if (it->second.find(key) == it->second.end() && + it->second.size() >= Caps::kConfigKeysPerTenantMax) { + return false; + } + it->second[key] = value; + return true; + } + + std::string get(const std::string& slug, + const std::string& key, + const std::string& fallback = "") const { + std::lock_guard lock(mu_); + auto it = tenants_.find(slug); + if (it != tenants_.end()) { + auto jt = it->second.find(key); + if (jt != it->second.end()) return jt->second; + } + auto gt = global_.find(key); + return gt == global_.end() ? fallback : gt->second; + } + + std::int64_t get_int(const std::string& slug, const std::string& key, std::int64_t fallback) const { + std::string raw = get(slug, key, ""); + if (raw.empty()) return fallback; + try { + std::size_t pos = 0; + std::int64_t v = std::stoll(raw, &pos); + return pos == raw.size() ? v : fallback; + } catch (...) { + return fallback; + } + } + + bool get_bool(const std::string& slug, const std::string& key, bool fallback) const { + std::string raw = get(slug, key, ""); + if (raw == "true" || raw == "1" || raw == "yes") return true; + if (raw == "false" || raw == "0" || raw == "no") return false; + return fallback; + } + + bool clear_tenant(const std::string& slug) { + std::lock_guard lock(mu_); + return tenants_.erase(slug) > 0; + } + + private: + mutable std::mutex mu_; + std::unordered_map global_; + std::unordered_map> tenants_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/context.h b/src/tenancy/context.h new file mode 100644 index 0000000..64a59ac --- /dev/null +++ b/src/tenancy/context.h @@ -0,0 +1,35 @@ +#pragma once +// Per-request tenant context: uuid plus slug plus strategy. Attached to the +// request by TenantMiddleware. data_tenant() renders the legacy tenant string +// carried by molo::data::Context. + +#include +#include + +#include "tenancy/strategy.h" +#include "tenancy/tenant.h" + +namespace molo::tenancy { + +class TenantContext { + public: + TenantContext(std::string uuid, std::string slug, Strategy strategy) + : uuid_(std::move(uuid)), slug_(std::move(slug)), strategy_(strategy) {} + + static TenantContext FromTenant(const Tenant& tenant) { + return TenantContext(tenant.id(), tenant.slug(), tenant.strategy()); + } + + const std::string& uuid() const { return uuid_; } + const std::string& slug() const { return slug_; } + Strategy strategy() const { return strategy_; } + + std::string data_tenant() const { return "tenant:" + slug_; } + + private: + std::string uuid_; + std::string slug_; + Strategy strategy_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/feature_flags.h b/src/tenancy/feature_flags.h new file mode 100644 index 0000000..5195ce0 --- /dev/null +++ b/src/tenancy/feature_flags.h @@ -0,0 +1,51 @@ +#pragma once +// FeatureFlags holds global flag defaults with per-tenant overrides. +// Bounded at kFlagsMax definitions; unknown flags read as false. + +#include +#include +#include +#include + +#include "tenancy/caps.h" + +namespace molo::tenancy { + +class FeatureFlags { + public: + bool define(const std::string& name, bool default_value) { + std::lock_guard lock(mu_); + if (defaults_.find(name) == defaults_.end() && defaults_.size() >= Caps::kFlagsMax) { + return false; + } + defaults_[name] = default_value; + return true; + } + + bool set_tenant(const std::string& slug, const std::string& name, bool value) { + std::lock_guard lock(mu_); + if (defaults_.find(name) == defaults_.end()) return false; + overrides_[slug + '\x1f' + name] = value; + return true; + } + + bool enabled(const std::string& slug, const std::string& name) const { + std::lock_guard lock(mu_); + auto ov = overrides_.find(slug + '\x1f' + name); + if (ov != overrides_.end()) return ov->second; + auto def = defaults_.find(name); + return def == defaults_.end() ? false : def->second; + } + + std::size_t count() const { + std::lock_guard lock(mu_); + return defaults_.size(); + } + + private: + mutable std::mutex mu_; + std::unordered_map defaults_; + std::unordered_map overrides_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/metrics.h b/src/tenancy/metrics.h new file mode 100644 index 0000000..9d4c9c9 --- /dev/null +++ b/src/tenancy/metrics.h @@ -0,0 +1,66 @@ +#pragma once +// TenantMetrics counts requests and errors per tenant with a bounded label +// set: the first kMetricTenantsMax slugs keep their own label, every later +// tenant folds into "other" so cardinality can never explode. + +#include +#include +#include +#include +#include +#include + +#include "tenancy/caps.h" + +namespace molo::tenancy { + +class TenantMetrics { + public: + struct Counters { + std::uint64_t requests = 0; + std::uint64_t errors = 0; + }; + + std::string label_for(const std::string& slug) { + std::lock_guard lock(mu_); + if (slug == kOther) return kOther; + if (labels_.find(slug) != labels_.end()) return slug; + if (labels_.size() >= Caps::kMetricTenantsMax) return kOther; + labels_.insert(slug); + return slug; + } + + void record_request(const std::string& slug) { + std::lock_guard lock(mu_); + counts_[LabelLocked(slug)].requests += 1; + } + + void record_error(const std::string& slug) { + std::lock_guard lock(mu_); + counts_[LabelLocked(slug)].errors += 1; + } + + std::map snapshot() const { + std::lock_guard lock(mu_); + return std::map(counts_.begin(), counts_.end()); + } + + private: + static const char* kOther; + + std::string LabelLocked(const std::string& slug) { + if (slug == kOther) return kOther; + if (labels_.find(slug) != labels_.end()) return slug; + if (labels_.size() >= Caps::kMetricTenantsMax) return kOther; + labels_.insert(slug); + return slug; + } + + mutable std::mutex mu_; + std::unordered_set labels_; + std::unordered_map counts_; +}; + +inline const char* TenantMetrics::kOther = "other"; + +} // namespace molo::tenancy diff --git a/src/tenancy/middleware.h b/src/tenancy/middleware.h new file mode 100644 index 0000000..2862a36 --- /dev/null +++ b/src/tenancy/middleware.h @@ -0,0 +1,61 @@ +#pragma once +// TenantMiddleware resolves the tenant, strips the /t/ path prefix and +// attaches a TenantContext plus the legacy data tenant string to the request +// as extensions. Category Security: register it first in that category so it +// runs right after observability and before all other security middleware. +// Unknown maps to 404 tenant_unknown, suspended to 403 tenant_suspended, +// conflicting sources to 400 tenant_conflict. + +#include + +#include "tenancy/adapters/http.h" +#include "tenancy/catalog.h" +#include "tenancy/context.h" +#include "tenancy/resolver.h" +#include "tenancy/tenant.h" + +namespace molo::tenancy { + +class TenantMiddleware : public adapters::MiddlewareBase { + public: + TenantMiddleware(Catalog& catalog, Resolver resolver) + : catalog_(catalog), resolver_(std::move(resolver)) {} + + adapters::Category category() const override { return adapters::Category::Security; } + std::string name() const override { return "tenant"; } + + adapters::Task handle(adapters::HttpRequest& req, + adapters::Next next) override { + ResolveRequest rreq{req.header("host"), req.path(), req.header("x-tenant")}; + ResolveOutcome outcome = resolver_.Resolve(rreq, catalog_); + if (outcome.status == ResolveStatus::Conflict) { + co_return adapters::HttpResponse::FromError( + adapters::HttpError::TenantConflict(outcome.detail)); + } + if (outcome.status == ResolveStatus::Unknown || !outcome.tenant.has_value()) { + co_return adapters::HttpResponse::FromError( + adapters::HttpError::TenantUnknown(outcome.detail)); + } + const Tenant& tenant = *outcome.tenant; + if (tenant.status() == TenantStatus::Suspended) { + co_return adapters::HttpResponse::FromError( + adapters::HttpError::TenantSuspended("tenant suspended: " + tenant.slug())); + } + if (tenant.status() == TenantStatus::Tombstone) { + co_return adapters::HttpResponse::FromError( + adapters::HttpError::TenantUnknown("tenant removed: " + tenant.slug())); + } + req.set_path(Resolver::StripTenantPrefix(req.path()).first); + TenantContext ctx = TenantContext::FromTenant(tenant); + req.set_ext("tenancy.context", ctx); + req.set_ext("tenancy.data_tenant", ctx.data_tenant()); + adapters::HttpResponse resp = co_await next(req); + co_return resp; + } + + private: + Catalog& catalog_; + Resolver resolver_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/orm_scope.h b/src/tenancy/orm_scope.h new file mode 100644 index 0000000..5eb7a7c --- /dev/null +++ b/src/tenancy/orm_scope.h @@ -0,0 +1,94 @@ +#pragma once +// OrmScope is the shared_schema scoping helper. ApplySelect/ApplyUpdate inject +// a tenant_id predicate unless the caller is privileged or the statement is +// already scoped. KeyPrefix/ScopedChannel render the tenant:: prefix used +// for cache keys and pubsub channels. + +#include +#include + +namespace molo::tenancy { + +class OrmScope { + public: + static std::string TenantColumn() { return "tenant_id"; } + + static std::string KeyPrefix(const std::string& slug) { return "tenant:" + slug + ":"; } + + static std::string ScopedChannel(const std::string& slug, const std::string& channel) { + return "tenant:" + slug + ":" + channel; + } + + static std::string FilterClause(const std::string& slug) { + return "tenant_id = '" + Escape(slug) + "'"; + } + + static std::string ApplySelect(const std::string& sql, + const std::string& slug, + bool privileged) { + return ApplyWhere(sql, slug, privileged); + } + + static std::string ApplyUpdate(const std::string& sql, + const std::string& slug, + bool privileged) { + return ApplyWhere(sql, slug, privileged); + } + + static std::string InsertWithTenant(const std::string& table, + const std::string& columns, + const std::string& placeholders, + const std::string& slug) { + return "INSERT INTO " + table + " (tenant_id, " + columns + ") VALUES ('" + + Escape(slug) + "', " + placeholders + ")"; + } + + private: + static std::string Escape(const std::string& raw) { + std::string out; + for (char c : raw) { + if (c == '\'') out += '\''; + out += c; + } + return out; + } + + static std::size_t FindKeyword(const std::string& sql, + const std::string& keyword, + std::size_t from) { + for (std::size_t i = from; i + keyword.size() <= sql.size(); ++i) { + bool match = true; + for (std::size_t k = 0; k < keyword.size(); ++k) { + if (std::tolower(static_cast(sql[i + k])) != keyword[k]) { + match = false; + break; + } + } + if (match) return i; + } + return std::string::npos; + } + + static std::string ApplyWhere(const std::string& sql, + const std::string& slug, + bool privileged) { + if (privileged) return sql; + if (FindKeyword(sql, "tenant_id", 0) != std::string::npos) return sql; + std::size_t where = FindKeyword(sql, "where", 0); + if (where != std::string::npos) { + std::string out = sql; + out.insert(where + 5, " " + FilterClause(slug) + " AND"); + return out; + } + std::size_t tail = sql.size(); + for (const char* kw : {"group by", "order by", "limit", "offset"}) { + std::size_t pos = FindKeyword(sql, kw, 0); + if (pos != std::string::npos && pos < tail) tail = pos; + } + std::string out = sql; + out.insert(tail, " WHERE " + FilterClause(slug)); + return out; + } +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/pool_factory.h b/src/tenancy/pool_factory.h new file mode 100644 index 0000000..c58b2e1 --- /dev/null +++ b/src/tenancy/pool_factory.h @@ -0,0 +1,160 @@ +#pragma once +// PoolFactory mirrors the Pool checkout discipline per tenant: acquire +// asserts an empty scope then sets it to the tenant slug (SET LOCAL +// app.current_tenant on pg, attached file on sqlite); the RAII Lease resets +// the scope on release. Registry capped at kPoolTenantsMax, reusable slots at +// kPoolSlotsMax. A dirty slot on acquire throws instead of leaking a tenant. + +#include +#include +#include +#include +#include +#include +#include + +#include "tenancy/adapters/db.h" +#include "tenancy/caps.h" +#include "tenancy/tenant_id.h" + +namespace molo::tenancy { + +class PoolFactory { + public: + using Backend = adapters::db::Backend; + + class Lease { + public: + Lease() : factory_(nullptr), slot_(0) {} + ~Lease() { release(); } + + Lease(Lease&& other) noexcept + : factory_(std::exchange(other.factory_, nullptr)), + slot_(other.slot_), + slug_(std::move(other.slug_)), + backend_(other.backend_) {} + Lease& operator=(Lease&& other) noexcept { + if (this != &other) { + release(); + factory_ = std::exchange(other.factory_, nullptr); + slot_ = other.slot_; + slug_ = std::move(other.slug_); + backend_ = other.backend_; + } + return *this; + } + Lease(const Lease&) = delete; + Lease& operator=(const Lease&) = delete; + + bool active() const { return factory_ != nullptr; } + const std::string& slug() const { return slug_; } + Backend backend() const { return backend_; } + + std::string guc() const { + if (factory_ == nullptr) return ""; + return factory_->SlotGuc(slot_); + } + + void release() { + if (factory_ != nullptr) { + factory_->ReleaseSlot(slot_); + factory_ = nullptr; + } + } + + private: + friend class PoolFactory; + Lease(PoolFactory* factory, std::size_t slot, std::string slug, Backend backend) + : factory_(factory), slot_(slot), slug_(std::move(slug)), backend_(backend) {} + + PoolFactory* factory_; + std::size_t slot_; + std::string slug_; + Backend backend_ = Backend::Postgres; + }; + + Lease acquire(const std::string& slug, Backend backend) { + if (!TenantId::TryParse(slug).has_value()) { + throw adapters::db::DbError(ToDbBackend(backend), "invalid_tenant", + "invalid tenant slug: " + slug); + } + std::lock_guard lock(mu_); + if (tenants_.find(slug) == tenants_.end()) { + if (tenants_.size() >= Caps::kPoolTenantsMax) { + throw adapters::db::DbError(ToDbBackend(backend), "too_many_tenants", + "tenant registry cap reached"); + } + tenants_.insert(slug); + } + std::size_t slot = slots_.size(); + for (std::size_t i = 0; i < slots_.size(); ++i) { + if (!slots_[i].busy) { + slot = i; + break; + } + } + if (slot == slots_.size()) { + if (slots_.size() >= Caps::kPoolSlotsMax) { + throw adapters::db::DbError(ToDbBackend(backend), "pool_exhausted", + "no free pool slot"); + } + slots_.push_back(Slot{}); + } + if (!slots_[slot].guc.empty()) { + throw adapters::db::DbError(ToDbBackend(backend), "scope_dirty", + "slot scope not reset, refusing tenant " + slug); + } + slots_[slot].busy = true; + slots_[slot].guc = slug; + slots_[slot].backend = backend; + ++active_; + return Lease(this, slot, slug, backend); + } + + std::size_t active_count() const { + std::lock_guard lock(mu_); + return active_; + } + std::size_t tenant_count() const { + std::lock_guard lock(mu_); + return tenants_.size(); + } + std::size_t slot_count() const { + std::lock_guard lock(mu_); + return slots_.size(); + } + + void DebugDirtySlotForTest(std::size_t slot, std::string guc) { + std::lock_guard lock(mu_); + if (slot >= slots_.size()) throw std::out_of_range("slot"); + slots_[slot].guc = std::move(guc); + } + + private: + struct Slot { + bool busy = false; + std::string guc; + Backend backend = Backend::Postgres; + }; + + static adapters::db::Backend ToDbBackend(Backend b) { return b; } + + void ReleaseSlot(std::size_t slot) { + std::lock_guard lock(mu_); + slots_[slot].guc.clear(); + slots_[slot].busy = false; + --active_; + } + + std::string SlotGuc(std::size_t slot) const { + std::lock_guard lock(mu_); + return slots_[slot].guc; + } + + mutable std::mutex mu_; + std::vector slots_; + std::unordered_set tenants_; + std::size_t active_ = 0; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/provisioner.h b/src/tenancy/provisioner.h new file mode 100644 index 0000000..943b7b2 --- /dev/null +++ b/src/tenancy/provisioner.h @@ -0,0 +1,78 @@ +#pragma once +// Provisioner creates and retires tenants. Backend hooks (schema, database, +// file creation) run inside provision; a failing hook rolls the catalog entry +// back so no half provisioned tenant survives. Purge delegates to the catalog +// gates and only then runs the drop hook. + +#include +#include +#include +#include +#include + +#include "tenancy/catalog.h" +#include "tenancy/strategy.h" +#include "tenancy/tenant.h" + +namespace molo::tenancy { + +class Provisioner { + public: + using Hook = std::function; + + struct Hooks { + Hook on_create; + Hook on_drop; + }; + + Provisioner(Catalog& catalog, Hooks hooks = {}) : catalog_(catalog), hooks_(std::move(hooks)) {} + + std::optional provision(const std::string& slug, + Strategy strategy, + std::int64_t now_unix) { + auto tenant = catalog_.create(slug, strategy, now_unix); + if (!tenant.has_value()) return std::nullopt; + if (hooks_.on_create && !hooks_.on_create(*tenant)) { + RollbackEntry(*tenant, now_unix); + return std::nullopt; + } + return tenant; + } + + bool rollback(const std::string& slug, std::int64_t now_unix) { + auto found = catalog_.find_by_slug(slug); + if (!found.has_value()) return false; + return RollbackEntry(*found, now_unix); + } + + bool suspend(const std::string& slug) { return catalog_.suspend(slug); } + + bool tombstone(const std::string& slug, std::int64_t now_unix) { + return catalog_.tombstone(slug, now_unix); + } + + PurgeResult purge(const std::string& slug, + const std::string& confirm, + std::int64_t now_unix, + std::int64_t retention_sec) { + auto found = catalog_.find_by_slug(slug); + PurgeResult gated = catalog_.purge(slug, confirm, now_unix, retention_sec); + if (gated == PurgeResult::Ok && hooks_.on_drop && found.has_value()) { + hooks_.on_drop(*found); + } + return gated; + } + + private: + bool RollbackEntry(const Tenant& tenant, std::int64_t now_unix) { + if (hooks_.on_drop) hooks_.on_drop(tenant); + catalog_.tombstone(tenant.slug(), now_unix); + return catalog_.purge(tenant.slug(), "PURGE:" + tenant.id(), now_unix, 0) == + PurgeResult::Ok; + } + + Catalog& catalog_; + Hooks hooks_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/quota.h b/src/tenancy/quota.h new file mode 100644 index 0000000..c3cfb27 --- /dev/null +++ b/src/tenancy/quota.h @@ -0,0 +1,112 @@ +#pragma once +// Quota enforces per-tenant request rate (token bucket: rate plus burst, +// 429 with retry_after on excess) and storage (507 insufficient_storage past +// the byte cap). Clock is injected in millis for deterministic tests. + +#include +#include +#include +#include +#include +#include + +#include "tenancy/caps.h" + +namespace molo::tenancy { + +struct QuotaLimits { + double requests_per_minute = Caps::kRequestsPerMinuteDefault; + double burst = Caps::kRequestBurstDefault; + std::int64_t storage_bytes = Caps::kStorageBytesDefault; +}; + +class Quota { + public: + using Limits = QuotaLimits; + + struct Verdict { + bool allowed = true; + int status = 200; + std::int64_t retry_after_sec = 0; + }; + + using Clock = std::function; + + explicit Quota(Limits limits = QuotaLimits{}, Clock clock = SystemClock) + : limits_(limits), clock_(std::move(clock)) {} + + static std::int64_t SystemClock() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + } + + Verdict check_request(const std::string& slug) { + std::lock_guard lock(mu_); + Bucket* bp = FindBucket(slug); + if (bp == nullptr) return Verdict{false, 429, 60}; + Bucket& b = *bp; + std::int64_t now = clock_(); + double rate_per_ms = limits_.requests_per_minute / 60000.0; + double capacity = limits_.burst > 0 ? limits_.burst : 1.0; + if (b.last_ms < 0) { + b.tokens = capacity; + } else if (now > b.last_ms) { + b.tokens += static_cast(now - b.last_ms) * rate_per_ms; + if (b.tokens > capacity) b.tokens = capacity; + } + b.last_ms = now; + if (b.tokens >= 1.0) { + b.tokens -= 1.0; + return Verdict{true, 200, 0}; + } + double deficit = 1.0 - b.tokens; + std::int64_t wait_ms = + rate_per_ms > 0 ? static_cast(deficit / rate_per_ms) : 60000; + return Verdict{false, 429, wait_ms / 1000 + 1}; + } + + Verdict add_storage(const std::string& slug, std::int64_t bytes) { + std::lock_guard lock(mu_); + Bucket* bp = FindBucket(slug); + if (bp == nullptr) return Verdict{false, 507, 0}; + Bucket& b = *bp; + if (b.storage_used + bytes > limits_.storage_bytes) { + return Verdict{false, 507, 0}; + } + b.storage_used += bytes; + return Verdict{true, 200, 0}; + } + + std::int64_t storage_used(const std::string& slug) { + std::lock_guard lock(mu_); + auto it = buckets_.find(slug); + return it == buckets_.end() ? 0 : it->second.storage_used; + } + + void reset(const std::string& slug) { + std::lock_guard lock(mu_); + buckets_.erase(slug); + } + + private: + struct Bucket { + double tokens = 0.0; + std::int64_t last_ms = -1; + std::int64_t storage_used = 0; + }; + + Bucket* FindBucket(const std::string& slug) { + auto it = buckets_.find(slug); + if (it != buckets_.end()) return &it->second; + if (buckets_.size() >= Caps::kPoolTenantsMax) return nullptr; + return &buckets_.emplace(slug, Bucket{}).first->second; + } + + Limits limits_; + Clock clock_; + std::mutex mu_; + std::unordered_map buckets_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/resolver.h b/src/tenancy/resolver.h new file mode 100644 index 0000000..12b0193 --- /dev/null +++ b/src/tenancy/resolver.h @@ -0,0 +1,113 @@ +#pragma once +// Resolver maps a request to a tenant from three sources: the X-Tenant +// header, the /t/ path prefix, and the host subdomain in the form +// . for a configured base. Matrix: exactly one distinct +// slug resolves; two distinct slugs conflict; none resolves to the dev +// default only when dev mode is on, else unknown. + +#include +#include +#include +#include + +#include "tenancy/adapters/http.h" +#include "tenancy/catalog.h" +#include "tenancy/tenant.h" +#include "tenancy/tenant_id.h" + +namespace molo::tenancy { + +struct ResolveRequest { + std::string host; + std::string path; + std::string tenant_header; +}; + +enum class ResolveStatus { Resolved, Conflict, Unknown }; + +struct ResolveOutcome { + ResolveStatus status = ResolveStatus::Unknown; + std::optional tenant; + bool dev_default = false; + std::string detail; +}; + +struct ResolverOptions { + bool dev_mode = false; + std::string dev_default_slug = "dev"; + std::vector base_domains = {"localhost"}; +}; + +class Resolver { + public: + using Options = ResolverOptions; + + explicit Resolver(Options opts = ResolverOptions{}) : opts_(std::move(opts)) {} + + static std::pair StripTenantPrefix(const std::string& path) { + if (path == "/t" || path.compare(0, 3, "/t/") != 0) return {path, ""}; + std::string rest = path.substr(3); + std::size_t slash = rest.find('/'); + std::string slug = slash == std::string::npos ? rest : rest.substr(0, slash); + if (!TenantId::TryParse(slug).has_value()) return {path, ""}; + std::string stripped = slash == std::string::npos ? "/" : rest.substr(slash); + if (stripped.empty()) stripped = "/"; + return {stripped, slug}; + } + + static std::string SlugFromHost(const std::string& host, + const std::vector& base_domains) { + std::string bare = host; + std::size_t colon = bare.find(':'); + if (colon != std::string::npos) bare = bare.substr(0, colon); + bare = adapters::Lower(bare); + for (const std::string& base : base_domains) { + std::string suffix = "." + adapters::Lower(base); + if (bare.size() <= suffix.size()) continue; + if (bare.compare(bare.size() - suffix.size(), suffix.size(), suffix) != 0) continue; + std::string label = bare.substr(0, bare.size() - suffix.size()); + if (label.find('.') != std::string::npos) continue; + if (!TenantId::TryParse(label).has_value()) continue; + return label; + } + return ""; + } + + ResolveOutcome Resolve(const ResolveRequest& req, Catalog& catalog) const { + std::string from_header = + TenantId::TryParse(req.tenant_header).has_value() ? req.tenant_header : ""; + std::string from_path = StripTenantPrefix(req.path).second; + std::string from_host = SlugFromHost(req.host, opts_.base_domains); + + std::string chosen; + for (const std::string* cand : {&from_header, &from_path, &from_host}) { + if (cand->empty()) continue; + if (chosen.empty()) { + chosen = *cand; + } else if (chosen != *cand) { + return ResolveOutcome{ResolveStatus::Conflict, std::nullopt, false, + "tenant sources disagree"}; + } + } + + bool dev_default = false; + if (chosen.empty()) { + if (!opts_.dev_mode) return ResolveOutcome{ResolveStatus::Unknown, std::nullopt, + false, "no tenant source"}; + chosen = opts_.dev_default_slug; + dev_default = true; + } + + auto tenant = catalog.find_by_slug(chosen); + if (!tenant.has_value()) { + return ResolveOutcome{ResolveStatus::Unknown, std::nullopt, false, + "unknown tenant: " + chosen}; + } + return ResolveOutcome{ResolveStatus::Resolved, tenant, dev_default, ""}; + } + + private: + Options opts_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/rls_policy.h b/src/tenancy/rls_policy.h new file mode 100644 index 0000000..8fa6db8 --- /dev/null +++ b/src/tenancy/rls_policy.h @@ -0,0 +1,40 @@ +#pragma once +// RlsPolicy renders Postgres row level security DDL for shared_schema tables. +// The policy reads app.current_tenant, which PoolFactory sets on every pinned +// checkout and resets on release. Table names are caller validated. + +#include + +namespace molo::tenancy { + +class RlsPolicy { + public: + static std::string SettingName() { return "app.current_tenant"; } + + static std::string EnableRls(const std::string& table) { + return "ALTER TABLE " + table + " ENABLE ROW LEVEL SECURITY"; + } + + static std::string PolicyDdl(const std::string& table) { + return "CREATE POLICY tenant_isolation ON " + table + + " USING (tenant_id = current_setting('app.current_tenant', true)) " + + "WITH CHECK (tenant_id = current_setting('app.current_tenant', true))"; + } + + static std::string DropPolicy(const std::string& table) { + return "DROP POLICY IF EXISTS tenant_isolation ON " + table; + } + + static std::string SetLocal(const std::string& slug) { + return "SET LOCAL app.current_tenant = '" + slug + "'"; + } + + static std::string ResetLocal() { return "RESET app.current_tenant"; } + + static std::string TenantIndexDdl(const std::string& table) { + return "CREATE INDEX IF NOT EXISTS " + table + + "_tenant_idx ON " + table + " (tenant_id)"; + } +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/store/catalog_store.h b/src/tenancy/store/catalog_store.h new file mode 100644 index 0000000..fdae805 --- /dev/null +++ b/src/tenancy/store/catalog_store.h @@ -0,0 +1,24 @@ +#pragma once +// Abstract tenant catalog storage. MemoryCatalog serves dev and tests; +// DbCatalog persists the same records in the tenants table. + +#include +#include +#include +#include + +#include "tenancy/tenant.h" + +namespace molo::tenancy::store { + +class CatalogStore { + public: + virtual ~CatalogStore() = default; + virtual bool upsert(const Tenant& tenant) = 0; + virtual std::optional get_by_id(const std::string& id) = 0; + virtual std::optional get_by_slug(const std::string& slug) = 0; + virtual bool remove(const std::string& id) = 0; + virtual std::vector list(std::size_t limit) = 0; +}; + +} // namespace molo::tenancy::store diff --git a/src/tenancy/store/db_catalog.h b/src/tenancy/store/db_catalog.h new file mode 100644 index 0000000..f035523 --- /dev/null +++ b/src/tenancy/store/db_catalog.h @@ -0,0 +1,119 @@ +#pragma once +// DbCatalog persists tenant records in a tenants table through the +// Connection::exec seam only. SQL uses positional ? placeholders accepted by +// both backends. Catalog reads run with an empty data tenant string: the +// catalog itself is never tenant scoped. + +#include +#include +#include +#include + +#include "tenancy/adapters/db.h" +#include "tenancy/store/catalog_store.h" +#include "tenancy/strategy.h" +#include "tenancy/tenant.h" + +namespace molo::tenancy::store { + +class DbCatalog : public CatalogStore { + public: + explicit DbCatalog(adapters::db::Connection& conn) : conn_(conn) {} + + void ensure_schema() { + adapters::db::Context ctx("tenancy-catalog", "", 30); + conn_.exec(ctx, adapters::db::BoundStatement::Raw( + "CREATE TABLE IF NOT EXISTS tenants (id TEXT PRIMARY KEY, " + "slug TEXT UNIQUE NOT NULL, strategy TEXT NOT NULL, " + "status TEXT NOT NULL, storage_bytes BIGINT NOT NULL, " + "created_unix BIGINT NOT NULL, tombstoned_unix BIGINT NOT NULL)")); + } + + bool upsert(const Tenant& tenant) override { + adapters::db::Context ctx("tenancy-catalog", "", 30); + adapters::db::Result updated = conn_.exec( + ctx, adapters::db::BoundStatement::Raw( + "UPDATE tenants SET slug=?, strategy=?, status=?, storage_bytes=?, " + "created_unix=?, tombstoned_unix=? WHERE id=?", + {adapters::db::MakeString(tenant.slug()), + adapters::db::MakeString(ToString(tenant.strategy())), + adapters::db::MakeString(ToString(tenant.status())), + adapters::db::MakeInt(tenant.storage_bytes()), + adapters::db::MakeInt(tenant.created_unix()), + adapters::db::MakeInt(tenant.tombstoned_unix()), + adapters::db::MakeString(tenant.id())})); + if (updated.affected > 0) return true; + adapters::db::Result inserted = conn_.exec( + ctx, adapters::db::BoundStatement::Raw( + "INSERT INTO tenants (id, slug, strategy, status, storage_bytes, " + "created_unix, tombstoned_unix) VALUES (?,?,?,?,?,?,?)", + {adapters::db::MakeString(tenant.id()), + adapters::db::MakeString(tenant.slug()), + adapters::db::MakeString(ToString(tenant.strategy())), + adapters::db::MakeString(ToString(tenant.status())), + adapters::db::MakeInt(tenant.storage_bytes()), + adapters::db::MakeInt(tenant.created_unix()), + adapters::db::MakeInt(tenant.tombstoned_unix())})); + return inserted.affected > 0; + } + + std::optional get_by_id(const std::string& id) override { + return SelectOne( + "SELECT id, slug, strategy, status, storage_bytes, created_unix, " + "tombstoned_unix FROM tenants WHERE id=?", + adapters::db::MakeString(id)); + } + + std::optional get_by_slug(const std::string& slug) override { + return SelectOne( + "SELECT id, slug, strategy, status, storage_bytes, created_unix, " + "tombstoned_unix FROM tenants WHERE slug=?", + adapters::db::MakeString(slug)); + } + + bool remove(const std::string& id) override { + adapters::db::Context ctx("tenancy-catalog", "", 30); + adapters::db::Result r = conn_.exec( + ctx, adapters::db::BoundStatement::Raw("DELETE FROM tenants WHERE id=?", + {adapters::db::MakeString(id)})); + return r.affected > 0; + } + + std::vector list(std::size_t limit) override { + adapters::db::Context ctx("tenancy-catalog", "", 30); + adapters::db::Result r = conn_.exec( + ctx, adapters::db::BoundStatement::Raw( + "SELECT id, slug, strategy, status, storage_bytes, created_unix, " + "tombstoned_unix FROM tenants ORDER BY id LIMIT ?", + {adapters::db::MakeInt(static_cast(limit))})); + std::vector out; + for (const auto& row : r.rows) out.push_back(FromRow(row)); + return out; + } + + private: + std::optional SelectOne(const std::string& sql, adapters::db::Value param) { + adapters::db::Context ctx("tenancy-catalog", "", 30); + adapters::db::Result r = conn_.exec( + ctx, adapters::db::BoundStatement::Raw(sql, {std::move(param)})); + if (r.rows.empty()) return std::nullopt; + return FromRow(r.rows.front()); + } + + static Tenant FromRow(const adapters::db::Row& row) { + auto get = [&row](const std::string& key) -> adapters::db::Value { + auto it = row.find(key); + return it == row.end() ? adapters::db::Value{nullptr} : it->second; + }; + return Tenant(adapters::db::AsString(get("id")), adapters::db::AsString(get("slug")), + ParseStrategy(adapters::db::AsString(get("strategy"))), + ParseStatus(adapters::db::AsString(get("status"))), + adapters::db::AsInt(get("storage_bytes")), + adapters::db::AsInt(get("created_unix")), + adapters::db::AsInt(get("tombstoned_unix"))); + } + + adapters::db::Connection& conn_; +}; + +} // namespace molo::tenancy::store diff --git a/src/tenancy/store/memory_catalog.h b/src/tenancy/store/memory_catalog.h new file mode 100644 index 0000000..b346234 --- /dev/null +++ b/src/tenancy/store/memory_catalog.h @@ -0,0 +1,81 @@ +#pragma once +// In-memory CatalogStore. Mutex guarded, bounded by kCatalogTenantsMax. +// New ids past the cap are rejected; updates to existing ids still succeed. + +#include +#include +#include +#include +#include +#include +#include + +#include "tenancy/caps.h" +#include "tenancy/store/catalog_store.h" +#include "tenancy/tenant.h" + +namespace molo::tenancy::store { + +class MemoryCatalog : public CatalogStore { + public: + bool upsert(const Tenant& tenant) override { + std::lock_guard lock(mu_); + auto it = by_id_.find(tenant.id()); + if (it == by_id_.end() && by_id_.size() >= Caps::kCatalogTenantsMax) return false; + if (it != by_id_.end() && it->second.slug() != tenant.slug()) { + slug_to_id_.erase(it->second.slug()); + } + by_id_.insert_or_assign(tenant.id(), tenant); + slug_to_id_[tenant.slug()] = tenant.id(); + return true; + } + + std::optional get_by_id(const std::string& id) override { + std::lock_guard lock(mu_); + auto it = by_id_.find(id); + if (it == by_id_.end()) return std::nullopt; + return it->second; + } + + std::optional get_by_slug(const std::string& slug) override { + std::lock_guard lock(mu_); + auto it = slug_to_id_.find(slug); + if (it == slug_to_id_.end()) return std::nullopt; + auto jt = by_id_.find(it->second); + if (jt == by_id_.end()) return std::nullopt; + return jt->second; + } + + bool remove(const std::string& id) override { + std::lock_guard lock(mu_); + auto it = by_id_.find(id); + if (it == by_id_.end()) return false; + slug_to_id_.erase(it->second.slug()); + by_id_.erase(it); + return true; + } + + std::vector list(std::size_t limit) override { + std::lock_guard lock(mu_); + std::vector out; + for (const auto& [id, tenant] : by_id_) { + if (out.size() >= limit) break; + out.push_back(tenant); + } + std::sort(out.begin(), out.end(), + [](const Tenant& a, const Tenant& b) { return a.id() < b.id(); }); + return out; + } + + std::size_t size() const { + std::lock_guard lock(mu_); + return by_id_.size(); + } + + private: + mutable std::mutex mu_; + std::unordered_map by_id_; + std::unordered_map slug_to_id_; +}; + +} // namespace molo::tenancy::store diff --git a/src/tenancy/strategy.h b/src/tenancy/strategy.h new file mode 100644 index 0000000..3f32182 --- /dev/null +++ b/src/tenancy/strategy.h @@ -0,0 +1,46 @@ +#pragma once +// Tenant isolation strategy. SharedSchema keeps one schema with a tenant_id +// column plus RLS; SchemaPerTenant isolates by schema; DatabasePerTenant by +// database (pg) ; FilePerTenant by SQLite file (bridge). + +#include +#include +#include + +namespace molo::tenancy { + +enum class Strategy { + SharedSchema, + SchemaPerTenant, + DatabasePerTenant, + FilePerTenant, +}; + +inline std::string ToString(Strategy s) { + switch (s) { + case Strategy::SharedSchema: + return "shared_schema"; + case Strategy::SchemaPerTenant: + return "schema_per_tenant"; + case Strategy::DatabasePerTenant: + return "database_per_tenant"; + case Strategy::FilePerTenant: + return "file_per_tenant"; + } + return "shared_schema"; +} + +inline std::optional TryParseStrategy(std::string_view raw) { + if (raw == "shared_schema") return Strategy::SharedSchema; + if (raw == "schema_per_tenant") return Strategy::SchemaPerTenant; + if (raw == "database_per_tenant") return Strategy::DatabasePerTenant; + if (raw == "file_per_tenant") return Strategy::FilePerTenant; + return std::nullopt; +} + +inline Strategy ParseStrategy(std::string_view raw) { + auto parsed = TryParseStrategy(raw); + return parsed.has_value() ? *parsed : Strategy::SharedSchema; +} + +} // namespace molo::tenancy diff --git a/src/tenancy/tenant.h b/src/tenancy/tenant.h new file mode 100644 index 0000000..c8c3870 --- /dev/null +++ b/src/tenancy/tenant.h @@ -0,0 +1,80 @@ +#pragma once +// Tenant record: identity, isolation strategy, lifecycle status and usage. +// Tombstone is a terminal marker: a tombstoned tenant serves nothing and can +// only be purged after the retention gate. + +#include +#include +#include + +#include "tenancy/strategy.h" + +namespace molo::tenancy { + +enum class TenantStatus { Active, Suspended, Tombstone }; + +inline std::string ToString(TenantStatus s) { + switch (s) { + case TenantStatus::Active: + return "active"; + case TenantStatus::Suspended: + return "suspended"; + case TenantStatus::Tombstone: + return "tombstone"; + } + return "active"; +} + +inline TenantStatus ParseStatus(const std::string& raw) { + if (raw == "suspended") return TenantStatus::Suspended; + if (raw == "tombstone") return TenantStatus::Tombstone; + return TenantStatus::Active; +} + +class Tenant { + public: + Tenant(std::string id, + std::string slug, + Strategy strategy, + TenantStatus status, + std::int64_t storage_bytes, + std::int64_t created_unix, + std::int64_t tombstoned_unix) + : id_(std::move(id)), + slug_(std::move(slug)), + strategy_(strategy), + status_(status), + storage_bytes_(storage_bytes), + created_unix_(created_unix), + tombstoned_unix_(tombstoned_unix) {} + + const std::string& id() const { return id_; } + const std::string& slug() const { return slug_; } + Strategy strategy() const { return strategy_; } + TenantStatus status() const { return status_; } + std::int64_t storage_bytes() const { return storage_bytes_; } + std::int64_t created_unix() const { return created_unix_; } + std::int64_t tombstoned_unix() const { return tombstoned_unix_; } + + bool is_active() const { return status_ == TenantStatus::Active; } + + void suspend() { + if (status_ == TenantStatus::Active) status_ = TenantStatus::Suspended; + } + void mark_tombstone(std::int64_t now_unix) { + status_ = TenantStatus::Tombstone; + tombstoned_unix_ = now_unix; + } + void add_storage(std::int64_t bytes) { storage_bytes_ += bytes; } + + private: + std::string id_; + std::string slug_; + Strategy strategy_; + TenantStatus status_; + std::int64_t storage_bytes_; + std::int64_t created_unix_; + std::int64_t tombstoned_unix_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/tenant_buckets.h b/src/tenancy/tenant_buckets.h new file mode 100644 index 0000000..400d5ed --- /dev/null +++ b/src/tenancy/tenant_buckets.h @@ -0,0 +1,78 @@ +#pragma once +// TenantBuckets is a named token bucket registry for tenant scoped rate +// limits (per route, per key, per operation). Named buckets carry their own +// capacity and refill rate; entries are tenant plus key pairs. Time is caller +// supplied millis so behavior stays deterministic in tests. + +#include +#include +#include +#include +#include + +#include "tenancy/caps.h" + +namespace molo::tenancy { + +class TenantBuckets { + public: + struct Spec { + double capacity = 100.0; + double refill_per_sec = 10.0; + }; + + bool configure(const std::string& bucket, Spec spec) { + std::lock_guard lock(mu_); + if (specs_.find(bucket) == specs_.end() && specs_.size() >= Caps::kRateBucketsMax) { + return false; + } + specs_[bucket] = spec; + return true; + } + + bool take(const std::string& bucket, + const std::string& key, + std::int64_t now_ms, + double count = 1.0) { + std::lock_guard lock(mu_); + auto spec = specs_.find(bucket); + if (spec == specs_.end()) return false; + std::string entry = bucket + '\x1f' + key; + auto it = entries_.find(entry); + if (it == entries_.end()) { + if (entries_.size() >= Caps::kBucketEntriesMax) return false; + it = entries_.emplace(entry, Entry{spec->second.capacity, now_ms}).first; + } + Entry& e = it->second; + if (now_ms > e.last_ms) { + e.tokens += static_cast(now_ms - e.last_ms) / 1000.0 * spec->second.refill_per_sec; + if (e.tokens > spec->second.capacity) e.tokens = spec->second.capacity; + e.last_ms = now_ms; + } + if (e.tokens < count) return false; + e.tokens -= count; + return true; + } + + void reset(const std::string& bucket, const std::string& key) { + std::lock_guard lock(mu_); + entries_.erase(bucket + '\x1f' + key); + } + + std::size_t entries() const { + std::lock_guard lock(mu_); + return entries_.size(); + } + + private: + struct Entry { + double tokens = 0.0; + std::int64_t last_ms = 0; + }; + + mutable std::mutex mu_; + std::unordered_map specs_; + std::unordered_map entries_; +}; + +} // namespace molo::tenancy diff --git a/src/tenancy/tenant_id.h b/src/tenancy/tenant_id.h new file mode 100644 index 0000000..10c2c3f --- /dev/null +++ b/src/tenancy/tenant_id.h @@ -0,0 +1,59 @@ +#pragma once +// Validated tenant slug. Lowercase DNS-safe: [a-z0-9-], 1..63 chars, +// must start and end with an alphanumeric. + +#include +#include +#include +#include +#include + +#include "tenancy/caps.h" + +namespace molo::tenancy { + +class TenantId { + public: + static std::optional TryParse(std::string_view raw) { + if (raw.empty() || raw.size() > Caps::kSlugMaxLen) return std::nullopt; + for (char c : raw) { + bool ok = (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-'; + if (!ok) return std::nullopt; + } + auto alnum = [](char c) { + return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); + }; + if (!alnum(raw.front()) || !alnum(raw.back())) return std::nullopt; + return TenantId(std::string(raw)); + } + + const std::string& slug() const { return slug_; } + + std::uint64_t hash() const { + std::uint64_t h = 1469598103934665603ULL; + for (unsigned char c : slug_) { + h ^= static_cast(c); + h *= 1099511628211ULL; + } + return h; + } + + bool operator==(const TenantId& other) const { return slug_ == other.slug_; } + bool operator!=(const TenantId& other) const { return !(*this == other); } + bool operator<(const TenantId& other) const { return slug_ < other.slug_; } + + private: + explicit TenantId(std::string slug) : slug_(std::move(slug)) {} + std::string slug_; +}; + +} // namespace molo::tenancy + +namespace std { +template <> +struct hash { + std::size_t operator()(const molo::tenancy::TenantId& id) const noexcept { + return static_cast(id.hash()); + } +}; +} // namespace std diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..c1f878e --- /dev/null +++ b/tests/README.md @@ -0,0 +1,65 @@ +# Test conventions (plan 10-workflow-example.md, section 6.3) + +Integration tests only. Every test exercises a URL end to end through real +HTTP handling, routing, middleware, views, and services. Isolated unit +tests are forbidden. + +## Placement + +Test file path and name derive from the URL under test; exactly one +structured place per route: + +| Route | Test file | +|---|---| +| `/health` | `tests/example/health_test.cpp` | +| `/users/:id` | `tests/example/users/id_test.cpp` | +| `/orm/users` | `tests/example/orm/users_test.cpp` | + +Shared helpers live in `tests/example/support/` and are not tests. + +## Case template + +Every test file follows Arrange / Act / Assert: + +```cpp +// Route: GET /orm/users (one-line behavior note) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; // Arrange: seed via app API + auto res = f.client->get("/orm/users"); // Act: one HTTP request + check(res.status == 200, "status 200"); // Assert: status ... + check(contains(res.body, "\"rows\""), "body shape"); + check(!f.services.sqlite->tables().empty(), "side effect: table exists"); + return finish("orm/users"); // names the route, never silent +} +``` + +Every route needs at minimum: happy path, auth failure (or 404/405 when +the route has no auth), validation failure, and one limit case (body too +large, rate limited, or not found). + +## Rules + +- Seed through the app API, never direct service writes, unless the test + targets self-healing. +- Assert status code, content-type, body shape, plus one side-effect check + (DB row, metric emitted, listener ran, file listed). +- Loud skips: if a fixture or the environment cannot be verified, fail + with a clear reason and counter. Never silently pass. +- Fixture services reset per test binary (fresh `Fixture` per main); + never share state across files. +- Caps: full suite max 10 min; single test max 30 s. Zero tolerance for + flakes: quarantine plus spec gap entry, fix within one session. + +## Running + +```sh +make -f example/example.mk example-test # this part, standalone +make test # whole repo (root Makefile) +``` + +Each binary prints `ok:` lines and ends with `PASS: ` or +`FAIL: (N failures)` with a nonzero exit. diff --git a/tests/example/auth/login_test.cpp b/tests/example/auth/login_test.cpp new file mode 100644 index 0000000..f163e15 --- /dev/null +++ b/tests/example/auth/login_test.cpp @@ -0,0 +1,22 @@ +// Route: POST /auth/login +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto ok = f.client->post("/auth/login", + "{\"username\":\"ada\",\"password\":" + "\"correct-horse\"}", + "application/json"); + check(ok.status == 200, "good credentials are 200"); + check(contains(ok.body, "\"user\":\"ada\""), "username echoed"); + auto bad = f.client->post("/auth/login", + "{\"username\":\"ada\",\"password\":\"wrong\"}", + "application/json"); + check(bad.status == 403, "bad password is 403"); + auto missing = f.client->post("/auth/login", "{\"username\":\"ada\"}", + "application/json"); + check(missing.status == 400, "missing field is 400"); + return finish("auth/login"); +} diff --git a/tests/example/auth/me_test.cpp b/tests/example/auth/me_test.cpp new file mode 100644 index 0000000..d254e23 --- /dev/null +++ b/tests/example/auth/me_test.cpp @@ -0,0 +1,21 @@ +// Route: GET /auth/me (Basic auth) +#include "support/Assert.hpp" + +using namespace molo::example::test; +using molo::example::fw::Method; + +int main() { + Fixture f; + auto anon = f.client->get("/auth/me"); + check(anon.status == 401, "no header is 401"); + auto bad = f.client->request( + Method::Get, "/auth/me", "", + {{"Authorization", "Basic YWRhOndyb25n"}}); + check(bad.status == 403, "bad password is 403"); + auto ok = f.client->request( + Method::Get, "/auth/me", "", + {{"Authorization", "Basic YWRhOmNvcnJlY3QtaG9yc2U="}}); + check(ok.status == 200, "good credentials are 200"); + check(contains(ok.body, "\"user\":\"ada\""), "identity returned"); + return finish("auth/me"); +} diff --git a/tests/example/cache/lookup_test.cpp b/tests/example/cache/lookup_test.cpp new file mode 100644 index 0000000..f430005 --- /dev/null +++ b/tests/example/cache/lookup_test.cpp @@ -0,0 +1,14 @@ +// Route: /cache/lookup (miss then hit) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto miss = f.client->get("/cache/lookup?key=k1"); + check(contains(miss.body, "\"hit\":false"), "first lookup misses"); + auto hit = f.client->get("/cache/lookup?key=k1"); + check(contains(hit.body, "\"hit\":true"), "second lookup hits"); + check(contains(hit.body, "computed-for-k1"), "same value served"); + return finish("cache/lookup"); +} diff --git a/tests/example/chain/deep_test.cpp b/tests/example/chain/deep_test.cpp new file mode 100644 index 0000000..652fa32 --- /dev/null +++ b/tests/example/chain/deep_test.cpp @@ -0,0 +1,14 @@ +// Route: /app1/app1/app1/whoami (application chaining, depth 3) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/app1/app1/app1/whoami"); + check(res.status == 200, "status 200"); + check(contains(res.body, "\"depth\":3"), "three mounts merged"); + auto shallow = f.client->get("/app1/whoami"); + check(shallow.status == 404, "partial chain is 404"); + return finish("chain/deep"); +} diff --git a/tests/example/cli/ai_test.cpp b/tests/example/cli/ai_test.cpp new file mode 100644 index 0000000..cfb1d53 --- /dev/null +++ b/tests/example/cli/ai_test.cpp @@ -0,0 +1,16 @@ +// Route: GET /cli/ai (AI harness sample, secrets redacted) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/cli/ai?prompt=add+a+route"); + check(res.status == 200, "status 200"); + check(contains(res.body, "devplace.net/openai/v1"), "devplace base url"); + check(contains(res.body, "\"model\":\"molodetz\""), "molodetz model"); + auto secret = f.client->get("/cli/ai?prompt=DEVPLACE_API_KEY%3Dabc123"); + check(!contains(secret.body, "abc123"), "secret redacted"); + check(contains(secret.body, "REDACTED"), "redaction marker present"); + return finish("cli/ai"); +} diff --git a/tests/example/embedded/assets_test.cpp b/tests/example/embedded/assets_test.cpp new file mode 100644 index 0000000..4f65c94 --- /dev/null +++ b/tests/example/embedded/assets_test.cpp @@ -0,0 +1,16 @@ +// Route: GET /embedded/{name} (embedded C++ assets) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/embedded/hello.txt"); + check(res.status == 200, "status 200"); + check(res.headers.get("X-Asset-Source") == "embedded", + "served from embedded bundle"); + check(contains(res.body, "embedded assets"), "bundle content served"); + auto missing = f.client->get("/embedded/nope.txt"); + check(missing.status == 404, "missing asset is 404"); + return finish("embedded/assets"); +} diff --git a/tests/example/events/fire_test.cpp b/tests/example/events/fire_test.cpp new file mode 100644 index 0000000..f4edd71 --- /dev/null +++ b/tests/example/events/fire_test.cpp @@ -0,0 +1,19 @@ +// Route: /events/fire (typed event bus) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + int seen = 0; + f.services.events->on("demo.ping", + [&](const molo::example::fw::Event&) { ++seen; }); + auto fire = f.client->post( + "/events/fire", "{\"type\":\"demo.ping\",\"payload\":\"p1\"}", + "application/json"); + check(fire.status == 200, "fire status 200"); + check(seen == 1, "side effect: listener ran"); + auto list = f.client->get("/events/fire"); + check(contains(list.body, "demo.ping"), "delivery recorded"); + return finish("events/fire"); +} diff --git a/tests/example/formats/data_test.cpp b/tests/example/formats/data_test.cpp new file mode 100644 index 0000000..4a13a77 --- /dev/null +++ b/tests/example/formats/data_test.cpp @@ -0,0 +1,26 @@ +// Route: GET /formats/data (content negotiation) +#include "support/Assert.hpp" + +using namespace molo::example::test; +using molo::example::fw::Method; + +int main() { + Fixture f; + auto def = f.client->get("/formats/data"); + check(contains(def.headers.get("Content-Type"), "application/json"), + "default is json"); + auto xml = f.client->request(Method::Get, "/formats/data", "", + {{"Accept", "application/xml"}}); + check(contains(xml.headers.get("Content-Type"), "application/xml"), + "xml negotiated"); + check(contains(xml.body, "42"), "xml body"); + auto weighted = f.client->request( + Method::Get, "/formats/data", "", + {{"Accept", "text/html;q=0.2, text/plain;q=0.9"}}); + check(contains(weighted.headers.get("Content-Type"), "text/plain"), + "q weights honored"); + auto nope = f.client->request(Method::Get, "/formats/data", "", + {{"Accept", "image/png"}}); + check(nope.status == 406, "no match is 406"); + return finish("formats/data"); +} diff --git a/tests/example/forms/contact_test.cpp b/tests/example/forms/contact_test.cpp new file mode 100644 index 0000000..6976f79 --- /dev/null +++ b/tests/example/forms/contact_test.cpp @@ -0,0 +1,21 @@ +// Route: /forms/contact (form render plus validation) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto form = f.client->get("/forms/contact"); + check(form.status == 200, "form renders 200"); + check(contains(form.body, "post("/forms/contact", + "name=ada&email=ada@x.io&message=hi", + "application/x-www-form-urlencoded"); + check(ok.status == 200, "valid post is 200"); + check(contains(ok.body, "Thanks ada"), "success page names user"); + auto bad = f.client->post("/forms/contact", "name=ada", + "application/x-www-form-urlencoded"); + check(bad.status == 422, "invalid post is 422"); + check(contains(bad.body, "validation_error"), "failures listed"); + return finish("forms/contact"); +} diff --git a/tests/example/health_test.cpp b/tests/example/health_test.cpp new file mode 100644 index 0000000..39bc214 --- /dev/null +++ b/tests/example/health_test.cpp @@ -0,0 +1,26 @@ +// Route: GET /health (probe plus real-HTTP loopback round-trip) +#include "support/Assert.hpp" + +#include "fw/Server.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/health"); + check(res.status == 200, "status 200"); + check(contains(res.body, "\"status\":\"ok\""), "ok body"); + + molo::example::fw::Server server(f.app); + check(server.start("127.0.0.1", 0), "loopback server starts"); + std::string wire = molo::example::fw::http_roundtrip( + "127.0.0.1", server.port(), + "GET /health HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + check(contains(wire, "200 OK"), "real HTTP status line"); + check(contains(wire, "\"status\":\"ok\""), "real HTTP body"); + server.stop(); + + auto missing = f.client->get("/no-such-route"); + check(missing.status == 404, "unknown route is 404"); + return finish("health"); +} diff --git a/tests/example/index_test.cpp b/tests/example/index_test.cpp new file mode 100644 index 0000000..937aa8e --- /dev/null +++ b/tests/example/index_test.cpp @@ -0,0 +1,18 @@ +// Route: GET / (index, template inheritance, coverage matrix) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/"); + check(res.status == 200, "status 200"); + check(contains(res.headers.get("Content-Type"), "text/html"), + "html content type"); + check(contains(res.body, "Living example"), "matrix heading"); + check(contains(res.body, "/routing/hello/world"), "matrix links a route"); + check(contains(res.body, "viewport"), "responsive viewport meta"); + check(contains(res.body, "nav-narrow"), "narrow nav dropdown present"); + check(res.headers.has("X-Request-ID"), "request id middleware ran"); + return finish("index"); +} diff --git a/tests/example/kv/items_test.cpp b/tests/example/kv/items_test.cpp new file mode 100644 index 0000000..0034fb1 --- /dev/null +++ b/tests/example/kv/items_test.cpp @@ -0,0 +1,19 @@ +// Route: /kv/items +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto missing = f.client->get("/kv/items?key=nope"); + check(missing.status == 404, "missing key is 404"); + auto bad = f.client->post("/kv/items", "{\"key\":\"a\"}", + "application/json"); + check(bad.status == 400, "key without value is 400"); + auto put = f.client->post("/kv/items", "{\"key\":\"a\",\"value\":\"b\"}", + "application/json"); + check(put.status == 201, "store status 201"); + auto get = f.client->get("/kv/items?key=a"); + check(contains(get.body, "\"value\":\"b\""), "round trip works"); + return finish("kv/items"); +} diff --git a/tests/example/metrics/counters_test.cpp b/tests/example/metrics/counters_test.cpp new file mode 100644 index 0000000..6c090e1 --- /dev/null +++ b/tests/example/metrics/counters_test.cpp @@ -0,0 +1,15 @@ +// Route: GET /metrics/counters (prometheus render) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto first = f.client->get("/metrics/counters"); + check(first.status == 200, "status 200"); + check(contains(first.body, "example.page.views 1"), "counter at 1"); + f.client->get("/metrics/counters"); + check(f.services.metrics->get("example.page.views") == 2.0, + "side effect: counter at 2"); + return finish("metrics/counters"); +} diff --git a/tests/example/openapi/docs_test.cpp b/tests/example/openapi/docs_test.cpp new file mode 100644 index 0000000..32ef6e7 --- /dev/null +++ b/tests/example/openapi/docs_test.cpp @@ -0,0 +1,16 @@ +// Routes: GET /openapi.json and GET /docs +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto spec = f.client->get("/openapi.json"); + check(spec.status == 200, "spec status 200"); + check(contains(spec.body, "\"openapi\":\"3.0.0\""), "openapi version"); + check(contains(spec.body, "/health"), "health path listed"); + auto docs = f.client->get("/docs"); + check(docs.status == 200, "docs status 200"); + check(contains(docs.body, "/openapi.json"), "docs link spec"); + return finish("openapi/docs"); +} diff --git a/tests/example/orm/users_test.cpp b/tests/example/orm/users_test.cpp new file mode 100644 index 0000000..9e9a5b3 --- /dev/null +++ b/tests/example/orm/users_test.cpp @@ -0,0 +1,24 @@ +// Route: /orm/users (dual backends, auto-sync self-healing) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto first = f.client->get("/orm/users?backend=sqlite"); + check(first.status == 200, "sqlite status 200"); + check(contains(first.body, "\"healed\":true"), "table auto-created"); + auto insert = f.client->post("/orm/users?backend=sqlite", + "{\"name\":\"ada\"}", "application/json"); + check(insert.status == 201, "insert status 201"); + auto widen = f.client->post("/orm/users?backend=sqlite", + "{\"name\":\"g\",\"email\":\"g@x.io\"}", + "application/json"); + check(contains(widen.body, "auto-added column email"), + "new column self-healed"); + auto pg = f.client->get("/orm/users?backend=postgres"); + check(contains(pg.body, "\"backend\":\"postgres\""), + "postgres backend selected"); + check(!f.services.sqlite->tables().empty(), "side effect: table exists"); + return finish("orm/users"); +} diff --git a/tests/example/pubsub/publish_test.cpp b/tests/example/pubsub/publish_test.cpp new file mode 100644 index 0000000..e832abb --- /dev/null +++ b/tests/example/pubsub/publish_test.cpp @@ -0,0 +1,20 @@ +// Route: /pubsub/publish +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + int seen = 0; + f.services.pubsub->subscribe("news", + [&](const std::string&) { ++seen; }); + auto pub = f.client->post("/pubsub/publish", + "{\"topic\":\"news\",\"message\":\"hi\"}", + "application/json"); + check(pub.status == 200, "publish status 200"); + check(contains(pub.body, "\"delivered\":1"), "one subscriber reached"); + check(seen == 1, "side effect: listener ran"); + auto hist = f.client->get("/pubsub/publish?topic=news"); + check(contains(hist.body, "hi"), "history kept"); + return finish("pubsub/publish"); +} diff --git a/tests/example/redirect/here_test.cpp b/tests/example/redirect/here_test.cpp new file mode 100644 index 0000000..5e4136d --- /dev/null +++ b/tests/example/redirect/here_test.cpp @@ -0,0 +1,14 @@ +// Route: GET /redirect/here (temporary vs permanent) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto temp = f.client->get("/redirect/here"); + check(temp.status == 302, "default is temporary"); + check(temp.headers.get("Location") == "/health", "location set"); + auto perm = f.client->get("/redirect/here?mode=perm"); + check(perm.status == 301, "perm mode is permanent"); + return finish("redirect/here"); +} diff --git a/tests/example/replication/state_test.cpp b/tests/example/replication/state_test.cpp new file mode 100644 index 0000000..25f2dd9 --- /dev/null +++ b/tests/example/replication/state_test.cpp @@ -0,0 +1,18 @@ +// Route: /replication/state (append plus since) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto add = f.client->post("/replication/state", + "{\"object\":\"doc-1\",\"data\":\"v1\"}", + "application/json"); + check(add.status == 201, "append is 201"); + check(contains(add.body, "last-writer-wins"), "conflict rule named"); + auto full = f.client->get("/replication/state?since=0"); + check(contains(full.body, "doc-1"), "entry replicated"); + auto empty = f.client->get("/replication/state?since=99"); + check(contains(empty.body, "\"entries\":[]"), "cursor filters"); + return finish("replication/state"); +} diff --git a/tests/example/restclient/self_test.cpp b/tests/example/restclient/self_test.cpp new file mode 100644 index 0000000..eae5e67 --- /dev/null +++ b/tests/example/restclient/self_test.cpp @@ -0,0 +1,12 @@ +// Route: GET /restclient/self (REST client self-call) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/restclient/self"); + check(res.status == 200, "status 200"); + check(contains(res.body, "\"self_call\":200"), "self call reached 200"); + return finish("restclient/self"); +} diff --git a/tests/example/routing/hello_test.cpp b/tests/example/routing/hello_test.cpp new file mode 100644 index 0000000..74e4575 --- /dev/null +++ b/tests/example/routing/hello_test.cpp @@ -0,0 +1,16 @@ +// Route: GET /routing/hello/{name} +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/routing/hello/ada"); + check(res.status == 200, "status 200"); + check(contains(res.body, "\"hello\":\"ada\""), "path param bound"); + auto encoded = f.client->get("/routing/hello/a%20b"); + check(contains(encoded.body, "\"hello\":\"a b\""), "param url-decoded"); + auto missing = f.client->get("/routing/hello"); + check(missing.status == 404, "missing segment is 404"); + return finish("routing/hello"); +} diff --git a/tests/example/routing/routers_test.cpp b/tests/example/routing/routers_test.cpp new file mode 100644 index 0000000..8bfbfe9 --- /dev/null +++ b/tests/example/routing/routers_test.cpp @@ -0,0 +1,14 @@ +// Route: /routing/routers/nested/deep (nested router include) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/routing/routers/nested/deep"); + check(res.status == 200, "status 200"); + check(contains(res.body, "\"nested\":true"), "nested router reached"); + auto shallow = f.client->get("/routing/routers"); + check(shallow.status == 404, "prefix alone is 404"); + return finish("routing/routers"); +} diff --git a/tests/example/routing/views_test.cpp b/tests/example/routing/views_test.cpp new file mode 100644 index 0000000..71579a3 --- /dev/null +++ b/tests/example/routing/views_test.cpp @@ -0,0 +1,21 @@ +// Route: /routing/views/greet (class-based view, GET plus POST) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto get = f.client->get("/routing/views/greet?name=grace"); + check(get.status == 200, "GET status 200"); + check(contains(get.body, "GreetView"), "view dispatched GET"); + check(contains(get.body, "grace"), "query value used"); + auto post = f.client->post("/routing/views/greet", "name=ada", + "application/x-www-form-urlencoded"); + check(post.status == 200, "POST status 200"); + check(contains(post.body, "\"method\":\"POST\""), "view dispatched POST"); + auto put = f.client->request(molo::example::fw::Method::Put, + "/routing/views/greet"); + check(put.status == 404 || put.status == 405, + "unregistered method rejected"); + return finish("routing/views"); +} diff --git a/tests/example/sessions/counter_test.cpp b/tests/example/sessions/counter_test.cpp new file mode 100644 index 0000000..1c263dd --- /dev/null +++ b/tests/example/sessions/counter_test.cpp @@ -0,0 +1,17 @@ +// Route: GET /sessions/counter (signed cookie session) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto first = f.client->get("/sessions/counter"); + check(first.status == 200, "status 200"); + check(contains(first.body, "\"count\":1"), "counter starts at 1"); + check(contains(first.body, "\"fresh\":true"), "first visit is fresh"); + check(first.headers.has("Set-Cookie"), "session cookie set"); + auto second = f.client->get("/sessions/counter"); + check(contains(second.body, "\"count\":2"), "jar keeps session"); + check(contains(second.body, "\"fresh\":false"), "second visit resumes"); + return finish("sessions/counter"); +} diff --git a/tests/example/socketio/rooms_test.cpp b/tests/example/socketio/rooms_test.cpp new file mode 100644 index 0000000..a416c58 --- /dev/null +++ b/tests/example/socketio/rooms_test.cpp @@ -0,0 +1,17 @@ +// Route: /socketio/rooms (socket.io skeleton endpoint) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto info = f.client->get("/socketio/rooms"); + check(info.status == 200, "status 200"); + check(contains(info.body, "socket.io"), "protocol named"); + check(contains(info.body, "long-polling"), "fallback named"); + auto emit = f.client->post( + "/socketio/rooms", "{\"room\":\"lobby\",\"event\":\"join\"}", + "application/json"); + check(contains(emit.body, "\"ack\":true"), "emit acknowledged"); + return finish("socketio/rooms"); +} diff --git a/tests/example/sse/events_test.cpp b/tests/example/sse/events_test.cpp new file mode 100644 index 0000000..61e18cc --- /dev/null +++ b/tests/example/sse/events_test.cpp @@ -0,0 +1,14 @@ +// Route: GET /sse/events +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/sse/events"); + check(res.status == 200, "status 200"); + check(contains(res.headers.get("Content-Type"), "text/event-stream"), + "sse content type"); + check(contains(res.body, "data: hello"), "event framed"); + return finish("sse/events"); +} diff --git a/tests/example/static/files_test.cpp b/tests/example/static/files_test.cpp new file mode 100644 index 0000000..994fb09 --- /dev/null +++ b/tests/example/static/files_test.cpp @@ -0,0 +1,17 @@ +// Route: GET /static/{name} (disk static, traversal guard) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto css = f.client->get("/static/app.css"); + check(css.status == 200, "css status 200"); + check(contains(css.headers.get("Content-Type"), "text/css"), + "css content type"); + check(css.headers.has("ETag"), "etag present"); + check(contains(css.body, "nav-narrow"), "stylesheet served"); + auto missing = f.client->get("/static/nope.css"); + check(missing.status == 404, "missing file is 404"); + return finish("static/files"); +} diff --git a/tests/example/streaming/feed_test.cpp b/tests/example/streaming/feed_test.cpp new file mode 100644 index 0000000..12544ab --- /dev/null +++ b/tests/example/streaming/feed_test.cpp @@ -0,0 +1,15 @@ +// Route: GET /streaming/feed +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/streaming/feed"); + check(res.status == 200, "status 200"); + check(res.headers.get("Transfer-Encoding") == "chunked", + "chunked encoding marked"); + check(contains(res.body, "chunk-1"), "first chunk present"); + check(contains(res.body, "chunk-3"), "last chunk present"); + return finish("streaming/feed"); +} diff --git a/tests/example/streaming/passthrough_test.cpp b/tests/example/streaming/passthrough_test.cpp new file mode 100644 index 0000000..c8f8a7d --- /dev/null +++ b/tests/example/streaming/passthrough_test.cpp @@ -0,0 +1,14 @@ +// Route: GET /streaming/passthrough +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/streaming/passthrough?target=/health"); + check(res.status == 200, "status 200"); + check(contains(res.body, "passthrough from /health"), + "upstream named"); + check(contains(res.body, "\"status\":\"ok\""), "upstream body relayed"); + return finish("streaming/passthrough"); +} diff --git a/tests/example/supervisor/tasks_test.cpp b/tests/example/supervisor/tasks_test.cpp new file mode 100644 index 0000000..9e36d44 --- /dev/null +++ b/tests/example/supervisor/tasks_test.cpp @@ -0,0 +1,18 @@ +// Route: /supervisor/tasks (enqueue plus list) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto add = f.client->post("/supervisor/tasks", + "{\"kind\":\"email\",\"payload\":\"{}\" }", + "application/json"); + check(add.status == 201, "enqueue is 201"); + check(contains(add.body, "\"id\":\"task-"), "task id issued"); + auto list = f.client->get("/supervisor/tasks"); + check(contains(list.body, "task-1"), "task listed"); + check(f.services.tasks->list().size() == 1, + "side effect: queue holds one"); + return finish("supervisor/tasks"); +} diff --git a/tests/example/supervisor/workers_test.cpp b/tests/example/supervisor/workers_test.cpp new file mode 100644 index 0000000..74f1bd0 --- /dev/null +++ b/tests/example/supervisor/workers_test.cpp @@ -0,0 +1,14 @@ +// Route: GET /supervisor/workers +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/supervisor/workers"); + check(res.status == 200, "status 200"); + check(contains(res.body, "\"name\":\"mailer\""), "mailer listed"); + check(contains(res.body, "\"name\":\"indexer\""), "indexer listed"); + check(contains(res.body, "\"running\":true"), "seed worker runs"); + return finish("supervisor/workers"); +} diff --git a/tests/example/support/Assert.hpp b/tests/example/support/Assert.hpp new file mode 100644 index 0000000..d79e2e4 --- /dev/null +++ b/tests/example/support/Assert.hpp @@ -0,0 +1,49 @@ +#pragma once + +// Shared test support for the living example suite. Not a test itself: every +// test file owns one main() and uses these helpers for loud assertions. + +#include +#include + +#include "app/App.hpp" +#include "fw/TestClient.hpp" + +namespace molo::example::test { + +inline int g_failures = 0; + +inline void check(bool cond, const std::string& what) { + if (cond) { + std::cout << " ok: " << what << "\n"; + } else { + ++g_failures; + std::cout << " FAIL: " << what << "\n"; + } +} + +inline bool contains(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +inline int finish(const std::string& name) { + if (g_failures == 0) { + std::cout << "PASS: " << name << "\n"; + return 0; + } + std::cout << "FAIL: " << name << " (" << g_failures << " failures)\n"; + return 1; +} + +struct Fixture { + molo::example::app::Services services; + std::shared_ptr app; + std::shared_ptr client; + + Fixture() { + app = molo::example::app::build_application(services); + client = std::make_shared(app); + } +}; + +} // namespace molo::example::test diff --git a/tests/example/telemetry/trace_test.cpp b/tests/example/telemetry/trace_test.cpp new file mode 100644 index 0000000..887b6bb --- /dev/null +++ b/tests/example/telemetry/trace_test.cpp @@ -0,0 +1,16 @@ +// Route: GET /telemetry/trace (span plus correlation id) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/telemetry/trace"); + check(res.status == 200, "status 200"); + check(contains(res.body, "\"trace\":\"trace-"), "trace id issued"); + check(contains(res.body, "\"correlation\":\"req-"), + "correlation id flows"); + check(f.services.telemetry->spans().size() == 1, + "side effect: span recorded"); + return finish("telemetry/trace"); +} diff --git a/tests/example/templates/page_test.cpp b/tests/example/templates/page_test.cpp new file mode 100644 index 0000000..88d8ffc --- /dev/null +++ b/tests/example/templates/page_test.cpp @@ -0,0 +1,18 @@ +// Route: GET /templates/page (extends, blocks, for, filters, escape) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/templates/page"); + check(res.status == 200, "status 200"); + check(contains(res.body, "Templates"), + "base layout applied"); + check(contains(res.body, "Author: ADA"), "upper filter ran"); + check(contains(res.body, "<b>bold</b>"), + "default output escaped"); + check(contains(res.body, "bold"), "safe opt-out kept markup"); + check(contains(res.body, "extends (inheritance)"), "for loop rendered"); + return finish("templates/page"); +} diff --git a/tests/example/uploads/file_test.cpp b/tests/example/uploads/file_test.cpp new file mode 100644 index 0000000..b61ff6b --- /dev/null +++ b/tests/example/uploads/file_test.cpp @@ -0,0 +1,23 @@ +// Route: POST /uploads/file +#include "support/Assert.hpp" + +using namespace molo::example::test; +using molo::example::fw::Method; + +int main() { + Fixture f; + auto ok = f.client->post("/uploads/file?filename=note.txt", "hello", + "text/plain"); + check(ok.status == 201, "allowed upload is 201"); + check(contains(ok.body, "\"size\":5"), "size reported"); + auto evil = f.client->request(Method::Post, + "/uploads/file?filename=x.bin", "data", + {{"Content-Type", "application/x-evil"}}); + check(evil.status == 415, "blocked content type is 415"); + auto traversal = f.client->post("/uploads/file?filename=../x.txt", "d", + "text/plain"); + check(traversal.status == 400, "unsafe filename is 400"); + check(f.services.uploads->list().size() == 1, + "side effect: one upload stored"); + return finish("uploads/file"); +} diff --git a/tests/example/validate/user_test.cpp b/tests/example/validate/user_test.cpp new file mode 100644 index 0000000..bf7a286 --- /dev/null +++ b/tests/example/validate/user_test.cpp @@ -0,0 +1,21 @@ +// Route: POST /validate/user (all-failures validation error) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto ok = f.client->post( + "/validate/user", + "{\"username\":\"ada\",\"email\":\"ada@x.io\",\"age\":\"36\"}", + "application/json"); + check(ok.status == 200, "valid model is 200"); + auto bad = f.client->post("/validate/user", + "{\"username\":\"ad\",\"age\":\"old\"}", + "application/json"); + check(bad.status == 422, "invalid model is 422"); + check(contains(bad.body, "username"), "username failure listed"); + check(contains(bad.body, "email"), "missing email listed"); + check(contains(bad.body, "age"), "bad age listed"); + return finish("validate/user"); +} diff --git a/tests/example/webdav/files_test.cpp b/tests/example/webdav/files_test.cpp new file mode 100644 index 0000000..dedb1ea --- /dev/null +++ b/tests/example/webdav/files_test.cpp @@ -0,0 +1,23 @@ +// Route: /webdav/files/ (PROPFIND, GET, PUT, DELETE) +#include "support/Assert.hpp" + +using namespace molo::example::test; +using molo::example::fw::Method; + +int main() { + Fixture f; + auto list = f.client->request(Method::Propfind, "/webdav/files/"); + check(list.status == 207, "propfind is 207"); + check(contains(list.body, "welcome.txt"), "seed file listed"); + auto put = f.client->request(Method::Put, "/webdav/files/new.txt", + "new-data", {{"Content-Type", "text/plain"}}); + check(put.status == 201, "put is 201"); + auto get = f.client->request(Method::Get, "/webdav/files/new.txt"); + check(contains(get.body, "new-data"), "round trip works"); + auto del = + f.client->request(Method::Delete, "/webdav/files/new.txt"); + check(del.status == 200, "delete is 200"); + auto gone = f.client->request(Method::Get, "/webdav/files/new.txt"); + check(gone.status == 404, "deleted file is 404"); + return finish("webdav/files"); +} diff --git a/tests/example/workflow/release_test.cpp b/tests/example/workflow/release_test.cpp new file mode 100644 index 0000000..1fcd22b --- /dev/null +++ b/tests/example/workflow/release_test.cpp @@ -0,0 +1,24 @@ +// Route: POST /workflow/release (bump, changelog, verify demo) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto feat = f.client->post("/workflow/release", + "commits=feat(routing): add hello", + "application/x-www-form-urlencoded"); + check(feat.status == 200, "status 200"); + check(contains(feat.body, "\"bump\":\"minor\""), "feat maps to minor"); + check(contains(feat.body, "\"version\":\"0.2.0\""), "version bumped"); + check(contains(feat.body, "\"green\":true"), "checks green"); + auto breaking = f.client->post( + "/workflow/release", "commits=feat!: drop old api", + "application/x-www-form-urlencoded"); + check(contains(breaking.body, "\"bump\":\"major\""), + "bang maps to major"); + auto bad = f.client->post("/workflow/release", "commits=nope", + "application/x-www-form-urlencoded"); + check(bad.status == 400, "malformed commit rejected"); + return finish("workflow/release"); +} diff --git a/tests/example/ws/chat_test.cpp b/tests/example/ws/chat_test.cpp new file mode 100644 index 0000000..06ab698 --- /dev/null +++ b/tests/example/ws/chat_test.cpp @@ -0,0 +1,15 @@ +// Route: /ws/chat (websocket skeleton endpoint) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto info = f.client->get("/ws/chat"); + check(info.status == 200, "status 200"); + check(contains(info.body, "\"protocol\":\"websocket\""), + "upgrade contract described"); + auto echo = f.client->post("/ws/chat", "ping", "text/plain"); + check(contains(echo.body, "\"echo\":\"ping\""), "frame echoed"); + return finish("ws/chat"); +} diff --git a/tests/src/app/00_compile_all_test.cpp b/tests/src/app/00_compile_all_test.cpp new file mode 100644 index 0000000..68e1732 --- /dev/null +++ b/tests/src/app/00_compile_all_test.cpp @@ -0,0 +1,62 @@ +// Compile-all: every header in this part must compile in one TU. +#include "app/app/application.h" +#include "app/app/plugin.h" +#include "app/async/task.h" +#include "app/caps.h" +#include "app/di/container.h" +#include "app/errors/handler.h" +#include "app/errors/http_error.h" +#include "app/forms/form.h" +#include "app/forms/multipart.h" +#include "app/forms/urlencoded.h" +#include "app/http_msg/cookies.h" +#include "app/http_msg/headers.h" +#include "app/http_msg/html.h" +#include "app/http_msg/method.h" +#include "app/http_msg/query.h" +#include "app/http_msg/request.h" +#include "app/http_msg/response.h" +#include "app/middleware/builtins.h" +#include "app/middleware/chain.h" +#include "app/middleware/middleware.h" +#include "app/negotiate/accept.h" +#include "app/negotiate/negotiator.h" +#include "app/routing/path_compiler.h" +#include "app/routing/route.h" +#include "app/routing/router.h" +#include "app/seams/request_id.h" +#include "app/seams/session.h" +#include "app/templates/ast.h" +#include "app/templates/cache.h" +#include "app/templates/context.h" +#include "app/templates/environment.h" +#include "app/templates/filters.h" +#include "app/templates/lexer.h" +#include "app/templates/loaders.h" +#include "app/templates/parser.h" +#include "app/templates/predicates.h" +#include "app/templates/renderer.h" +#include "app/templates/response.h" +#include "app/validate/errors.h" +#include "app/validate/field.h" +#include "app/validate/json.h" +#include "app/validate/model.h" +#include "app/validate/value.h" +#include "app/views/adapters.h" +#include "app/views/flash.h" +#include "app/views/view.h" + +#include +#include + +// R4: raw strings never convert implicitly to a 200 text response. +static_assert(!std::is_convertible_v); +static_assert(!std::is_convertible_v); +static_assert(!std::is_constructible_v); + +int main() { + molo::app::app::Application app; + (void)app.total_routes(); + std::cout << "PASS compile_all\n"; + return 0; +} diff --git a/tests/src/app/_testlib.h b/tests/src/app/_testlib.h new file mode 100644 index 0000000..62f9f12 --- /dev/null +++ b/tests/src/app/_testlib.h @@ -0,0 +1,92 @@ +#pragma once +// Shared integration-test helper: every test drives Application::handle +// (real routing, middleware, validation, negotiation, errors). Test files +// mirror URL paths under tests/src/app/. + +#include +#include +#include + +#include "app/app/application.h" +#include "app/async/task.h" +#include "app/http_msg/request.h" +#include "app/http_msg/response.h" +#include "app/validate/json.h" +#include "app/validate/value.h" +#include "app/views/adapters.h" + +static int g_failures = 0; +static int g_checks = 0; + +#define CHECK(cond, msg) \ + do { \ + ++g_checks; \ + if (!(cond)) { \ + ++g_failures; \ + std::cerr << "FAIL line " << __LINE__ << ": " << msg << "\n"; \ + } \ + } while (0) + +#define CHECK_EQ_STR(a, b, msg) \ + do { \ + ++g_checks; \ + std::string _a = (a); \ + std::string _b = (b); \ + if (_a != _b) { \ + ++g_failures; \ + std::cerr << "FAIL line " << __LINE__ << ": " << msg << "\n got: [" << _a \ + << "]\n want: [" << _b << "]\n"; \ + } \ + } while (0) + +#define CHECK_EQ_INT(a, b, msg) \ + do { \ + ++g_checks; \ + long long _a = (long long)(a); \ + long long _b = (long long)(b); \ + if (_a != _b) { \ + ++g_failures; \ + std::cerr << "FAIL line " << __LINE__ << ": " << msg << " got " << _a \ + << " want " << _b << "\n"; \ + } \ + } while (0) + +inline molo::app::http_msg::Request make_request( + const std::string& method, const std::string& target, + const std::map& headers = {}, const std::string& body = "") { + using namespace molo::app::http_msg; + Method m = parse_method(method).value(); + std::string path = target; + std::string qraw; + std::size_t q = target.find('?'); + if (q != std::string::npos) { + path = target.substr(0, q); + qraw = target.substr(q + 1); + } + Headers h; + for (const auto& kv : headers) h.set(kv.first, kv.second); + Request req(m, path, Query::parse(qraw), h, body); + req.set_query_raw(qraw); + return req; +} + +inline molo::app::http_msg::Response run_request(molo::app::app::Application& app, + molo::app::http_msg::Request& req) { + return molo::app::async::run(app.handle(req)); +} + +inline molo::app::http_msg::Response get(molo::app::app::Application& app, + const std::string& target, + const std::map& headers = {}) { + molo::app::http_msg::Request req = make_request("GET", target, headers); + return run_request(app, req); +} + +inline int test_report(const std::string& name) { + if (g_failures == 0) { + std::cout << "PASS " << name << " (" << g_checks << " checks)\n"; + return 0; + } + std::cout << "FAIL " << name << " (" << g_failures << "/" << g_checks << " failed)\n"; + return 1; +} diff --git a/tests/src/app/di/scopes_test.cpp b/tests/src/app/di/scopes_test.cpp new file mode 100644 index 0000000..e4ba63f --- /dev/null +++ b/tests/src/app/di/scopes_test.cpp @@ -0,0 +1,63 @@ +// V9: app scope shared, request scope isolated, cycles fail quoting. +#include "../_testlib.h" + +#include + +using namespace molo::app; + +int main() { + app::Application app; + app.di().provide_value("greeting", std::string("hello")); + std::atomic builds{0}; + app.di().provide_typed( + "ticket", + [&builds](di::Resolver&) { + return builds.fetch_add(1) + 1; + }, + di::Scope::Request); + app.router().get("/who", views::adapt([](http_msg::Request& req) { + di::RequestScope* scope = app::Application::di_scope(req); + std::string greeting = scope->get_as("greeting"); + int first = scope->get_as("ticket"); + int second = scope->get_as("ticket"); + std::string out = greeting + " " + std::to_string(first) + " " + std::to_string(second); + return http_msg::TextResponse(out); + })); + app.startup(); + + http_msg::Response r1 = get(app, "/who"); + http_msg::Response r2 = get(app, "/who"); + CHECK_EQ_STR(r1.body(), "hello 1 1", "request scope caches within one request"); + CHECK_EQ_STR(r2.body(), "hello 2 2", "request scope is fresh per request"); + CHECK_EQ_INT(builds.load(), 2, "request factory ran once per request"); + + app::Application cyc; + cyc.di().provide("a", [](di::Resolver& r) -> std::any { + return r.get("b"); + }, di::Scope::App, {"b"}); + cyc.di().provide("b", [](di::Resolver& r) -> std::any { + return r.get("a"); + }, di::Scope::App, {"a"}); + bool threw = false; + std::string msg; + try { + cyc.startup(); + } catch (const di::CycleError& e) { + threw = true; + msg = e.what(); + } + CHECK(threw, "dependency cycle fails startup"); + CHECK(msg.find("'a'") != std::string::npos, "cycle quotes first provider"); + CHECK(msg.find("'b'") != std::string::npos, "cycle quotes second provider"); + + app::Application dup; + dup.di().provide_value("x", 1); + bool dup_threw = false; + try { + dup.di().provide_value("x", 2); + } catch (const di::DiError&) { + dup_threw = true; + } + CHECK(dup_threw, "duplicate DI provider fails loudly"); + return test_report("di/scopes"); +} diff --git a/tests/src/app/errors/render_test.cpp b/tests/src/app/errors/render_test.cpp new file mode 100644 index 0000000..c25ca68 --- /dev/null +++ b/tests/src/app/errors/render_test.cpp @@ -0,0 +1,47 @@ +// V5: mapped exceptions render exact JSON and HTML shapes; 500 hides +// internals; request_id is present in both. +#include "../_testlib.h" + +using namespace molo::app; + +int main() { + app::Application app; + app.router().get("/bad", views::adapt([](http_msg::Request&) -> http_msg::Response { + throw errors::BadRequest("custom bad"); + })); + app.router().get("/deny", views::adapt([](http_msg::Request&) -> http_msg::Response { + throw errors::Forbidden(); + })); + app.router().get("/leak", views::adapt([](http_msg::Request&) -> http_msg::Response { + throw std::runtime_error("secret db password hunter2"); + })); + app.startup(); + + http_msg::Response j = get(app, "/bad"); + CHECK_EQ_INT(j.status(), 400, "BadRequest maps to 400"); + CHECK_EQ_STR(j.content_type(), "application/json", "API error is JSON"); + validate::Value parsed = validate::parse_json(j.body()); + const auto* root = std::get_if(&parsed.storage()); + const auto* err = std::get_if(&root->at("error").storage()); + CHECK_EQ_STR(err->at("code").as_text(), "bad_request", "error code field"); + CHECK_EQ_STR(err->at("message").as_text(), "custom bad", "error message field"); + CHECK(!err->at("request_id").as_text().empty(), "JSON error carries request_id"); + + http_msg::Response page = get(app, "/deny", {{"accept", "text/html"}}); + CHECK_EQ_INT(page.status(), 403, "Forbidden maps to 403"); + CHECK(page.content_type().find("text/html") != std::string::npos, "browser error is HTML"); + CHECK(page.body().find("viewport") != std::string::npos, "error page is responsive"); + CHECK(page.body().find("forbidden") != std::string::npos, "error page shows code"); + CHECK(page.body().find("req-") != std::string::npos, "HTML error carries request_id"); + + http_msg::Response leak = get(app, "/leak"); + CHECK_EQ_INT(leak.status(), 500, "unknown exception maps to 500"); + CHECK(leak.body().find("hunter2") == std::string::npos, "500 hides internals"); + CHECK(leak.body().find("internal server error") != std::string::npos, + "500 shows fixed message"); + + http_msg::Response leak_html = get(app, "/leak", {{"accept", "text/html"}}); + CHECK_EQ_INT(leak_html.status(), 500, "HTML unknown exception maps to 500"); + CHECK(leak_html.body().find("hunter2") == std::string::npos, "HTML 500 hides internals"); + return test_report("errors/render"); +} diff --git a/tests/src/app/forms/bind_test.cpp b/tests/src/app/forms/bind_test.cpp new file mode 100644 index 0000000..21ff126 --- /dev/null +++ b/tests/src/app/forms/bind_test.cpp @@ -0,0 +1,92 @@ +// V10: JSON, urlencoded and multipart bind the same model; cap fixtures +// are rejected with the documented status. +#include "../_testlib.h" + +using namespace molo::app; + +static std::shared_ptr user_model() { + auto m = std::make_shared("user"); + validate::Field name("username", validate::FieldType::String); + name.required().min_len(3); + validate::Field age("age", validate::FieldType::Int); + age.required().ge(0); + m->field(name).field(age); + return m; +} + +int main() { + app::Application app; + routing::RouteOptions o; + o.name = "create"; + o.body_model = user_model(); + app.router().post("/users", views::adapt([](http_msg::Request& req) { + const validate::Object* b = req.validated("body"); + std::string out = b->at("username").as_text() + ":" + + std::to_string((long long)b->at("age").as_number()); + return http_msg::TextResponse(out); + }), o); + routing::RouteOptions tiny; + tiny.name = "tiny"; + tiny.body_model = user_model(); + tiny.max_body = 10; + app.router().post("/tiny", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("tiny"); + }), tiny); + app.startup(); + + http_msg::Request json = + make_request("POST", "/users", {{"content-type", "application/json"}}, + R"({"username":"ada","age":36})"); + CHECK_EQ_STR(run_request(app, json).body(), "ada:36", "JSON binds to model"); + + http_msg::Request url = + make_request("POST", "/users", {{"content-type", "application/x-www-form-urlencoded"}}, + "username=ada&age=36"); + CHECK_EQ_STR(run_request(app, url).body(), "ada:36", "urlencoded binds to model"); + + std::string mp = + "--B\r\nContent-Disposition: form-data; name=\"username\"\r\n\r\nada\r\n" + "--B\r\nContent-Disposition: form-data; name=\"age\"\r\n\r\n36\r\n" + "--B--\r\n"; + http_msg::Request multi = + make_request("POST", "/users", {{"content-type", "multipart/form-data; boundary=B"}}, mp); + CHECK_EQ_STR(run_request(app, multi).body(), "ada:36", "multipart binds to model"); + + http_msg::Request bad = + make_request("POST", "/users", {{"content-type", "application/json"}}, + R"({"username":"al","age":-1})"); + http_msg::Response v = run_request(app, bad); + CHECK_EQ_INT(v.status(), 422, "bad body fields yield one 422"); + CHECK(v.body().find("[\"body\",\"username\"]") != std::string::npos, "username failure listed"); + CHECK(v.body().find("[\"body\",\"age\"]") != std::string::npos, "age failure listed"); + + // 1001 urlencoded fields exceed the form cap. + std::string many; + for (int i = 0; i < 1001; ++i) { + if (i) many += '&'; + many += "f" + std::to_string(i) + "=v"; + } + http_msg::Request flood = + make_request("POST", "/users", {{"content-type", "application/x-www-form-urlencoded"}}, many); + CHECK_EQ_INT(run_request(app, flood).status(), 413, "1001 fields rejected with 413"); + + // JSON depth 33 exceeds the depth cap. + std::string deep(33, '['); + deep += std::string(33, ']'); + http_msg::Request deep_req = + make_request("POST", "/users", {{"content-type", "application/json"}}, deep); + CHECK_EQ_INT(run_request(app, deep_req).status(), 400, "depth 33 rejected with 400"); + + // Oversized single value exceeds the 1 MiB cap. + std::string big(1048577, 'x'); + http_msg::Request big_req = make_request( + "POST", "/users", {{"content-type", "application/x-www-form-urlencoded"}}, "v=" + big); + CHECK_EQ_INT(run_request(app, big_req).status(), 413, "oversized value rejected with 413"); + + // Per-route document cap. + http_msg::Request over = + make_request("POST", "/tiny", {{"content-type", "application/json"}}, + R"({"username":"ada","age":36})"); + CHECK_EQ_INT(run_request(app, over).status(), 413, "per-route max_body enforced"); + return test_report("forms/bind"); +} diff --git a/tests/src/app/health/ping_test.cpp b/tests/src/app/health/ping_test.cpp new file mode 100644 index 0000000..3a246d4 --- /dev/null +++ b/tests/src/app/health/ping_test.cpp @@ -0,0 +1,65 @@ +// V1: response subclass round-trips through the real stack. +#include "../_testlib.h" + +using namespace molo::app; + +int main() { + app::Application app; + routing::RouteOptions o; + app.router().get("/ping", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("pong"); + })); + app.router().get("/user", views::adapt([](http_msg::Request&) { + validate::Object u; + u["name"] = validate::Value("ada"); + u["id"] = validate::Value(7); + return http_msg::Response(http_msg::JsonResponse(validate::Value(u))); + })); + app.router().get("/page", views::adapt([](http_msg::Request&) { + return http_msg::Response(http_msg::HtmlResponse("

    hi

    ")); + })); + app.router().get("/feed", views::adapt([](http_msg::Request&) { + return http_msg::Response(http_msg::XmlResponse("")); + })); + app.router().get("/old", views::adapt([](http_msg::Request&) { + return http_msg::Response(http_msg::RedirectResponse::temporary("/page")); + })); + app.router().get("/moved", views::adapt([](http_msg::Request&) { + return http_msg::Response(http_msg::RedirectResponse::permanent("/page")); + })); + app.startup(); + + http_msg::Response t = get(app, "/ping"); + CHECK_EQ_INT(t.status(), 200, "text status"); + CHECK_EQ_STR(t.body(), "pong", "text body"); + CHECK_EQ_STR(t.content_type(), "text/plain; charset=utf-8", "text content type"); + + http_msg::Response j = get(app, "/user"); + CHECK_EQ_INT(j.status(), 200, "json status"); + CHECK_EQ_STR(j.content_type(), "application/json", "json content type"); + validate::Value parsed = validate::parse_json(j.body()); + const auto* obj = std::get_if(&parsed.storage()); + CHECK(obj != nullptr, "json body parses to object"); + CHECK(obj != nullptr && obj->at("name") == validate::Value("ada"), "json name field"); + CHECK(obj != nullptr && obj->at("id") == validate::Value(7), "json id field"); + + http_msg::Response h = get(app, "/page"); + CHECK_EQ_INT(h.status(), 200, "html status"); + CHECK_EQ_STR(h.content_type(), "text/html; charset=utf-8", "html content type"); + CHECK_EQ_STR(h.body(), "

    hi

    ", "html body"); + + http_msg::Response x = get(app, "/feed"); + CHECK_EQ_INT(x.status(), 200, "xml status"); + CHECK_EQ_STR(x.content_type(), "application/xml; charset=utf-8", "xml content type"); + + http_msg::Response r307 = get(app, "/old"); + CHECK_EQ_INT(r307.status(), 307, "temporary redirect keeps method with 307"); + CHECK_EQ_STR(r307.headers().get("location"), "/page", "307 location"); + + http_msg::Response r308 = get(app, "/moved"); + CHECK_EQ_INT(r308.status(), 308, "permanent redirect keeps method with 308"); + CHECK_EQ_STR(r308.headers().get("location"), "/page", "308 location"); + + CHECK(!t.headers().get("x-request-id").empty(), "request id echoed on response"); + return test_report("health/ping"); +} diff --git a/tests/src/app/middleware/order_test.cpp b/tests/src/app/middleware/order_test.cpp new file mode 100644 index 0000000..bf73c9f --- /dev/null +++ b/tests/src/app/middleware/order_test.cpp @@ -0,0 +1,96 @@ +// V8: onion order, double-Next failure, short-circuit. +#include "../_testlib.h" + +using namespace molo::app; + +struct Probe : public middleware::Middleware { + std::string id; + std::vector* log; + Probe(std::string i, std::vector* l) : id(std::move(i)), log(l) {} + middleware::Category category() const override { return middleware::Category::User; } + std::string name() const override { return "probe_" + id; } + async::Task handle(http_msg::Request& req, + middleware::Next next) override { + log->push_back("in_" + id); + http_msg::Response res = co_await next(req); + log->push_back("out_" + id); + co_return res; + } +}; + +struct ShortCircuit : public middleware::Middleware { + middleware::Category category() const override { return middleware::Category::User; } + std::string name() const override { return "short"; } + async::Task handle(http_msg::Request&, + middleware::Next) override { + co_return http_msg::TextResponse("blocked"); + } +}; + +struct DoubleNext : public middleware::Middleware { + middleware::Category category() const override { return middleware::Category::User; } + std::string name() const override { return "double"; } + async::Task handle(http_msg::Request& req, + middleware::Next next) override { + http_msg::Response first = co_await next(req); + http_msg::Response second = co_await next(req); + (void)second; + co_return first; + } +}; + +struct WrongOrder : public middleware::Middleware { + middleware::Category category() const override { return middleware::Category::Security; } + std::string name() const override { return "late_security"; } + async::Task handle(http_msg::Request& req, + middleware::Next next) override { + co_return co_await next(req); + } +}; + +int main() { + std::vector log; + app::Application app; + app.chain().use(std::make_shared("A", &log)); + app.chain().use(std::make_shared("B", &log)); + app.router().get("/x", views::adapt([&log](http_msg::Request&) { + log.push_back("handler"); + return http_msg::TextResponse("x"); + })); + app.startup(); + CHECK_EQ_STR(get(app, "/x").body(), "x", "chain reaches handler"); + std::string seq; + for (const auto& e : log) seq += e + ","; + CHECK_EQ_STR(seq, "in_A,in_B,handler,out_B,out_A,", "middleware onion order"); + + bool order_threw = false; + try { + app.chain().use(std::make_shared()); + } catch (const middleware::ChainOrderError& e) { + order_threw = true; + CHECK(std::string(e.what()).find("late_security") != std::string::npos, + "order error names the layer"); + } + CHECK(order_threw, "cross-category reorder fails loudly"); + + app::Application cut; + cut.chain().use(std::make_shared()); + bool reached = false; + cut.router().get("/x", views::adapt([&reached](http_msg::Request&) { + reached = true; + return http_msg::TextResponse("x"); + })); + cut.startup(); + CHECK_EQ_STR(get(cut, "/x").body(), "blocked", "short-circuit answers directly"); + CHECK(!reached, "short-circuit never reaches the handler"); + + app::Application dbl; + dbl.chain().use(std::make_shared()); + dbl.router().get("/x", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("x"); + })); + dbl.startup(); + http_msg::Response res = get(dbl, "/x"); + CHECK_EQ_INT(res.status(), 500, "double Next fails loudly as 500"); + return test_report("middleware/order"); +} diff --git a/tests/src/app/mount/nested_test.cpp b/tests/src/app/mount/nested_test.cpp new file mode 100644 index 0000000..8d56d4a --- /dev/null +++ b/tests/src/app/mount/nested_test.cpp @@ -0,0 +1,102 @@ +// V3: prefixed includes, triple-nested mount, startup diagnostics. +#include "../_testlib.h" + +using namespace molo::app; + +int main() { + // Prefixed include_router. + app::Application app; + routing::Router users; + routing::RouteOptions o; + o.name = "user_list"; + users.get("/users", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("list"); + }), o); + app.include_router("/api", users); + app.startup(); + CHECK_EQ_STR(get(app, "/api/users").body(), "list", "prefixed include serves"); + CHECK_EQ_STR(app.url_for("user_list"), "/api/users", "url_for sees merged path"); + + // Triple-nested mount: /app1/app1/app1. + auto inner = std::make_shared(); + routing::RouteOptions deep; + deep.name = "deep"; + inner->router().get("/app1", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("deep"); + }), deep); + auto mid = std::make_shared(); + mid->mount("/app1", inner); + app::Application top; + top.mount("/app1", mid); + top.startup(); + CHECK_EQ_STR(get(top, "/app1/app1/app1").body(), "deep", "triple mount merges paths"); + CHECK_EQ_STR(top.url_for("app1:app1:deep"), "/app1/app1/app1", "url_for crosses mounts"); + + // Duplicate names fail startup with quoted paths. + app::Application dup; + routing::RouteOptions d1; + d1.name = "dup"; + dup.router().get("/a", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("a"); + }), d1); + routing::RouteOptions d2; + d2.name = "dup"; + dup.router().get("/b", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("b"); + }), d2); + bool dup_threw = false; + std::string dup_msg; + try { + dup.startup(); + } catch (const app::StartupError& e) { + dup_threw = true; + dup_msg = e.what(); + } + CHECK(dup_threw, "duplicate route names fail startup"); + CHECK(dup_msg.find("'/a'") != std::string::npos, "diagnostic quotes first path"); + CHECK(dup_msg.find("'/b'") != std::string::npos, "diagnostic quotes second path"); + CHECK(dup_msg.find("dup") != std::string::npos, "diagnostic names the route"); + + // Unknown converter fails fast naming the route. + routing::Router bad; + bool conv_threw = false; + std::string conv_msg; + try { + routing::RouteOptions bo; + bo.name = "broken"; + bad.get("/x/{id:uuid}", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("x"); + }), bo); + } catch (const routing::PathCompileError& e) { + conv_threw = true; + conv_msg = e.what(); + } + CHECK(conv_threw, "unknown converter is a startup error"); + CHECK(conv_msg.find("uuid") != std::string::npos, "diagnostic names the converter"); + CHECK(conv_msg.find("broken") != std::string::npos, "diagnostic names the route"); + + // slash_insensitive registers both forms with 308 to the declared form. + app::Application slash; + routing::RouteOptions so; + so.name = "canon"; + so.slash_insensitive = true; + slash.router().get("/canon", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("canon"); + }), so); + slash.startup(); + CHECK_EQ_STR(get(slash, "/canon").body(), "canon", "declared slash form serves"); + http_msg::Response redir = get(slash, "/canon/"); + CHECK_EQ_INT(redir.status(), 308, "alternate slash form 308-redirects"); + CHECK_EQ_STR(redir.headers().get("location"), "/canon", "redirect targets declared form"); + + bool frozen_threw = false; + try { + slash.router().get("/late", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("late"); + })); + } catch (const routing::PathCompileError& e) { + frozen_threw = std::string(e.what()).find("frozen") != std::string::npos; + } + CHECK(frozen_threw, "routes freeze at startup"); + return test_report("mount/nested"); +} diff --git a/tests/src/app/negotiate/select_test.cpp b/tests/src/app/negotiate/select_test.cpp new file mode 100644 index 0000000..fc0eddd --- /dev/null +++ b/tests/src/app/negotiate/select_test.cpp @@ -0,0 +1,74 @@ +// V6: q-weighted Accept picks the format; 406 lists available types; +// wrong Content-Type yields 415; format override is opt-in. +#include "../_testlib.h" + +using namespace molo::app; + +static http_msg::Response variant_data() { + validate::Object d; + d["name"] = validate::Value("ada"); + return http_msg::Response(http_msg::VariantResponse( + validate::Value(d), {"application/json", "text/html"})); +} + +int main() { + app::Application app; + routing::RouteOptions o; + o.name = "data"; + o.produces = {"application/json", "text/html"}; + app.router().get("/data", views::adapt([](http_msg::Request&) { return variant_data(); }), o); + routing::RouteOptions f; + f.name = "flex"; + f.produces = {"application/json", "text/html"}; + f.allow_format_override = true; + app.router().get("/flex", views::adapt([](http_msg::Request&) { return variant_data(); }), f); + routing::RouteOptions s; + s.name = "submit"; + s.consumes = {"application/json"}; + app.router().post("/submit", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("stored"); + }), s); + app.startup(); + + http_msg::Response def = get(app, "/data"); + CHECK_EQ_STR(def.content_type(), "application/json", "missing Accept means first produces"); + + http_msg::Response html = get(app, "/data", {{"accept", "text/html"}}); + CHECK(html.content_type().find("text/html") != std::string::npos, "exact match wins"); + CHECK(html.body().find("ada") != std::string::npos, "html variant renders data"); + + http_msg::Response q = + get(app, "/data", {{"accept", "text/html;q=0.5, application/json;q=0.9"}}); + CHECK_EQ_STR(q.content_type(), "application/json", "higher q wins"); + + http_msg::Response star = get(app, "/data", {{"accept", "text/*"}}); + CHECK(star.content_type().find("text/html") != std::string::npos, "type star matches"); + + http_msg::Response none = get(app, "/data", {{"accept", "image/png"}}); + CHECK_EQ_INT(none.status(), 406, "no match yields 406"); + validate::Value parsed = validate::parse_json(none.body()); + const auto* root = std::get_if(&parsed.storage()); + CHECK(root != nullptr && root->count("Detail"), "406 uses capital Detail shape"); + const auto* inner = std::get_if(&root->at("Detail").storage()); + const auto* avail = std::get_if(&inner->at("available").storage()); + CHECK(avail != nullptr && avail->size() == 2, "406 lists available types"); + + http_msg::Response over = + get(app, "/flex?format=html", {{"accept", "application/json"}}); + CHECK(over.content_type().find("text/html") != std::string::npos, + "format override wins when allowed"); + http_msg::Response ignored = + get(app, "/data?format=html", {{"accept", "application/json"}}); + CHECK_EQ_STR(ignored.content_type(), "application/json", + "format override ignored when not allowed"); + + http_msg::Request bad = + make_request("POST", "/submit", {{"content-type", "text/plain"}}, "x"); + http_msg::Response r415 = run_request(app, bad); + CHECK_EQ_INT(r415.status(), 415, "unconsumed content type yields 415"); + + http_msg::Request good = + make_request("POST", "/submit", {{"content-type", "application/json"}}, "{}"); + CHECK_EQ_INT(run_request(app, good).status(), 200, "consumed content type passes"); + return test_report("negotiate/select"); +} diff --git a/tests/src/app/plugins/hooks_test.cpp b/tests/src/app/plugins/hooks_test.cpp new file mode 100644 index 0000000..7c8dabc --- /dev/null +++ b/tests/src/app/plugins/hooks_test.cpp @@ -0,0 +1,75 @@ +// V12: plugin hooks fire in order; undeclared access is denied. +#include "../_testlib.h" + +using namespace molo::app; + +int main() { + std::vector events; + app::Application app; + app::Plugin plugin; + plugin.name = "watcher"; + plugin.capabilities = {"routes", "di", "errors"}; + plugin.hooks.route_added = [&events](app::PluginContext& ctx, const routing::Route& route) { + events.push_back("route_added:" + route.options().name + ":" + + std::to_string(ctx.route_count())); + }; + plugin.hooks.app_started = [&events](app::PluginContext& ctx) { + events.push_back(std::string("app_started:") + (ctx.di_has("nope") ? "yes" : "no")); + }; + plugin.hooks.error_render = [&events](app::PluginContext& ctx, int status, + const std::string& code) { + events.push_back("error_render:" + std::to_string(status) + ":" + code + ":" + + ctx.last_error_code()); + }; + app.plugins().register_plugin(plugin); + routing::RouteOptions a; + a.name = "first"; + app.router().get("/a", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("a"); + }), a); + routing::RouteOptions b; + b.name = "second"; + app.router().get("/b", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("b"); + }), b); + app.startup(); + CHECK_EQ_INT((long long)events.size(), 3, "startup fires route hooks plus started"); + CHECK(events.size() > 0 && events[0] == "route_added:first:2", "first route hook with count"); + CHECK(events.size() > 1 && events[1] == "route_added:second:2", "second route hook"); + CHECK(events.size() > 2 && events[2] == "app_started:no", "started hook after routes"); + + CHECK_EQ_INT(get(app, "/missing").status(), 404, "missing route is 404"); + CHECK(events.size() == 4 && events[3] == "error_render:404:not_found:not_found", + "error hook fires with status and code"); + + // Undeclared access is denied. + app::Application locked; + app::Plugin nosy; + nosy.name = "nosy"; + nosy.capabilities = {}; + nosy.hooks.app_started = [](app::PluginContext& ctx) { (void)ctx.route_count(); }; + locked.plugins().register_plugin(nosy); + bool denied = false; + try { + locked.startup(); + } catch (const app::PluginAccessError& e) { + denied = true; + CHECK(std::string(e.what()).find("nosy") != std::string::npos, "denial names plugin"); + CHECK(std::string(e.what()).find("routes") != std::string::npos, "denial names capability"); + } + CHECK(denied, "undeclared capability access throws"); + + // Unknown capabilities are rejected at registration. + app::Application reg; + app::Plugin bad; + bad.name = "bad"; + bad.capabilities = {"kernel"}; + bool reg_threw = false; + try { + reg.plugins().register_plugin(bad); + } catch (const app::PluginAccessError&) { + reg_threw = true; + } + CHECK(reg_threw, "unknown capability rejected at registration"); + return test_report("plugins/hooks"); +} diff --git a/tests/src/app/templates/render_test.cpp b/tests/src/app/templates/render_test.cpp new file mode 100644 index 0000000..e10226e --- /dev/null +++ b/tests/src/app/templates/render_test.cpp @@ -0,0 +1,220 @@ +// V11: template engine fixtures through the real stack. +#include "../_testlib.h" + +#include +#include + +#include "app/templates/response.h" + +using namespace molo::app; + +static templates::Environment make_env() { + templates::Environment env; + auto dict = std::make_shared(); + dict->set("base.html", + "{% block title %}base{% endblock %}" + "{% block body %}empty{% endblock %}"); + dict->set("child.html", + "{% extends \"base.html\" %}{% block title %}kid{% endblock %}" + "{% block body %}Hello {{ name }}!{% endblock %}"); + dict->set("head.html", "
    HEAD
    "); + dict->set("page.html", "{% include \"head.html\" %}
    {{ n }}
    "); + dict->set("cond.html", "{% if n > 1 %}big{% elif n == 1 %}one{% else %}none{% endif %}"); + dict->set("loop.html", + "{% for u in users %}{{ loop.index }}:{{ u }}{% if not loop.last %},{% endif %}" + "{% else %}nobody{% endfor %}"); + dict->set("filt.html", "{{ name|upper }}|{{ items|join(\", \") }}|{{ missing|default(\"d\") }}"); + dict->set("tests.html", + "{% if x is defined %}D{% endif %}{% if y is none %}N{% endif %}" + "{% if n is even %}E{% endif %}{% if s is string %}S{% endif %}"); + dict->set("mac.html", "{% macro greet(who, punct=\"!\") %}Hi {{ who }}{{ punct }}{% endmacro %}" + "{{ greet(\"ada\") }}{{ greet(\"bob\", punct=\"?\") }}"); + dict->set("xss.html", "A{{ name }}B{{ name|safe }}C"); + dict->set("raw.txt", "{{ name }}"); + dict->set("loopvar.html", "{{ loop.length }}"); // unused outside loops + dict->set("set.html", "{% set t = a ~ b %}{{ t }}"); + dict->set("esc.html", "{% autoescape false %}{{ name }}{% endautoescape %}"); + env.add_loader(dict); + env.set_undefined_policy(templates::UndefinedPolicy::Chain); + return env; +} + +int main() { + templates::Environment env = make_env(); + + validate::Object ctx; + ctx["name"] = validate::Value("ada"); + CHECK(env.render("child.html", ctx).find("kid") != std::string::npos, + "extends overrides title block"); + CHECK(env.render("child.html", ctx).find("Hello ada!") != std::string::npos, + "extends overrides body block"); + CHECK(env.render("child.html", ctx).find("empty") == std::string::npos, + "parent default body replaced"); + + CHECK(env.render("page.html", ctx).find("
    HEAD
    ") != std::string::npos, + "include renders partial"); + + validate::Object c2; + c2["n"] = validate::Value(5); + CHECK_EQ_STR(env.render("cond.html", c2), "big", "if branch"); + c2["n"] = validate::Value(1); + CHECK_EQ_STR(env.render("cond.html", c2), "one", "elif branch"); + c2["n"] = validate::Value(0); + CHECK_EQ_STR(env.render("cond.html", c2), "none", "else branch"); + + validate::Object loop; + validate::Array users; + users.emplace_back("a"); + users.emplace_back("b"); + loop["users"] = validate::Value(users); + CHECK_EQ_STR(env.render("loop.html", loop), "1:a,2:b", "for with loop.index and loop.last"); + validate::Object empty; + empty["users"] = validate::Value(validate::Array{}); + CHECK_EQ_STR(env.render("loop.html", empty), "nobody", "for else on empty"); + + validate::Object flt; + flt["name"] = validate::Value("ada"); + validate::Array items; + items.emplace_back("x"); + items.emplace_back("y"); + flt["items"] = validate::Value(items); + CHECK_EQ_STR(env.render("filt.html", flt), "ADA|x, y|d", "filters upper join default"); + + validate::Object tst; + tst["x"] = validate::Value(1); + tst["y"] = validate::Value::null(); + tst["n"] = validate::Value(4); + tst["s"] = validate::Value("hi"); + CHECK_EQ_STR(env.render("tests.html", tst), "DNES", "tests defined none even string"); + + CHECK(env.render("mac.html", {}).find("Hi ada!") != std::string::npos, "macro call"); + CHECK(env.render("mac.html", {}).find("Hi bob?") != std::string::npos, "macro kwarg"); + + validate::Object evil; + evil["name"] = validate::Value(""); + std::string xss = env.render("xss.html", evil); + CHECK(xss.find("<script>") != std::string::npos, "autoescape neutralizes markup"); + CHECK(xss.find("BC") != std::string::npos, "safe opts out"); + CHECK_EQ_STR(env.render("raw.txt", evil), "", + "txt template leaves text raw"); + CHECK_EQ_STR(env.render("esc.html", evil), "", + "autoescape block opts out"); + + validate::Object ab; + ab["a"] = validate::Value("x"); + ab["b"] = validate::Value("y"); + CHECK_EQ_STR(env.render("set.html", ab), "xy", "set with concat"); + + // Undefined policies. + templates::Environment strict = make_env(); + strict.set_undefined_policy(templates::UndefinedPolicy::Strict); + bool strict_threw = false; + try { + strict.render_source("{{ nope }}", "t.html", {}); + } catch (const templates::UndefinedError&) { + strict_threw = true; + } + CHECK(strict_threw, "strict undefined throws"); + CHECK_EQ_STR(env.render_source("{{ nope }}", "t.html", {}), "", "chain renders empty"); + + // TemplateResponse through the stack. + app::Application app; + app.templates_env().add_loader(std::make_shared( + std::map{{"hi.html", "

    Hi {{ name }}!

    "}})); + app.router().get("/hi", views::adapt([&app](http_msg::Request&) { + validate::Object c; + c["name"] = validate::Value("ada"); + return http_msg::Response(templates::TemplateResponse(app.templates_env(), "hi.html", c)); + })); + app.startup(); + http_msg::Response tr = get(app, "/hi"); + CHECK_EQ_INT(tr.status(), 200, "template route serves"); + CHECK(tr.body().find("Hi ada!") != std::string::npos, "template route renders context"); + + // Include depth cap aborts safely. + templates::Environment deep = make_env(); + auto chain = std::make_shared(); + for (int i = 0; i < 18; ++i) { + chain->set("d" + std::to_string(i) + ".html", + "{% include \"d" + std::to_string(i + 1) + ".html\" %}"); + } + chain->set("d18.html", "bottom"); + deep.add_loader(chain); + bool depth_threw = false; + try { + deep.render("d0.html", {}); + } catch (const templates::TemplateError&) { + depth_threw = true; + } + CHECK(depth_threw, "include depth 17 aborts safely"); + + // Loop iteration cap aborts safely. + validate::Object big; + validate::Array many; + many.reserve(100001); + for (int i = 0; i < 100001; ++i) many.emplace_back(i); + big["items"] = validate::Value(many); + bool loop_threw = false; + try { + env.render_source("{% for i in items %}{{ i }}{% endfor %}", "t.html", big); + } catch (const templates::TemplateError&) { + loop_threw = true; + } + CHECK(loop_threw, "100001 loop iterations abort safely"); + + // Output cap aborts safely. + std::string wide; + for (int i = 0; i < 400; ++i) wide += "{{ 'x' * 100000 }}"; + bool out_threw = false; + try { + env.render_source(wide, "t.html", {}); + } catch (const templates::TemplateError&) { + out_threw = true; + } + CHECK(out_threw, "render output over 32 MiB aborts safely"); + + // Traversal protection. + bool trav_threw = false; + try { + templates::FileLoader::sanitize("../../etc/passwd"); + if (templates::FileLoader::sanitize("../../etc/passwd").empty()) trav_threw = true; + } catch (...) { + trav_threw = true; + } + CHECK(trav_threw, "template loader rejects traversal"); + + // Dev reload vs prod cache through real files. + std::filesystem::create_directories("build/tmp_templates"); + { + std::ofstream f("build/tmp_templates/live.html"); + f << "v1"; + } + templates::Environment prod; + prod.add_loader(std::make_shared("build/tmp_templates")); + CHECK_EQ_STR(prod.render("live.html", {}), "v1", "prod renders file"); + { + std::ofstream f("build/tmp_templates/live.html"); + f << "v2"; + } + CHECK_EQ_STR(prod.render("live.html", {}), "v1", "prod serves compiled cache"); + templates::Environment dev; + dev.add_loader(std::make_shared("build/tmp_templates")); + dev.set_dev_reload(true); + CHECK_EQ_STR(dev.render("live.html", {}), "v2", "dev reload picks up edits"); + + // Cache holds at most 512 entries. + templates::Environment cache_env = make_env(); + for (int i = 0; i < 520; ++i) { + cache_env.render_source("x", "t" + std::to_string(i) + ".html", {}); + } + (void)cache_env; + templates::Environment cache2; + auto many_loader = std::make_shared(); + for (int i = 0; i < 520; ++i) { + many_loader->set("k" + std::to_string(i) + ".html", "x"); + } + cache2.add_loader(many_loader); + for (int i = 0; i < 520; ++i) cache2.render("k" + std::to_string(i) + ".html", {}); + CHECK_EQ_INT((long long)cache2.cache().size(), 512, "compiled cache capped at 512 LRU"); + return test_report("templates/render"); +} diff --git a/tests/src/app/users/detail_test.cpp b/tests/src/app/users/detail_test.cpp new file mode 100644 index 0000000..c13f207 --- /dev/null +++ b/tests/src/app/users/detail_test.cpp @@ -0,0 +1,75 @@ +// V2: static/converter/catch-all matches, 404 shape, 405 Allow, url_for. +#include "../_testlib.h" + +using namespace molo::app; + +int main() { + app::Application app; + app.router().get("/users/me", views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("me"); + })); + routing::RouteOptions detail; + detail.name = "user_detail"; + app.router().get("/users/{id:int}", views::adapt([](http_msg::Request& req) { + return http_msg::TextResponse("user " + req.path_param("id")); + }), detail); + routing::RouteOptions file; + file.name = "file_catch"; + app.router().get("/files/{rest:path}", views::adapt([](http_msg::Request& req) { + return http_msg::TextResponse("file " + req.path_param("rest")); + }), file); + routing::RouteOptions multi; + multi.name = "multi"; + app.router().add({http_msg::Method::GET, http_msg::Method::POST}, "/multi", + views::adapt([](http_msg::Request&) { + return http_msg::TextResponse("m"); + }), + multi); + app.startup(); + + CHECK_EQ_STR(get(app, "/users/me").body(), "me", "static route wins over converter"); + CHECK_EQ_STR(get(app, "/users/42").body(), "user 42", "int converter binds"); + CHECK_EQ_STR(get(app, "/files/a/b/c").body(), "file a/b/c", "path catch-all binds rest"); + + http_msg::Response nf = get(app, "/nope"); + CHECK_EQ_INT(nf.status(), 404, "missing route is 404"); + validate::Value body = validate::parse_json(nf.body()); + const auto* root = std::get_if(&body.storage()); + CHECK(root != nullptr && root->count("error"), "404 has error shape"); + const auto* inner = std::get_if(&root->at("error").storage()); + CHECK(inner != nullptr && inner->at("code") == validate::Value("not_found"), + "404 code is not_found"); + CHECK(inner != nullptr && !inner->at("request_id").as_text().empty(), + "404 carries request_id"); + + http_msg::Request bad = make_request("POST", "/users/me"); + http_msg::Response m = run_request(app, bad); + CHECK_EQ_INT(m.status(), 405, "method mismatch is 405"); + CHECK_EQ_STR(m.headers().get("allow"), "GET", "405 Allow lists GET"); + + http_msg::Request bad2 = make_request("DELETE", "/multi"); + http_msg::Response m2 = run_request(app, bad2); + CHECK_EQ_INT(m2.status(), 405, "multi-method mismatch is 405"); + CHECK_EQ_STR(m2.headers().get("allow"), "GET, POST", "405 Allow exact value"); + + CHECK_EQ_STR(app.url_for("user_detail", {{"id", "7"}}), "/users/7", "url_for converter"); + CHECK_EQ_STR(app.url_for("file_catch", {{"rest", "a/b"}}), "/files/a/b", "url_for catch-all"); + CHECK_EQ_STR(app.url_for("multi", {}, "a=1"), "/multi?a=1", "url_for with query"); + + bool threw = false; + try { + app.url_for("no_such_route"); + } catch (const errors::NotFound&) { + threw = true; + } + CHECK(threw, "url_for unknown name throws NotFound"); + + bool slash_404 = (get(app, "/users/me/").status() == 404); + CHECK(slash_404, "trailing slash is strict by default"); + + std::string long_path = "/" + std::string(2048, 'x'); + http_msg::Response too_long = get(app, long_path); + CHECK_EQ_INT(too_long.status(), 414, "overlong path yields 414"); + CHECK(!too_long.headers().get("x-request-id").empty(), "414 echoes request id"); + return test_report("users/detail"); +} diff --git a/tests/src/app/validate/input_test.cpp b/tests/src/app/validate/input_test.cpp new file mode 100644 index 0000000..445d2d9 --- /dev/null +++ b/tests/src/app/validate/input_test.cpp @@ -0,0 +1,75 @@ +// V4: one 422 listing every bad field across path, query, header, cookie. +#include "../_testlib.h" + +using namespace molo::app; + +static std::shared_ptr query_model() { + auto m = std::make_shared("q"); + validate::Field age("age", validate::FieldType::Int); + age.required().ge(0); + validate::Field name("name", validate::FieldType::String); + name.required().min_len(2); + m->field(age).field(name); + return m; +} + +int main() { + app::Application app; + routing::RouteOptions o; + o.name = "search"; + o.query_model = query_model(); + auto hm = std::make_shared("h"); + validate::Field token("x-token", validate::FieldType::String); + token.required(); + hm->field(token); + o.header_model = hm; + auto cm = std::make_shared("c"); + validate::Field sess("session", validate::FieldType::String); + sess.required(); + cm->field(sess); + o.cookie_model = cm; + app.router().get("/search", views::adapt([](http_msg::Request& req) { + const validate::Object* q = req.validated("query"); + return http_msg::TextResponse("ok " + q->at("name").as_text()); + }), o); + routing::RouteOptions u; + u.name = "user"; + app.router().get("/users/{id:int}", views::adapt([](http_msg::Request& req) { + return http_msg::TextResponse("user " + req.path_param("id")); + }), u); + app.startup(); + + http_msg::Response good = + get(app, "/search?age=3&name=ada", {{"x-token", "t"}, {"cookie", "session=s"}}); + CHECK_EQ_INT(good.status(), 200, "valid input binds"); + CHECK_EQ_STR(good.body(), "ok ada", "validated value reaches handler"); + + // Four violations at once: age, name, header, cookie. + http_msg::Response bad = get(app, "/search?age=-1&name=x"); + CHECK_EQ_INT(bad.status(), 422, "invalid input yields 422"); + validate::Value parsed = validate::parse_json(bad.body()); + const auto* root = std::get_if(&parsed.storage()); + CHECK(root != nullptr && root->count("detail"), "422 has detail shape"); + const auto* detail = std::get_if(&root->at("detail").storage()); + CHECK(detail != nullptr && detail->size() == 4, "422 lists every bad field at once"); + std::string joined; + for (const auto& item : *detail) { + const auto* f = std::get_if(&item.storage()); + const auto* loc = std::get_if(&f->at("loc").storage()); + for (const auto& part : *loc) joined += part.as_text() + "."; + joined += ";"; + CHECK(!f->at("msg").as_text().empty(), "failure carries msg"); + CHECK(!f->at("type").as_text().empty(), "failure carries type"); + } + CHECK(joined.find("query.age") != std::string::npos, "query age failure listed"); + CHECK(joined.find("query.name") != std::string::npos, "query name failure listed"); + CHECK(joined.find("header.x-token") != std::string::npos, "header failure listed"); + CHECK(joined.find("cookie.session") != std::string::npos, "cookie failure listed"); + + // Path converter failure is a 422 with path loc. + http_msg::Response pint = get(app, "/users/abc"); + CHECK_EQ_INT(pint.status(), 422, "bad path converter yields 422"); + CHECK(pint.body().find("\"path\"") != std::string::npos, "path loc present"); + CHECK(pint.body().find("\"id\"") != std::string::npos, "path param name present"); + return test_report("validate/input"); +} diff --git a/tests/src/app/views/dispatch_test.cpp b/tests/src/app/views/dispatch_test.cpp new file mode 100644 index 0000000..4f1a9d2 --- /dev/null +++ b/tests/src/app/views/dispatch_test.cpp @@ -0,0 +1,54 @@ +// V7: view method dispatch, 405 on unimplemented, redirects, flash-once. +#include "../_testlib.h" + +using namespace molo::app; + +class UserView : public views::View { + public: + using View::View; + async::Task get() override { + std::string out = "got " + request().path_param("id"); + co_return http_msg::TextResponse(out); + } + async::Task post() override { + co_return http_msg::TextResponse("posted"); + } + std::vector allowed_methods() const override { return {"GET", "POST"}; } +}; + +int main() { + app::Application app; + app.router().add({http_msg::Method::GET, http_msg::Method::POST, http_msg::Method::PUT}, + "/users/{id:int}", views::view_handler()); + app.router().get("/flash", views::adapt([](http_msg::Request& req) { + http_msg::Response res = http_msg::TextResponse("flashed"); + views::Flash::flash(req, res, "info", "hello once"); + return res; + })); + app.router().get("/show", views::adapt([](http_msg::Request& req) { + std::vector msgs = views::Flash::take(req); + std::string out; + for (const auto& m : msgs) out += m.level + ":" + m.text + ";"; + return http_msg::TextResponse(out); + })); + app.startup(); + + CHECK_EQ_STR(get(app, "/users/9").body(), "got 9", "view dispatches GET"); + http_msg::Request post = make_request("POST", "/users/9"); + CHECK_EQ_STR(run_request(app, post).body(), "posted", "view dispatches POST"); + http_msg::Request put = make_request("PUT", "/users/9"); + http_msg::Response m = run_request(app, put); + CHECK_EQ_INT(m.status(), 405, "unimplemented view method yields 405"); + CHECK_EQ_STR(m.headers().get("allow"), "GET, POST", "view 405 Allow value"); + + http_msg::Response f1 = get(app, "/flash"); + CHECK_EQ_INT(f1.status(), 200, "flash route serves"); + std::string set_cookie = f1.headers().get("set-cookie"); + CHECK(set_cookie.find("molo_session=") != std::string::npos, "flash sets session cookie"); + std::string cookie = set_cookie.substr(0, set_cookie.find(';')); + http_msg::Response s1 = get(app, "/show", {{"cookie", cookie}}); + CHECK_EQ_STR(s1.body(), "info:hello once;", "flash appears on next request"); + http_msg::Response s2 = get(app, "/show", {{"cookie", cookie}}); + CHECK_EQ_STR(s2.body(), "", "flash appears exactly once"); + return test_report("views/dispatch"); +} diff --git a/tests/src/audit/chain_tamper.cpp b/tests/src/audit/chain_tamper.cpp new file mode 100644 index 0000000..048b274 --- /dev/null +++ b/tests/src/audit/chain_tamper.cpp @@ -0,0 +1,61 @@ +// chain_tamper: 10k chained records verify clean; a 1-byte tamper names the +// exact seq; a dropped record reports a gap; seals round-trip under HMAC. +#include +#include +#include + +#include "audit/log/appender.h" +#include "audit/log/chain.h" +#include "audit/log/store.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL chain_tamper: " << msg << "\n"; \ + return 1; \ + } + +int main() { + using namespace molo::audit::log; + MemoryStore store; + Appender app(store); + for (int i = 0; i < 10000; ++i) { + Appender::Write w; + w.actor = "tester"; + w.action = "orm.update"; + w.table = "users"; + w.pk = std::to_string(i); + w.op = "update"; + w.after_json = "{\"i\":" + std::to_string(i) + "}"; + w.tenant = "t1"; + app.Append(std::move(w)); + } + CHECK(store.LatestSeq() == 10000, "10k appended"); + CHECK(app.appended() == 10000, "appended counter"); + CHECK(store.Seals().size() == 10, "seal every 1000 records"); + + std::vector all; + for (std::uint64_t s = 1; s <= 10000;) { + auto page = store.ReadRange(s, 1000); + CHECK(!page.empty(), "page nonempty"); + for (auto& r : page) all.push_back(r); + s = all.back().seq + 1; + } + VerifyResult ok = Chain::Verify(all); + CHECK(ok.ok, "clean chain verifies"); + + std::vector tampered = all; + tampered[4999].after_json[5] ^= 0x01; // 1-byte flip at seq 5000 + VerifyResult bad = Chain::Verify(tampered); + CHECK(!bad.ok && bad.bad_seq == 5000, "tamper names seq 5000"); + + std::vector gapped = all; + gapped.erase(gapped.begin() + 1234); // drop seq 1235 + VerifyResult gap = Chain::Verify(gapped); + CHECK(!gap.ok && gap.reason == "gap" && gap.bad_seq == 1235, "gap names seq 1235"); + + Seal s = Chain::MakeSeal(1, store.Tip(), 1700000000, "audit-key"); + CHECK(Chain::CheckSeal(s, "audit-key"), "seal verifies"); + CHECK(!Chain::CheckSeal(s, "wrong-key"), "seal rejects wrong key"); + std::cout << "PASS chain_tamper\n"; + return 0; +} diff --git a/tests/src/audit/erasure_cascade.cpp b/tests/src/audit/erasure_cascade.cpp new file mode 100644 index 0000000..2a1dac8 --- /dev/null +++ b/tests/src/audit/erasure_cascade.cpp @@ -0,0 +1,82 @@ +// erasure_cascade: all seven stores run, held subjects erase nothing, +// exempted subjects report Exempted, audit index keeps erased: marker, +// and the audit chain stays valid throughout. +#include +#include +#include + +#include "audit/log/appender.h" +#include "audit/log/chain.h" +#include "audit/log/store.h" +#include "audit/rights/eraser.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL erasure_cascade: " << msg << "\n"; \ + return 1; \ + } + +int main() { + using namespace molo::audit::rights; + using molo::audit::log::Appender; + using molo::audit::log::Chain; + using molo::audit::log::MemoryStore; + using molo::audit::retention::Hold; + + MemoryStore store; + Appender app(store); + RightsRequest req; + req.id = "dsar-9"; + req.kind = RequestKind::kErasure; + req.subject = Subject::User("t1", "42"); + + std::map stock = {{"sessions", 2}, {"cache_tags", 3}, {"kv_prefix", 5}, + {"orm_rows", 4}, {"uploads", 1}, {"tasks", 2}}; + std::string audit_index = "user:42"; + CascadeDeleters d; + auto mk = [&](const std::string& name) { + return [&, name](const Subject&) -> std::uint64_t { + std::uint64_t n = stock[name]; + stock[name] = 0; + return n; + }; + }; + d.sessions = mk("sessions"); + d.cache_tags = mk("cache_tags"); + d.kv_prefix = mk("kv_prefix"); + d.orm_rows = mk("orm_rows"); + d.uploads = mk("uploads"); + d.tasks = mk("tasks"); + d.audit_index = [&](const Subject&, const std::string& id) { audit_index = "erased:" + id; }; + + Hold hold; + EraseReport r = Eraser::Erase(req, hold, d); + CHECK(r.outcome == EraseOutcome::kDone, "outcome done"); + CHECK(r.erased_total == 17, "all stores erased"); + CHECK(r.per_store.size() == 6, "six store counts"); + CHECK(audit_index == "erased:dsar-9", "audit index marker"); + for (auto& [_, left] : stock) CHECK(left == 0, "rescan zero left"); + + Appender::Write w; + w.actor = "dsar"; + w.action = "rights.erasure"; + w.table = "subjects"; + w.pk = "user:42"; + w.op = "erase"; + w.tenant = "t1"; + app.Append(std::move(w)); + CHECK(Chain::Verify(store.ReadRange(1, 100)).ok, "chain valid after erasure"); + + stock["orm_rows"] = 4; + Hold hold2; + hold2.Add(Hold::SubjectKey("t1", "user:42"), "court order"); + EraseReport h = Eraser::Erase(req, hold2, d); + CHECK(h.outcome == EraseOutcome::kHeld && h.erased_total == 0, "held erases nothing"); + CHECK(stock["orm_rows"] == 4, "held rows untouched"); + + d.has_exemption = [](const Subject&) { return true; }; + EraseReport x = Eraser::Erase(req, Hold{}, d); + CHECK(x.outcome == EraseOutcome::kExempted, "exemption reported"); + std::cout << "PASS erasure_cascade\n"; + return 0; +} diff --git a/tests/src/audit/export_bundle.cpp b/tests/src/audit/export_bundle.cpp new file mode 100644 index 0000000..7f6575b --- /dev/null +++ b/tests/src/audit/export_bundle.cpp @@ -0,0 +1,59 @@ +// export_bundle: manifest schema, per-chunk checksums, byte determinism +// regardless of input row order. +#include +#include +#include + +#include "audit/rights/exporter.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL export_bundle: " << msg << "\n"; \ + return 1; \ + } + +static molo::audit::rights::ExportBundle Run(const std::vector& in, + const molo::audit::rights::RightsRequest& req) { + using namespace molo::audit::rights; + std::size_t i = 0; + return Exporter::Build(req, [&](ExportRow& out) -> bool { + if (i >= in.size()) return false; + out = in[i++]; + return true; + }); +} + +static std::string Sha(const std::string& s) { + return molo::security::crypto::Sha256::hex(molo::security::crypto::Sha256::hash(s)); +} + +int main() { + using namespace molo::audit::rights; + RightsRequest req; + req.id = "dsar-1"; + req.subject = Subject::User("t1", "42"); + + std::vector rows = {{"users", "42", "{\"name\":\"ada\"}"}, + {"orders", "7", "{\"total\":9}"}, + {"orders", "3", "{\"total\":1}"}}; + ExportBundle b = Run(rows, req); + CHECK(b.rows == 3, "row count"); + CHECK(b.manifest_json.find("\"request\":\"dsar-1\"") != std::string::npos, "manifest id"); + CHECK(b.manifest_json.find("\"rows\":3") != std::string::npos, "manifest rows"); + CHECK(b.manifest_json.find("\"sha256\":\"" + b.bundle_sha256 + "\"") != std::string::npos, + "manifest checksum"); + std::string hashes; + for (auto& c : b.chunks) { + CHECK(Sha(c.bytes) == c.sha256, "chunk checksum verifies"); + hashes += c.sha256; + } + CHECK(Sha(hashes) == b.bundle_sha256, "bundle checksum verifies"); + + std::vector shuffled = {rows[2], rows[0], rows[1]}; + ExportBundle b2 = Run(shuffled, req); + CHECK(b2.bundle_sha256 == b.bundle_sha256, "deterministic across input order"); + CHECK(b2.chunks.size() == b.chunks.size() && b2.chunks[0].bytes == b.chunks[0].bytes, + "deterministic bytes"); + std::cout << "PASS export_bundle\n"; + return 0; +} diff --git a/tests/src/audit/mask_fixture.cpp b/tests/src/audit/mask_fixture.cpp new file mode 100644 index 0000000..cf89255 --- /dev/null +++ b/tests/src/audit/mask_fixture.cpp @@ -0,0 +1,47 @@ +// mask_fixture: multi-chunk scrub (PII straddling the 1MB boundary) leaves +// zero residuals, masks are deterministic per seed, join keys stay stable. +#include +#include + +#include "audit/mask/fixture.h" +#include "audit/mask/masker.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL mask_fixture: " << msg << "\n"; \ + return 1; \ + } + +int main() { + using namespace molo::audit::mask; + Masker m("seed-1"); + CHECK(m.Mask("ada@example.com", molo::audit::pii::PiiClass::kEmail) == + m.Mask("ada@example.com", molo::audit::pii::PiiClass::kEmail), + "mask deterministic"); + CHECK(m.Mask("a", molo::audit::pii::PiiClass::kName) != + m.Mask("b", molo::audit::pii::PiiClass::kName), + "mask differs per value"); + CHECK(m.Mask("x", molo::audit::pii::PiiClass::kName) != + Masker("seed-2").Mask("x", molo::audit::pii::PiiClass::kName), + "mask differs per seed"); + CHECK(m.JoinKey("user:42") == m.JoinKey("user:42"), "join key stable"); + + // 2.5MB blob with PII placed to straddle the 1MB chunk boundary. + std::string blob; + blob.reserve(3 * 1024 * 1024); + std::string pad(molo::audit::kFixtureChunkBytes - 10, '~'); + blob += pad + "ada@example.com" + pad + "card 4111 1111 1111 1111" + pad; + std::string scrubbed; + ScrubStats st = Fixture::Scrub(blob, m, [&](std::string_view piece) { scrubbed += piece; }); + CHECK(st.chunks == 4, "four chunks streamed"); + CHECK(st.masked >= 2, "both PII hits masked"); + CHECK(scrubbed.find("ada@example.com") == std::string::npos, "no raw email left"); + CHECK(scrubbed.find("4111") == std::string::npos, "no raw card left"); + CHECK(Fixture::Rescan(scrubbed) == 0, "rescan zero"); + + std::string again; + Fixture::Scrub(blob, Masker("seed-1"), [&](std::string_view p) { again += p; }); + CHECK(again == scrubbed, "scrub deterministic"); + std::cout << "PASS mask_fixture\n"; + return 0; +} diff --git a/tests/src/audit/orm_hook.cpp b/tests/src/audit/orm_hook.cpp new file mode 100644 index 0000000..64c4ddd --- /dev/null +++ b/tests/src/audit/orm_hook.cpp @@ -0,0 +1,94 @@ +// orm_hook: Model save()/remove() reach the audit Appender through the +// PATCHES section 2 hook. Without an installed Appender the hook no-ops; +// with one installed, insert/update/delete append self-describing records. +#include +#include +#include + +#include "audit/log/appender.h" +#include "audit/log/store.h" +#include "data/orm/model.hpp" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL orm_hook: " << msg << "\n"; \ + return 1; \ + } + +namespace { + +using molo::data::orm::FieldDef; +using molo::data::orm::Model; + +struct Item : public Model { + static std::string table_name() { return "items"; } + static std::string primary_key() { return "id"; } + static std::vector fields() { + return {FieldDef::primary_key("id"), FieldDef::text("name")}; + } +}; + +class FakeConn : public molo::data::Connection { + public: + molo::data::Backend backend() const override { return molo::data::Backend::Sqlite; } + std::string backend_id() const override { return "fake"; } + molo::data::Result exec(const molo::data::Context&, + const molo::data::BoundStatement&) override { + return molo::data::Result::command(0); + } + molo::data::Prepared prepare(const molo::data::Context&, + const molo::data::BoundStatement&) override { + return {}; + } + molo::data::Result exec_prepared(const molo::data::Context&, const molo::data::Prepared&, + const std::vector&) override { + return molo::data::Result::command(0); + } + void ping(const molo::data::Context&) override {} + void close() override {} +}; + +} // namespace + +int main() { + using namespace molo::audit::log; + FakeConn conn; + molo::data::Context ctx("req-1", "t1", 30); + + // Without an installed Appender the hook no-ops (no crash, no store). + { + Item it; + it.set("id", molo::data::make_int(7)); + it.set("name", molo::data::make_text("n1")); + it.save(conn, ctx); + it.set("name", molo::data::make_text("n2")); + it.save(conn, ctx); + it.remove(conn, ctx); + } + + MemoryStore store; + Appender app(store); + Appender::Install(&app); + { + Item it; + it.set("id", molo::data::make_int(7)); + it.set("name", molo::data::make_text("n1")); + it.save(conn, ctx); + it.set("name", molo::data::make_text("n2")); + it.save(conn, ctx); + it.remove(conn, ctx); + } + Appender::Install(nullptr); + + auto rows = store.ReadRange(1, 10); + CHECK(rows.size() == 3, "three hook records"); + CHECK(rows[0].op == "insert" && rows[0].table == "items" && rows[0].pk == "7", + "insert record shape"); + CHECK(rows[0].tenant == "t1" && rows[0].request_id == "req-1", "context carried"); + CHECK(rows[0].after_json.find("n1") != std::string::npos, "insert after image"); + CHECK(rows[1].op == "update", "update op"); + CHECK(rows[2].op == "delete" && rows[2].after_json.empty(), "delete op"); + CHECK(rows[2].before_json.find("n2") != std::string::npos, "delete before image"); + std::cout << "PASS orm_hook\n"; + return 0; +} diff --git a/tests/src/audit/pii_boot.cpp b/tests/src/audit/pii_boot.cpp new file mode 100644 index 0000000..7f7508c --- /dev/null +++ b/tests/src/audit/pii_boot.cpp @@ -0,0 +1,46 @@ +// pii_boot: registry builds when every field is marked, boot-fails naming +// the unmarked fields otherwise. +#include +#include +#include +#include + +#include "audit/pii/registry.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL pii_boot: " << msg << "\n"; \ + return 1; \ + } + +int main() { + using namespace molo::audit::pii; + std::map> schema = { + {"users", {"id", "email", "name"}}, {"orders", {"id", "user_id", "total"}}}; + std::map marks = { + {"users.id", {PiiClass::kNone, true, ErasureRule::kKeep, "pk"}}, + {"users.email", {PiiClass::kEmail, true, ErasureRule::kDelete, "consent"}}, + {"users.name", {PiiClass::kName, true, ErasureRule::kDelete, "consent"}}, + {"orders.id", {PiiClass::kNone, true, ErasureRule::kKeep, "pk"}}, + {"orders.user_id", {PiiClass::kNationalId, false, ErasureRule::kMask, "contract"}}, + {"orders.total", {PiiClass::kFinancial, true, ErasureRule::kDelete, "contract"}}, + }; + Registry r = Registry::Build(schema, marks); + CHECK(r.size() == 6, "registry size"); + CHECK(r.Lookup("users", "email").cls == PiiClass::kEmail, "lookup class"); + CHECK(!r.Lookup("orders", "user_id").exportable, "lookup exportable"); + CHECK(r.Lookup("nope", "nope").cls == PiiClass::kNone, "lookup default none"); + + marks.erase("orders.total"); + bool threw = false; + try { + Registry::Build(schema, marks); + } catch (const std::runtime_error& e) { + threw = true; + CHECK(std::string(e.what()).find("orders.total") != std::string::npos, + "names missing field"); + } + CHECK(threw, "unmarked field must boot-fail"); + std::cout << "PASS pii_boot\n"; + return 0; +} diff --git a/tests/src/audit/query_guard.cpp b/tests/src/audit/query_guard.cpp new file mode 100644 index 0000000..90e6d66 --- /dev/null +++ b/tests/src/audit/query_guard.cpp @@ -0,0 +1,64 @@ +// query_guard: queries without audit.read get 403 and are self-audited; +// granted queries page rows and are self-audited too. +#include +#include +#include + +#include "audit/log/appender.h" +#include "audit/log/query.h" +#include "audit/log/store.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL query_guard: " << msg << "\n"; \ + return 1; \ + } + +int main() { + using namespace molo::audit::log; + using namespace molo::security::auth; + MemoryStore store; + Appender app(store); + for (int i = 0; i < 5; ++i) { + Appender::Write w; + w.actor = "writer"; + w.action = "orm.insert"; + w.table = "orders"; + w.pk = std::to_string(i); + w.op = "insert"; + w.tenant = "t1"; + app.Append(std::move(w)); + } + Query q(store, &app); + + // INTEGRATION: real Session (security/session) plus explicit roles. + molo::security::session::Session anon; + anon.user_id = "anon"; + Filter f; + f.tenant = "t1"; + QueryPage denied = q.Run(f, anon, {}); + CHECK(denied.http_status == 403 && denied.code == "forbidden", "anon gets 403"); + CHECK(denied.rows.empty(), "denied returns no rows"); + auto trail = store.ReadRange(6, 10); + CHECK(trail.size() == 1 && trail[0].action == "query.denied", "denial self-audited"); + + molo::security::session::Session reader; + reader.user_id = "reader"; + reader.privilege = molo::security::session::Session::Privilege::User; + std::vector roles = {Role{"auditor", {Permission{"audit.read"}}}}; + QueryPage allowed = q.Run(f, reader, roles); + CHECK(allowed.http_status == 200, "reader gets 200"); + CHECK(allowed.rows.size() == 6, "reader sees 5 rows plus denial self-audit"); + CHECK(allowed.next_cursor == "6", "cursor at last row"); + auto trail2 = store.ReadRange(7, 10); + CHECK(trail2.size() == 1 && trail2[0].action == "query.allowed", "grant self-audited"); + + f.limit = 2; + QueryPage p1 = q.Run(f, reader, roles); + CHECK(p1.rows.size() == 2 && p1.next_cursor == "2", "page 1"); + f.cursor = p1.next_cursor; + QueryPage p2 = q.Run(f, reader, roles); + CHECK(p2.rows.size() == 2 && p2.rows[0].seq == 3, "page 2 resumes"); + std::cout << "PASS query_guard\n"; + return 0; +} diff --git a/tests/src/audit/redactor.cpp b/tests/src/audit/redactor.cpp new file mode 100644 index 0000000..c564eb8 --- /dev/null +++ b/tests/src/audit/redactor.cpp @@ -0,0 +1,50 @@ +// redactor: PII shapes replaced with [redacted:], sensitive log keys +// replaced wholesale, oversize input capped. +#include +#include +#include + +#include "audit/redact/redactor.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL redactor: " << msg << "\n"; \ + return 1; \ + } + +int main() { + using molo::audit::redact::Redactor; + std::string mail = Redactor::Redact("contact ada@example.com please"); + CHECK(mail.find("ada@example.com") == std::string::npos, "email removed"); + CHECK(mail.find("[redacted:email]") != std::string::npos, "email tag"); + + std::string cred = Redactor::Redact("login password: s3cret hunter2"); + CHECK(cred.find("s3cret") == std::string::npos, "credential removed"); + CHECK(cred.find("[redacted:credential]") != std::string::npos, "credential tag"); + + std::string ssn = Redactor::Redact("ssn 123-45-6789 on file"); + CHECK(ssn.find("123-45-6789") == std::string::npos, "national id removed"); + CHECK(ssn.find("[redacted:national_id]") != std::string::npos, "national id tag"); + + std::string clean = Redactor::Redact("nothing sensitive here"); + CHECK(clean == "nothing sensitive here", "clean text untouched"); + + molo::log::Record rec; + rec.message = "user ada@example.com logged in"; + // INTEGRATION: real FieldValue is a variant; spell strings explicitly + // (bare literals would bind the bool alternative). + rec.fields.push_back({"api_key", std::string("abcdef")}); + rec.fields.push_back({"route", std::string("/v1/items")}); + Redactor::RedactRecord(rec); + CHECK(rec.message.find("[redacted:email]") != std::string::npos, "record message"); + CHECK(std::get(rec.fields[0].value) == "[redacted:credential]", + "sensitive key wholesale"); + CHECK(std::get(rec.fields[1].value) == "/v1/items", "plain field kept"); + + std::string big(2 * 1024 * 1024, 'x'); + big += "ada@example.com"; + std::string out = Redactor::Redact(big); + CHECK(out.size() <= molo::audit::kRedactMaxBytesPerCall + 64, "oversize capped"); + std::cout << "PASS redactor\n"; + return 0; +} diff --git a/tests/src/audit/retention_hold.cpp b/tests/src/audit/retention_hold.cpp new file mode 100644 index 0000000..13daf4f --- /dev/null +++ b/tests/src/audit/retention_hold.cpp @@ -0,0 +1,59 @@ +// retention_hold: TTL boundary (89 kept, 91 deleted), grace days, legal holds +// block deletion, apply honors the pinned policy version only. +#include +#include +#include + +#include "audit/retention/enforcer.h" +#include "audit/retention/hold.h" +#include "audit/retention/policy.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL retention_hold: " << msg << "\n"; \ + return 1; \ + } + +int main() { + using namespace molo::audit::retention; + Policy policy(7, {{"users", TableRule{90, 0}}, {"logs", TableRule{90, 5}}}); + std::vector rows = { + {"t1", "users", "a", 89}, {"t1", "users", "b", 91}, {"t1", "logs", "c", 93}}; + Hold hold; + + auto scan = [&](ScannedRow& out) -> bool { + static std::size_t i = 0; + if (i >= rows.size()) { + i = 0; + return false; + } + out = rows[i++]; + return true; + }; + DryRunReport dry = Enforcer::DryRun(policy, hold, scan); + CHECK(dry.scanned == 3 && dry.kept == 2 && dry.would_delete == 1 && dry.held == 0, + "89 kept, 91 deleted, 93 in grace"); + CHECK(dry.deletions.size() == 1 && dry.deletions[0].pk == "b", "deletion names pk b"); + + hold.Add(Hold::RowKey("t1", "users", "b"), "litigation"); + DryRunReport dry2 = Enforcer::DryRun(policy, hold, scan); + CHECK(dry2.held == 1 && dry2.would_delete == 0, "hold blocks deletion"); + + hold.Remove(Hold::RowKey("t1", "users", "b")); + std::vector deleted; + ApplyResult applied = + Enforcer::Apply(policy, hold, dry, [&](const ScannedRow& r) { deleted.push_back(r.pk); }); + CHECK(applied.deleted == 1 && deleted[0] == "b", "apply deletes pinned rows"); + + hold.Add(Hold::RowKey("t1", "users", "b"), "late hold"); + ApplyResult held = + Enforcer::Apply(policy, hold, dry, [&](const ScannedRow&) { deleted.push_back("x"); }); + CHECK(held.deleted == 0 && held.skipped_hold == 1, "late hold skips apply"); + + Policy newer(8, {{"users", TableRule{90, 0}}}); + ApplyResult stale = + Enforcer::Apply(newer, Hold{}, dry, [&](const ScannedRow&) { deleted.push_back("x"); }); + CHECK(stale.deleted == 0 && stale.skipped_policy == 1, "version change skips apply"); + std::cout << "PASS retention_hold\n"; + return 0; +} diff --git a/tests/src/audit/sql_store.cpp b/tests/src/audit/sql_store.cpp new file mode 100644 index 0000000..ac5ebef --- /dev/null +++ b/tests/src/audit/sql_store.cpp @@ -0,0 +1,107 @@ +// sql_store: SqlStore round-trips records and seals through the SqlExec seam +// under both placeholder styles (Postgres $n and SQLite ?), and the stored +// chain verifies. +#include +#include +#include + +#include "audit/log/chain.h" +#include "audit/log/sql_store.h" + +#define CHECK(c, msg) \ + if (!(c)) { \ + std::cerr << "FAIL sql_store: " << msg << "\n"; \ + return 1; \ + } + +// In-memory SqlExec honoring the exact statements SqlStore emits. +class FakeExec : public molo::audit::log::SqlExec { + public: + explicit FakeExec(bool pg) : pg_(pg) {} + void Exec(const std::string& sql, const std::vector& p) override { + if (sql.rfind("CREATE TABLE", 0) == 0) return; + if (sql.rfind("INSERT INTO audit_records", 0) == 0) { + if (p.size() != 14) throw std::runtime_error("records INSERT arity"); + records_.push_back(p); + return; + } + if (sql.rfind("INSERT INTO audit_seals", 0) == 0) { + if (p.size() != 4) throw std::runtime_error("seals INSERT arity"); + seals_.push_back(p); + return; + } + throw std::runtime_error("unexpected Exec: " + sql); + } + std::vector> Query(const std::string& sql, + const std::vector& p, + std::size_t max_rows) override { + if (sql == "SELECT seq,hash FROM audit_records ORDER BY seq DESC LIMIT 1") { + if (records_.empty()) return {}; + auto& r = records_.back(); + return {{r[0], r[13]}}; + } + if (sql == "SELECT seq FROM audit_records ORDER BY seq DESC LIMIT 1") { + if (records_.empty()) return {}; + return {{records_.back()[0]}}; + } + if (sql == "SELECT hash FROM audit_records ORDER BY seq DESC LIMIT 1") { + if (records_.empty()) return {}; + return {{records_.back()[13]}}; + } + if (sql.rfind("SELECT seq,ts,tenant,", 0) == 0) { + std::uint64_t from = std::stoull(p.at(0)); + std::vector> out; + for (auto& r : records_) { + if (out.size() >= max_rows) break; + if (std::stoull(r[0]) >= from) out.push_back(r); + } + return out; + } + if (sql.rfind("SELECT seal_seq,root,ts,mac", 0) == 0) return seals_; + throw std::runtime_error("unexpected Query: " + sql); + } + bool IsPostgres() const override { return pg_; } + + private: + bool pg_; + std::vector> records_, seals_; +}; + +static int RunBoth(bool pg) { + using namespace molo::audit::log; + FakeExec db(pg); + SqlStore store(db); + CHECK(store.LatestSeq() == 0, "empty latest 0"); + CHECK(store.Tip() == GenesisHash(), "empty tip genesis"); + for (int i = 0; i < 50; ++i) { + AuditRecord r; + r.ts = 1700000000 + i; + r.tenant = "t1"; + r.actor = "w"; + r.action = "orm.insert"; + r.table = "users"; + r.pk = std::to_string(i); + r.op = "insert"; + r.after_json = "{\"i\":" + std::to_string(i) + "}"; + CHECK(store.Append(r) == (std::uint64_t)(i + 1), "seq assigned"); + } + CHECK(store.LatestSeq() == 50, "latest 50"); + auto page = store.ReadRange(11, 10); + CHECK(page.size() == 10 && page.front().seq == 11 && page.back().seq == 20, "range page"); + CHECK(page[0].after_json == "{\"i\":10}", "payload round-trip"); + auto all = store.ReadRange(1, 1000); + CHECK(Chain::Verify(all).ok, "stored chain verifies"); + Seal s = Chain::MakeSeal(1, store.Tip(), 1700000600, "k"); + store.SaveSeal(s); + auto seals = store.Seals(); + CHECK(seals.size() == 1 && seals[0].mac == s.mac, "seal round-trip"); + CHECK(Chain::CheckSeal(seals[0], "k"), "seal mac verifies"); + return 0; +} + +int main() { + if (RunBoth(false) != 0) return 1; + if (RunBoth(true) != 0) return 1; + std::cout << "PASS sql_store\n"; + return 0; +} diff --git a/tests/src/cli/test_ai.cpp b/tests/src/cli/test_ai.cpp new file mode 100644 index 0000000..8c67655 --- /dev/null +++ b/tests/src/cli/test_ai.cpp @@ -0,0 +1,73 @@ +// AI harness: SSE framing, redaction, plan-then-apply, stub client. +#include +#include +#include + +#include "ai/ai_client.hpp" +#include "ai/ai_session.hpp" +#include "ai/redactor.hpp" +#include "ai/sse_stream_parser.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::ai; + SseStreamParser parser; + parser.feed("data: {\"a\":1}\n\n"); + parser.feed("data: {\"b\":2}\n\n"); + auto events = parser.take_events(); + CHECK(events.size() == 2 && !parser.done(), "two SSE chunks parsed"); + parser.feed("data: [DONE]\n\n"); + CHECK(parser.done() && parser.take_events().empty(), "[DONE] ends stream"); + + CHECK(Redactor::key_is_secret("DEVPLACE_API_KEY"), "api key detected"); + CHECK(!Redactor::key_is_secret("username"), "plain key kept"); + Redactor::Result redacted = Redactor::redact("{\"API_KEY\": \"hunter2\", \"n\": 1}"); + CHECK(redacted.redacted == 1, "one value redacted"); + CHECK(redacted.text.find("hunter2") == std::string::npos, "secret gone from text"); + + AiSession session("t"); + std::string error; + CHECK(session.append("user", "hi", error), "session append"); + namespace fs = std::filesystem; + fs::path target = fs::temp_directory_path() / "molo-ai-nowrite.txt"; + fs::remove(target); + std::size_t index = session.request_write(target.string(), "data"); + CHECK(index != static_cast(-1), "write requested"); + CHECK(!fs::exists(target), "no file written without confirmation"); + CHECK(session.confirm_write(index, error), "confirmed write applies: " + error); + CHECK(fs::exists(target), "file exists after confirmation"); + fs::remove(target); + + // Client over stub transport with http base URL (https needs wolfSSL). + setenv("DEVPLACE_API_KEY", "test-key", 1); + std::string sse = + "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n" + "data: {\"choices\":[{\"delta\":{\"content\":\"!\"}}]}\n\n" + "data: [DONE]\n\n"; + StubAiTransport stub({sse}); + AiClientOptions opts; + opts.base_url = "http://127.0.0.1:9"; + opts.model = "molodetz"; + AiClient client(opts, stub); + std::string deltas; + std::string full = client.chat_stream({{"user", "hi"}}, + [&](const std::string& d) { deltas += d; }, error); + CHECK(error.empty(), "stub chat ok: " + error); + CHECK(full == "Hello!", "transcript assembled"); + CHECK(stub.seen().size() == 1, "one request sent"); + CHECK(stub.seen()[0].body_json.find("\"stream\":true") != std::string::npos, + "stream flag in request"); + CHECK(AiClient::extract_delta("{\"choices\":[{\"delta\":{\"content\":\"x\\ny\"}}]}") == "x\ny", + "delta unescapes"); + unsetenv("DEVPLACE_API_KEY"); + std::cout << "ALL_AI_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_cli.cpp b/tests/src/cli/test_cli.cpp new file mode 100644 index 0000000..6b1c741 --- /dev/null +++ b/tests/src/cli/test_cli.cpp @@ -0,0 +1,82 @@ +// Unit checks for CLI parsing, tree, help, writers, exit codes. +#include +#include + +#include "cli/arg_parser.hpp" +#include "cli/command.hpp" +#include "cli/command_tree.hpp" +#include "cli/exit_code.hpp" +#include "cli/flag.hpp" +#include "cli/flag_set.hpp" +#include "cli/help_renderer.hpp" +#include "cli/json_writer.hpp" +#include "cli/output.hpp" +#include "cli/table_writer.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << " (line " << __LINE__ << ")\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::cli; + CHECK(to_int(ExitCode::Ok) == 0, "exit ok is 0"); + CHECK(to_int(ExitCode::Runtime) == 1, "exit runtime is 1"); + CHECK(to_int(ExitCode::Usage) == 2, "exit usage is 2"); + CHECK(to_int(ExitCode::Unavailable) == 69, "exit unavailable is 69"); + + FlagSet flags; + flags.add(Flag("port", "p", "listen port", true, "8000")); + CHECK(flags.declares("port"), "flagset declares port"); + CHECK(flags.get("port") == "8000", "flagset default value"); + flags.set("port", "9000"); + CHECK(flags.get("port") == "9000", "flagset set/get"); + + ArgParser parser; + ParsedArgs parsed = parser.parse({"molo", "-vv", "--format", "json", "serve", "--port", "9000"}); + CHECK(parsed.ok(), "arg parse ok"); + CHECK(parsed.global.verbose == 2, "repeatable verbose counts"); + CHECK(parsed.global.format == "json", "format flag parsed"); + CHECK(parsed.flags["port"] == "9000", "command flag value parsed"); + ParsedArgs bad = parser.parse({"molo", "--format", "yaml"}); + CHECK(!bad.ok(), "bad format rejected"); + + CommandTree tree; + auto serve = std::make_shared("serve", "run server"); + serve->usage("[--port N]").example("serve --reload"); + tree.add(serve); + ResolveResult found = tree.resolve({"serve"}); + CHECK(found.command != nullptr && found.consumed == 1, "tree resolves serve"); + ResolveResult missing = tree.resolve({"nosuch"}); + CHECK(missing.command == nullptr, "tree rejects unknown command"); + + HelpRenderer help; + std::string concise = help.render_concise(*serve, "molo"); + CHECK(concise.find("usage: molo serve") != std::string::npos, "concise help has usage"); + std::string full = help.render_full(*serve, "molo", "serve"); + CHECK(full.find("examples:") != std::string::npos, "full help has examples"); + std::string root = help.render_root(tree, "molo"); + CHECK(root.find("--format") != std::string::npos, "root help lists global flags"); + + CHECK(JsonWriter::quote("a\"b") == "\"a\\\"b\"", "json escapes quotes"); + std::map fields; + fields["ok"] = "true"; + fields["n"] = "3"; + CHECK(JsonWriter::object(fields) == "{\"n\":3,\"ok\":true}", "json object renders"); + + TableWriter table; + table.header({"a", "bb"}); + table.row({"1", "2"}); + std::string rendered = table.render(); + CHECK(rendered.find("a") != std::string::npos && rendered.find("bb") != std::string::npos, + "table renders header"); + + Output out(true, 0, false, false); + CHECK(out.json_mode() && !out.tty(), "output flags stored"); + std::cout << "ALL_CLI_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_cli_binary.sh b/tests/src/cli/test_cli_binary.sh new file mode 100755 index 0000000..275c1f2 --- /dev/null +++ b/tests/src/cli/test_cli_binary.sh @@ -0,0 +1,86 @@ +#!/bin/sh +# Integration tests through the real `molo` binary. Fails fast on first error. +set -u +MOLO="$1" +case "$MOLO" in + /*) ;; + *) MOLO="$(pwd)/$MOLO" ;; +esac +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +fail() { echo "FAIL: $1"; exit 1; } +pass() { echo "PASS: $1"; } + +"$MOLO" version >/dev/null 2>&1 || fail "version exits 0" +[ "$("$MOLO" version --format json | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["version"])')" = "0.1.0" ] || fail "version json parses" +pass "version text+json" + +"$MOLO" --help >/dev/null 2>&1 || fail "--help exits 0" +"$MOLO" nosuch >/dev/null 2>&1 && fail "nosuch must fail" +[ "$("$MOLO" nosuch >/dev/null 2>&1; echo $?)" = "2" ] || fail "nosuch exits 2" +pass "help + unknown command exit codes" + +# No color codes when piped. +"$MOLO" --help | grep -q "$(printf '\033')" && fail "color leaked to pipe" +pass "no color when piped" + +# `molo new` scaffolds, re-run without --force fails. +(cd "$TMP" && "$MOLO" new shop >/dev/null 2>&1) || fail "molo new fails" +[ -f "$TMP/shop/Makefile" ] && [ -f "$TMP/shop/.env.json" ] && [ -f "$TMP/shop/.gitignore" ] && [ -d "$TMP/shop/.git" ] || fail "scaffold files missing" +(cd "$TMP" && "$MOLO" new shop >/dev/null 2>&1) && fail "re-run without --force must fail" +[ "$(cd "$TMP" && "$MOLO" new shop >/dev/null 2>&1; echo $?)" = "1" ] || fail "re-run exits 1" +pass "molo new + force guard" + +# serve refuses prod reload without --allow-prod. +mkdir -p "$TMP/prod" && printf '{"MOLO_ENV":"prod"}' > "$TMP/prod/.env.json" +[ "$(cd "$TMP/prod" && MOLO_ENV=prod "$MOLO" serve --reload --port 8123 >/dev/null 2>&1; echo $?)" = "2" ] || fail "prod reload must exit 2" +pass "serve prod guard" + +# health json parses. +"$MOLO" health --format json | python3 -c 'import json,sys; json.load(sys.stdin)' || fail "health json parses" +pass "health json" + +# supervisor/task structured output with unavailable backend. +"$MOLO" supervisor status --format json | python3 -c 'import json,sys; d=json.load(sys.stdin); assert d["ok"] is False' || fail "supervisor json shape" +[ "$("$MOLO" supervisor status --format json >/dev/null 2>&1; echo $?)" = "69" ] || fail "supervisor exits 69" +"$MOLO" task inspect --id x --format json | python3 -c 'import json,sys; json.load(sys.stdin)' || fail "task json parses" +pass "supervisor/task json + 69" + +# embed fixture twice, byte-identical hpp. +mkdir -p "$TMP/fx/css" && printf '

    x

    ' > "$TMP/fx/index.html" && printf 'b{}' > "$TMP/fx/css/a.css" +"$MOLO" embed "$TMP/fx" --out "$TMP/e1.cpp" --gzip >/dev/null 2>&1 || fail "embed run 1" +"$MOLO" embed "$TMP/fx" --out "$TMP/e2.cpp" --gzip >/dev/null 2>&1 || fail "embed run 2" +cmp -s "$TMP/e1.hpp" "$TMP/e2.hpp" || fail "embed hpp byte-identical" +grep -q "molo-embed-1" "$TMP/e1.hpp" || fail "embed tool version stamped" +pass "embed reproducibility" + +# `molo ai` against a local stub SSE server. +export DEVPLACE_API_KEY="stub-key" +python3 - "$TMP" <<'EOF' & +import http.server, sys +tmp = sys.argv[1] +BODY = ('data: {"choices":[{"delta":{"content":"Hi"}}]}\n\n' + 'data: {"choices":[{"delta":{"content":"!"}}]}\n\n' + 'data: [DONE]\n\n') +class H(http.server.BaseHTTPRequestHandler): + def do_POST(self): + ln = int(self.headers.get('Content-Length', 0)) + self.rfile.read(ln) + payload = BODY.encode() + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Content-Length', str(len(payload))) + self.end_headers() + self.wfile.write(payload) + def log_message(self, *a): + pass +http.server.HTTPServer(('127.0.0.1', 18321), H).serve_forever() +EOF +SRV=$! +sleep 1 +"$MOLO" ai new --base-url http://127.0.0.1:18321 --model molodetz --prompt "say hi" >"$TMP/ai.txt" 2>"$TMP/ai.err" || fail "molo ai new against stub" +grep -q "Hi!" "$TMP/ai.txt" || fail "ai transcript assembled" +kill $SRV 2>/dev/null +pass "ai stub streaming" + +echo "ALL_BINARY_OK" diff --git a/tests/src/cli/test_csp.cpp b/tests/src/cli/test_csp.cpp new file mode 100644 index 0000000..3ed60b3 --- /dev/null +++ b/tests/src/cli/test_csp.cpp @@ -0,0 +1,48 @@ +// CSP: deny-first defaults, nonce plumbing, report endpoint caps. +#include + +#include "csp/csp_builder.hpp" +#include "csp/csp_report_handler.hpp" +#include "csp/nonce.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::csp; + CspPolicy def = CspBuilder().build(); + CHECK(def.report_only(), "report-only by default"); + CHECK(def.header_name() == "Content-Security-Policy-Report-Only", "report-only header name"); + CHECK(def.header_value().find("default-src 'none'") != std::string::npos, "deny-first default"); + CHECK(def.header_value().find("frame-ancestors 'none'") != std::string::npos, + "frame-ancestors none"); + + std::string nonce = Nonce::generate(); + CHECK(nonce.size() == 24, "nonce is 16 bytes base64 (24 chars)"); + CHECK(Nonce::generate() != nonce, "nonces differ per call"); + CspPolicy with_nonce = CspBuilder().script_nonce(nonce).build(); + CHECK(with_nonce.header_value().find("'nonce-" + nonce + "'") != std::string::npos, + "nonce plumbed into script-src"); + + CspPolicy enforced = CspBuilder().enforce().build(); + CHECK(!enforced.report_only() && enforced.header_name() == "Content-Security-Policy", + "enforce() switches header"); + + // POST /csp-report + CspReportHandler handler; + CspReportResult accepted = handler.handle("10.0.0.1", "{\"blocked-uri\":\"x\"}", 1000); + CHECK(accepted.status == 202, "report accepted 202"); + CspReportResult too_big = handler.handle("10.0.0.1", std::string(70000, 'x'), 1000); + CHECK(too_big.status == 413, "oversize report 413"); + for (int i = 0; i < 100; ++i) handler.handle("10.0.0.2", "{}", 2000); + CspReportResult limited = handler.handle("10.0.0.2", "{}", 2001); + CHECK(limited.status == 429, "report rate limit 429"); + std::cout << "ALL_CSP_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_dev.cpp b/tests/src/cli/test_dev.cpp new file mode 100644 index 0000000..d34fca4 --- /dev/null +++ b/tests/src/cli/test_dev.cpp @@ -0,0 +1,67 @@ +// Dev server: config split enforcement, backoff math, polling watcher. +#include +#include +#include +#include +#include + +#include "dev/dev_config.hpp" +#include "dev/dev_server.hpp" +#include "dev/polling_watcher.hpp" +#include "dev/reloader.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::dev; + namespace fs = std::filesystem; + + CHECK(DevServer::should_refuse(true, true, false), "prod+reload refused"); + CHECK(!DevServer::should_refuse(true, true, true), "allow-prod passes"); + CHECK(!DevServer::should_refuse(false, true, false), "dev reload passes"); + + CHECK(Reloader::backoff_for(1) == 1, "backoff 1s"); + CHECK(Reloader::backoff_for(2) == 2, "backoff 2s"); + CHECK(Reloader::backoff_for(3) == 4, "backoff 4s"); + CHECK(Reloader::backoff_for(99) == 30, "backoff holds at 30s"); + + fs::path dir = fs::temp_directory_path() / "molo-dev-test"; + fs::remove_all(dir); + fs::create_directories(dir); + { std::ofstream(dir / ".env.json") << "{\"MOLO_ENV\": \"prod\", \"PORT\": 9000}"; } + unsetenv("MOLO_ENV"); + ConfigLoader loader; + LoadedConfig prod = loader.load((dir / ".env.json").string(), true, false); + CHECK(!prod.ok(), "prod reload without allow-prod errors"); + LoadedConfig prod_ok = loader.load((dir / ".env.json").string(), true, true); + CHECK(prod_ok.ok() && prod_ok.is_prod() && prod_ok.dev.port == 9000, "allow-prod loads prod"); + setenv("MOLO_ENV", "dev", 1); + LoadedConfig dev = loader.load((dir / ".env.json").string(), true, false); + CHECK(dev.ok() && !dev.is_prod(), "MOLO_ENV overrides file"); + unsetenv("MOLO_ENV"); + + PollingFileWatcher watcher; + std::string error; + CHECK(watcher.start({dir.string()}, error), "watcher starts: " + error); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + { std::ofstream(dir / "t.txt") << "x"; } + bool saw = false; + for (int i = 0; i < 20 && !saw; ++i) { + auto events = watcher.poll(200); + for (const auto& event : events) { + if (event.path.find("t.txt") != std::string::npos) saw = true; + } + } + CHECK(saw, "watcher sees new file"); + watcher.stop(); + fs::remove_all(dir); + std::cout << "ALL_DEV_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_embed.cpp b/tests/src/cli/test_embed.cpp new file mode 100644 index 0000000..d6a0c8a --- /dev/null +++ b/tests/src/cli/test_embed.cpp @@ -0,0 +1,81 @@ +// Embedder: scan caps, emitter reproducibility, bundle reader, sha/mime/gzip. +#include +#include +#include + +#include "embed/asset_scanner.hpp" +#include "embed/embed_emitter.hpp" +#include "embed/embedded_bundle.hpp" +#include "embed/gzip_stored.hpp" +#include "embed/mime.hpp" +#include "embed/sha256.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::embed; + namespace fs = std::filesystem; + + CHECK(Sha256::hex_of_string("abc") == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "sha256 known vector"); + CHECK(Mime::of_path("x/app.css") == "text/css", "mime by extension"); + CHECK(Mime::is_textual("text/html") && !Mime::is_textual("image/png"), "textual detection"); + + std::vector raw = {'h', 'i'}; + auto gz = GzipStored::compress(raw); + CHECK(gz.size() > 10 && gz[0] == 0x1F && gz[1] == 0x8B, "gzip stored magic"); + + // Fixture tree. + fs::path root = fs::temp_directory_path() / "molo-embed-test"; + fs::remove_all(root); + fs::create_directories(root / "css"); + { std::ofstream(root / "index.html") << "

    hi

    "; } + { std::ofstream(root / "css" / "a.css") << "body{}"; } + { std::ofstream(root / ".hidden") << "skip me"; } + + AssetScanner scanner; + ScanResult scanned = scanner.scan(root.string()); + CHECK(scanned.ok(), "scan ok"); + CHECK(scanned.files.size() == 2, "dotfile skipped"); + CHECK(scanned.files[0].rel_path < scanned.files[1].rel_path, "scan sorted"); + + // Emit twice: byte-identical. + EmitOptions opts; + opts.namespc = "molo::assets"; + opts.gzip = true; + EmbedEmitter emitter; + std::string error; + std::string cpp1 = (root / "a1.cpp").string(), hpp1 = (root / "a1.hpp").string(); + std::string cpp2 = (root / "a2.cpp").string(), hpp2 = (root / "a2.hpp").string(); + CHECK(emitter.emit(scanned.files, cpp1, hpp1, opts, error), "emit run 1: " + error); + CHECK(emitter.emit(scanned.files, cpp2, hpp2, opts, error), "emit run 2: " + error); + auto slurp = [](const std::string& p) { + std::ifstream in(p, std::ios::binary); + return std::string(std::istreambuf_iterator(in), std::istreambuf_iterator()); + }; + // Note: cpp embeds its own hpp basename, so compare normalized. + std::string c1 = slurp(cpp1), c2 = slurp(cpp2); + CHECK(slurp(hpp1) == slurp(hpp2), "hpp byte-identical"); + CHECK(c1.size() == c2.size(), "cpp same size across runs"); + + EmbeddedBundle bundle; + EmbeddedAsset asset; + asset.path = "index.html"; + asset.data = {'<', 'h', '1', '>'}; + asset.sha256 = "deadbeef"; + asset.mime = "text/html"; + bundle.add(asset); + CHECK(bundle.find("index.html") != nullptr, "bundle find hit"); + CHECK(bundle.find("nope") == nullptr, "bundle find miss"); + fs::remove_all(root); + std::cout << "ALL_EMBED_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_healthz.cpp b/tests/src/cli/test_healthz.cpp new file mode 100644 index 0000000..6d53e5a --- /dev/null +++ b/tests/src/cli/test_healthz.cpp @@ -0,0 +1,38 @@ +// /healthz: liveness stays 200 unless shutdown is stuck. +#include + +#include "ops/health_registry.hpp" +#include "ops/liveness_check.hpp" +#include "ops/ops_routes.hpp" +#include "ops/prometheus_exporter.hpp" +#include "ops/readiness_check.hpp" +#include "ops/startup_gate.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::ops; + auto live = std::make_shared(); + HealthRegistry registry; + registry.add(live); + ReadinessCheck readiness; + StartupGate gate; + gate.mark_boot_complete(); + PrometheusExporter exporter; + OpsRoutes routes(registry, readiness, gate, exporter); + + OpsResponse hz = routes.healthz(); + CHECK(hz.status == 200 && hz.body == "{\"status\":\"ok\"}", "healthz 200 ok"); + live->set_shutdown_stuck(true); + OpsResponse stuck = routes.healthz(); + CHECK(stuck.status == 503, "healthz 503 when shutdown stuck"); + std::cout << "ALL_HEALTHZ_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_i18n.cpp b/tests/src/cli/test_i18n.cpp new file mode 100644 index 0000000..8132aa7 --- /dev/null +++ b/tests/src/cli/test_i18n.cpp @@ -0,0 +1,60 @@ +// i18n: locale chain, PO loader with plurals, translator fallback, resolution. +#include + +#include "i18n/locale.hpp" +#include "i18n/plural_rule.hpp" +#include "i18n/po_loader.hpp" +#include "i18n/translator.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::i18n; + Locale nl = Locale::parse("nl-NL"); + CHECK(nl.language() == "nl" && nl.region() == "NL", "locale normalizes"); + auto chain = nl.fallback_chain(); + CHECK(chain.size() == 3 && chain[0].str() == "nl-NL" && chain[1].str() == "nl" && + chain[2].str() == "en", + "fallback chain nl-NL to nl to en"); + + CHECK(PluralRule::form_index(1, "en") == 0, "en singular"); + CHECK(PluralRule::form_index(2, "en") == 1, "en plural"); + CHECK(PluralRule::form_index(5, "ru") == 2, "ru many form"); + + std::string po = + "msgid \"Hello\"\nmsgstr \"Hallo\"\n\n" + "msgid \"apple\"\nmsgid_plural \"apples\"\nmsgstr[0] \"appel\"\nmsgstr[1] " + "\"appels\"\n"; + PoLoadResult loaded = PoLoader::load_text(po, Locale::parse("nl")); + CHECK(loaded.ok(), "po loads"); + CHECK(loaded.catalog.size() == 2, "po has two entries"); + + Translator translator; + translator.add_catalog(std::move(loaded.catalog)); + unsigned long missing_notes = 0; + translator.set_missing_sink([&](const std::string&) { ++missing_notes; }); + CHECK(translator.translate("Hello", Locale::parse("nl-NL")) == "Hallo", "nl-NL translates"); + CHECK(translator.translate("apple", Locale::parse("nl"), 2) == "appels", "plural form renders"); + CHECK(translator.translate("apple", Locale::parse("nl"), 1) == "appel", + "singular form renders"); + CHECK(translator.translate("Gone", Locale::parse("nl")) == "Gone", "missing key falls back"); + CHECK(translator.translate("Gone", Locale::parse("nl")) == "Gone", "missing key repeats msgid"); + CHECK(missing_notes == 1, "missing key logs once"); + + RequestLocale req; + req.route_prefix = "nl"; + req.accept_language = "fr;q=0.9,en;q=0.1"; + CHECK(Translator::resolve(req).language() == "nl", "route prefix wins"); + RequestLocale req2; + req2.accept_language = "fr;q=0.9,en;q=0.1"; + CHECK(Translator::resolve(req2).language() == "fr", "q-values honored"); + std::cout << "ALL_I18N_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_metrics.cpp b/tests/src/cli/test_metrics.cpp new file mode 100644 index 0000000..d9c4a45 --- /dev/null +++ b/tests/src/cli/test_metrics.cpp @@ -0,0 +1,48 @@ +// /metrics: Prometheus 0.0.4 format, caps, gzip parity, route templates. +#include + +#include "ops/prometheus_exporter.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::ops; + PrometheusExporter exporter; + MetricSample sample; + sample.name = "molo_requests_total"; + sample.labels = {{"route", "/users/{id}"}, {"code", "200"}}; + sample.value = 42; + sample.type = MetricType::Counter; + sample.help = "total requests"; + CHECK(exporter.add(sample), "add sample under caps"); + + std::string text = exporter.render(); + CHECK(!text.empty() && text.back() == '\n', "trailing newline on last line"); + CHECK(text.find("# HELP molo_requests_total") != std::string::npos, "HELP line present"); + CHECK(text.find("# TYPE molo_requests_total counter") != std::string::npos, "TYPE line present"); + CHECK(text.find("molo_metrics_dropped_total 0") != std::string::npos, "dropped counter present"); + + // Label overflow drops and counts. + MetricSample flooded; + flooded.name = "flood"; + for (int i = 0; i < 100; ++i) flooded.labels["l" + std::to_string(i)] = "v"; + CHECK(!exporter.add(flooded), "64 label cap enforced"); + CHECK(exporter.dropped() == 1, "overflow bumps dropped counter"); + + // gzip variant: valid gzip magic, decodes by length sanity. + auto gz = exporter.render_gzip(); + CHECK(gz.size() > 10 && gz[0] == 0x1F && gz[1] == 0x8B, "gzip magic present"); + + CHECK(PrometheusExporter::normalize_route("/users/123") == "/users/{id}", + "numeric segment templated"); + CHECK(PrometheusExporter::normalize_route("/healthz") == "/healthz", "plain path kept"); + std::cout << "ALL_METRICS_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_readyz.cpp b/tests/src/cli/test_readyz.cpp new file mode 100644 index 0000000..ff2031b --- /dev/null +++ b/tests/src/cli/test_readyz.cpp @@ -0,0 +1,72 @@ +// /readyz: 200 when deps pass, 503 degraded with failed list otherwise. +#include +#include +#include + +#include "ops/health_registry.hpp" +#include "ops/liveness_check.hpp" +#include "ops/ops_routes.hpp" +#include "ops/prometheus_exporter.hpp" +#include "ops/readiness_check.hpp" +#include "ops/startup_gate.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::ops; + auto live = std::make_shared(); + HealthRegistry registry; + registry.add(live); + ReadinessCheck readiness; + bool db_up = true; + readiness.add_probe({"db", [&]() { + HealthResult r; + r.state = db_up ? HealthState::Ok : HealthState::Failed; + return r; + }}); + readiness.add_probe({"kv", []() { + HealthResult r; + r.state = HealthState::Ok; + return r; + }}); + StartupGate gate; + gate.mark_boot_complete(); + PrometheusExporter exporter; + OpsRoutes routes(registry, readiness, gate, exporter); + + OpsResponse ready = routes.readyz(); + CHECK(ready.status == 200, "readyz 200 when deps pass"); + db_up = false; // kill the DB + OpsResponse degraded = routes.readyz(); + CHECK(degraded.status == 503, "readyz 503 when db down"); + CHECK(degraded.body.find("degraded") != std::string::npos, "readyz body says degraded"); + CHECK(degraded.body.find("\"db\"") != std::string::npos, "readyz names failed dep"); + OpsResponse hz = routes.healthz(); + CHECK(hz.status == 200, "healthz stays 200 while readyz degraded"); + + // Registry short-circuits slow checks at the timeout. + HealthRegistry slow_registry; + class Slow : public HealthCheck { + public: + std::string name() const override { return "slow"; } + HealthResult check() override { + std::this_thread::sleep_for(std::chrono::milliseconds(1500)); + return HealthResult(); + } + }; + slow_registry.add(std::make_shared()); + auto start = std::chrono::steady_clock::now(); + auto results = slow_registry.run_all(200); + auto elapsed = std::chrono::steady_clock::now() - start; + CHECK(results.size() == 1 && results[0].result.timed_out, "slow check times out"); + CHECK(elapsed < std::chrono::seconds(2), "timeout short-circuits under 2s"); + std::cout << "ALL_READYZ_OK\n"; + return 0; +} diff --git a/tests/src/cli/test_startupz.cpp b/tests/src/cli/test_startupz.cpp new file mode 100644 index 0000000..3e6e882 --- /dev/null +++ b/tests/src/cli/test_startupz.cpp @@ -0,0 +1,37 @@ +// /startupz: 503 until boot completes, then 200. +#include + +#include "ops/health_registry.hpp" +#include "ops/liveness_check.hpp" +#include "ops/ops_routes.hpp" +#include "ops/prometheus_exporter.hpp" +#include "ops/readiness_check.hpp" +#include "ops/startup_gate.hpp" + +#define CHECK(cond, label) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL: " << label << "\n"; \ + return 1; \ + } \ + std::cout << "PASS: " << label << "\n"; \ + } while (0) + +int main() { + using namespace molo::ops; + auto live = std::make_shared(); + HealthRegistry registry; + registry.add(live); + ReadinessCheck readiness; + StartupGate gate; + PrometheusExporter exporter; + OpsRoutes routes(registry, readiness, gate, exporter); + + OpsResponse starting = routes.startupz(); + CHECK(starting.status == 503, "startupz 503 before boot"); + gate.mark_boot_complete(); + OpsResponse booted = routes.startupz(); + CHECK(booted.status == 200 && booted.body == "{\"status\":\"ok\"}", "startupz 200 after boot"); + std::cout << "ALL_STARTUPZ_OK\n"; + return 0; +} diff --git a/tests/src/clients/framework.h b/tests/src/clients/framework.h new file mode 100644 index 0000000..a00c2e6 --- /dev/null +++ b/tests/src/clients/framework.h @@ -0,0 +1,106 @@ +#pragma once + +// Minimal dependency-free test framework for the clients part. + +#include +#include +#include +#include +#include + +namespace clients_test { + +struct Failure : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +struct Case { + std::string suite; + std::string name; + std::function fn; +}; + +inline std::vector& registry() { + static std::vector cases; + return cases; +} + +struct Registrar { + Registrar(const std::string& suite, const std::string& name, + std::function fn) { + registry().push_back(Case{suite, name, std::move(fn)}); + } +}; + +int run_all(); + +template +std::string show(const T& value) { + std::ostringstream out; + out << value; + return out.str(); +} + +inline std::string show(const std::string& value) { return "'" + value + "'"; } +inline std::string show(const char* value) { + return value ? "'" + std::string(value) + "'" : ""; +} +inline std::string show(bool value) { return value ? "true" : "false"; } + +} // namespace clients_test + +#define TEST(suite, name) \ + static void clients_test_fn_##suite##_##name(); \ + static ::clients_test::Registrar clients_test_reg_##suite##_##name( \ + #suite, #name, clients_test_fn_##suite##_##name); \ + static void clients_test_fn_##suite##_##name() + +#define ASSERT_TRUE(cond) \ + do { \ + if (!(cond)) { \ + std::ostringstream clients_test_msg; \ + clients_test_msg << __FILE__ << ":" << __LINE__ \ + << ": ASSERT_TRUE failed: " #cond; \ + throw ::clients_test::Failure(clients_test_msg.str()); \ + } \ + } while (0) + +#define ASSERT_FALSE(cond) ASSERT_TRUE(!(cond)) + +#define ASSERT_EQ(a, b) \ + do { \ + auto clients_test_a = (a); \ + auto clients_test_b = (b); \ + if (!(clients_test_a == clients_test_b)) { \ + std::ostringstream clients_test_msg; \ + clients_test_msg << __FILE__ << ":" << __LINE__ << ": ASSERT_EQ (" \ + << #a << " == " << #b << ") failed: got " \ + << ::clients_test::show(clients_test_a) \ + << ", want " << ::clients_test::show(clients_test_b); \ + throw ::clients_test::Failure(clients_test_msg.str()); \ + } \ + } while (0) + +#define ASSERT_THROW(stmt, exc_type) \ + do { \ + bool clients_test_caught = false; \ + try { \ + stmt; \ + } catch (const exc_type&) { \ + clients_test_caught = true; \ + } \ + if (!clients_test_caught) { \ + std::ostringstream clients_test_msg; \ + clients_test_msg << __FILE__ << ":" << __LINE__ \ + << ": ASSERT_THROW failed: " #stmt \ + << " did not throw " #exc_type; \ + throw ::clients_test::Failure(clients_test_msg.str()); \ + } \ + } while (0) + +#define FAIL(msg) \ + do { \ + std::ostringstream clients_test_msg; \ + clients_test_msg << __FILE__ << ":" << __LINE__ << ": " << msg; \ + throw ::clients_test::Failure(clients_test_msg.str()); \ + } while (0) diff --git a/tests/src/clients/main.cpp b/tests/src/clients/main.cpp new file mode 100644 index 0000000..e9890ba --- /dev/null +++ b/tests/src/clients/main.cpp @@ -0,0 +1,44 @@ +#include "framework.h" + +#include +#include + +namespace clients_test { + +int run_all() { + // Loopback tests must never be proxied; proxy tests set their own env. + unsetenv("HTTP_PROXY"); + unsetenv("HTTPS_PROXY"); + unsetenv("NO_PROXY"); + unsetenv("http_proxy"); + unsetenv("https_proxy"); + unsetenv("no_proxy"); + int passed = 0; + int failed = 0; + for (const auto& test : registry()) { + try { + test.fn(); + ++passed; + std::cout << "ok " << test.suite << "." << test.name << "\n"; + } catch (const Failure& err) { + ++failed; + std::cout << "FAIL " << test.suite << "." << test.name << ": " << err.what() + << "\n"; + } catch (const std::exception& err) { + ++failed; + std::cout << "FAIL " << test.suite << "." << test.name + << ": uncaught std::exception: " << err.what() << "\n"; + } catch (...) { + ++failed; + std::cout << "FAIL " << test.suite << "." << test.name + << ": uncaught unknown exception\n"; + } + } + std::cout << "passed " << passed << " failed " << failed << " total " + << (passed + failed) << "\n"; + return failed == 0 ? 0 : 1; +} + +} // namespace clients_test + +int main() { return clients_test::run_all(); } diff --git a/tests/src/clients/test_bodies.cpp b/tests/src/clients/test_bodies.cpp new file mode 100644 index 0000000..a874538 --- /dev/null +++ b/tests/src/clients/test_bodies.cpp @@ -0,0 +1,141 @@ +#include "framework.h" + +#include + +#include "clients/body/form.h" +#include "clients/body/json.h" +#include "clients/body/stream.h" +#include "clients/body/xml.h" + +using namespace molo::clients::body; + +TEST(bodies, json_roundtrip) { + std::string text = + R"({"name":"ada","age":36,"admin":true,"tags":["a","b"],"addr":{"city":"x"},"nil":null})"; + Json parsed = Json::parse(text); + ASSERT_TRUE(parsed.is_object()); + ASSERT_EQ(parsed.find("name")->as_string(), "ada"); + ASSERT_EQ(parsed.find("age")->as_int(), 36); + ASSERT_TRUE(parsed.find("admin")->as_bool()); + ASSERT_EQ(parsed.find("tags")->as_array().size(), 2u); + ASSERT_TRUE(parsed.find("nil")->is_null()); + Json again = Json::parse(parsed.dump()); + ASSERT_TRUE(again.equals(parsed)); + // Deterministic key order: sorted. + ASSERT_TRUE(parsed.dump().rfind("{\"addr\":", 0) == 0); +} + +TEST(bodies, json_escapes_and_numbers) { + Json parsed = Json::parse(R"({"s":"a\"b\\c\nd\u00e9","n":-12.5e2})"); + ASSERT_EQ(parsed.find("s")->as_string(), "a\"b\\c\nd\xc3\xa9"); + ASSERT_EQ(parsed.find("n")->as_number(), -1250.0); + ASSERT_THROW(Json::parse("{bad}"), JsonError); + ASSERT_THROW(Json::parse("[1,]"), JsonError); + ASSERT_THROW(Json::parse("\"unterminated"), JsonError); +} + +TEST(bodies, json_depth_cap) { + std::string deep(40, '['); + deep += std::string(40, ']'); + ASSERT_THROW(Json::parse(deep), JsonError); + // Exactly at the cap parses. + std::string ok(32, '['); + ok += std::string(32, ']'); + ASSERT_TRUE(Json::parse(ok).is_array()); +} + +TEST(bodies, yaml_dump) { + Json doc = Json::object(); + doc.set("openapi", Json("3.1.0")); + Json info = Json::object(); + info.set("title", Json("demo")); + doc.set("info", info); + Json list = Json::array(); + list.push(Json("a")); + list.push(Json(1)); + doc.set("list", list); + std::string yaml = yaml_dump(doc); + ASSERT_TRUE(yaml.find("openapi: 3.1.0") != std::string::npos); + ASSERT_TRUE(yaml.find("title: demo") != std::string::npos); + ASSERT_TRUE(yaml.find("- a") != std::string::npos); +} + +TEST(bodies, form_urlencoded) { + UrlEncodedForm form; + form.add("name", "a b"); + form.add("q", "x+y&z="); + std::string enc = form.encode(); + ASSERT_TRUE(enc.find("name=a%20b") != std::string::npos); + UrlEncodedForm back = UrlEncodedForm::decode(enc); + ASSERT_EQ(back.fields().size(), 2u); + ASSERT_EQ(back.fields()[1].second, "x+y&z="); +} + +TEST(bodies, multipart) { + MultipartBody body("testboundary"); + body.add_text("field", "value"); + body.add_file("file", "a.txt", "text/plain", "hello"); + std::string enc = body.encode(); + ASSERT_EQ(body.content_type(), + "multipart/form-data; boundary=testboundary"); + ASSERT_TRUE(enc.find("--testboundary\r\n") != std::string::npos); + ASSERT_TRUE(enc.find("filename=\"a.txt\"") != std::string::npos); + ASSERT_TRUE(enc.find("\r\nhello\r\n") != std::string::npos); + ASSERT_TRUE(enc.find("--testboundary--\r\n") != std::string::npos); + ASSERT_TRUE(body.encoded_size() > 0); +} + +TEST(bodies, xml_builder_and_parse) { + XmlBuilder builder("user"); + builder.attr("id", "7").child("name", "a") != std::string::npos); + ASSERT_TRUE(text.find("<") != std::string::npos); + auto flat = xml_parse_flat(text); + ASSERT_EQ(flat["name"], "a    "), XmlError); + ASSERT_THROW(xml_parse_flat("nope"), XmlError); +} + +TEST(bodies, stream_readers) { + StringReader reader("0123456789"); + ASSERT_EQ(reader.read_some(4), "0123"); + ASSERT_EQ(reader.read_some(100), "456789"); + ASSERT_TRUE(reader.eof()); + ASSERT_TRUE(reader.known_size().has_value()); + + bool eof = false; + CallbackReader callback( + [&](std::size_t max_bytes, bool& done) { + (void)max_bytes; + done = true; + return std::string("cb"); + }, + 2u); + ASSERT_EQ(callback.read_some(64), "cb"); + ASSERT_TRUE(callback.eof()); + ASSERT_TRUE(callback.known_size().has_value()); + (void)eof; + + const char* path = "build/clients_test_tmp.bin"; + { + std::FILE* file = std::fopen(path, "wb"); + ASSERT_TRUE(file != nullptr); + std::fwrite("filedata", 1, 8, file); + std::fclose(file); + } + FileReader file_reader(path); + ASSERT_EQ(file_reader.read_some(4), "file"); + ASSERT_EQ(file_reader.read_some(4), "data"); + ASSERT_TRUE(file_reader.eof()); + std::remove(path); + ASSERT_THROW(FileReader("build/does-not-exist-molo.bin"), StreamError); +} + +TEST(bodies, capped_collector) { + CappedCollector collector(4); + ASSERT_TRUE(collector.push("ab")); + ASSERT_TRUE(collector.push("cd")); + ASSERT_EQ(collector.data(), "abcd"); + ASSERT_THROW(collector.push("e"), StreamError); +} diff --git a/tests/src/clients/test_client_mock.cpp b/tests/src/clients/test_client_mock.cpp new file mode 100644 index 0000000..9eaf2d9 --- /dev/null +++ b/tests/src/clients/test_client_mock.cpp @@ -0,0 +1,375 @@ +#include "framework.h" + +#include +#include +#include + +#include "clients/auth.h" +#include "clients/client.h" +#include "clients/transports/mock_transport.h" + +using namespace molo::clients; +using molo::clients::transports::MockTransport; + +namespace { + +struct RecordingReporter : Reporter { + std::vector spans; + void on_request(const RequestSpan& span) override { spans.push_back(span); } +}; + +ClientResponse text_response(int status, const std::string& body) { + HeaderMap headers; + headers.set("Content-Type", "text/plain"); + return ClientResponse(status, std::move(headers), body); +} + +} // namespace + +TEST(client, verbs_over_mock) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + mock->expect_status(Method::kGet, "/v1/users", 200, "[]", + "application/json"); + mock->expect_status(Method::kPost, "/v1/users", 201, "{}", + "application/json"); + Client client(std::move(mock)); + ClientResponse list = client.get_blocking("http://api.test/v1/users"); + ASSERT_EQ(list.status(), 200); + ASSERT_EQ(list.body(), "[]"); + ClientResponse created = + client.post_blocking("http://api.test/v1/users", "{}", "application/json"); + ASSERT_EQ(created.status(), 201); + raw->verify(); + ASSERT_EQ(raw->send_count(), 2u); +} + +TEST(client, template_and_predicates) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + transports::MockExpectation exp; + exp.method = Method::kGet; + exp.path = "/v1/users/{id}"; + exp.required_headers["X-Tenant"] = "acme"; + exp.response = text_response(200, "user"); + raw->expect(std::move(exp)); + Client client(std::move(mock)); + ClientRequest req = ClientRequest::get("http://api.test/v1/users/42"); + req.header("X-Tenant", "acme"); + ASSERT_EQ(client.send_blocking(std::move(req)).status(), 200); + raw->verify(); +} + +TEST(client, request_helpers_and_json) { + ClientRequest req = ClientRequest::post("http://h/x"); + body::Json doc = body::Json::object(); + doc.set("a", body::Json(1)); + req.json(doc); + ASSERT_EQ(req.headers().get_or("Content-Type", ""), "application/json"); + ASSERT_TRUE(req.body_bytes().find("\"a\":1") != std::string::npos); + ClientRequest form = ClientRequest::post("http://h/f"); + body::UrlEncodedForm fields; + fields.add("k", "v"); + form.form(fields); + ASSERT_EQ(form.body_bytes(), "k=v"); +} + +TEST(client, connect_error_retries_then_succeeds) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_error(Method::kGet, "/flaky", + std::make_exception_ptr(TimeoutError(TimeoutPhase::kConnect))); + raw->expect_status(Method::kGet, "/flaky", 200, "ok"); + ClientConfig config; + std::vector sleeps; + config.sleeper = [&](std::chrono::milliseconds ms) { sleeps.push_back(ms); }; + Client client(std::move(mock), config); + ClientResponse resp = client.get_blocking("http://h/flaky"); + ASSERT_EQ(resp.status(), 200); + ASSERT_EQ(resp.attempts(), 2); + ASSERT_EQ(sleeps.size(), 1u); + ASSERT_EQ(client.stats().retries, 1u); + raw->verify(); +} + +TEST(client, retryable_status_matrix) { + for (int status : {429, 502, 503, 504}) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_status(Method::kGet, "/r", status, "bad"); + raw->expect_status(Method::kGet, "/r", 200, "good"); + ClientConfig config; + config.sleeper = [](std::chrono::milliseconds) {}; + Client client(std::move(mock), config); + ClientResponse resp = client.get_blocking("http://h/r"); + ASSERT_EQ(resp.status(), 200); + ASSERT_EQ(resp.attempts(), 2); + raw->verify(); + } + // Non-retryable status passes through untouched. + auto mock = std::make_unique(); + mock->expect_status(Method::kGet, "/r", 500, "bad"); + Client client(std::move(mock)); + ASSERT_EQ(client.get_blocking("http://h/r").attempts(), 1); +} + +TEST(client, post_not_retried_without_idempotency) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_status(Method::kPost, "/orders", 503, "busy"); + ClientConfig config; + config.sleeper = [](std::chrono::milliseconds) {}; + Client client(std::move(mock), config); + ClientResponse resp = client.post_blocking("http://h/orders", "{}"); + ASSERT_EQ(resp.status(), 503); + ASSERT_EQ(resp.attempts(), 1); + ASSERT_EQ(raw->send_count(), 1u); + raw->verify(); +} + +TEST(client, post_retried_with_idempotency_key) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_status(Method::kPost, "/orders", 503, "busy"); + raw->expect_status(Method::kPost, "/orders", 200, "done"); + ClientConfig config; + config.sleeper = [](std::chrono::milliseconds) {}; + Client client(std::move(mock), config); + ClientRequest req = ClientRequest::post("http://h/orders"); + req.json_text("{}"); + req.idempotency_key("key-123"); + ClientResponse resp = client.send_blocking(std::move(req)); + ASSERT_EQ(resp.status(), 200); + ASSERT_EQ(resp.attempts(), 2); + raw->verify(); + // The key was actually sent. + auto seen = raw->seen(); + ASSERT_EQ(seen.size(), 2u); + ASSERT_EQ(seen[0].headers.get_or("Idempotency-Key", ""), "key-123"); +} + +TEST(client, retry_after_honored) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + HeaderMap headers; + headers.set("Content-Type", "text/plain"); + headers.set("Retry-After", "2"); + transports::MockExpectation exp; + exp.method = Method::kGet; + exp.path = "/limited"; + exp.response = ClientResponse(429, headers, "slow down"); + raw->expect(std::move(exp)); + raw->expect_status(Method::kGet, "/limited", 200, "ok"); + ClientConfig config; + std::vector sleeps; + config.sleeper = [&](std::chrono::milliseconds ms) { sleeps.push_back(ms); }; + Client client(std::move(mock), config); + ASSERT_EQ(client.get_blocking("http://h/limited").status(), 200); + ASSERT_EQ(sleeps.size(), 1u); + ASSERT_EQ(sleeps[0].count(), 2000); + raw->verify(); +} + +TEST(client, retry_after_capped) { + ASSERT_EQ(RetryPolicy::parse_retry_after("120")->count(), 5000); + ASSERT_TRUE(!RetryPolicy::parse_retry_after("garbage").has_value()); + ASSERT_TRUE(!RetryPolicy::parse_retry_after("").has_value()); + // HTTP-date in the past yields zero delay. + auto past = + RetryPolicy::parse_retry_after("Wed, 21 Oct 2015 07:28:00 GMT"); + ASSERT_TRUE(past.has_value()); + ASSERT_EQ(past->count(), 0); +} + +TEST(client, backoff_bounds) { + RetryPolicy::set_test_seed(42); + RetryPolicy policy; + for (int i = 0; i < 50; ++i) { + auto d1 = policy.backoff_for(1); + ASSERT_TRUE(d1.count() >= 0 && d1.count() <= 200); + auto d3 = policy.backoff_for(3); + ASSERT_TRUE(d3.count() >= 0 && d3.count() <= 800); + auto d30 = policy.backoff_for(30); + ASSERT_TRUE(d30.count() >= 0 && d30.count() <= 5000); + } + RetryPolicy::set_test_seed(-1); +} + +TEST(client, redirect_follow_relative) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + HeaderMap headers; + headers.set("Location", "/v1/target"); + transports::MockExpectation exp; + exp.method = Method::kGet; + exp.path = "/v1/start"; + exp.response = ClientResponse(302, headers, ""); + raw->expect(std::move(exp)); + raw->expect_status(Method::kGet, "/v1/target", 200, "there"); + Client client(std::move(mock)); + ClientResponse resp = client.get_blocking("http://h/v1/start"); + ASSERT_EQ(resp.status(), 200); + ASSERT_EQ(resp.body(), "there"); + ASSERT_EQ(client.stats().redirect_hops, 1u); + raw->verify(); +} + +TEST(client, redirect_strips_auth_cross_host) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + HeaderMap headers; + headers.set("Location", "http://other.test/v1/target"); + transports::MockExpectation exp; + exp.method = Method::kGet; + exp.path = "/v1/start"; + exp.response = ClientResponse(302, headers, ""); + raw->expect(std::move(exp)); + raw->expect_status(Method::kGet, "/v1/target", 200, "there"); + ClientConfig config; + config.auth = std::make_shared("secret"); + Client client(std::move(mock), config); + ASSERT_EQ(client.get_blocking("http://h/v1/start").status(), 200); + auto seen = raw->seen(); + ASSERT_EQ(seen.size(), 2u); + ASSERT_EQ(seen[0].headers.get_or("Authorization", ""), "Bearer secret"); + ASSERT_FALSE(seen[1].headers.has("Authorization")); + raw->verify(); +} + +TEST(client, redirect_loop_bounded) { + auto mock = std::make_unique(); + HeaderMap headers; + headers.set("Location", "/loop"); + for (int i = 0; i < 6; ++i) { + transports::MockExpectation exp; + exp.method = Method::kGet; + exp.path = "/loop"; + exp.response = ClientResponse(302, headers, ""); + mock->expect(std::move(exp)); + } + Client client(std::move(mock)); + ASSERT_THROW(client.get_blocking("http://h/loop"), RedirectError); +} + +TEST(client, post_redirect_not_followed) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + HeaderMap headers; + headers.set("Location", "/elsewhere"); + transports::MockExpectation exp; + exp.method = Method::kPost; + exp.path = "/submit"; + exp.response = ClientResponse(302, headers, ""); + raw->expect(std::move(exp)); + Client client(std::move(mock)); + ClientResponse resp = client.post_blocking("http://h/submit", "x"); + ASSERT_EQ(resp.status(), 302); + ASSERT_EQ(raw->send_count(), 1u); + raw->verify(); +} + +TEST(client, cookie_jar_roundtrip) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + HeaderMap login_headers; + login_headers.set("Content-Type", "text/plain"); + login_headers.add("Set-Cookie", "session=abc; Path=/; HttpOnly"); + transports::MockExpectation login; + login.method = Method::kPost; + login.path = "/login"; + login.response = ClientResponse(200, login_headers, "ok"); + raw->expect(std::move(login)); + transports::MockExpectation me; + me.method = Method::kGet; + me.path = "/me"; + me.required_headers["Cookie"] = "session=abc"; + me.response = text_response(200, "me"); + raw->expect(std::move(me)); + Client client(std::move(mock)); + ASSERT_EQ(client.post_blocking("http://h/login", "").status(), 200); + ASSERT_EQ(client.get_blocking("http://h/me").body(), "me"); + ASSERT_EQ(client.jar().size(), 1u); + raw->verify(); +} + +TEST(client, secure_cookie_not_sent_over_http) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + HeaderMap login_headers; + login_headers.add("Set-Cookie", "s=1; Secure; Path=/"); + transports::MockExpectation login; + login.method = Method::kGet; + login.path = "/login"; + login.response = ClientResponse(200, login_headers, "ok"); + raw->expect(std::move(login)); + transports::MockExpectation next; + next.method = Method::kGet; + next.path = "/next"; + next.predicate = [](const ClientRequest& req) { + return !req.headers().has("Cookie"); + }; + next.response = text_response(200, "ok"); + raw->expect(std::move(next)); + Client client(std::move(mock)); + client.get_blocking("http://h/login"); + client.get_blocking("http://h/next"); + raw->verify(); +} + +TEST(client, auth_basic_and_apikey) { + BasicAuth basic("user", "pass"); + ClientRequest req = ClientRequest::get("http://h/x"); + basic.apply(req); + ASSERT_EQ(req.headers().get_or("Authorization", ""), + "Basic dXNlcjpwYXNz"); + ApiKeyAuth key("k", "X-Api-Key"); + ClientRequest req2 = ClientRequest::get("http://h/x"); + key.apply(req2); + ASSERT_EQ(req2.headers().get_or("X-Api-Key", ""), "k"); + ApiKeyAuth query_key("k", "X-Api-Key", "api_key"); + ClientRequest req3 = ClientRequest::get("http://h/x"); + query_key.apply(req3); + ASSERT_TRUE(req3.url().target().find("api_key=k") != std::string::npos); +} + +TEST(client, base_url_and_reporter) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_status(Method::kGet, "/v1/ping", 200, "pong"); + auto reporter = std::make_shared(); + ClientConfig config; + config.base_url = "http://api.test"; + config.reporter = reporter; + Client client(std::move(mock), config); + ClientResponse resp = client.get_blocking("/v1/ping"); + ASSERT_EQ(resp.body(), "pong"); + ASSERT_EQ(reporter->spans.size(), 1u); + ASSERT_EQ(reporter->spans[0].status, 200); + ASSERT_EQ(reporter->spans[0].method, "GET"); + ASSERT_EQ(client.stats().requests, 1u); + ASSERT_EQ(client.stats().responses_2xx, 1u); + raw->verify(); +} + +TEST(client, unmatched_and_unconsumed) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_status(Method::kGet, "/known", 200, "x"); + Client client(std::move(mock)); + ASSERT_THROW(client.get_blocking("http://h/unknown"), + transports::MockError); + ASSERT_THROW(raw->verify(), transports::MockError); +} + +TEST(client, stream_upload_mock) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_status(Method::kPost, "/upload", 200, "stored"); + Client client(std::move(mock)); + auto reader = std::make_shared("chunked-data"); + ClientRequest req = ClientRequest::post("http://h/upload"); + req.stream(reader, std::nullopt, "application/octet-stream"); + ASSERT_TRUE(req.is_stream_body()); + ASSERT_EQ(client.send_blocking(std::move(req)).status(), 200); + raw->verify(); +} diff --git a/tests/src/clients/test_http_transport.cpp b/tests/src/clients/test_http_transport.cpp new file mode 100644 index 0000000..cb00e7f --- /dev/null +++ b/tests/src/clients/test_http_transport.cpp @@ -0,0 +1,375 @@ +#include "framework.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "clients/client.h" +#include "clients/transports/http_transport.h" +#include "clients/transports/tls.h" + +using namespace molo::clients; +using molo::clients::transports::BodyTooLargeError; +using molo::clients::transports::H2StubError; +using molo::clients::transports::HttpTransport; +using molo::clients::transports::TlsStubError; + +namespace { + +std::string to_lower(std::string s) { + for (auto& c : s) c = (char)tolower((unsigned char)c); + return s; +} + +bool recv_until(int fd, std::string& buf, const std::string& marker) { + char chunk[4096]; + while (buf.find(marker) == std::string::npos) { + struct pollfd pfd {fd, POLLIN, 0}; + if (::poll(&pfd, 1, 5000) <= 0) return false; + ssize_t r = ::recv(fd, chunk, sizeof(chunk), 0); + if (r <= 0) return false; + buf.append(chunk, (std::size_t)r); + if (buf.size() > 1024 * 1024) return false; + } + return true; +} + +bool recv_line(int fd, std::string& buf, std::string& line) { + if (!recv_until(fd, buf, "\r\n")) return false; + auto at = buf.find("\r\n"); + line = buf.substr(0, at); + buf.erase(0, at + 2); + return true; +} + +void send_all(int fd, const std::string& data) { + std::size_t sent = 0; + while (sent < data.size()) { + ssize_t n = ::send(fd, data.data() + sent, data.size() - sent, MSG_NOSIGNAL); + if (n <= 0) return; + sent += (std::size_t)n; + } +} + +// Minimal loopback HTTP/1.1 server for real-transport tests. +class LoopServer { + public: + LoopServer() { + listen_fd_ = ::socket(AF_INET, SOCK_STREAM, 0); + int one = 1; + ::setsockopt(listen_fd_, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + struct sockaddr_in addr {}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + ::bind(listen_fd_, (struct sockaddr*)&addr, sizeof(addr)); + ::listen(listen_fd_, 16); + socklen_t len = sizeof(addr); + ::getsockname(listen_fd_, (struct sockaddr*)&addr, &len); + port_ = ntohs(addr.sin_port); + thread_ = std::thread([this] { accept_loop(); }); + } + + ~LoopServer() { + stop_ = true; + ::shutdown(listen_fd_, SHUT_RDWR); + ::close(listen_fd_); + if (thread_.joinable()) thread_.join(); + } + + int port() const { return port_; } + int connections() const { return connections_; } + int requests() const { return requests_; } + + std::string base() const { + return "http://127.0.0.1:" + std::to_string(port_); + } + + private: + void accept_loop() { + while (!stop_) { + struct pollfd pfd {listen_fd_, POLLIN, 0}; + if (::poll(&pfd, 1, 100) <= 0) continue; + int fd = ::accept(listen_fd_, nullptr, nullptr); + if (fd < 0) continue; + connections_++; + handle_connection(fd); + ::close(fd); + } + } + + void handle_connection(int fd) { + std::string buf; + for (int i = 0; i < 50 && !stop_; ++i) { + if (!recv_until(fd, buf, "\r\n\r\n")) return; + auto at = buf.find("\r\n\r\n"); + std::string head = buf.substr(0, at); + buf.erase(0, at + 4); + std::string request_line = head.substr(0, head.find("\r\n")); + std::map headers; + std::size_t pos = head.find("\r\n"); + while (pos != std::string::npos) { + std::size_t next = head.find("\r\n", pos + 2); + std::string line = next == std::string::npos + ? head.substr(pos + 2) + : head.substr(pos + 2, next - pos - 2); + pos = next; + auto colon = line.find(':'); + if (colon == std::string::npos) continue; + headers[to_lower(line.substr(0, colon))] = line.substr(colon + 1); + } + // Trim header values. + for (auto& kv : headers) { + while (!kv.second.empty() && kv.second.front() == ' ') + kv.second.erase(kv.second.begin()); + while (!kv.second.empty() && kv.second.back() == '\r') + kv.second.pop_back(); + } + std::string body; + auto te = headers.find("transfer-encoding"); + auto cl = headers.find("content-length"); + if (te != headers.end() && te->second.find("chunked") != std::string::npos) { + while (true) { + std::string line; + if (!recv_line(fd, buf, line)) return; + long size = strtol(line.c_str(), nullptr, 16); + if (size == 0) { + std::string trailer; + if (!recv_line(fd, buf, trailer)) return; + break; + } + while ((long)buf.size() < size) { + char chunk[4096]; + struct pollfd pfd {fd, POLLIN, 0}; + if (::poll(&pfd, 1, 5000) <= 0) return; + ssize_t r = ::recv(fd, chunk, sizeof(chunk), 0); + if (r <= 0) return; + buf.append(chunk, (std::size_t)r); + } + body.append(buf.data(), (std::size_t)size); + buf.erase(0, (std::size_t)size); + std::string crlf; + if (!recv_line(fd, buf, crlf)) return; + } + } else if (cl != headers.end()) { + std::size_t need = (std::size_t)std::stoul(cl->second); + while (buf.size() < need) { + char chunk[4096]; + struct pollfd pfd {fd, POLLIN, 0}; + if (::poll(&pfd, 1, 5000) <= 0) return; + ssize_t r = ::recv(fd, chunk, sizeof(chunk), 0); + if (r <= 0) return; + buf.append(chunk, (std::size_t)r); + } + body = buf.substr(0, need); + buf.erase(0, need); + } + requests_++; + std::string method = request_line.substr(0, request_line.find(' ')); + std::string target = ""; + { + auto sp1 = request_line.find(' '); + auto sp2 = request_line.find(' ', sp1 + 1); + if (sp1 != std::string::npos && sp2 != std::string::npos) + target = request_line.substr(sp1 + 1, sp2 - sp1 - 1); + } + auto q = target.find('?'); + std::string path = q == std::string::npos ? target : target.substr(0, q); + // Absolute-form targets (proxied): reduce to origin-form. + if (path.rfind("http://", 0) == 0) { + auto slash = path.find('/', 7); + path = slash == std::string::npos ? "/" : path.substr(slash); + } + std::string payload; + std::string content_type = "text/plain"; + std::string extra_headers = "Connection: keep-alive\r\n"; + bool chunked_response = false; + if (path == "/echo") { + payload = method + " " + target + "\n" + body; + } else if (path == "/keep") { + payload = "keep-alive-ok"; + } else if (path == "/close") { + payload = "closing"; + extra_headers = "Connection: close\r\n"; + } else if (path == "/chunked") { + chunked_response = true; + } else if (path == "/big") { + payload = std::string(256 * 1024, 'x'); + } else if (path == "/slow") { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + payload = "slow-ok"; + } else { + std::string not_found = "HTTP/1.1 404 Not Found\r\nContent-Length: 9\r\n" + + extra_headers + "\r\nnot found"; + send_all(fd, not_found); + continue; + } + if (chunked_response) { + send_all(fd, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n" + + extra_headers + "Content-Type: text/plain\r\n\r\n" + "5\r\nhello\r\n6\r\n world\r\n1\r\n!\r\n0\r\n\r\n"); + } else { + std::string response = "HTTP/1.1 200 OK\r\nContent-Length: " + + std::to_string(payload.size()) + "\r\n" + + extra_headers + "Content-Type: " + content_type + + "\r\n\r\n" + payload; + send_all(fd, response); + } + if (extra_headers.find("close") != std::string::npos) return; + (void)content_type; + } + } + + int listen_fd_ = -1; + int port_ = 0; + std::atomic stop_{false}; + std::atomic connections_{0}; + std::atomic requests_{0}; + std::thread thread_; +}; + +} // namespace + +TEST(http, echo_roundtrip) { + LoopServer server; + Client client(std::make_unique()); + ClientResponse get = client.get_blocking(server.base() + "/echo?m=1"); + ASSERT_EQ(get.status(), 200); + ASSERT_TRUE(get.body().find("GET /echo?m=1") != std::string::npos); + ClientResponse post = + client.post_blocking(server.base() + "/echo", "ping-body"); + ASSERT_EQ(post.status(), 200); + ASSERT_TRUE(post.body().find("ping-body") != std::string::npos); + ASSERT_EQ(server.requests(), 2); +} + +TEST(http, keepalive_reuse) { + LoopServer server; + Client client(std::make_unique()); + for (int i = 0; i < 3; ++i) { + ClientResponse resp = client.get_blocking(server.base() + "/keep"); + ASSERT_EQ(resp.body(), "keep-alive-ok"); + if (i > 0) ASSERT_TRUE(resp.from_pool_reuse()); + } + ASSERT_EQ(server.connections(), 1); + ASSERT_TRUE(client.pool().stats().reused >= 2); +} + +TEST(http, chunked_response) { + LoopServer server; + Client client(std::make_unique()); + ClientResponse resp = client.get_blocking(server.base() + "/chunked"); + ASSERT_EQ(resp.status(), 200); + ASSERT_EQ(resp.body(), "hello world!"); +} + +TEST(http, streamed_download) { + LoopServer server; + Client client(std::make_unique()); + std::size_t bytes = 0; + int chunks = 0; + ClientRequest req = ClientRequest::get(server.base() + "/big"); + client.send_stream_blocking(std::move(req), [&](const std::string& chunk) { + bytes += chunk.size(); + chunks++; + return true; + }); + ASSERT_EQ(bytes, 256u * 1024u); + ASSERT_TRUE(chunks > 1); + // Cancellation stops the transfer without an error. + ClientRequest req2 = ClientRequest::get(server.base() + "/big"); + int seen = 0; + client.send_stream_blocking(std::move(req2), [&](const std::string&) { + return ++seen < 2; + }); + ASSERT_EQ(seen, 2); +} + +TEST(http, close_response_not_reused) { + LoopServer server; + Client client(std::make_unique()); + ASSERT_EQ(client.get_blocking(server.base() + "/close").status(), 200); + ASSERT_EQ(client.get_blocking(server.base() + "/keep").status(), 200); + ASSERT_EQ(server.connections(), 2); +} + +TEST(http, read_timeout_phase) { + LoopServer server; + auto transport = std::make_unique(); + Client client(std::move(transport)); + Timeouts timeouts; + timeouts.read = std::chrono::milliseconds(100); + timeouts.total = std::chrono::milliseconds(10000); + ClientRequest req = ClientRequest::get(server.base() + "/slow"); + req.timeouts(timeouts); + req.mark_idempotent(false); + try { + client.send_blocking(std::move(req)); + FAIL("expected TimeoutError"); + } catch (const TimeoutError& err) { + ASSERT_TRUE(err.phase() == TimeoutPhase::kRead || + err.phase() == TimeoutPhase::kTotal); + } +} + +TEST(http, connect_refused_is_connect_phase) { + int probe = ::socket(AF_INET, SOCK_STREAM, 0); + struct sockaddr_in addr {}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + ::bind(probe, (struct sockaddr*)&addr, sizeof(addr)); + socklen_t len = sizeof(addr); + ::getsockname(probe, (struct sockaddr*)&addr, &len); + int closed_port = ntohs(addr.sin_port); + ::close(probe); + Client client(std::make_unique()); + try { + client.get_blocking("http://127.0.0.1:" + std::to_string(closed_port) + + "/"); + FAIL("expected TransportError"); + } catch (const TransportError& err) { + ASSERT_TRUE(err.connect_phase()); + } +} + +TEST(http, https_needs_tls_stub) { + LoopServer server; + Client client(std::make_unique()); + ASSERT_THROW(client.get_blocking("https://127.0.0.1:" + + std::to_string(server.port()) + "/"), + TlsStubError); +} + +TEST(http, h2c_stub) { + LoopServer server; + Client client(std::make_unique()); + ClientRequest req = ClientRequest::get(server.base() + "/keep"); + req.h2(true); + ASSERT_THROW(client.send_blocking(std::move(req)), H2StubError); +} + +TEST(http, buffered_body_cap) { + Client client(std::make_unique()); + ClientRequest req = ClientRequest::post("http://127.0.0.1:9/echo"); + req.bytes(std::string(11 * 1024 * 1024, 'y')); + ASSERT_THROW(client.send_blocking(std::move(req)), BodyTooLargeError); +} + +TEST(http, not_found_status) { + LoopServer server; + Client client(std::make_unique()); + ClientResponse resp = client.get_blocking(server.base() + "/missing"); + ASSERT_EQ(resp.status(), 404); + ASSERT_EQ(resp.body(), "not found"); +} diff --git a/tests/src/clients/test_openapi.cpp b/tests/src/clients/test_openapi.cpp new file mode 100644 index 0000000..b610e86 --- /dev/null +++ b/tests/src/clients/test_openapi.cpp @@ -0,0 +1,161 @@ +#include "framework.h" + +#include "clients/openapi/builder.h" +#include "clients/openapi/document.h" +#include "clients/openapi/extensions.h" +#include "clients/openapi/routes.h" +#include "clients/openapi/schema.h" + +using namespace molo::clients::openapi; +using molo::clients::body::Json; + +namespace { + +ModelDecl user_model() { + ModelDecl model; + model.name = "User"; + model.description = "A user."; + FieldDecl id; + id.name = "id"; + id.type = "integer"; + id.required = true; + FieldDecl name; + name.name = "name"; + name.type = "string"; + name.required = true; + name.min_length = 1; + name.max_length = 80; + FieldDecl role; + role.name = "role"; + role.type = "string"; + role.enum_values = {"admin", "member"}; + model.fields = {id, name, role}; + return model; +} + +std::vector sample_routes() { + RouteDecl list; + list.method = "GET"; + list.path = "/v1/users"; + list.operation_id = "listUsers"; + list.summary = "List users"; + list.tags = {"users"}; + ParameterDecl page; + page.name = "page"; + page.in = "query"; + page.schema = Json::object(); + page.schema.set("type", Json("integer")); + list.parameters = {page}; + list.responses = {{"200", "UserList"}}; + + RouteDecl create; + create.method = "POST"; + create.path = "/v1/users"; + create.operation_id = "createUser"; + create.request_model = "UserCreate"; + create.responses = {{"201", "User"}, {"422", ""}}; + create.security = {"bearerAuth"}; + return {list, create}; +} + +} // namespace + +TEST(openapi, schema_converter) { + Json schema = SchemaConverter::convert(user_model()); + ASSERT_EQ(schema.find("type")->as_string(), "object"); + const Json* required = schema.find("required"); + ASSERT_TRUE(required && required->is_array()); + ASSERT_EQ(required->as_array().size(), 2u); + const Json* props = schema.find("properties"); + ASSERT_TRUE(props && props->is_object()); + ASSERT_EQ(props->find("role")->find("enum")->as_array().size(), 2u); + FieldDecl owner; + owner.name = "owner"; + owner.ref = "User"; + ASSERT_EQ(SchemaConverter::field_schema(owner).find("$ref")->as_string(), + "#/components/schemas/User"); +} + +TEST(openapi, build_and_snapshot) { + BuilderInput input; + input.info.title = "Demo"; + input.info.version = "1.0.0"; + input.servers = {ServerDecl{"https://api.test", "prod"}}; + input.routes = sample_routes(); + input.models = {user_model()}; + SecuritySchemeDecl bearer; + bearer.type = "http"; + bearer.scheme = "bearer"; + bearer.bearer_format = "JWT"; + input.security_schemes["bearerAuth"] = bearer; + input.tags = {"users"}; + Builder builder; + Document doc = builder.build(input); + std::string json = doc.to_json_string(); + ASSERT_TRUE(json.find("\"openapi\":\"3.1.0\"") != std::string::npos); + ASSERT_TRUE(json.find("/v1/users") != std::string::npos); + ASSERT_TRUE(json.find("listUsers") != std::string::npos); + ASSERT_TRUE(json.find("#/components/schemas/User") != std::string::npos); + ASSERT_TRUE(json.find("x-molo-framework-version") != std::string::npos); + ASSERT_TRUE(json.find("bearerAuth") != std::string::npos); + // Undeclared descriptions render as TBD, never invented. + ASSERT_TRUE(json.find("\"description\":\"TBD\"") != std::string::npos); + Json parsed = Json::parse(json); + ASSERT_TRUE(parsed.find("paths")->find("/v1/users")->has("get")); + ASSERT_TRUE(parsed.find("paths")->find("/v1/users")->has("post")); +} + +TEST(openapi, deterministic_bytes) { + Builder builder; + auto routes = sample_routes(); + auto models = std::vector{user_model()}; + std::string first = builder.from_routes(routes, models).to_json_string(); + std::string second = builder.from_routes(routes, models).to_json_string(); + ASSERT_EQ(first, second); + std::string yaml_first = + builder.from_routes(routes, models).to_yaml_string(); + std::string yaml_second = + builder.from_routes(routes, models).to_yaml_string(); + ASSERT_EQ(yaml_first, yaml_second); + ASSERT_TRUE(yaml_first.find("openapi:") != std::string::npos); +} + +TEST(openapi, caps_error) { + BuilderInput input; + for (int i = 0; i < 5001; ++i) { + RouteDecl route; + route.method = "GET"; + route.path = "/r" + std::to_string(i); + input.routes.push_back(route); + } + Builder builder; + ASSERT_THROW(builder.build(input), OpenApiError); +} + +TEST(openapi, routes_table) { + Builder builder; + Document doc = + builder.from_routes(sample_routes(), {user_model()}); + auto table = doc_route_table(doc); + ASSERT_EQ(table.size(), 4u); + ASSERT_TRUE(table.count("/openapi.json") == 1); + ASSERT_TRUE(table.count("/openapi.yaml") == 1); + ASSERT_TRUE(table.count("/docs") == 1); + ASSERT_TRUE(table.count("/redoc") == 1); + ASSERT_EQ(table["/openapi.json"].content_type, "application/json"); + ASSERT_TRUE(table["/openapi.json"].body.find("\"openapi\"") != + std::string::npos); + ASSERT_TRUE(table["/docs"].body.find("MOLO_OPENAPI_URL") != std::string::npos); +} + +TEST(openapi, extend_hook_and_extensions) { + Builder builder; + builder.extend_hook([](Document& doc) { + doc.extensions()["x-team"] = Json("clients"); + }); + Document doc = builder.from_routes(sample_routes(), {}); + ASSERT_TRUE(doc.to_json_string().find("x-team") != std::string::npos); + apply_overrides(doc, nullptr); + doc.extensions()["bad-key"] = Json(1); + ASSERT_THROW(apply_overrides(doc, nullptr), OpenApiError); +} diff --git a/tests/src/clients/test_proxy.cpp b/tests/src/clients/test_proxy.cpp new file mode 100644 index 0000000..34c4c9b --- /dev/null +++ b/tests/src/clients/test_proxy.cpp @@ -0,0 +1,67 @@ +#include "framework.h" + +#include + +#include "clients/proxy.h" + +using namespace molo::clients; + +TEST(proxy, no_proxy_match_matrix) { + ASSERT_TRUE(no_proxy_match("*", "anything.example.com")); + ASSERT_TRUE(no_proxy_match("example.com", "example.com")); + ASSERT_TRUE(no_proxy_match("example.com", "a.example.com")); + ASSERT_FALSE(no_proxy_match("example.com", "example.com.evil.com")); + ASSERT_TRUE(no_proxy_match(".example.com", "a.example.com")); + ASSERT_TRUE(no_proxy_match(".example.com", "example.com")); + ASSERT_TRUE(no_proxy_match("*.example.com", "a.example.com")); + ASSERT_FALSE(no_proxy_match("*.example.com", "example.com.evil.com")); + ASSERT_TRUE(no_proxy_match("10.0.0.0/8", "10.1.2.3")); + ASSERT_FALSE(no_proxy_match("10.0.0.0/8", "11.0.0.1")); + ASSERT_TRUE(no_proxy_match("192.168.1.0/24", "192.168.1.200")); + ASSERT_FALSE(no_proxy_match("192.168.1.0/24", "192.168.2.1")); + ASSERT_TRUE(no_proxy_match("127.0.0.1/32", "127.0.0.1")); + ASSERT_FALSE(no_proxy_match("not a cidr/8", "10.0.0.1")); + ASSERT_FALSE(no_proxy_match("", "example.com")); +} + +TEST(proxy, ipv4_parse) { + std::uint32_t out = 0; + ASSERT_TRUE(parse_ipv4("127.0.0.1", out)); + ASSERT_EQ(out, 0x7F000001u); + ASSERT_FALSE(parse_ipv4("999.1.1.1", out)); + ASSERT_FALSE(parse_ipv4("1.2.3", out)); + ASSERT_FALSE(parse_ipv4("::1", out)); +} + +TEST(proxy, select_and_bypass) { + ProxyConfig config; + config.http_proxy = "http://proxy:8080"; + config.https_proxy = "http://proxy:8443"; + config.no_proxy = {"internal.test", "10.0.0.0/8"}; + ProxySelector selector(config); + auto plain = selector.select(Url::parse("http://example.com/x")); + ASSERT_TRUE(plain.has_value()); + ASSERT_EQ(*plain, "http://proxy:8080"); + auto secure = selector.select(Url::parse("https://example.com/x")); + ASSERT_TRUE(secure.has_value()); + ASSERT_EQ(*secure, "http://proxy:8443"); + ASSERT_FALSE(selector.select(Url::parse("http://internal.test/x")).has_value()); + ASSERT_FALSE(selector.select(Url::parse("http://10.9.9.9/x")).has_value()); + ASSERT_TRUE(selector.is_bypassed("internal.test:8080")); +} + +TEST(proxy, from_env) { + setenv("HTTP_PROXY", "http://env:8080", 1); + setenv("NO_PROXY", "localhost, 127.0.0.1", 1); + unsetenv("HTTPS_PROXY"); + unsetenv("https_proxy"); + ProxyConfig config = ProxyConfig::from_env(); + ASSERT_EQ(config.http_proxy, "http://env:8080"); + ASSERT_EQ(config.no_proxy.size(), 2u); + ProxySelector selector(config); + ASSERT_FALSE(selector.select(Url::parse("http://localhost/x")).has_value()); + auto other = selector.select(Url::parse("http://example.com/x")); + ASSERT_TRUE(other.has_value()); + unsetenv("HTTP_PROXY"); + unsetenv("NO_PROXY"); +} diff --git a/tests/src/clients/test_rest.cpp b/tests/src/clients/test_rest.cpp new file mode 100644 index 0000000..fe567f2 --- /dev/null +++ b/tests/src/clients/test_rest.cpp @@ -0,0 +1,143 @@ +#include "framework.h" + +#include +#include + +#include "clients/client.h" +#include "clients/rest/errors.h" +#include "clients/rest/idempotency.h" +#include "clients/rest/paginator.h" +#include "clients/rest/resource.h" +#include "clients/transports/mock_transport.h" + +using namespace molo::clients; +using molo::clients::transports::MockTransport; + +TEST(rest, problem_details_golden) { + ClientResponse resp = + rest::problem_response(422, "Validation failed", "bad age", "/v1/users"); + ASSERT_EQ(resp.status(), 422); + ASSERT_EQ(resp.content_type(), "application/problem+json"); + body::Json parsed = resp.json(); + ASSERT_EQ(parsed.find("title")->as_string(), "Validation failed"); + ASSERT_EQ(parsed.find("detail")->as_string(), "bad age"); + ASSERT_EQ(parsed.find("status")->as_int(), 422); + ASSERT_THROW(rest::raise_for_status(resp), rest::RestError); + try { + rest::raise_for_status(resp); + } catch (const rest::RestError& err) { + ASSERT_EQ(err.status(), 422); + ASSERT_TRUE(std::string(err.what()).find("Validation failed") != + std::string::npos); + } + ClientResponse ok = ClientResponse(200, HeaderMap{}, "fine"); + rest::raise_for_status(ok); // Must not throw. +} + +TEST(rest, validation_error_shape) { + body::Json doc = body::Json::object(); + doc.set("title", body::Json("Validation failed")); + doc.set("status", body::Json(422)); + body::Json errors = body::Json::array(); + body::Json field = body::Json::object(); + field.set("field", body::Json("age")); + field.set("message", body::Json("must be positive")); + errors.push(field); + doc.set("errors", errors); + rest::ProblemDetails problem = rest::ProblemDetails::parse(doc); + ASSERT_EQ(problem.errors.size(), 1u); + ASSERT_EQ(problem.errors[0].field, "age"); + body::Json roundtrip = problem.to_json(); + ASSERT_TRUE(roundtrip.equals(doc) || roundtrip.has("errors")); +} + +TEST(rest, resource_crud_paths) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_status(Method::kGet, "/v1/users", 200, "[]"); + raw->expect_status(Method::kGet, "/v1/users/7", 200, "{}"); + raw->expect_status(Method::kPost, "/v1/users", 201, "{}"); + raw->expect_status(Method::kPut, "/v1/users/7", 200, "{}"); + raw->expect_status(Method::kPatch, "/v1/users/7", 200, "{}"); + raw->expect_status(Method::kDelete, "/v1/users/7", 204, ""); + Client client(std::move(mock)); + rest::Resource users(client, "http://h", "/v1/users"); + ASSERT_EQ(users.list_blocking().status(), 200); + ASSERT_EQ(users.get_blocking("7").status(), 200); + body::Json doc = body::Json::object(); + doc.set("name", body::Json("ada")); + ASSERT_EQ(users.create_blocking(doc).status(), 201); + ASSERT_EQ(users.update_blocking("7", doc).status(), 200); + ASSERT_EQ(users.patch_blocking("7", doc).status(), 200); + ASSERT_EQ(users.remove_blocking("7").status(), 204); + raw->verify(); + auto seen = raw->seen(); + ASSERT_EQ(seen.size(), 6u); + ASSERT_EQ(seen[0].target, "/v1/users"); + ASSERT_TRUE(seen[2].headers.has("Idempotency-Key")); + ASSERT_TRUE(seen[2].body.find("ada") != std::string::npos); +} + +TEST(rest, paginator_walks_250_items) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + auto page_body = [](int page, int per_page, int total) { + int start = (page - 1) * per_page; + int end = start + per_page < total ? start + per_page : total; + std::ostringstream out; + out << "{\"data\":["; + for (int i = start; i < end; ++i) { + if (i > start) out << ","; + out << "{\"id\":" << i << "}"; + } + out << "],\"meta\":{\"page\":" << page << ",\"per_page\":" << per_page + << ",\"total\":" << total << "},\"links\":{}}"; + return out.str(); + }; + raw->expect_status(Method::kGet, "/v1/users", 200, page_body(1, 100, 250), + "application/json"); + raw->expect_status(Method::kGet, "/v1/users", 200, page_body(2, 100, 250), + "application/json"); + raw->expect_status(Method::kGet, "/v1/users", 200, page_body(3, 100, 250), + "application/json"); + Client client(std::move(mock)); + rest::Paginator paginator(client, "http://h/v1/users"); + paginator.per_page(100); + std::vector all = paginator.fetch_all_blocking(); + ASSERT_EQ(all.size(), 250u); + ASSERT_EQ(all[0].find("id")->as_int(), 0); + ASSERT_EQ(all[249].find("id")->as_int(), 249); + raw->verify(); +} + +TEST(rest, paginator_cursor_style) { + auto mock = std::make_unique(); + MockTransport* raw = mock.get(); + raw->expect_status(Method::kGet, "/v1/feed", 200, + R"({"data":[{"id":1}],"meta":{"next_cursor":"c2"},"links":{}})", + "application/json"); + raw->expect_status(Method::kGet, "/v1/feed", 200, + R"({"data":[{"id":2}],"meta":{},"links":{}})", + "application/json"); + Client client(std::move(mock)); + rest::Paginator paginator(client, "http://h/v1/feed"); + std::vector all = paginator.fetch_all_blocking(); + ASSERT_EQ(all.size(), 2u); + raw->verify(); +} + +TEST(rest, idempotency_store_replay) { + rest::InMemoryIdempotencyStore store; + ASSERT_TRUE(!store.lookup("k1").has_value()); + rest::IdempotentResult result; + result.status = 201; + result.body = "{}"; + result.content_type = "application/json"; + store.store("k1", result); + auto replay = store.lookup("k1"); + ASSERT_TRUE(replay.has_value()); + ASSERT_EQ(replay->status, 201); + ASSERT_EQ(store.size(), 1u); + std::string key = rest::IdempotencyKey::generate(); + ASSERT_EQ(key.size(), 32u); +} diff --git a/tests/src/clients/test_testing.cpp b/tests/src/clients/test_testing.cpp new file mode 100644 index 0000000..e9e6aec --- /dev/null +++ b/tests/src/clients/test_testing.cpp @@ -0,0 +1,150 @@ +#include "framework.h" + +#include "clients/openapi/builder.h" +#include "clients/openapi/schema.h" +#include "clients/testing/assertions.h" +#include "clients/testing/clock.h" +#include "clients/testing/fixtures.h" +#include "clients/testing/test_client.h" + +using namespace molo::clients; +using namespace molo::clients::testing; + +namespace { + +ClientResponse echo_app(const ClientRequest& req) { + HeaderMap headers; + headers.set("Content-Type", "application/json"); + headers.set("X-Echo-Path", req.url().path()); + body::Json doc = body::Json::object(); + doc.set("method", body::Json(method_name(req.method()))); + doc.set("path", body::Json(req.url().path())); + doc.set("now", body::Json((int)Clock::now())); + return ClientResponse(200, std::move(headers), doc.dump()); +} + +} // namespace + +TEST(testing, client_verbs_like_production) { + TestClient client(echo_app); + auto guard = client.lifespan(); + ClientResponse resp = client.get_blocking("http://app.test/v1/echo"); + testing::expect_status(resp, 200); + testing::expect_header(resp, "X-Echo-Path", "/v1/echo"); + body::Json parsed = resp.json(); + ASSERT_EQ(parsed.find("method")->as_string(), "GET"); + ClientResponse posted = + client.post_blocking("http://app.test/v1/echo", "{}"); + ASSERT_EQ(posted.json().find("method")->as_string(), "POST"); + ASSERT_EQ(client.dispatch_count(), 2u); + // Zero real sockets: the pool never created a connection. + ASSERT_EQ(client.pool().stats().created, 0u); +} + +TEST(testing, lifespan_hooks_fire) { + bool started = false; + bool stopped = false; + TestClient client(echo_app); + client.app_transport().on_startup([&] { started = true; }); + client.app_transport().on_shutdown([&] { stopped = true; }); + { + auto guard = client.lifespan(); + ASSERT_TRUE(started); + ASSERT_TRUE(client.app_transport().started()); + } + ASSERT_TRUE(stopped); + ASSERT_FALSE(client.app_transport().started()); +} + +TEST(testing, fixtures_and_db) { + FixtureRegistry registry; + std::vector order; + FixtureHooks hooks; + hooks.setup = [&] { order.push_back("setup"); }; + hooks.teardown = [&] { order.push_back("teardown"); }; + registry.define("db", hooks); + registry.setup_all(); + InMemoryDb db; + SeededFactory factory; + factory.define("user", [](int seq) { + body::Json row = body::Json::object(); + row.set("id", body::Json(seq)); + return row; + }); + for (auto& row : factory.create_many("user", 3)) db.insert("users", row); + ASSERT_EQ(db.count("users"), 3u); + ASSERT_EQ(db.all("users")[2].find("id")->as_int(), 3); + db.reset(); + ASSERT_EQ(db.count("users"), 0u); + registry.teardown_all(); + ASSERT_EQ(order.size(), 2u); + ASSERT_EQ(order[0], "setup"); + ASSERT_EQ(order[1], "teardown"); + ASSERT_THROW(registry.setup_one("missing"), FixtureError); +} + +TEST(testing, frozen_clock) { + Clock::freeze(1700000000); + ASSERT_TRUE(Clock::frozen()); + ASSERT_EQ(Clock::now(), 1700000000); + TestClient client(echo_app); + body::Json first = + client.get_blocking("http://app.test/t").json(); + body::Json second = + client.get_blocking("http://app.test/t").json(); + ASSERT_EQ(first.find("now")->as_int(), second.find("now")->as_int()); + Clock::advance(std::chrono::seconds(60)); + ASSERT_EQ(Clock::now(), 1700000060); + Clock::unfreeze(); + ASSERT_FALSE(Clock::frozen()); +} + +TEST(testing, assertion_failures) { + ClientResponse resp = ClientResponse(200, HeaderMap{}, R"({"a":1})"); + testing::expect_status(resp, 200); + ASSERT_THROW(testing::expect_status(resp, 404), ExpectError); + ASSERT_THROW(testing::expect_header(resp, "X-Missing", "v"), ExpectError); + body::Json want = body::Json::object(); + want.set("a", body::Json(2)); + ASSERT_THROW(testing::expect_json(resp, want), ExpectError); + body::Json subset = body::Json::object(); + subset.set("a", body::Json(1)); + testing::expect_json_subset(resp.json(), subset); + body::Json bad_subset = body::Json::object(); + bad_subset.set("b", body::Json(1)); + ASSERT_THROW(testing::expect_json_subset(resp.json(), bad_subset), + ExpectError); +} + +TEST(testing, schema_conformance) { + openapi::ModelDecl user; + user.name = "User"; + openapi::FieldDecl id; + id.name = "id"; + id.type = "integer"; + id.required = true; + openapi::FieldDecl name; + name.name = "name"; + name.type = "string"; + name.required = true; + user.fields = {id, name}; + openapi::Builder builder; + openapi::Document doc = builder.from_routes({}, {user}); + + body::Json good = body::Json::object(); + good.set("id", body::Json(7)); + good.set("name", body::Json("ada")); + testing::expect_schema_by_name(good, "User", doc); + + body::Json missing = body::Json::object(); + missing.set("id", body::Json(7)); + ASSERT_THROW(testing::expect_schema_by_name(missing, "User", doc), + ExpectError); + + body::Json wrong_type = body::Json::object(); + wrong_type.set("id", body::Json("seven")); + wrong_type.set("name", body::Json("ada")); + ASSERT_THROW(testing::expect_schema_by_name(wrong_type, "User", doc), + ExpectError); + ASSERT_THROW(testing::expect_schema_by_name(good, "Nope", doc), ExpectError); +} diff --git a/tests/src/clients/test_url.cpp b/tests/src/clients/test_url.cpp new file mode 100644 index 0000000..8da6791 --- /dev/null +++ b/tests/src/clients/test_url.cpp @@ -0,0 +1,75 @@ +#include "framework.h" + +#include "clients/url.h" + +using molo::clients::Url; +using molo::clients::UrlError; + +TEST(url, parse_full) { + Url url = Url::parse("https://user:pass@example.com:8443/v1/users?page=2&x=a%20b#frag"); + ASSERT_EQ(url.scheme(), "https"); + ASSERT_EQ(url.host(), "example.com"); + ASSERT_EQ(url.port(), 8443); + ASSERT_EQ(url.path(), "/v1/users"); + ASSERT_EQ(url.fragment(), "frag"); + ASSERT_TRUE(url.is_https()); + auto page = url.query_value("page"); + ASSERT_TRUE(page.has_value()); + ASSERT_EQ(*page, "2"); + auto x = url.query_value("x"); + ASSERT_TRUE(x.has_value()); + ASSERT_EQ(*x, "a b"); + ASSERT_EQ(url.target(), "/v1/users?page=2&x=a%20b"); + ASSERT_EQ(url.host_port(), "example.com:8443"); + ASSERT_EQ(url.pool_key(), "https://example.com:8443"); +} + +TEST(url, default_ports) { + ASSERT_EQ(Url::parse("http://h/").port(), 80); + ASSERT_EQ(Url::parse("https://h/").port(), 443); + ASSERT_EQ(Url::parse("http://h/").origin(), "http://h"); + ASSERT_EQ(Url::parse("http://h:8080/").origin(), "http://h:8080"); +} + +TEST(url, case_and_ipv6) { + Url url = Url::parse("HTTP://EXAMPLE.COM/Path"); + ASSERT_EQ(url.scheme(), "http"); + ASSERT_EQ(url.host(), "example.com"); + Url v6 = Url::parse("http://[::1]:8080/x"); + ASSERT_EQ(v6.host(), "[::1]"); + ASSERT_EQ(v6.port(), 8080); +} + +TEST(url, join_matrix) { + Url base = Url::parse("http://h:8080/a/b/c?x=1"); + ASSERT_EQ(base.join("http://other/x").str(), "http://other/x"); + ASSERT_EQ(base.join("/abs?q=2").str(), "http://h:8080/abs?q=2"); + ASSERT_EQ(base.join("rel").str(), "http://h:8080/a/b/rel"); + ASSERT_EQ(base.join("../up").str(), "http://h:8080/a/up"); + ASSERT_EQ(base.join("./dot").str(), "http://h:8080/a/b/dot"); + ASSERT_EQ(base.join("?only=query").str(), "http://h:8080/a/b/c?only=query"); + ASSERT_EQ(base.join("#frag").str(), "http://h:8080/a/b/c?x=1"); + ASSERT_EQ(base.join("").str(), "http://h:8080/a/b/c?x=1"); +} + +TEST(url, encode_roundtrip) { + std::string raw = "a b/c?d=e&f+g"; + std::string enc = Url::encode_component(raw); + ASSERT_EQ(Url::decode_component(enc), raw); + ASSERT_EQ(Url::decode_component("a+b"), "a b"); +} + +TEST(url, query_mutation) { + Url url = Url::parse("http://h/items"); + url.set_query_arg("page", "1"); + url.set_query_arg("per_page", "20"); + url.set_query_arg("page", "3"); + ASSERT_EQ(url.query_string(), "page=3&per_page=20"); +} + +TEST(url, errors) { + ASSERT_THROW(Url::parse(""), UrlError); + ASSERT_THROW(Url::parse("http:///no-host"), UrlError); + ASSERT_THROW(Url::decode_component("%zz"), UrlError); + ASSERT_THROW(Url::parse(std::string(9000, 'a')), UrlError); +} diff --git a/tests/src/data/mini.hpp b/tests/src/data/mini.hpp new file mode 100644 index 0000000..c207e3a --- /dev/null +++ b/tests/src/data/mini.hpp @@ -0,0 +1,33 @@ +#pragma once +// Minimal test helper: stdlib only, loud failures with line numbers. +#include +#include + +static int g_checks = 0; + +#define CHECK(cond) \ + do { \ + ++g_checks; \ + if (!(cond)) { \ + std::cerr << "FAIL " << __FILE__ << ":" << __LINE__ << ": " #cond \ + << std::endl; \ + return 1; \ + } \ + } while (0) + +#define CHECK_EQ(a, b) \ + do { \ + ++g_checks; \ + if (!((a) == (b))) { \ + std::cerr << "FAIL " << __FILE__ << ":" << __LINE__ << ": " #a \ + << " != " #b << std::endl; \ + return 1; \ + } \ + } while (0) + +#define SUITE_DONE(name) \ + do { \ + std::cout << "PASS " << name << " (" << g_checks << " checks)" \ + << std::endl; \ + return 0; \ + } while (0) diff --git a/tests/src/data/test_kv_cache.cpp b/tests/src/data/test_kv_cache.cpp new file mode 100644 index 0000000..a6a1f98 --- /dev/null +++ b/tests/src/data/test_kv_cache.cpp @@ -0,0 +1,217 @@ +// Step 8: KV store plus cache with TTL, tags, LRU, singleflight and +// ORM write-through invalidation. +#include +#include +#include +#include + +#include "cache/cache.hpp" +#include "cache/policy.hpp" +#include "cache/singleflight.hpp" +#include "context.hpp" +#include "drivers/sqlite/db.hpp" +#include "drivers/sqlite/store.hpp" +#include "kv/memory.hpp" +#include "kv/store.hpp" +#include "mini.hpp" +#include "orm/invalidation.hpp" +#include "orm/model.hpp" +#include "orm/sync.hpp" + +using namespace molo::data; +using namespace molo::data::kv; +using namespace molo::data::cache; + +class Doc : public orm::Model { + public: + static std::string table_name() { return "docs"; } + static std::string primary_key() { return "id"; } + static std::vector fields() { + return {orm::FieldDef::primary_key(), + orm::FieldDef::varchar("title", 32)}; + } +}; + +static Bytes to_bytes(const std::string& s) { + return Bytes(s.begin(), s.end()); +} + +int main() { + // Basic set/get/del/exists. + { + MemoryKv kv; + CHECK(!kv.get("a").has_value()); + kv.set("a", to_bytes("1"), 60); + CHECK(kv.exists("a")); + CHECK(kv.get("a") == to_bytes("1")); + CHECK(kv.del("a")); + CHECK(!kv.exists("a")); + } + + // TTL expiry. + { + MemoryKv kv; + kv.set("short", to_bytes("x"), 1); + CHECK(kv.exists("short")); + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + CHECK(!kv.get("short").has_value()); + } + + // Scan pagination. + { + MemoryKv kv; + for (int i = 0; i < 25; ++i) + kv.set("user:" + std::to_string(i), to_bytes("v"), 60); + kv.set("other:1", to_bytes("v"), 60); + ScanPage p1 = kv.scan("user:", "", 10); + CHECK(p1.keys.size() == 10 && !p1.done); + ScanPage p2 = kv.scan("user:", p1.next_cursor, 10); + CHECK(p2.keys.size() == 10 && !p2.done); + ScanPage p3 = kv.scan("user:", p2.next_cursor, 10); + CHECK(p3.keys.size() == 5 && p3.done); + } + + // LRU eviction past the key cap. + { + MemoryKv kv(4); + kv.set("a", to_bytes("1"), 60); + kv.set("b", to_bytes("1"), 60); + kv.set("c", to_bytes("1"), 60); + kv.set("d", to_bytes("1"), 60); + (void)kv.get("a"); // refresh a + kv.set("e", to_bytes("1"), 60); + CHECK(kv.exists("a")); + CHECK(!kv.exists("b")); // least recently used evicted + CHECK(kv.evictions() == 1); + } + + // Key and value caps enforced. + { + MemoryKv kv; + bool threw = false; + try { + kv.set("", to_bytes("x"), 60); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(threw); + threw = false; + try { + kv.set("k", Bytes(17 * 1024 * 1024, 0), 60); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(threw); + } + + // Cache get-or-compute with tags and explicit invalidation. + { + MemoryKv kv; + Cache cache(kv); + int computes = 0; + Bytes v = cache.get_or_compute( + "u:1", {tag_for_table("users")}, + [&] { + ++computes; + return to_bytes("ada"); + }); + CHECK(v == to_bytes("ada") && computes == 1); + Bytes v2 = cache.get_or_compute( + "u:1", {tag_for_table("users")}, + [&] { + ++computes; + return to_bytes("ada"); + }); + CHECK(v2 == to_bytes("ada") && computes == 1); // served from cache + CHECK(cache.hits() == 1); // second call hits; first call missed twice + cache.invalidate("u:1"); + CHECK(!cache.get("u:1").has_value()); + cache.put("t:1", to_bytes("x"), {tag_for_table("teams")}); + cache.put("t:2", to_bytes("y"), {tag_for_table("teams")}); + CHECK(cache.purge_tag(tag_for_table("teams")) == 2); + CHECK(!cache.get("t:1").has_value()); + } + + // Cache LRU at entry cap. + { + MemoryKv kv; + CachePolicy policy; + policy.max_entries = 3; + Cache cache(kv, policy); + cache.put("a", to_bytes("1")); + cache.put("b", to_bytes("1")); + cache.put("c", to_bytes("1")); + (void)cache.get("a"); + cache.put("d", to_bytes("1")); + CHECK(cache.get("a").has_value()); + CHECK(!cache.get("b").has_value()); + } + + // Stampede: 100 concurrent computes collapse to 1. + { + MemoryKv kv; + Cache cache(kv); + std::atomic computes = 0; + std::vector threads; + std::atomic ok = 0; + for (int i = 0; i < 100; ++i) { + threads.emplace_back([&] { + Bytes v = cache.get_or_compute( + "hot", {}, [&] { + ++computes; + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + return to_bytes("value"); + }); + if (v == to_bytes("value")) ++ok; + }); + } + for (auto& t : threads) t.join(); + CHECK(ok == 100); + CHECK(computes == 1); + } + + // SingleFlight shares results directly too. + { + SingleFlight flight; + std::atomic runs = 0; + std::atomic bad = 0; + std::vector threads; + for (int i = 0; i < 10; ++i) { + threads.emplace_back([&] { + const int v = flight.compute("k", [&] { + ++runs; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + return 7; + }); + if (v != 7) ++bad; + }); + } + for (auto& t : threads) t.join(); + CHECK(runs == 1); + CHECK(bad == 0); + } + + // ORM write invalidates the cached read across the bus. + { + auto store = std::make_shared(); + sqlite::SqliteDb db(sqlite::SqliteConfig{}, store); + db.open(Context::background("test")); + Context ctx = Context::background("test"); + orm::Syncer().sync(ctx, db, {orm::describe_model()}); + orm::InvalidationBus bus; + MemoryKv kv; + Cache cache(kv); + cache.attach_bus(bus); + + Doc d; + d.set("title", make_text("v1")); + d.save(db, ctx, &bus, 1); + cache.put("doc:list", to_bytes("v1"), {tag_for_table("docs")}); + CHECK(cache.get("doc:list").has_value()); + d.set("title", make_text("v2")); + d.save(db, ctx, &bus, 2); // write-through purge + CHECK(!cache.get("doc:list").has_value()); + } + + SUITE_DONE("kv_cache"); +} diff --git a/tests/src/data/test_orm.cpp b/tests/src/data/test_orm.cpp new file mode 100644 index 0000000..f8e02c4 --- /dev/null +++ b/tests/src/data/test_orm.cpp @@ -0,0 +1,218 @@ +// Step 5: ORM core against the real SQLite path, plus Postgres SQL +// generation proof through a scripted connection. +#include + +#include "connection.hpp" +#include "context.hpp" +#include "drivers/sqlite/db.hpp" +#include "drivers/sqlite/store.hpp" +#include "mini.hpp" +#include "orm/invalidation.hpp" +#include "orm/model.hpp" +#include "orm/query.hpp" +#include "orm/relations.hpp" +#include "orm/sync.hpp" + +using namespace molo::data; +using namespace molo::data::orm; +using namespace molo::data::sqlite; + +class User : public Model { + public: + static std::string table_name() { return "users"; } + static std::string primary_key() { return "id"; } + static std::vector fields() { + auto id = FieldDef::primary_key(); + auto name = FieldDef::varchar("name", 32); + name.indexed = true; + auto email = FieldDef::text("email"); + email.unique = true; + return {id, name, email}; + } +}; + +class Todo : public Model { + public: + static std::string table_name() { return "todos"; } + static std::string primary_key() { return "id"; } + static std::vector fields() { + auto id = FieldDef::primary_key(); + auto user_id = FieldDef::bigint("user_id"); + user_id.indexed = true; + auto title = FieldDef::varchar("title", 64); + auto done = FieldDef::boolean("done"); + return {id, user_id, title, done}; + } +}; + +class Membership : public Model { + public: + static std::string table_name() { return "memberships"; } + static std::string primary_key() { return "id"; } + static std::vector fields() { + return {FieldDef::primary_key(), FieldDef::bigint("user_id"), + FieldDef::bigint("team_id")}; + } +}; + +class Team : public Model { + public: + static std::string table_name() { return "teams"; } + static std::string primary_key() { return "id"; } + static std::vector fields() { + return {FieldDef::primary_key(), FieldDef::varchar("name", 32)}; + } +}; + +// Scripted Postgres-side connection: records SQL, returns canned results. +class ScriptedPg : public Connection { + public: + Backend backend() const override { return Backend::Postgres; } + std::string backend_id() const override { return "pg:scripted"; } + Result exec(const Context&, const BoundStatement& stmt) override { + seen.push_back(stmt.sql()); + return Result{}; + } + Prepared prepare(const Context&, const BoundStatement& stmt) override { + return Prepared{"p", stmt.sql()}; + } + Result exec_prepared(const Context&, const Prepared&, + const std::vector&) override { + return Result{}; + } + void ping(const Context&) override {} + void close() override {} + std::vector seen; +}; + +int main() { + auto store = std::make_shared(); + SqliteDb db(SqliteConfig{}, store); + db.open(Context::background("test")); + Context ctx = Context::background("test"); + + // Syncer boots the schema from models. + Syncer syncer; + SyncReport rep = syncer.sync(ctx, db, + {describe_model(), describe_model(), + describe_model(), + describe_model()}); + CHECK(rep.tables_created == 4); + CHECK(rep.indexes_added == 2); // users.name, todos.user_id + CHECK(store->has_table("users") && store->has_table("todos")); + + // Second sync is a no-op diff. + SyncReport rep2 = + syncer.sync(ctx, db, {describe_model(), describe_model()}); + CHECK(rep2.tables_created == 0 && rep2.columns_added == 0); + + // CRUD with hooks and invalidation events. + InvalidationBus bus; + std::vector events; + bus.subscribe([&](const Invalidation& inv) { events.push_back(inv); }); + + User ada; + ada.set("name", make_text("ada")); + ada.set("email", make_text("ada@ex.dev")); + ada.save(db, ctx, &bus, 41); + CHECK(ada.persisted()); + CHECK(std::get(ada.get("id")) == 1); + CHECK(events.size() == 1 && events[0].op == "insert" && + events[0].txid == 41 && events[0].table == "users"); + + ada.set("email", make_text("ada2@ex.dev")); + ada.save(db, ctx, &bus, 42); + CHECK(events.size() == 2 && events[1].op == "update"); + + std::vector found = + Query(db).filter("name", make_text("ada")).all(ctx); + CHECK(found.size() == 1); + CHECK(std::get(found[0].get("email")) == "ada2@ex.dev"); + CHECK(Query(db).filter("name", make_text("ada")).exists(ctx)); + CHECK(!Query(db).filter("name", make_text("nobody")).exists(ctx)); + CHECK(Query(db).count(ctx) == 1); + + // Injection filter binds literally. + CHECK(Query(db).filter("name", make_text("' OR '1'='1")).count(ctx) == + 0); + + ada.remove(db, ctx, &bus, 43); + CHECK(events.size() == 3 && events[2].op == "delete"); + CHECK(Query(db).count(ctx) == 0); + + // Relations: HasMany lazy and eager, BelongsTo, ManyToMany. + User grace; + grace.set("name", make_text("grace")); + grace.set("email", make_text("g@ex.dev")); + grace.save(db, ctx); + User ed; + ed.set("name", make_text("edsger")); + ed.set("email", make_text("e@ex.dev")); + ed.save(db, ctx); + for (int i = 0; i < 3; ++i) { + Todo t; + t.set("user_id", grace.get("id")); + t.set("title", make_text("g" + std::to_string(i))); + t.set("done", make_bool(false)); + t.save(db, ctx); + } + Todo solo; + solo.set("user_id", ed.get("id")); + solo.set("title", make_text("solo")); + solo.set("done", make_bool(true)); + solo.save(db, ctx); + + HasMany user_todos{"user_id", "id"}; + CHECK(user_todos.load(db, ctx, grace).size() == 3); + BelongsTo todo_user{"user_id", "id"}; + std::vector one = + Query(db).filter("title", make_text("solo")).all(ctx); + CHECK(todo_user.load(db, ctx, one[0]).size() == 1); + + std::vector users = Query(db).order("id").all(ctx); + auto grouped = user_todos.preload(db, ctx, users); + CHECK(grouped["2"].size() == 3); // grace id 2 + CHECK(grouped["3"].size() == 1); // ed id 3 + + Team a; + a.set("name", make_text("core")); + a.save(db, ctx); + Membership m; + m.set("user_id", grace.get("id")); + m.set("team_id", a.get("id")); + m.save(db, ctx); + ManyToMany user_teams{"memberships", "user_id", "team_id", "id", + "id"}; + std::vector teams = user_teams.load(db, ctx, grace); + CHECK(teams.size() == 1); + CHECK(std::get(teams[0].get("name")) == "core"); + + // Pagination and ordering. + { + std::vector page = + Query(db).order("title").limit(2).offset(1).all(ctx); + CHECK(page.size() == 2); + CHECK(std::get(page[0].get("title")) == "g1"); + } + + // Postgres SQL generation uses $n placeholders and PG types. + { + ScriptedPg pg; + std::vector none = + Query(pg).filter("name", make_text("x")).limit(5).all(ctx); + CHECK(pg.seen.size() == 1); + CHECK(pg.seen[0].find("WHERE name = $1") != std::string::npos); + CHECK(pg.seen[0].find("LIMIT $2") != std::string::npos); + User u; + u.set("name", make_text("n")); + u.set("email", make_text("e")); + u.save(pg, ctx); + CHECK(pg.seen.back().find("VALUES ($1, $2)") != std::string::npos); + CHECK(create_table_sql(describe_model(), Backend::Postgres).find( + "BIGINT GENERATED ALWAYS AS IDENTITY") != std::string::npos); + CHECK(create_table_sql(describe_model(), Backend::Sqlite).find( + "PRIMARY KEY") != std::string::npos); + } + + SUITE_DONE("orm"); +} diff --git a/tests/src/data/test_pg_wire.cpp b/tests/src/data/test_pg_wire.cpp new file mode 100644 index 0000000..902f76d --- /dev/null +++ b/tests/src/data/test_pg_wire.cpp @@ -0,0 +1,313 @@ +// Step 2: Postgres wire client through a loopback fake server, plus +// codecs, messages and SCRAM vectors. +#include +#include +#include +#include + +#include +#include + +#include "drivers/pg/auth.hpp" +#include "drivers/pg/codecs.hpp" +#include "drivers/pg/conn.hpp" +#include "drivers/pg/extended.hpp" +#include "drivers/pg/message.hpp" +#include "drivers/pg/secret.hpp" +#include "drivers/pg/sha256.hpp" +#include "mini.hpp" + +using namespace molo::data; +using namespace molo::data::pg; + +static std::string hex_of(const std::array& h) { + static const char* hex = "0123456789abcdef"; + std::string out; + for (auto c : h) { + out.push_back(hex[c >> 4]); + out.push_back(hex[c & 15]); + } + return out; +} + +static std::vector recv_exact_fd(int fd, std::size_t n) { + std::vector out(n); + std::size_t got = 0; + while (got < n) { + const ssize_t r = recv(fd, out.data() + got, n - got, 0); + if (r <= 0) throw std::runtime_error("fake server recv failed"); + got += static_cast(r); + } + return out; +} + +static void send_all_fd(int fd, const std::vector& b) { + std::size_t sent = 0; + while (sent < b.size()) { + const ssize_t n = send(fd, b.data() + sent, b.size() - sent, 0); + if (n <= 0) throw std::runtime_error("fake server send failed"); + sent += static_cast(n); + } +} + +static std::vector row_desc_msg() { + Writer b; + b.put_i16(2); + b.put_cstr("id"); + b.put_i32(0); + b.put_i16(0); + b.put_i32(oid::kInt4); + b.put_i16(4); + b.put_i32(-1); + b.put_i16(0); + b.put_cstr("name"); + b.put_i32(0); + b.put_i16(0); + b.put_i32(oid::kText); + b.put_i16(-1); + b.put_i32(-1); + b.put_i16(0); + return envelope('T', b.take()); +} + +// Minimal fake Postgres: trust auth, canned SELECT 1 and extended rows. +static void fake_server(int listen_fd, std::atomic& stop) { + const int fd = accept(listen_fd, nullptr, nullptr); + if (fd < 0) return; + try { + // Startup: length-prefixed, no type byte. + const auto len_b = recv_exact_fd(fd, 4); + const std::int32_t len = get_net_i32(len_b.data()); + (void)recv_exact_fd(fd, static_cast(len - 4)); + Writer auth; + auth.put_i32(0); + send_all_fd(fd, envelope('R', auth.take())); + Writer ps; + ps.put_cstr("client_encoding"); + ps.put_cstr("UTF8"); + send_all_fd(fd, envelope('S', ps.take())); + Writer kd; + kd.put_i32(1234); + kd.put_i32(5678); + send_all_fd(fd, envelope('K', kd.take())); + send_all_fd(fd, envelope('Z', {static_cast('I')})); + while (!stop) { + const auto hdr = recv_exact_fd(fd, 5); + const char type = static_cast(hdr[0]); + const std::int32_t mlen = get_net_i32(hdr.data() + 1); + const auto body = recv_exact_fd(fd, static_cast(mlen - 4)); + if (type == 'X' || type == 0) break; + if (type == 'Q') { + Reader r(body); + const std::string sql = r.get_cstr(); + if (sql == "SELECT 1") { + Writer d; + d.put_i16(1); + d.put_cstr("?column?"); + d.put_i32(0); + d.put_i16(0); + d.put_i32(oid::kInt4); + d.put_i16(4); + d.put_i32(-1); + d.put_i16(0); + send_all_fd(fd, envelope('T', d.take())); + Writer row; + row.put_i16(1); + row.put_i32(1); + row.put_u8('1'); + send_all_fd(fd, envelope('D', row.take())); + Writer cc; + cc.put_cstr("SELECT 1"); + send_all_fd(fd, envelope('C', cc.take())); + } else { + Writer cc; + cc.put_cstr("SELECT 0"); + send_all_fd(fd, envelope('C', cc.take())); + } + send_all_fd(fd, envelope('Z', {static_cast('I')})); + } else if (type == 'P') { + send_all_fd(fd, envelope('1', {})); + } else if (type == 'B') { + send_all_fd(fd, envelope('2', {})); + } else if (type == 'D') { + send_all_fd(fd, row_desc_msg()); + } else if (type == 'E') { + Writer row; + row.put_i16(2); + row.put_i32(2); + row.put_str("42"); + row.put_i32(3); + row.put_str("ada"); + send_all_fd(fd, envelope('D', row.take())); + Writer cc; + cc.put_cstr("SELECT 1"); + send_all_fd(fd, envelope('C', cc.take())); + } else if (type == 'S' || type == 'H') { + send_all_fd(fd, envelope('Z', {static_cast('I')})); + } + } + } catch (...) { + } + close(fd); +} + +int main() { + // SHA-256 NIST vector for "abc". + CHECK(hex_of(Sha256::hash("abc")) == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + // HMAC-SHA-256 RFC 4231 case 1. + { + const std::vector key(20, 0x0b); + CHECK(hex_of(hmac_sha256(key, "Hi There")) == + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7"); + } + // Base64 roundtrip. + { + const std::vector raw = {1, 2, 3, 250, 0, 17}; + const auto enc = base64_encode(raw); + CHECK(base64_decode(enc) == raw); + } + // SCRAM-SHA-256 exchange vector, cross-checked byte for byte against + // an independent Python hashlib implementation. + { + ScramSha256Client scram("user", "pencil", "rOprNGfwEbeRWgbNEkqO"); + CHECK(scram.client_first_message() == + "n,,n=user,r=rOprNGfwEbeRWgbNEkqO"); + const std::string final = scram.process_server_first( + "r=rOprNGfwEbeRWgbNEkqO%hvYDpWUa2RaTCAfuxFIlj)hNlF$k0," + "s=W22ZaJ0SNY7soEsUEjb6gQ==,i=4096"); + CHECK(final == + "c=biws,r=rOprNGfwEbeRWgbNEkqO%hvYDpWUa2RaTCAfuxFIlj)hNlF$k0," + "p=dHzbZapWIk4jUhN+Ute9ytag9zjfMHgsqmmiz7AndVQ="); + scram.verify_server_final("v=6rriTRBi23WpRR/wtup+mMhUZUn/dB5nLTJRsjl95G4="); + CHECK(scram.done()); + } + // SASLprep-lite rejects controls. + { + bool threw = false; + try { + saslprep_lite(std::string("ab\x01")); + } catch (const std::runtime_error&) { + threw = true; + } + CHECK(threw); + } + + // Message framing roundtrips. + { + const auto st = msg::startup("u", "d"); + Reader r(st); + const std::int32_t len = r.get_i32(); + CHECK(len == static_cast(st.size())); + CHECK(r.get_i32() == 196608); + const auto q = msg::simple_query("SELECT 1"); + CHECK(q[0] == 'Q'); + const auto ef = msg::parse_error([] { + Writer b; + b.put_u8('S'); + b.put_cstr("ERROR"); + b.put_u8('C'); + b.put_cstr("42P01"); + b.put_u8('M'); + b.put_cstr("nope"); + b.put_u8(0); + return b.take(); + }()); + CHECK(ef.code == "42P01" && ef.message == "nope"); + } + + // Codecs roundtrip text and binary. + { + CHECK(encode_text(make_int(-5)) == "-5"); + CHECK(encode_text(make_bool(true)) == "t"); + CHECK(encode_text(make_bytes({0xab, 0x01})) == "\\xab01"); + const auto bin = encode_binary(make_int(1234567890123LL)); + CHECK(bin.size() == 8); + Value v = decode_cell(bin, true, oid::kInt8, 1); + CHECK(std::get(v) == 1234567890123LL); + const std::vector tb = {'4', '2'}; + Value t = decode_cell(tb, true, oid::kInt4, 0); + CHECK(std::get(t) == 42); + const std::vector bb = {'\\', 'x', 'a', 'b'}; + Value bv = decode_cell(bb, true, oid::kBytea, 0); + CHECK(std::get(bv) == Bytes{0xab}); + const auto bd = encode_binary(make_double(1.5)); + CHECK(std::get(decode_cell(bd, true, oid::kFloat8, 1)) == 1.5); + const auto now = std::chrono::system_clock::now(); + const auto bt = encode_binary(make_time(now)); + Value tv = decode_cell(bt, true, oid::kTimestampTz, 1); + const auto back = std::get(tv); + const auto diff = std::chrono::duration_cast( + now - back) + .count(); + CHECK(diff >= 0 && diff < 1000000); + } + + // Extended script carries params out of band. + { + BoundStatement stmt("SELECT $1", {make_text("oops' OR '1'='1")}); + ExtendedScript script = build_extended(stmt, false, 7); + CHECK(script.statement_name == "molo_s7"); + CHECK(!script.bytes.empty()); + // SQL text and value both present but framed separately: Parse holds + // the SQL, Bind holds the value. + const std::string blob(script.bytes.begin(), script.bytes.end()); + CHECK(blob.find("SELECT $1") != std::string::npos); + CHECK(blob.find("oops' OR '1'='1") != std::string::npos); + } + + // Secrets and redaction. + { + Secret s = Secret::from_string("hunter2"); + CHECK(!s.empty() && s.view() == "hunter2"); + const std::string red = + redact_conninfo("host=db password=hunter2 dbname=app"); + CHECK(red.find("hunter2") == std::string::npos); + CHECK(red.find("***") != std::string::npos); + } + + // Live wire test against the loopback fake server. + { + const int listen_fd = socket(AF_INET, SOCK_STREAM, 0); + CHECK(listen_fd >= 0); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + CHECK(bind(listen_fd, reinterpret_cast(&addr), sizeof(addr)) == + 0); + CHECK(listen(listen_fd, 1) == 0); + socklen_t slen = sizeof(addr); + CHECK(getsockname(listen_fd, reinterpret_cast(&addr), &slen) == + 0); + const int port = ntohs(addr.sin_port); + std::atomic stop = false; + std::thread server(fake_server, listen_fd, std::ref(stop)); + + Context ctx("pg-test", "", 10); + PgConnection conn; + PgConnectParams params; + params.host = "127.0.0.1"; + params.port = port; + params.user = "u"; + params.database = "d"; + conn.connect(ctx, std::move(params)); + CHECK(conn.backend() == Backend::Postgres); + Result one = conn.simple(ctx, "SELECT 1"); + CHECK(one.size() == 1); + CHECK(std::get(one.raw(0)[0]) == 1); + Result ext = + conn.exec(ctx, BoundStatement("SELECT $1", {make_int(5)})); + CHECK(ext.size() == 1); + CHECK(ext.columns()[0] == "id" && ext.columns()[1] == "name"); + CHECK(std::get(ext.raw(0)[0]) == 42); + CHECK(std::get(ext.raw(0)[1]) == "ada"); + conn.ping(ctx); + conn.close(); + stop = true; + close(listen_fd); + server.join(); + } + + SUITE_DONE("pg_wire"); +} diff --git a/tests/src/data/test_pool_txn.cpp b/tests/src/data/test_pool_txn.cpp new file mode 100644 index 0000000..3db9b29 --- /dev/null +++ b/tests/src/data/test_pool_txn.cpp @@ -0,0 +1,187 @@ +// Step 4: pool and transaction behavior against the real SQLite path. +#include +#include +#include +#include + +#include "context.hpp" +#include "drivers/sqlite/db.hpp" +#include "drivers/sqlite/store.hpp" +#include "mini.hpp" +#include "pool.hpp" +#include "pool_config.hpp" +#include "transaction.hpp" + +using namespace molo::data; +using namespace molo::data::sqlite; + +static std::unique_ptr make_db( + std::shared_ptr store) { + auto db = std::make_unique(SqliteConfig{}, store); + db->open(Context::background("pool")); + return db; +} + +int main() { + // Basic acquire, use, return, reuse. + { + auto store = std::make_shared(); + PoolConfig cfg = PoolConfig::sqlite(); + Pool pool(cfg, [&] { return make_db(store); }); + Context ctx = Context::background("test"); + { + auto c = pool.acquire(ctx); + c->exec(ctx, BoundStatement::raw("CREATE TABLE t (id INTEGER)")); + CHECK(pool.total_count() == 1); + } + CHECK(pool.idle_count() == 1); + { + auto c = pool.acquire(ctx); + c->exec(ctx, + BoundStatement("INSERT INTO t (id) VALUES (?)", {make_int(1)})); + } + CHECK(pool.total_count() == 1); // reused, not regrown + } + + // Exhaustion fails fast with structured error. + { + auto store = std::make_shared(); + PoolConfig cfg = PoolConfig::sqlite(); + cfg.max_size = 1; + cfg.acquire_timeout_sec = 1; + Pool pool(cfg, [&] { return make_db(store); }); + Context ctx = Context::background("test"); + auto held = pool.acquire(ctx); + bool threw = false; + try { + auto extra = pool.acquire(ctx); + } catch (const DbError& e) { + threw = e.code() == "MOLO_POOL"; + } + CHECK(threw); + CHECK(pool.exhausted_count() >= 1); + } + + // Waiter cap overflow fails fast too. + { + auto store = std::make_shared(); + PoolConfig cfg = PoolConfig::sqlite(); + cfg.max_size = 1; + cfg.max_waiters = 1; + cfg.acquire_timeout_sec = 5; + Pool pool(cfg, [&] { return make_db(store); }); + Context ctx = Context::background("test"); + auto held = pool.acquire(ctx); + std::atomic waiter_failed = false; + std::atomic waiter_started = false; + std::thread waiter([&] { + waiter_started = true; + try { + auto c = pool.acquire(Context::background("w")); + } catch (const DbError& e) { + if (e.code() == "MOLO_POOL") waiter_failed = true; + } + }); + while (!waiter_started) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + bool threw = false; + try { + auto extra = pool.acquire(ctx); + } catch (const DbError& e) { + threw = e.code() == "MOLO_POOL"; + } + CHECK(threw); + held = decltype(held)(); // release, waiter proceeds + waiter.join(); + CHECK(!waiter_failed); + } + + // Concurrent checkouts across threads. + { + auto store = std::make_shared(); + PoolConfig cfg = PoolConfig::sqlite(); + cfg.max_size = 4; + Pool pool(cfg, [&] { return make_db(store); }); + { + auto c = pool.acquire(Context::background("test")); + c->exec(Context::background("test"), + BoundStatement::raw("CREATE TABLE t (id INTEGER)")); + } + std::vector threads; + for (int w = 0; w < 4; ++w) { + threads.emplace_back([&, w] { + for (int i = 0; i < 10; ++i) { + auto c = pool.acquire(Context::background("w")); + c->exec(Context::background("w"), + BoundStatement("INSERT INTO t (id) VALUES (?)", + {make_int(w * 100 + i)})); + } + }); + } + for (auto& t : threads) t.join(); + auto c = pool.acquire(Context::background("test")); + Result r = + c->exec(Context::background("test"), BoundStatement("SELECT * FROM t")); + CHECK(r.size() == 40); + } + + // Transactions: commit, rollback, savepoints, scope rollback. + { + auto store = std::make_shared(); + SqliteDb db = [&] { + SqliteDb d(SqliteConfig{}, store); + d.open(Context::background("test")); + return d; + }(); + Context ctx = Context::background("test"); + db.exec(ctx, BoundStatement::raw("CREATE TABLE t (id INTEGER)")); + + { + Transaction txn(db, ctx); + db.exec(ctx, + BoundStatement("INSERT INTO t (id) VALUES (?)", {make_int(1)})); + txn.commit(); + CHECK(!txn.active()); + } + CHECK(db.exec(ctx, BoundStatement("SELECT * FROM t")).size() == 1); + + { + Transaction txn(db, ctx); + db.exec(ctx, + BoundStatement("INSERT INTO t (id) VALUES (?)", {make_int(2)})); + txn.rollback(); + } + CHECK(db.exec(ctx, BoundStatement("SELECT * FROM t")).size() == 1); + + { + Transaction txn(db, ctx); + db.exec(ctx, + BoundStatement("INSERT INTO t (id) VALUES (?)", {make_int(3)})); + const std::string sp = txn.savepoint(); + CHECK(sp == "molo_sp_1_1"); + db.exec(ctx, + BoundStatement("INSERT INTO t (id) VALUES (?)", {make_int(4)})); + txn.rollback_to(sp); + txn.release_savepoint(sp); + txn.commit(); + } + Result r = db.exec(ctx, BoundStatement("SELECT id FROM t ORDER BY id")); + CHECK(r.size() == 2); + CHECK(std::get(r.raw(1)[0]) == 3); + + { + Transaction txn(db, ctx); + db.exec(ctx, + BoundStatement("INSERT INTO t (id) VALUES (?)", {make_int(9)})); + } // scope exit rolls back + CHECK(db.exec(ctx, BoundStatement("SELECT * FROM t")).size() == 2); + + // Serializable opt-in accepted on the SQLite path too. + { + Transaction txn(db, ctx, Isolation::Serializable); + txn.commit(); + } + } + + SUITE_DONE("pool_txn"); +} diff --git a/tests/src/data/test_pubsub.cpp b/tests/src/data/test_pubsub.cpp new file mode 100644 index 0000000..def6425 --- /dev/null +++ b/tests/src/data/test_pubsub.cpp @@ -0,0 +1,137 @@ +// Step 9: pubsub broker fan-out, ordering, drops, patterns. +#include +#include +#include +#include + +#include "mini.hpp" +#include "pubsub/broker.hpp" +#include "pubsub/channel.hpp" + +using namespace molo::data::pubsub; + +static Payload to_payload(int n) { + const std::string s = std::to_string(n); + return Payload(s.begin(), s.end()); +} + +static int from_payload(const Payload& p) { + return std::stoi(std::string(p.begin(), p.end())); +} + +int main() { + // Pattern matching rules. + CHECK(pattern_match("orders:created", "orders:created")); + CHECK(pattern_match("orders:created", "orders:*")); + CHECK(pattern_match("orders:created", "*:created")); + CHECK(pattern_match("a:b:c", "*:b:*")); + CHECK(!pattern_match("orders:created", "orders:deleted")); + CHECK(!pattern_match("a:b", "a:b:c")); + CHECK(!pattern_match("a:b:c", "a:*")); + + // 10k messages fan out to 100 subscribers in order. + { + Broker broker; + const int kSubs = 100; + const int kMsgs = 10000; + std::vector> received(kSubs); + std::vector ids; + for (int s = 0; s < kSubs; ++s) + ids.push_back(broker.subscribe( + "events:all", [&, s](const Message& m) { + received[s].push_back(from_payload(m.payload)); + })); + for (int i = 0; i < kMsgs; ++i) { + DeliveryReport r = broker.publish("events:all", to_payload(i), "text"); + CHECK(!r.no_subscribers && r.callbacks == kSubs); + } + CHECK(broker.published() == kMsgs); + for (int s = 0; s < kSubs; ++s) { + CHECK(static_cast(received[s].size()) == kMsgs); + for (int i = 0; i < kMsgs; ++i) + if (received[s][i] != i) { + CHECK(false); + break; + } + } + for (int id : ids) broker.unsubscribe(id); + DeliveryReport r = broker.publish("events:all", to_payload(0)); + CHECK(r.no_subscribers && broker.no_subscriber() == 1); + } + + // Slow queued subscriber drops oldest, counted; publisher never blocks. + { + Broker broker; + auto sub = broker.subscribe_queued("slow:work"); + const int kMsgs = 3000; + int dropped_reports = 0; + for (int i = 0; i < kMsgs; ++i) { + DeliveryReport r = broker.publish("slow:work", to_payload(i)); + dropped_reports += r.dropped; + } + CHECK(sub->depth() == 1024); + CHECK(sub->dropped() == kMsgs - 1024); + CHECK(dropped_reports == kMsgs - 1024); + auto batch = sub->poll(); + CHECK(static_cast(batch.size()) == 1024); + // Oldest surviving message is kMsgs - 1024. + CHECK(from_payload(batch.front().payload) == kMsgs - 1024); + CHECK(from_payload(batch.back().payload) == kMsgs - 1); + } + + // Patterns route, exact names do not leak across topics. + { + Broker broker; + std::atomic wild = 0; + std::atomic exact = 0; + broker.subscribe("orders:*", + [&](const Message&) { ++wild; }); + broker.subscribe("orders:created", + [&](const Message&) { ++exact; }); + broker.publish("orders:created", to_payload(1)); + broker.publish("orders:deleted", to_payload(2)); + broker.publish("users:created", to_payload(3)); + CHECK(wild == 2 && exact == 1); + } + + // Payload cap enforced, channel names validated. + { + Broker broker; + bool threw = false; + try { + broker.publish("big:one", Payload(1024 * 1024 + 1, 0)); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(threw); + threw = false; + try { + broker.subscribe("bad name", [](const Message&) {}); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(threw); + } + + // Concurrent publishers do not lose inline deliveries. + { + Broker broker; + std::atomic total = 0; + std::mutex mutex; + broker.subscribe("race:x", [&](const Message&) { + std::lock_guard lock(mutex); + ++total; + }); + std::vector threads; + for (int t = 0; t < 8; ++t) { + threads.emplace_back([&, t] { + for (int i = 0; i < 500; ++i) + broker.publish("race:x", to_payload(t * 1000 + i)); + }); + } + for (auto& th : threads) th.join(); + CHECK(total == 4000); + } + + SUITE_DONE("pubsub"); +} diff --git a/tests/src/data/test_sqlite.cpp b/tests/src/data/test_sqlite.cpp new file mode 100644 index 0000000..82096c2 --- /dev/null +++ b/tests/src/data/test_sqlite.cpp @@ -0,0 +1,235 @@ +// Step 3: SQLite handling through the real SQL layer: CRUD, constraints, +// injection-proof binding, pragmas, busy retry, single-writer gate, +// checkpoints, file persistence and path refusal. +#include +#include +#include + +#include "drivers/sqlite/busy.hpp" +#include "drivers/sqlite/db.hpp" +#include "drivers/sqlite/store.hpp" +#include "mini.hpp" + +using namespace molo::data; +using namespace molo::data::sqlite; + +static SqliteDb fresh_db(std::shared_ptr store = nullptr) { + SqliteConfig cfg; + if (!store) store = std::make_shared(); + SqliteDb db(cfg, store); + db.open(Context::background("test")); + return db; +} + +int main() { + // Pragma sequence applied at open. + { + auto store = std::make_shared(); + SqliteDb db = fresh_db(store); + const auto& pragmas = store->applied_pragmas(); + CHECK(pragmas.size() == 4); + CHECK(pragmas[0] == "journal_mode=WAL"); + CHECK(pragmas[1] == "synchronous=NORMAL"); + CHECK(pragmas[2] == "foreign_keys=ON"); + CHECK(pragmas[3].rfind("busy_timeout=", 0) == 0); + CHECK(db.backend() == Backend::Sqlite); + db.ping(Context::background("test")); + } + + // Network paths refused. + { + SqliteConfig cfg; + cfg.path = "//remote/share/db.sqlite"; + SqliteDb db(cfg); + bool threw = false; + try { + db.open(Context::background("test")); + } catch (const DbError& e) { + threw = e.code() == "MOLO_PATH"; + } + CHECK(threw); + } + + // CRUD plus constraints. + { + SqliteDb db = fresh_db(); + Context ctx = Context::background("test"); + db.exec(ctx, BoundStatement::raw( + "CREATE TABLE users (id INTEGER PRIMARY KEY, " + "name VARCHAR(32) NOT NULL, age INTEGER)")); + Result ins = db.exec(ctx, BoundStatement( + "INSERT INTO users (name, age) VALUES (?, ?)", + {make_text("ada"), make_int(36)})); + CHECK(ins.affected() == 1 && ins.last_insert_id() == 1); + Result sel = db.exec( + ctx, BoundStatement("SELECT id, name, age FROM users WHERE id = ?", + {make_int(1)})); + CHECK(sel.size() == 1); + CHECK(std::get(sel.raw(0)[1]) == "ada"); + + // Injection string binds literally, never executes. + db.exec(ctx, BoundStatement("INSERT INTO users (name) VALUES (?)", + {make_text("x' OR '1'='1")})); + Result all = + db.exec(ctx, BoundStatement("SELECT name FROM users ORDER BY id")); + CHECK(all.size() == 2); + CHECK(std::get(all.raw(1)[0]) == "x' OR '1'='1"); + + // Length enforcement is SQLITE_TOOBIG with SQLite shape. + bool threw = false; + try { + db.exec(ctx, BoundStatement( + "INSERT INTO users (name) VALUES (?)", + {make_text("way-too-long-name-for-varchar-32!!")})); + } catch (const DbError& e) { + threw = e.code() == "SQLITE_TOOBIG" && e.healable(); + } + CHECK(threw); + + // NOT NULL enforcement. + threw = false; + try { + db.exec(ctx, BoundStatement("INSERT INTO users (age) VALUES (?)", + {make_int(1)})); + } catch (const DbError& e) { + threw = e.code() == "SQLITE_CONSTRAINT" && !e.healable(); + } + CHECK(threw); + + // Missing table/column shapes. + threw = false; + try { + db.exec(ctx, BoundStatement("SELECT * FROM missing_tbl")); + } catch (const DbError& e) { + threw = e.code() == "SQLITE_NO_TABLE" && e.healable() && + parse_missing_table(e.detail()) == "missing_tbl"; + } + CHECK(threw); + threw = false; + try { + db.exec(ctx, BoundStatement("SELECT nope FROM users")); + } catch (const DbError& e) { + threw = e.code() == "SQLITE_NO_COLUMN" && + parse_missing_column(e.detail()) == "nope"; + } + CHECK(threw); + + // UPDATE and DELETE with affected counts. + Result upd = db.exec( + ctx, BoundStatement("UPDATE users SET age = ? WHERE id = ?", + {make_int(37), make_int(1)})); + CHECK(upd.affected() == 1); + Result del = db.exec( + ctx, BoundStatement("DELETE FROM users WHERE id = ?", {make_int(2)})); + CHECK(del.affected() == 1); + CHECK(db.exec(ctx, BoundStatement("SELECT * FROM users")).size() == 1); + } + + // JOIN, ORDER, LIMIT/OFFSET, COUNT. + { + SqliteDb db = fresh_db(); + Context ctx = Context::background("test"); + db.exec(ctx, BoundStatement::raw( + "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)")); + db.exec(ctx, BoundStatement::raw("CREATE TABLE todos (id INTEGER PRIMARY " + "KEY, user_id INTEGER, title TEXT)")); + for (int i = 1; i <= 3; ++i) + db.exec(ctx, BoundStatement("INSERT INTO users (name) VALUES (?)", + {make_text("u" + std::to_string(i))})); + for (int i = 1; i <= 5; ++i) + db.exec(ctx, BoundStatement( + "INSERT INTO todos (user_id, title) VALUES (?, ?)", + {make_int(1 + (i % 3)), + make_text("t" + std::to_string(i))})); + Result j = db.exec(ctx, + BoundStatement("SELECT users.name, todos.title FROM " + "users JOIN todos ON users.id = " + "todos.user_id ORDER BY todos.title " + "LIMIT 2 OFFSET 1")); + CHECK(j.size() == 2); + CHECK(std::get(j.raw(0)[1]) == "t2"); + Result cnt = + db.exec(ctx, BoundStatement("SELECT COUNT(*) FROM todos WHERE " + "user_id = ?", + {make_int(2)})); + CHECK(std::get(cnt.raw(0)[0]) == 2); + } + + // Busy retry then loud failure. + { + auto store = std::make_shared(); + SqliteDb db = fresh_db(store); + Context ctx = Context::background("test"); + db.exec(ctx, BoundStatement::raw("CREATE TABLE t (id INTEGER)")); + store->inject_busy(2); + db.exec(ctx, + BoundStatement("INSERT INTO t (id) VALUES (?)", {make_int(1)})); + CHECK(db.exec(ctx, BoundStatement("SELECT * FROM t")).size() == 1); + store->inject_busy(100); + bool threw = false; + try { + db.exec(ctx, + BoundStatement("INSERT INTO t (id) VALUES (?)", {make_int(2)})); + } catch (const DbError& e) { + threw = e.code() == "SQLITE_BUSY"; + } + CHECK(threw); + } + + // Single-writer gate: concurrent writers serialize without loss. + { + auto store = std::make_shared(); + SqliteDb db = fresh_db(store); + Context ctx = Context::background("test"); + db.exec(ctx, BoundStatement::raw("CREATE TABLE t (id INTEGER)")); + std::vector threads; + for (int w = 0; w < 8; ++w) { + threads.emplace_back([&, w] { + for (int i = 0; i < 25; ++i) + db.exec(Context::background("w"), + BoundStatement("INSERT INTO t (id) VALUES (?)", + {make_int(w * 100 + i)})); + }); + } + for (auto& t : threads) t.join(); + CHECK(db.exec(ctx, BoundStatement("SELECT COUNT(*) FROM t")).size() == 1); + Result c = db.exec(ctx, BoundStatement("SELECT COUNT(*) FROM t")); + CHECK(std::get(c.raw(0)[0]) == 200); + } + + // Checkpoint scheduler and file persistence roundtrip. + { + const std::string path = "/tmp/molo_data_test.sqlite snap"; + std::remove(path.c_str()); + { + SqliteConfig cfg; + cfg.path = "/tmp/molo_data_test.sqlite"; + std::remove(cfg.path.c_str()); + SqliteDb db(cfg); + db.open(Context::background("test")); + Context ctx = Context::background("test"); + db.exec(ctx, BoundStatement::raw("CREATE TABLE k (id INTEGER PRIMARY " + "KEY, v VARCHAR(16))")); + for (int i = 0; i < 50; ++i) + db.exec(ctx, BoundStatement("INSERT INTO k (v) VALUES (?)", + {make_text("v" + std::to_string(i))})); + db.checkpoint(); + CHECK(db.checkpoints_run() >= 1); + CHECK(db.wal().total_pages() > 0); + db.close(); + } + { + SqliteConfig cfg; + cfg.path = "/tmp/molo_data_test.sqlite"; + SqliteDb db(cfg); + db.open(Context::background("test")); + Result c = db.exec(Context::background("test"), + BoundStatement("SELECT COUNT(*) FROM k")); + CHECK(std::get(c.raw(0)[0]) == 50); + std::remove(cfg.path.c_str()); + } + (void)path; + } + + SUITE_DONE("sqlite"); +} diff --git a/tests/src/data/test_sync_heal_migrate.cpp b/tests/src/data/test_sync_heal_migrate.cpp new file mode 100644 index 0000000..55dfe51 --- /dev/null +++ b/tests/src/data/test_sync_heal_migrate.cpp @@ -0,0 +1,232 @@ +// Steps 6-7: sync plus migration journal plus healer matrix on both +// backends (SQLite live, Postgres through a scripted connection). +#include +#include + +#include "connection.hpp" +#include "context.hpp" +#include "drivers/sqlite/db.hpp" +#include "drivers/sqlite/store.hpp" +#include "mini.hpp" +#include "orm/healer.hpp" +#include "orm/migration.hpp" +#include "orm/migrator.hpp" +#include "orm/model.hpp" +#include "orm/query.hpp" +#include "orm/sync.hpp" + +using namespace molo::data; +using namespace molo::data::orm; +using namespace molo::data::sqlite; + +class Widget : public Model { + public: + static std::string table_name() { return "widgets"; } + static std::string primary_key() { return "id"; } + static std::vector fields() { + auto id = FieldDef::primary_key(); + auto sku = FieldDef::varchar("sku", 8); + auto qty = FieldDef::bigint("qty"); + auto note = FieldDef::text("note"); + return {id, sku, qty, note}; + } +}; + +// Scripted connection failing on demand with Postgres codes. +class ScriptedPg : public Connection { + public: + Backend backend() const override { return Backend::Postgres; } + std::string backend_id() const override { return "pg:scripted"; } + Result exec(const Context&, const BoundStatement& stmt) override { + seen.push_back(stmt.sql()); + if (!script.empty()) { + const bool fail = script.front(); + script.erase(script.begin()); + if (fail) throw DbError(Backend::Postgres, fail_code, fail_detail); + return Result{}; + } + if (fail_next) { + fail_next = false; + throw DbError(Backend::Postgres, fail_code, fail_detail); + } + if (always_fail) + throw DbError(Backend::Postgres, fail_code, fail_detail); + return Result{}; + } + Prepared prepare(const Context&, const BoundStatement& stmt) override { + return Prepared{"p", stmt.sql()}; + } + Result exec_prepared(const Context&, const Prepared&, + const std::vector&) override { + return Result{}; + } + void ping(const Context&) override {} + void close() override {} + + std::vector seen; + bool fail_next = false; + bool always_fail = false; + std::vector script; // per-call fail flags, consumed in order + std::string fail_code; + std::string fail_detail; +}; + +static std::string fresh_dir(const std::string& name) { + const std::string dir = "/tmp/molo_" + name; + std::filesystem::remove_all(dir); + return dir; +} + +int main() { + auto store = std::make_shared(); + SqliteDb db(SqliteConfig{}, store); + db.open(Context::background("test")); + Context ctx = Context::background("test"); + Syncer syncer; + syncer.sync(ctx, db, {describe_model()}); + + Widget w; + w.set("sku", make_text("abc")); + w.set("qty", make_int(5)); + w.set("note", make_text("first")); + w.save(db, ctx); + + // Dropped column heals: ADD COLUMN plus journal plus one retry. + { + store->debug_drop_column("widgets", "note"); + Healer healer({describe_model()}, fresh_dir("heal_col")); + Result r = healer.execute_with_heal( + db, ctx, BoundStatement("SELECT note FROM widgets")); + CHECK(r.size() == 1); + CHECK(store->table("widgets").cols.size() == 4); + bool journaled = false; + for (const auto& e : + std::filesystem::directory_iterator("/tmp/molo_heal_col")) + if (e.path().extension() == ".sql") journaled = true; + CHECK(journaled); + } + + // Dropped table heals: CREATE TABLE plus retry. + { + store->debug_drop_table("widgets"); + Healer healer({describe_model()}, fresh_dir("heal_table")); + Result r = healer.execute_with_heal( + db, ctx, BoundStatement("SELECT * FROM widgets")); + CHECK(r.empty()); + CHECK(store->has_table("widgets")); + } + + // Shortened VARCHAR heals via rebuild: length x4, rows preserved. + { + Widget w2; + w2.set("sku", make_text("ok")); + w2.set("qty", make_int(1)); + w2.set("note", make_text("n")); + w2.save(db, ctx); + store->debug_shorten_varchar("widgets", "sku", 4); + Healer healer({describe_model()}, fresh_dir("heal_varchar")); + BoundStatement stmt("INSERT INTO widgets (sku, qty) VALUES (?, ?)", + {make_text("longer-than-4"), make_int(2)}); + Result r = healer.execute_with_heal(db, ctx, stmt); + CHECK(r.affected() == 1); + Result all = db.exec(ctx, BoundStatement("SELECT sku FROM widgets")); + CHECK(all.size() == 2); // rebuild preserved the old row + // New width is 16: a 12-char value now fits. + db.exec(ctx, BoundStatement("INSERT INTO widgets (sku) VALUES (?)", + {make_text("twelve-chars")})); + } + + // Narrowed int heals via rebuild to BIGINT. + { + store->debug_narrow_int("widgets", "qty"); + Healer healer({describe_model()}, fresh_dir("heal_int")); + BoundStatement stmt("INSERT INTO widgets (sku, qty) VALUES (?, ?)", + {make_text("s"), make_int(9999999999LL)}); + Result r = healer.execute_with_heal(db, ctx, stmt); + CHECK(r.affected() == 1); + } + + // Postgres matrix through the scripted connection. + { + ScriptedPg pg; + Healer healer({describe_model()}, fresh_dir("heal_pg")); + pg.fail_next = true; + pg.fail_code = "42P01"; + pg.fail_detail = "relation \"widgets\" does not exist"; + Result r = healer.execute_with_heal( + pg, ctx, BoundStatement("SELECT * FROM widgets")); + (void)r; + CHECK(pg.seen.size() == 3); // failing stmt, CREATE TABLE, retry + CHECK(pg.seen[1].find("CREATE TABLE") != std::string::npos); + CHECK(pg.seen[2] == "SELECT * FROM widgets"); + // 42703 plans ADD COLUMN. + DbError e(Backend::Postgres, "42703", + "column \"note\" of relation \"widgets\" does not exist"); + HealPlan p = healer.plan(e, BoundStatement("SELECT note FROM widgets")); + CHECK(p.valid && p.table == "widgets" && p.column == "note"); + CHECK(p.ddl[0].find("ADD COLUMN") != std::string::npos); + // 22001 plans VARCHAR growth. + DbError t(Backend::Postgres, "22001", + "value too long for type character varying(8)"); + HealPlan pv = healer.plan( + t, BoundStatement("INSERT INTO widgets (sku) VALUES ($1)", + {make_text("way-too-long-sku-value")})); + CHECK(pv.valid); + CHECK(pv.ddl[0].find("VARCHAR(32)") != std::string::npos); + // Fatal codes never plan. + DbError f(Backend::Postgres, "23505", "duplicate key"); + CHECK(!healer.plan(f, BoundStatement("SELECT 1")).valid); + } + + // Retry failure propagates with the record attached. + { + ScriptedPg pg; + pg.script = {true, false, true}; // stmt fails, DDL ok, retry fails + pg.fail_code = "42P01"; + pg.fail_detail = "relation \"widgets\" does not exist"; + Healer healer({describe_model()}, fresh_dir("heal_fail")); + bool threw = false; + try { + healer.execute_with_heal(pg, ctx, + BoundStatement("SELECT * FROM widgets")); + } catch (const DbError& e) { + threw = !e.migration_record().empty(); + } + CHECK(threw); + } + + // Migrator: apply, list, checksum verify, pending. + { + Migrator migrator; + Migration m1 = + Migration::make("0001_widgets", "CREATE TABLE mig_a (id INTEGER);", + "test migration"); + migrator.apply(ctx, db, m1); + CHECK(store->has_table("mig_a")); + CHECK(migrator.is_applied(ctx, db, "0001_widgets")); + auto applied = migrator.list_applied(ctx, db); + CHECK(!applied.empty()); + // Re-apply is idempotent when the checksum matches. + migrator.apply(ctx, db, m1); + // Tampered checksum aborts loudly. + Migration tampered = m1; + tampered.checksum = "deadbeefdeadbeef"; + bool threw = false; + try { + migrator.apply(ctx, db, tampered); + } catch (const DbError& e) { + threw = e.code() == "MOLO_MIGRATION"; + } + CHECK(threw); + Migration m2 = Migration::make("0002_more", "CREATE TABLE mig_b (id INTEGER);", + "second"); + auto pend = migrator.pending(ctx, db, {m1, m2}); + CHECK(pend.size() == 1 && pend[0].id == "0002_more"); + // Journal file write works. + const std::string dir = fresh_dir("journal"); + const std::string path = Migrator::write_auto_journal(dir, m2); + CHECK(std::filesystem::exists(path)); + } + + SUITE_DONE("sync_heal_migrate"); +} diff --git a/tests/src/data/test_value_error.cpp b/tests/src/data/test_value_error.cpp new file mode 100644 index 0000000..d86f35b --- /dev/null +++ b/tests/src/data/test_value_error.cpp @@ -0,0 +1,102 @@ +// Step 1: Value/Row/Result/Error plus classification table. +#include "mini.hpp" +#include "value.hpp" +#include "row.hpp" +#include "result.hpp" +#include "error.hpp" +#include "statement.hpp" +#include "context.hpp" + +using namespace molo::data; + +int main() { + // Value factories and names. + CHECK(is_null(make_null())); + CHECK(!is_null(make_int(1))); + CHECK(std::string(value_type_name(make_bool(true))) == "bool"); + CHECK(std::string(value_type_name(make_text("x"))) == "text"); + + // Row access. + Row row({"a", "b"}, {make_int(7), make_text("hi")}); + CHECK(row.size() == 2); + CHECK(std::get(row.get("a")) == 7); + CHECK(row.has("b") && !row.has("zzz")); + bool threw = false; + try { + row.get("zzz"); + } catch (const std::out_of_range&) { + threw = true; + } + CHECK(threw); + + // Result shape. + Result r({"x"}, {{make_int(1)}, {make_int(2)}}, 2, 9); + CHECK(r.size() == 2 && r.affected() == 2 && r.last_insert_id() == 9); + CHECK(std::get(r.row(1).get("x")) == 2); + CHECK(Result{}.empty()); + + // Classification matrix: Postgres. + CHECK(classify(Backend::Postgres, "42P01").kind == HealKind::CreateTable); + CHECK(!classify(Backend::Postgres, "42P01").fatal); + CHECK(classify(Backend::Postgres, "42703").kind == HealKind::AddColumn); + CHECK(classify(Backend::Postgres, "22001").kind == HealKind::ExtendVarchar); + CHECK(classify(Backend::Postgres, "42804").kind == HealKind::WidenInt); + CHECK(classify(Backend::Postgres, "40001").retryable_txn); + CHECK(classify(Backend::Postgres, "40001").fatal); + CHECK(classify(Backend::Postgres, "23505").fatal); + CHECK(classify(Backend::Postgres, "23505").kind == HealKind::None); + + // Classification matrix: SQLite. + CHECK(classify(Backend::Sqlite, "SQLITE_NO_TABLE").kind == + HealKind::CreateTable); + CHECK(classify(Backend::Sqlite, "SQLITE_NO_COLUMN").kind == + HealKind::AddColumn); + CHECK(classify(Backend::Sqlite, "SQLITE_TOOBIG").kind == + HealKind::ExtendVarchar); + CHECK(classify(Backend::Sqlite, "SQLITE_MISMATCH").kind == + HealKind::WidenInt); + CHECK(classify(Backend::Sqlite, "SQLITE_CONSTRAINT").fatal); + + // DbError flags follow classification. + DbError healable(Backend::Postgres, "42P01", "nope"); + CHECK(healable.healable() && !healable.retryable_txn()); + DbError fatal(Backend::Postgres, "0A000", "nope"); + CHECK(!fatal.healable()); + DbError ser(Backend::Postgres, "40001", "retry me"); + CHECK(ser.retryable_txn()); + + // Message parsers. + CHECK(parse_missing_table("no such table: users") == "users"); + CHECK(parse_missing_table("other") == ""); + CHECK(parse_missing_column("no such column: age") == "age"); + + // Statements: param cap and raw flag. + BoundStatement s("SELECT $1", {make_int(1)}); + CHECK(!s.has_raw() && s.params().size() == 1); + CHECK(BoundStatement::raw("SELECT 1").has_raw()); + check_identifier("users.name_1"); + threw = false; + try { + check_identifier("a-b"); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(threw); + threw = false; + try { + check_identifier(std::string(64, 'x')); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(threw); + + // Context deadlines. + Context ctx("corr-1", "tenant-a", 30); + CHECK(ctx.correlation_id() == "corr-1"); + CHECK(ctx.tenant() == "tenant-a"); + CHECK(!ctx.expired() && ctx.remaining_ms() > 0); + Context bg = Context::background(); + CHECK(!bg.expired()); + + SUITE_DONE("value_error"); +} diff --git a/tests/src/governance/deprecation_headers_test.cpp b/tests/src/governance/deprecation_headers_test.cpp new file mode 100644 index 0000000..029362f --- /dev/null +++ b/tests/src/governance/deprecation_headers_test.cpp @@ -0,0 +1,65 @@ +// deprecation_headers_test: Deprecation/Sunset/Link headers on responses. +#include +#include +#include + +#include "governance/adapters/BaseAdapters.h" +#include "governance/versions/DeprecationMiddleware.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) +} // namespace + +using molo::govern::adapters::HttpResponse; +using molo::govern::versions::DeprecationMiddleware; + +int main() { + CHECK(DeprecationMiddleware::FormatImf(0) == "Thu, 01 Jan 1970 00:00:00 GMT"); + + DeprecationMiddleware mw; + DeprecationMiddleware::Rule rule; + rule.prefix = "/v1"; + rule.deprecation_unix = 1700000000; + rule.sunset_httpdate = DeprecationMiddleware::FormatImf(1800000000); + rule.successor_prefix = "/v2"; + CHECK(mw.AddRule(rule)); + CHECK(mw.RuleCount() == 1); + DeprecationMiddleware::Rule bad; + bad.prefix = "v9"; + CHECK(!mw.AddRule(bad)); + + const std::map h = mw.HeadersFor("/v1/users", 1750000000); + CHECK(h.at("Deprecation") == "@1700000000"); + CHECK(h.at("Sunset") == DeprecationMiddleware::FormatImf(1800000000)); + CHECK(h.at("Link").find("rel=\"successor-version\"") != std::string::npos); + CHECK(h.at("Link").find("/v2") != std::string::npos); + CHECK(h.at("Link").find("rel=\"deprecation\"") != std::string::npos); + + // Non-matching prefix yields no headers. + CHECK(mw.HeadersFor("/v2/users", 1750000000).empty()); + + // Longest-prefix rule wins. + DeprecationMiddleware::Rule sub; + sub.prefix = "/v1/legacy"; + sub.deprecation_unix = 1710000000; + CHECK(mw.AddRule(sub)); + CHECK(mw.HeadersFor("/v1/legacy/x", 1750000000).at("Deprecation") == "@1710000000"); + CHECK(mw.HeadersFor("/v1/users", 1750000000).at("Deprecation") == "@1700000000"); + + // Apply writes headers onto a real response. + HttpResponse res; + res.status = 200; + mw.Apply(res, "/v1/users", 1750000000); + CHECK(res.headers.get("Deprecation") == "@1700000000"); + CHECK(res.headers.get("Sunset") == DeprecationMiddleware::FormatImf(1800000000)); + + if (g_fail == 0) std::printf("PASS deprecation_headers_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/governance/differ_vectors_test.cpp b/tests/src/governance/differ_vectors_test.cpp new file mode 100644 index 0000000..ace3933 --- /dev/null +++ b/tests/src/governance/differ_vectors_test.cpp @@ -0,0 +1,139 @@ +// differ_vectors_test: golden ERR/WARN/INFO vectors plus determinism. +#include +#include +#include + +#include "governance/diff/BreakingRules.h" +#include "governance/diff/SpecDiffer.h" +#include "governance/diff/SpecSnapshot.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) + +bool HasRule(const std::vector& changes, + const std::string& rule) { + for (const auto& c : changes) { + if (c.rule == rule) return true; + } + return false; +} +} // namespace + +using molo::govern::diff::BreakingRules; +using molo::govern::diff::SpecDiffer; +using molo::govern::diff::SpecSnapshot; + +namespace { + +const char* kOld = R"JSON({ + "version": 1, + "endpoints": [ + {"method": "GET", "path": "/v1/users", "operation_id": "listUsers", + "summary": "List users", "example": "GET /v1/users", "deprecated": false, + "security": "key", + "params": [{"name": "page", "required": false, "type": "int"}], + "responses": {"200": "UserList", "404": "Problem"}}, + {"method": "GET", "path": "/v1/users/:id", "operation_id": "getUser", + "summary": "Get user", "example": "x", "deprecated": false, "security": "key", + "params": [{"name": "id", "required": true, "type": "string"}, + {"name": "verbose", "required": false, "type": "bool"}], + "responses": {"200": "User"}}, + {"method": "DELETE", "path": "/v1/users/:id", "operation_id": "delUser", + "summary": "Delete", "example": "x", "deprecated": false, "security": "key", + "params": [], "responses": {"204": "Empty"}} + ] +})JSON"; + +const char* kNew = R"JSON({ + "version": 1, + "endpoints": [ + {"method": "GET", "path": "/v1/users", "operation_id": "listUsersV2", + "summary": "List all users", "example": "GET /v1/users", "deprecated": true, + "security": "key+scopes", + "params": [{"name": "page", "required": false, "type": "string"}, + {"name": "tenant", "required": true, "type": "string"}], + "responses": {"200": "UserListV2"}}, + {"method": "GET", "path": "/v1/users/:id", "operation_id": "getUser", + "summary": "Get user", "example": "x", "deprecated": false, "security": "key", + "params": [{"name": "id", "required": true, "type": "string"}], + "responses": {"200": "User", "201": "User"}}, + {"method": "POST", "path": "/v1/users", "operation_id": "createUser", + "summary": "Create", "example": "x", "deprecated": false, "security": "key", + "params": [{"name": "dry_run", "required": false, "type": "bool"}], + "responses": {"201": "User"}} + ] +})JSON"; + +} // namespace + +int main() { + SpecSnapshot old_snap; + SpecSnapshot new_snap; + std::string error; + CHECK(SpecSnapshot::Parse(kOld, old_snap, error)); + CHECK(SpecSnapshot::Parse(kNew, new_snap, error)); + CHECK(old_snap.endpoints.size() == 3); + CHECK(new_snap.endpoints.size() == 3); + + // Bad inputs rejected loudly. + SpecSnapshot junk; + CHECK(!SpecSnapshot::Parse("{nope", junk, error)); + CHECK(!SpecSnapshot::Parse("[]", junk, error)); + CHECK(!SpecSnapshot::Parse("{\"endpoints\":[{\"method\":\"GET\"}]}", junk, error)); + + const std::vector changes = SpecDiffer::Diff(old_snap, new_snap); + + // ERR goldens. + CHECK(HasRule(changes, "removed-endpoint")); // DELETE /v1/users/:id + CHECK(HasRule(changes, "new-required-param")); // tenant + CHECK(HasRule(changes, "param-type-changed")); // page int -> string + CHECK(HasRule(changes, "removed-response")); // 404 gone + CHECK(HasRule(changes, "auth-added")); // key -> key+scopes + // WARN goldens. + CHECK(HasRule(changes, "deprecated")); // GET /v1/users + CHECK(HasRule(changes, "operation-id-changed")); // listUsers -> listUsersV2 + CHECK(HasRule(changes, "removed-optional-param")); // verbose + CHECK(HasRule(changes, "response-type-changed")); // 200 UserList -> UserListV2 + // INFO goldens. + CHECK(HasRule(changes, "added-endpoint")); // POST /v1/users + CHECK(HasRule(changes, "added-response")); // 201 on getUser + CHECK(HasRule(changes, "summary-changed")); + + // Severity names. + CHECK(BreakingRules::SeverityName(BreakingRules::Severity::kError) == "ERR"); + CHECK(BreakingRules::SeverityName(BreakingRules::Severity::kWarning) == "WARN"); + CHECK(BreakingRules::SeverityName(BreakingRules::Severity::kInfo) == "INFO"); + CHECK(BreakingRules::IsBreaking(BreakingRules::Severity::kError)); + CHECK(!BreakingRules::IsBreaking(BreakingRules::Severity::kWarning)); + CHECK(!BreakingRules::IsBreaking(BreakingRules::Severity::kInfo)); + + // Determinism: snapshot twice, diff empty. + const std::string canon1 = old_snap.CanonicalJson(); + SpecSnapshot reparsed; + CHECK(SpecSnapshot::Parse(canon1, reparsed, error)); + CHECK(reparsed.CanonicalJson() == canon1); + CHECK(SpecDiffer::Diff(old_snap, reparsed).empty()); + CHECK(SpecDiffer::Diff(old_snap, old_snap).empty()); + // Diff output order is stable across runs. + const std::string run1 = [&] { + std::string s; + for (const auto& c : SpecDiffer::Diff(old_snap, new_snap)) s += c.rule + c.target; + return s; + }(); + const std::string run2 = [&] { + std::string s; + for (const auto& c : SpecDiffer::Diff(old_snap, new_snap)) s += c.rule + c.target; + return s; + }(); + CHECK(run1 == run2); + + if (g_fail == 0) std::printf("PASS differ_vectors_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/governance/gate_test.cpp b/tests/src/governance/gate_test.cpp new file mode 100644 index 0000000..a9ae1d1 --- /dev/null +++ b/tests/src/governance/gate_test.cpp @@ -0,0 +1,120 @@ +// gate_test: GovernGate verdict/changelog/diff plus the apidiff CLI. +#include +#include +#include +#include +#include + +#include "governance/diff/ApiDiffCli.h" +#include "governance/diff/GovernGate.h" +#include "governance/diff/SpecDiffer.h" +#include "governance/diff/SpecSnapshot.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) +} // namespace + +using molo::govern::diff::ApiDiffCli; +using molo::govern::diff::BreakingRules; +using molo::govern::diff::GovernGate; +using molo::govern::diff::SpecDiffer; +using molo::govern::diff::SpecSnapshot; + +namespace { + +const char* kOld = R"JSON({"version": 1, "endpoints": [ + {"method": "GET", "path": "/v1/users", "operation_id": "a", "summary": "s", + "example": "e", "deprecated": false, "security": "key", + "params": [], "responses": {"200": "U"}}, + {"method": "GET", "path": "/v1/gone", "operation_id": "g", "summary": "s", + "example": "e", "deprecated": false, "security": "key", + "params": [], "responses": {"200": "U"}} +]})JSON"; + +const char* kNew = R"JSON({"version": 1, "endpoints": [ + {"method": "GET", "path": "/v1/users", "operation_id": "a", "summary": "s2", + "example": "e", "deprecated": true, "security": "key", + "params": [{"name": "opt", "required": false, "type": "string"}], + "responses": {"200": "U"}} +]})JSON"; + +std::vector SampleChanges() { + SpecSnapshot o; + SpecSnapshot n; + std::string err; + CHECK(SpecSnapshot::Parse(kOld, o, err)); + CHECK(SpecSnapshot::Parse(kNew, n, err)); + return SpecDiffer::Diff(o, n); +} + +} // namespace + +int main() { + const std::vector changes = SampleChanges(); + CHECK(!changes.empty()); + CHECK(GovernGate::Evaluate(changes) == GovernGate::Verdict::kFail); + + const std::string summary = GovernGate::Summary(changes); + CHECK(summary.find("ERR=1") != std::string::npos); + CHECK(summary.find("FAIL") != std::string::npos); + + const std::string changelog = GovernGate::Changelog(changes); + CHECK(changelog.find("## Breaking") != std::string::npos); + CHECK(changelog.find("## Warnings") != std::string::npos); + CHECK(changelog.find("## Changes") != std::string::npos); + CHECK(changelog.find("removed-endpoint") != std::string::npos); + + SpecSnapshot o; + SpecSnapshot n; + std::string err; + CHECK(SpecSnapshot::Parse(kOld, o, err)); + CHECK(SpecSnapshot::Parse(kNew, n, err)); + const std::string diff = GovernGate::DiffText(o, n); + CHECK(diff.find("ERR removed-endpoint") != std::string::npos); + CHECK(GovernGate::DiffText(o, o) == "No changes.\n"); + CHECK(GovernGate::Evaluate({}) == GovernGate::Verdict::kPass); + CHECK(GovernGate::Changelog({}) == "No changes.\n"); + + // CLI with an in-memory file reader. + std::map files = {{"old.json", kOld}, {"new.json", kNew}}; + ApiDiffCli::Reader reader = [&](const std::string& path, std::string& text, + std::string& error) { + auto it = files.find(path); + if (it == files.end()) { + error = "missing"; + return false; + } + text = it->second; + return true; + }; + std::ostringstream out; + std::ostringstream cerr; + CHECK(ApiDiffCli::Execute({"apidiff", "breaking", "old.json", "new.json"}, out, cerr, + reader) == 1); + CHECK(out.str().find("FAIL") != std::string::npos); + out.str(""); + CHECK(ApiDiffCli::Execute({"apidiff", "changelog", "old.json", "new.json"}, out, cerr, + reader) == 0); + CHECK(out.str().find("## Breaking") != std::string::npos); + out.str(""); + CHECK(ApiDiffCli::Execute({"apidiff", "diff", "old.json", "new.json"}, out, cerr, + reader) == 0); + CHECK(out.str().find("removed-endpoint") != std::string::npos); + CHECK(ApiDiffCli::Execute({"apidiff", "breaking", "old.json", "old.json"}, out, cerr, + reader) == 0); + CHECK(ApiDiffCli::Execute({"apidiff", "nope", "old.json", "new.json"}, out, cerr, + reader) == 2); + CHECK(ApiDiffCli::Execute({"apidiff"}, out, cerr, reader) == 2); + CHECK(ApiDiffCli::Execute({"apidiff", "diff", "missing", "new.json"}, out, cerr, + reader) == 2); + + if (g_fail == 0) std::printf("PASS gate_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/governance/keyauth_http_test.cpp b/tests/src/governance/keyauth_http_test.cpp new file mode 100644 index 0000000..75fa857 --- /dev/null +++ b/tests/src/governance/keyauth_http_test.cpp @@ -0,0 +1,101 @@ +// keyauth_http_test: KeyAuth through real HTTP request/response structs. +#include +#include + +#include "governance/adapters/BaseAdapters.h" +#include "governance/keys/KeyAuth.h" +#include "governance/keys/KeyIssuer.h" +#include "governance/keys/MemoryKeyStore.h" +#include "governance/problem.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) +} // namespace + +using molo::govern::adapters::HttpRequest; +using molo::govern::adapters::HttpResponse; +using molo::govern::keys::KeyAuth; +using molo::govern::keys::KeyIssuer; +using molo::govern::keys::MemoryKeyStore; +using molo::govern::Problem; + +int main() { + MemoryKeyStore store; + KeyIssuer issuer(store, 1700000000); + KeyIssuer::Issued k = issuer.Mint("pro", {"read:users"}, 3600); + KeyAuth auth(store, 1700000000); + + // Bearer header path. + HttpRequest bearer; + bearer.method = "GET"; + bearer.path = "/v1/users"; + bearer.headers.set("Authorization", "Bearer " + k.token); + KeyAuth::AuthResult ok = auth.AuthenticateRequest(bearer); + CHECK(ok.ok && ok.http_status == 200 && ok.key.has_value()); + CHECK(ok.key->id == k.record.id); + + // X-Api-Key header path. + HttpRequest xkey; + xkey.headers.set("X-Api-Key", k.token); + CHECK(auth.AuthenticateRequest(xkey).ok); + + // Missing credentials: 401 + problem body. + HttpRequest anon; + KeyAuth::AuthResult miss = auth.AuthenticateRequest(anon); + CHECK(!miss.ok && miss.http_status == 401 && miss.code == "unauthorized"); + Problem p = KeyAuth::ToProblem(miss); + CHECK(p.status() == 401); + CHECK(p.to_json_string().find("unauthorized") != std::string::npos); + CHECK(Problem::kContentType == "application/problem+json"); + HttpResponse denied; + denied.status = p.status(); + denied.headers.set("Content-Type", std::string(Problem::kContentType)); + denied.body = p.to_json_string(); + CHECK(denied.headers.get("Content-Type") == "application/problem+json"); + + // Wrong secret for a known id: 401. + std::string id; + std::string secret; + CHECK(KeyIssuer::ParseToken(k.token, id, secret)); + std::string forged = KeyIssuer::MakeToken(id, std::string(43, 'A')); + if (forged == k.token) forged[10] = 'B'; + HttpRequest bad; + bad.headers.set("Authorization", "Bearer " + forged); + CHECK(!auth.AuthenticateRequest(bad).ok); + + // Malformed token: 401. + HttpRequest malformed; + malformed.headers.set("Authorization", "Bearer junk"); + CHECK(!auth.AuthenticateRequest(malformed).ok); + + // Scope gate: 403 insufficient_scope + problem body. + KeyAuth::AuthResult scoped = KeyAuth::RequireScopes(ok, {"admin:users"}); + CHECK(!scoped.ok && scoped.http_status == 403 && scoped.code == "insufficient_scope"); + Problem sp = KeyAuth::ToProblem(scoped); + CHECK(sp.status() == 403 && sp.type().find("insufficient_scope") != std::string::npos); + CHECK(KeyAuth::RequireScopes(ok, {"read:users"}).ok); + + // Revoked key: 401 expired_key. + issuer.Revoke(k.record.id); + KeyAuth::AuthResult rev = auth.AuthenticateRequest(bearer); + CHECK(!rev.ok && rev.code == "expired_key"); + CHECK(KeyAuth::ToProblem(rev).status() == 401); + + // Expired key: 401 expired_key. + KeyIssuer::Issued short_lived = issuer.Mint("free", {}, 10); + auth.set_now(1700000000 + 11); + HttpRequest exp; + exp.headers.set("X-Api-Key", short_lived.token); + KeyAuth::AuthResult ex = auth.AuthenticateRequest(exp); + CHECK(!ex.ok && ex.code == "expired_key"); + + if (g_fail == 0) std::printf("PASS keyauth_http_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/governance/keys_issuer_test.cpp b/tests/src/governance/keys_issuer_test.cpp new file mode 100644 index 0000000..500e057 --- /dev/null +++ b/tests/src/governance/keys_issuer_test.cpp @@ -0,0 +1,85 @@ +// keys_issuer_test: mint/rotate/revoke roundtrips plus hash stability. +#include +#include +#include + +#include "governance/keys/KeyIssuer.h" +#include "governance/keys/MemoryKeyStore.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) +} // namespace + +using molo::govern::keys::KeyIssuer; +using molo::govern::keys::MemoryKeyStore; + +int main() { + // SHA-256 known vector. + CHECK(KeyIssuer::HashToken("abc") == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + CHECK(KeyIssuer::HashToken("abc") == KeyIssuer::HashToken("abc")); + + MemoryKeyStore store; + KeyIssuer issuer(store, 1700000000); + + KeyIssuer::Issued k = issuer.Mint("pro", {"read:users", "write:users"}, 3600); + CHECK(!k.token.empty()); + CHECK(k.token.substr(0, 5) == "molo_"); + CHECK(k.token.size() == 5 + 16 + 1 + 43); + std::string id; + std::string secret; + CHECK(KeyIssuer::ParseToken(k.token, id, secret)); + CHECK(id == k.record.id); + CHECK(secret.size() == 43); + CHECK(k.record.key_hash == KeyIssuer::HashToken(k.token)); + CHECK(k.record.tier == "pro"); + CHECK(k.record.scopes.size() == 2); + CHECK(k.record.expires_unix == 1700003600); + CHECK(k.record.Live(1700000000)); + CHECK(!k.record.Live(1700003600)); + CHECK(store.Size() == 1); + + CHECK(!KeyIssuer::ParseToken("nope", id, secret)); + CHECK(!KeyIssuer::ParseToken(k.token + "x", id, secret)); + CHECK(KeyIssuer::TokenId("junk") == ""); + + // Mint twice: ids and tokens differ. + KeyIssuer::Issued k2 = issuer.Mint("free", {}, 0); + CHECK(k2.token != k.token); + CHECK(k2.record.id != k.record.id); + CHECK(k2.record.expires_unix == 0); + CHECK(k2.record.Live(9999999999L)); + + // Rotate keeps the old key alive through the overlap window. + KeyIssuer::Issued rot = issuer.Rotate(k.record.id, 100); + CHECK(!rot.token.empty()); + CHECK(rot.record.id != k.record.id); + auto old = store.Find(k.record.id); + CHECK(old.has_value()); + CHECK(old->expires_unix == 1700000100); + CHECK(old->Live(1700000050)); + CHECK(!old->Live(1700000100)); + + // Revoke ends the key immediately. + CHECK(issuer.Revoke(k2.record.id)); + auto gone = store.Find(k2.record.id); + CHECK(gone.has_value() && !gone->Live(1700000000)); + CHECK(!issuer.Revoke("missing0000000000")); + CHECK(issuer.Rotate("missing0000000000").token.empty()); + + // Scope cap enforced at mint. + std::vector many; + for (int i = 0; i < 100; ++i) many.push_back("s:x" + std::to_string(i)); + KeyIssuer::Issued capped = issuer.Mint("free", many, 0); + CHECK(capped.record.scopes.size() == 64); + + if (g_fail == 0) std::printf("PASS keys_issuer_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/governance/keys_scope_test.cpp b/tests/src/governance/keys_scope_test.cpp new file mode 100644 index 0000000..39cb39d --- /dev/null +++ b/tests/src/governance/keys_scope_test.cpp @@ -0,0 +1,63 @@ +// keys_scope_test: Scope parse/valid/allows incl. full-segment wildcard. +#include +#include +#include + +#include "governance/keys/Scope.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) +} // namespace + +using molo::govern::keys::Scope; + +int main() { + CHECK(Scope::Valid("read:users")); + CHECK(Scope::Valid("read:*")); + CHECK(Scope::Valid("*")); + CHECK(!Scope::Valid("")); + CHECK(!Scope::Valid("read:")); + CHECK(!Scope::Valid(":read")); + CHECK(!Scope::Valid("read::users")); + CHECK(!Scope::Valid("read*")); + CHECK(!Scope::Valid("re*ad:users")); + CHECK(!Scope::Valid("read users")); + CHECK(!Scope::Valid(std::string(65, 'a'))); + + CHECK(Scope::Allows("read:users", "read:users")); + CHECK(Scope::Allows("read:*", "read:users")); + CHECK(!Scope::Allows("read:*", "write:users")); + CHECK(!Scope::Allows("read:users", "read:*")); + CHECK(!Scope::Allows("read", "read:users")); + CHECK(!Scope::Allows("read:users", "read")); + CHECK(Scope::Allows("*", "read")); + CHECK(!Scope::Allows("*", "read:users")); + + const std::vector parsed = Scope::Parse("read:users, write:users read:*"); + CHECK(parsed.size() == 3); + CHECK(Scope::AllowsAny(parsed, "write:users")); + CHECK(Scope::AllowsAny(parsed, "read:orders")); + CHECK(!Scope::AllowsAny(parsed, "admin:users")); + + const std::vector bad = Scope::Parse("ok:scope, bad scope!, :x"); + CHECK(bad.size() == 2); + CHECK(bad[0] == "ok:scope"); + CHECK(bad[1] == "bad"); + + std::string many; + for (int i = 0; i < 100; ++i) { + if (i) many += ","; + many += "s:scope" + std::to_string(i); + } + CHECK(Scope::Parse(many).size() == 64); + + if (g_fail == 0) std::printf("PASS keys_scope_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/governance/surface_lint_test.cpp b/tests/src/governance/surface_lint_test.cpp new file mode 100644 index 0000000..c0cf1ac --- /dev/null +++ b/tests/src/governance/surface_lint_test.cpp @@ -0,0 +1,115 @@ +// surface_lint_test: RouteLinter S1-S10 findings. +#include +#include +#include + +#include "governance/surface/RouteLinter.h" +#include "governance/surface/SurfaceRules.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) + +bool HasRule(const std::vector& fs, + const std::string& rule) { + for (const auto& f : fs) { + if (f.rule == rule) return true; + } + return false; +} +} // namespace + +using molo::govern::surface::RouteLinter; +using molo::govern::surface::SurfaceRules; + +namespace { + +RouteLinter::Route Clean() { + RouteLinter::Route r; + r.method = "GET"; + r.path = "/v1/users"; + r.statuses = {200, 404}; + r.auth = "key"; + r.paginated = true; + r.has_page_params = true; + r.has_envelope = true; + r.has_filters = true; + r.filter_in_query = true; + r.has_problem_body = true; + r.has_summary = true; + r.has_example = true; + return r; +} + +} // namespace + +int main() { + CHECK(SurfaceRules::RuleIds().size() == 10); + CHECK(SurfaceRules::Describe("S1") != "unknown rule"); + CHECK(SurfaceRules::SeverityOf("S10") == "ERR"); + CHECK(SurfaceRules::SeverityOf("S9") == "WARN"); + + const RouteLinter::Route clean = Clean(); + CHECK(RouteLinter::Lint({&clean, 1}).empty()); + + RouteLinter::Route s1 = Clean(); + s1.path = "/v1/user"; + CHECK(HasRule(RouteLinter::Lint({&s1, 1}), "S1")); + + RouteLinter::Route s2 = Clean(); + s2.has_body = true; + CHECK(HasRule(RouteLinter::Lint({&s2, 1}), "S2")); + RouteLinter::Route s2b = Clean(); + s2b.method = "BREW"; + CHECK(HasRule(RouteLinter::Lint({&s2b, 1}), "S2")); + + RouteLinter::Route s3 = Clean(); + s3.statuses = {404}; + CHECK(HasRule(RouteLinter::Lint({&s3, 1}), "S3")); + RouteLinter::Route s3b = Clean(); + s3b.statuses = {}; + CHECK(HasRule(RouteLinter::Lint({&s3b, 1}), "S3")); + + RouteLinter::Route s4 = Clean(); + s4.has_problem_body = false; + CHECK(HasRule(RouteLinter::Lint({&s4, 1}), "S4")); + + RouteLinter::Route s5 = Clean(); + s5.has_envelope = false; + CHECK(HasRule(RouteLinter::Lint({&s5, 1}), "S5")); + + RouteLinter::Route s6 = Clean(); + s6.filter_in_query = false; + CHECK(HasRule(RouteLinter::Lint({&s6, 1}), "S6")); + + RouteLinter::Route s7 = Clean(); + s7.auth = ""; + CHECK(HasRule(RouteLinter::Lint({&s7, 1}), "S7")); + RouteLinter::Route s7b = Clean(); + s7b.auth = "key+scopes"; + CHECK(HasRule(RouteLinter::Lint({&s7b, 1}), "S7")); + + RouteLinter::Route s8 = Clean(); + s8.method = "POST"; + s8.statuses = {201}; + CHECK(HasRule(RouteLinter::Lint({&s8, 1}), "S8")); + + RouteLinter::Route s9 = Clean(); + s9.has_summary = false; + s9.has_example = false; + const auto f9 = RouteLinter::Lint({&s9, 1}); + CHECK(HasRule(f9, "S9") && f9.size() == 2); + + RouteLinter::Route s10 = Clean(); + s10.path = "/users"; + CHECK(HasRule(RouteLinter::Lint({&s10, 1}), "S10")); + + if (g_fail == 0) std::printf("PASS surface_lint_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/governance/tiers_test.cpp b/tests/src/governance/tiers_test.cpp new file mode 100644 index 0000000..2d54d92 --- /dev/null +++ b/tests/src/governance/tiers_test.cpp @@ -0,0 +1,95 @@ +// tiers_test: tier table, downward-only policy, ledger, headers. +#include +#include +#include + +#include "governance/adapters/BaseAdapters.h" +#include "governance/tiers/QuotaLedger.h" +#include "governance/tiers/TierHeaders.h" +#include "governance/tiers/TierPolicy.h" +#include "governance/tiers/TierTable.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) +} // namespace + +using molo::govern::adapters::HttpResponse; +using molo::govern::tiers::QuotaLedger; +using molo::govern::tiers::TierHeaders; +using molo::govern::tiers::TierPolicy; +using molo::govern::tiers::TierTable; + +int main() { + const TierTable::Limits free = TierTable::ForTier("free"); + CHECK(free.per_min == 60 && free.per_day == 10000 && free.concurrency == 4); + const TierTable::Limits pro = TierTable::ForTier("pro"); + CHECK(pro.per_min == 600 && pro.per_day == 1000000 && pro.concurrency == 32); + const TierTable::Limits ent = TierTable::ForTier("enterprise"); + CHECK(ent.per_min == 6000 && ent.per_day == 50000000 && ent.concurrency == 256); + CHECK(TierTable::ForTier("ent").per_min == 6000); + CHECK(TierTable::ForTier("unknown").per_min == 60); + CHECK(TierTable::Canonical("ent") == "enterprise"); + CHECK(TierTable::Canonical("bogus") == "free"); + + // Downward-only override. + const TierTable::Limits down = TierPolicy::Resolve("pro", "free"); + CHECK(down.per_min == 60 && down.per_day == 10000); + const TierTable::Limits up = TierPolicy::Resolve("free", "enterprise"); + CHECK(up.per_min == 60 && up.per_day == 10000); + const TierTable::Limits none = TierPolicy::Resolve("pro", ""); + CHECK(none.per_min == 600); + const TierTable::Limits bogus = TierPolicy::Resolve("pro", "bogus"); + CHECK(bogus.per_min == 600); + + CHECK(QuotaLedger::LedgerKey("abc", "2024-01-01T00:00") == "gov:quota:abc:2024-01-01T00:00"); + + QuotaLedger ledger; + const long now = 1700000010; // 10s into a minute + long retry = 0; + for (int i = 0; i < 60; ++i) CHECK(ledger.Check("k1", free, now, retry)); + CHECK(retry == 0); + CHECK(!ledger.Check("k1", free, now, retry)); + CHECK(retry == 60 - (now % 60)); + // Next minute the budget resets. + CHECK(ledger.Check("k1", free, now + 60, retry)); + CHECK(ledger.Peek("k1", now + 60).minute_count == 1); + CHECK(ledger.Peek("unknown", now).minute_count == 0); + + // Retry-After is capped at 300. + QuotaLedger day_ledger; + TierTable::Limits tiny{1000000, 2, 1000, 1000}; + long r2 = 0; + CHECK(day_ledger.Check("k2", tiny, now, r2)); + CHECK(day_ledger.Check("k2", tiny, now, r2)); + CHECK(!day_ledger.Check("k2", tiny, now, r2)); + CHECK(r2 == 300); + + // Headers on a map and on a real response. + std::map h; + TierHeaders::Apply(h, free, 59, 9999, 50, 0); + CHECK(h["X-RateLimit-Limit"] == "60"); + CHECK(h["X-RateLimit-Remaining"] == "59"); + CHECK(h["X-RateLimit-Reset"] == "50"); + CHECK(h["RateLimit"].find("limit=60") != std::string::npos); + CHECK(h["RateLimit"].find("remaining=59") != std::string::npos); + CHECK(h["RateLimit-Policy"].find("w=60") != std::string::npos); + CHECK(h.find("Retry-After") == h.end()); + + TierHeaders::Apply(h, free, 0, 0, 50, 9999); + CHECK(h["Retry-After"] == "300"); + + HttpResponse res; + TierHeaders::Apply(res, pro, 10, 20, 5, 0); + CHECK(res.headers.get("X-RateLimit-Limit") == "600"); + CHECK(res.headers.get("RateLimit").find("reset=5") != std::string::npos); + + if (g_fail == 0) std::printf("PASS tiers_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/governance/versions_test.cpp b/tests/src/governance/versions_test.cpp new file mode 100644 index 0000000..3edddac --- /dev/null +++ b/tests/src/governance/versions_test.cpp @@ -0,0 +1,81 @@ +// versions_test: ApiVersion lifecycle plus VersionedRouter match/gone rules. +#include +#include + +#include "governance/versions/ApiVersion.h" +#include "governance/versions/VersionedRouter.h" + +namespace { +int g_fail = 0; +void Check(bool cond, int line, const char* expr) { + if (!cond) { + std::printf("FAIL line %d: %s\n", line, expr); + ++g_fail; + } +} +#define CHECK(c) Check((c), __LINE__, #c) +} // namespace + +using molo::govern::versions::ApiVersion; +using molo::govern::versions::VersionedRouter; + +int main() { + CHECK(ApiVersion::ParseMajor("/v1/users") == 1); + CHECK(ApiVersion::ParseMajor("/v99") == 99); + CHECK(ApiVersion::ParseMajor("/v2/") == 2); + CHECK(ApiVersion::ParseMajor("/users") == 0); + CHECK(ApiVersion::ParseMajor("/v0/users") == 0); + CHECK(ApiVersion::ParseMajor("/v100/users") == 0); + CHECK(ApiVersion::ParseMajor("/v01/users") == 0); + CHECK(ApiVersion::ParseMajor("/vx/users") == 0); + CHECK(ApiVersion::Prefix(3) == "/v3"); + + ApiVersion v; + v.major = 1; + v.deprecation_unix = 1700000000; + v.sunset_unix = 1800000000; + v.successor = 2; + CHECK(v.DatesConsistent()); + CHECK(v.At(1600000000) == ApiVersion::Status::kCurrent); + CHECK(v.At(1750000000) == ApiVersion::Status::kDeprecated); + CHECK(v.At(1850000000) == ApiVersion::Status::kSunset); + ApiVersion bad; + bad.deprecation_unix = 1800000000; + bad.sunset_unix = 1700000000; + CHECK(!bad.DatesConsistent()); + + VersionedRouter router; + CHECK(router.Add("GET", "/v1/users", "list_users")); + CHECK(router.Add("GET", "/v1/users/:id", "get_user")); + CHECK(!router.Add("GET", "/v1/users", "dup_pattern")); + CHECK(!router.Add("POST", "/v1/orders", "list_users")); + CHECK(!router.Add("GET", "/users", "no_version")); + CHECK(router.Size() == 2); + + auto hit = router.Lookup("GET", "/v1/users", 1700000000); + CHECK(hit.found && !hit.gone && hit.name == "list_users" && hit.major == 1); + auto param = router.Lookup("GET", "/v1/users/42", 1700000000); + CHECK(param.found && param.params["id"] == "42"); + auto miss = router.Lookup("GET", "/v1/missing", 1700000000); + CHECK(!miss.found && !miss.gone); + auto wrong_method = router.Lookup("DELETE", "/v1/users", 1700000000); + CHECK(!wrong_method.found && !wrong_method.gone); + + // Sunset match reports gone so the caller answers 410, not 404. + router.SetSunset(1, 1800000000); + auto live = router.Lookup("GET", "/v1/users", 1750000000); + CHECK(live.found && !live.gone); + auto gone = router.Lookup("GET", "/v1/users", 1850000000); + CHECK(!gone.found && gone.gone && gone.major == 1); + + // At most 3 live majors. + CHECK(router.Add("GET", "/v2/users", "v2_users")); + CHECK(router.Add("GET", "/v3/users", "v3_users")); + CHECK(!router.Add("GET", "/v4/users", "v4_users")); + + router.Freeze(); + CHECK(!router.Add("GET", "/v1/frozen", "frozen")); + + if (g_fail == 0) std::printf("PASS versions_test\n"); + return g_fail ? 1 : 0; +} diff --git a/tests/src/observability/correlation_test.cpp b/tests/src/observability/correlation_test.cpp new file mode 100644 index 0000000..bee24fd --- /dev/null +++ b/tests/src/observability/correlation_test.cpp @@ -0,0 +1,140 @@ +// Correlation: request ID generation and validation, RequestContext +// flow into logs, events and hooks (mirrors X-Request-ID behavior). + +#include +#include + +#include "observability/correlation/redaction.h" +#include "observability/correlation/request_context.h" +#include "observability/correlation/request_id.h" +#include "observability/events/event.h" +#include "observability/hooks.h" +#include "observability/logging/logger.h" +#include "observability/logging/registry.h" +#include "observability/logging/ring_sink.h" +#include "observability/tracing/ids.h" +#include "observability/tracing/span_context.h" +#include "test_helper.h" + +int main() { + TestCtx ctx; + + // Request IDs are unique 128-bit hex strings. + { + std::set seen; + for (int i = 0; i < 1000; ++i) { + std::string id = molo::corr::generate_request_id(); + CHECK(ctx, id.size() == 32); + seen.insert(id); + } + CHECK(ctx, seen.size() == 1000); + } + + // Validation and inbound normalization. + CHECK(ctx, molo::corr::is_valid_request_id("abc-123_XYZ")); + CHECK(ctx, !molo::corr::is_valid_request_id("")); + CHECK(ctx, !molo::corr::is_valid_request_id("has space")); + CHECK(ctx, !molo::corr::is_valid_request_id(std::string(65, 'a'))); + CHECK(ctx, molo::corr::normalize_request_id("good-id") == "good-id"); + CHECK(ctx, molo::corr::is_valid_request_id(molo::corr::normalize_request_id("bad id!"))); + + // RequestContext defaults and span attachment. + { + molo::corr::RequestContext fresh; + CHECK(ctx, molo::corr::is_valid_request_id(fresh.request_id())); + CHECK(ctx, !fresh.span_context().is_valid()); + molo::corr::RequestContext named("req-ctx-1"); + CHECK(ctx, named.request_id() == "req-ctx-1"); + molo::corr::RequestContext bad("bad id!"); + CHECK(ctx, bad.request_id() != "bad id!"); + + molo::trace::SpanContext span; + span.trace_id = molo::trace::TraceId::generate(); + span.span_id = molo::trace::SpanId::generate(); + named.set_span_context(span); + CHECK(ctx, named.span_context().trace_id == span.trace_id); + molo::corr::RequestContext grown = named.with_span(molo::trace::SpanContext::invalid()); + CHECK(ctx, grown.request_id() == "req-ctx-1"); + CHECK(ctx, !grown.span_context().is_valid()); + } + + // Context flows into log records. + { + molo::log::Registry registry; + auto ring = std::make_shared(16); + registry.add_sink(ring); + auto logger = registry.get("molo.http"); + molo::corr::RequestContext req("req-log-1"); + molo::trace::SpanContext span; + span.trace_id = molo::trace::TraceId::generate(); + span.span_id = molo::trace::SpanId::generate(); + req.set_span_context(span); + logger->set_request_id(req.request_id()); + logger->set_span(span.trace_id.to_hex(), span.span_id.to_hex()); + logger->log(molo::log::Level::Info, "request.finished"); + auto records = ring->snapshot(); + CHECK(ctx, records.size() == 1); + CHECK(ctx, records[0].request_id == "req-log-1"); + CHECK(ctx, records[0].trace_id == span.trace_id.to_hex()); + CHECK(ctx, records[0].span_id == span.span_id.to_hex()); + } + + // Context flows into events. + { + molo::corr::RequestContext req("req-evt-1"); + molo::trace::SpanContext span; + span.trace_id = molo::trace::TraceId::generate(); + span.span_id = molo::trace::SpanId::generate(); + req.set_span_context(span); + molo::events::Event event = molo::events::Event::from_context("custom", req); + CHECK(ctx, event.request_id() == "req-evt-1"); + CHECK(ctx, event.trace_id() == span.trace_id.to_hex()); + CHECK(ctx, event.span_id() == span.span_id.to_hex()); + } + + // Redaction helper behavior. + CHECK(ctx, molo::corr::is_sensitive_key("password")); + CHECK(ctx, molo::corr::is_sensitive_key("X-Auth-Token")); + CHECK(ctx, molo::corr::is_sensitive_key("session_state")); + CHECK(ctx, molo::corr::is_sensitive_key("request_body")); + CHECK(ctx, !molo::corr::is_sensitive_key("table")); + CHECK(ctx, !molo::corr::is_sensitive_key("monkey")); + CHECK(ctx, !molo::corr::is_sensitive_key("keyboard")); + CHECK(ctx, molo::corr::is_sensitive_key("api_key")); + CHECK(ctx, molo::corr::is_sensitive_key("key")); + CHECK(ctx, molo::corr::is_sensitive_key("key-id")); + CHECK(ctx, molo::corr::is_sensitive_key("cacheKey")); + CHECK(ctx, molo::corr::redact_value("token", "abc") == "***"); + CHECK(ctx, molo::corr::redact_value("table", "users") == "users"); + + // Lifecycle hooks are safe unset and fire when set. + { + molo::corr::RequestContext req("req-hook-1"); + molo::obs::fire_server_start(); + molo::obs::fire_server_stop(); + molo::obs::fire_request_start(req); + molo::obs::fire_request_finish(req); + int starts = 0; + int finishes = 0; + int server_starts = 0; + molo::obs::LifecycleHooks::global().on_server_start = [&server_starts] { ++server_starts; }; + molo::obs::LifecycleHooks::global().on_request_start = + [&](const molo::corr::RequestContext& context) { + ++starts; + CHECK(ctx, context.request_id() == "req-hook-1"); + }; + molo::obs::LifecycleHooks::global().on_request_finish = + [&](const molo::corr::RequestContext&) { ++finishes; }; + molo::obs::fire_server_start(); + molo::obs::fire_request_start(req); + molo::obs::fire_request_finish(req); + CHECK(ctx, server_starts == 1); + CHECK(ctx, starts == 1); + CHECK(ctx, finishes == 1); + molo::obs::LifecycleHooks::global() = molo::obs::LifecycleHooks{}; + molo::obs::fire_server_start(); + CHECK(ctx, server_starts == 1); + } + + return test_summary(ctx, "correlation"); +} diff --git a/tests/src/observability/events_test.cpp b/tests/src/observability/events_test.cpp new file mode 100644 index 0000000..7c27e1d --- /dev/null +++ b/tests/src/observability/events_test.cpp @@ -0,0 +1,120 @@ +// Events pillar: dispatch, zero-listener fast path, async delivery, +// lifecycle event set, pubsub bridge hook. + +#include +#include +#include +#include +#include +#include + +#include "observability/correlation/request_context.h" +#include "observability/events/dispatcher.h" +#include "observability/events/event.h" +#include "observability/events/lifecycle_events.h" +#include "test_helper.h" + +int main() { + TestCtx ctx; + molo::events::Dispatcher dispatcher; + + // Sync dispatch to multiple listeners. + int calls_a = 0; + int calls_b = 0; + std::uint64_t id_a = dispatcher.on("request.started", [&](const molo::events::Event& event) { + ++calls_a; + CHECK(ctx, event.type() == "request.started"); + }); + dispatcher.on("request.started", [&](const molo::events::Event&) { ++calls_b; }); + CHECK(ctx, dispatcher.has_listeners("request.started")); + CHECK(ctx, !dispatcher.has_listeners("nope")); + molo::corr::RequestContext req("req-events-1"); + molo::events::RequestStartedEvent started("GET", "/x", req); + dispatcher.emit(started); + CHECK(ctx, calls_a == 1); + CHECK(ctx, calls_b == 1); + CHECK(ctx, dispatcher.emitted_total() == 1); + + // Removal. + CHECK(ctx, dispatcher.off(id_a)); + CHECK(ctx, !dispatcher.off(id_a)); + dispatcher.emit(started); + CHECK(ctx, calls_a == 1); + CHECK(ctx, calls_b == 2); + + // Zero-listener fast path: emits without listeners are cheap and safe. + { + molo::events::Event lonely("nothing.listens"); + auto begin = std::chrono::steady_clock::now(); + const int iters = 100000; + for (int i = 0; i < iters; ++i) dispatcher.emit(lonely); + auto end = std::chrono::steady_clock::now(); + double ns = + std::chrono::duration(end - begin).count() / static_cast(iters); + std::printf("info: zero-listener dispatch %.1f ns/call\n", ns); + CHECK(ctx, ns < 5000.0); + } + + // Publisher bridge into the pubsub core. + int published = 0; + dispatcher.set_publisher([&](const molo::events::Event&) { ++published; }); + dispatcher.emit(started); + CHECK(ctx, published == 1); + dispatcher.set_publisher(nullptr); + dispatcher.emit(started); + CHECK(ctx, published == 1); + + // Async delivery on the background worker. + dispatcher.start(); + std::atomic async_calls{0}; + std::string seen_task; + dispatcher.on_async("task.finished", [&](std::shared_ptr event) { + async_calls.fetch_add(1); + auto typed = std::dynamic_pointer_cast(event); + if (typed) seen_task = typed->task; + }); + molo::events::TaskFinishedEvent done("backup", "t-1", 42); + dispatcher.emit(done); + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (async_calls.load() == 0 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + CHECK(ctx, async_calls.load() == 1); + CHECK(ctx, seen_task == "backup"); + CHECK(ctx, dispatcher.dropped_async() == 0); + + // Async drops when the worker was never started. + { + molo::events::Dispatcher unstated; + unstated.on_async("x", [](std::shared_ptr) {}); + molo::events::Event event("x"); + unstated.emit(event); + CHECK(ctx, unstated.dropped_async() == 1); + } + dispatcher.stop(); + + // Lifecycle event set: types and clone without slicing. + CHECK(ctx, std::string(molo::events::ServerStartedEvent::kType) == "server.started"); + CHECK(ctx, std::string(molo::events::ServerStoppedEvent::kType) == "server.stopped"); + CHECK(ctx, std::string(molo::events::ConnectionOpenedEvent::kType) == "connection.opened"); + CHECK(ctx, std::string(molo::events::ConnectionClosedEvent::kType) == "connection.closed"); + CHECK(ctx, std::string(molo::events::RequestStartedEvent::kType) == "request.started"); + CHECK(ctx, std::string(molo::events::RequestFinishedEvent::kType) == "request.finished"); + CHECK(ctx, std::string(molo::events::TaskEnqueuedEvent::kType) == "task.enqueued"); + CHECK(ctx, std::string(molo::events::TaskStartedEvent::kType) == "task.started"); + CHECK(ctx, std::string(molo::events::TaskFinishedEvent::kType) == "task.finished"); + CHECK(ctx, std::string(molo::events::TaskFailedEvent::kType) == "task.failed"); + + molo::events::RequestFinishedEvent finished("POST", "/items", 201, 12, req); + CHECK(ctx, finished.request_id() == "req-events-1"); + CHECK(ctx, finished.status == 201); + auto cloned = finished.clone(); + auto typed = std::dynamic_pointer_cast(cloned); + CHECK(ctx, typed != nullptr); + CHECK(ctx, typed->route == "/items"); + + molo::events::TaskFailedEvent failed("backup", "t-2", "timeout"); + CHECK(ctx, failed.error_kind == "timeout"); + + return test_summary(ctx, "events"); +} diff --git a/tests/src/observability/logging_test.cpp b/tests/src/observability/logging_test.cpp new file mode 100644 index 0000000..5af5d48 --- /dev/null +++ b/tests/src/observability/logging_test.cpp @@ -0,0 +1,208 @@ +// Logging pillar: levels, registry, loggers, sinks, JSON schema, macros, +// redaction, truncation, async queue drops, file rotation. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "observability/logging/async_sink.h" +#include "observability/logging/console_sink.h" +#include "observability/logging/file_sink.h" +#include "observability/logging/json_format.h" +#include "observability/logging/level.h" +#include "observability/logging/logger.h" +#include "observability/logging/macros.h" +#include "observability/logging/null_sink.h" +#include "observability/logging/registry.h" +#include "observability/logging/ring_sink.h" +#include "test_helper.h" + +namespace { + +bool contains(const std::string& haystack, const std::string& needle) { + return haystack.find(needle) != std::string::npos; +} + +} // namespace + +int main() { + TestCtx ctx; + + // Levels. + CHECK(ctx, molo::log::to_string(molo::log::Level::Info) == "info"); + CHECK(ctx, molo::log::from_string("warn") == molo::log::Level::Warn); + CHECK(ctx, molo::log::from_string("warning") == molo::log::Level::Warn); + CHECK(ctx, !molo::log::from_string("bogus").has_value()); + + // Registry and named loggers. + molo::log::Registry registry; + auto ring = std::make_shared(64); + registry.add_sink(ring); + registry.add_sink(std::make_shared()); + CHECK(ctx, registry.sink_count() == 2); + auto logger = registry.get("molo.http"); + auto same = registry.get("molo.http"); + CHECK(ctx, logger.get() == same.get()); + CHECK(ctx, logger->name() == "molo.http"); + + // Basic record flow with correlation context. + logger->set_request_id("req-1"); + logger->set_span("trace-1", "span-1"); + MOLO_LOG_INFO(*logger, "request.finished"); + auto records = ring->snapshot(); + CHECK(ctx, records.size() == 1); + CHECK(ctx, records[0].logger == "molo.http"); + CHECK(ctx, records[0].level == molo::log::Level::Info); + CHECK(ctx, records[0].message == "request.finished"); + CHECK(ctx, records[0].request_id == "req-1"); + CHECK(ctx, records[0].trace_id == "trace-1"); + CHECK(ctx, records[0].span_id == "span-1"); + CHECK(ctx, !records[0].file.empty()); + CHECK(ctx, records[0].line > 0); + CHECK(ctx, registry.records_total(molo::log::Level::Info) == 1); + + // Structured fields over string interpolation. + std::vector fields = {{"table", std::string("users")}, + {"rows", std::int64_t(3)}, + {"ratio", 0.5}, + {"cached", true}}; + MOLO_LOG_INFO(logger, "db.query", fields); + records = ring->snapshot(); + CHECK(ctx, records.size() == 2); + CHECK(ctx, records[1].fields.size() == 4); + + // JSON schema shape. + std::string json = molo::log::to_json(records[1]); + CHECK(ctx, contains(json, "\"level\":\"info\"")); + CHECK(ctx, contains(json, "\"logger\":\"molo.http\"")); + CHECK(ctx, contains(json, "\"msg\":\"db.query\"")); + CHECK(ctx, contains(json, "\"request_id\":\"req-1\"")); + CHECK(ctx, contains(json, "\"trace_id\":\"trace-1\"")); + CHECK(ctx, contains(json, "\"span_id\":\"span-1\"")); + CHECK(ctx, contains(json, "\"table\":\"users\"")); + CHECK(ctx, contains(json, "\"rows\":3")); + CHECK(ctx, contains(json, "\"cached\":true")); + CHECK(ctx, contains(json, "\"ts\":\"")); + CHECK(ctx, contains(json, "Z\"")); + + // Redaction of secrets in fields. + ring->clear(); + MOLO_LOG_INFO(*logger, "auth.login", + std::vector{{"user", std::string("u")}, + {"password", std::string("hunter2")}, + {"token", std::string("abc")}, + {"session_state", std::string("x")}}); + records = ring->snapshot(); + CHECK(ctx, records.size() == 1); + std::string redacted = molo::log::to_json(records[0]); + CHECK(ctx, contains(redacted, "\"password\":\"***\"")); + CHECK(ctx, contains(redacted, "\"token\":\"***\"")); + CHECK(ctx, contains(redacted, "\"session_state\":\"***\"")); + CHECK(ctx, !contains(redacted, "hunter2")); + + // Truncation of oversized lines. + ring->clear(); + std::string big(9000, 'x'); + MOLO_LOG_INFO(*logger, big); + records = ring->snapshot(); + CHECK(ctx, records.size() == 1); + CHECK(ctx, records[0].truncated); + CHECK(ctx, records[0].message.size() == molo::obs::Caps::kMaxLogLine); + CHECK(ctx, contains(molo::log::to_json(records[0]), "\"truncated\":true")); + + // Runtime level gating. + ring->clear(); + registry.set_global_level(molo::log::Level::Error); + MOLO_LOG_INFO(*logger, "suppressed"); + MOLO_LOG_ERROR(*logger, "kept"); + records = ring->snapshot(); + CHECK(ctx, records.size() == 1); + CHECK(ctx, records[0].message == "kept"); + registry.set_global_level(molo::log::Level::Debug); + + // Child loggers inherit context. + auto kid = logger->child("router"); + CHECK(ctx, kid->name() == "molo.http.router"); + CHECK(ctx, kid->request_id() == "req-1"); + + // Async sink delivers without blocking and drops under saturation. + { + auto inner = std::make_shared(4096); + molo::log::AsyncSink async(inner, 1024); + for (int i = 0; i < 200; ++i) { + molo::log::Record record; + record.logger = "async"; + record.message = "m" + std::to_string(i); + async.write(record); + } + async.flush(); + CHECK(ctx, inner->size() == 200); + CHECK(ctx, async.dropped() == 0); + } + { + molo::log::AsyncQueue queue([](const molo::log::Record&) {}, 8); + queue.stop(); + for (int i = 0; i < 100; ++i) { + molo::log::Record record; + CHECK(ctx, !queue.try_push(record)); + } + CHECK(ctx, queue.dropped() == 100); + } + + // Flood through a live small queue: bounded memory, drops counted. + { + std::atomic slow{0}; + molo::log::AsyncQueue queue( + [&slow](const molo::log::Record&) { + slow.fetch_add(1); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + }, + 8); + int accepted = 0; + for (int i = 0; i < 200; ++i) { + molo::log::Record record; + if (queue.try_push(record)) ++accepted; + } + queue.stop(); + CHECK(ctx, queue.dropped() > 0); + CHECK(ctx, accepted + static_cast(queue.dropped()) == 200); + } + + // File sink rotation. + { + std::filesystem::create_directories("build"); + std::string path = "build/obs_file_sink_test.log"; + std::remove(path.c_str()); + molo::log::FileSink sink(path, 1024, 3); + for (int i = 0; i < 100; ++i) { + molo::log::Record record; + record.logger = "file"; + record.message = "line-" + std::to_string(i) + "-" + std::string(60, 'y'); + sink.write(record); + } + sink.flush(); + CHECK(ctx, std::filesystem::exists(path)); + bool rotated = std::filesystem::exists(path + ".1") || std::filesystem::exists(path + ".2"); + CHECK(ctx, rotated); + } + + // Console sink does not throw. + { + molo::log::ConsoleSink text(false); + molo::log::ConsoleSink as_json(true); + molo::log::Record record; + record.logger = "console"; + record.message = "hello"; + text.write(record); + as_json.write(record); + text.flush(); + CHECK(ctx, true); + } + + return test_summary(ctx, "logging"); +} diff --git a/tests/src/observability/metrics_endpoint_test.cpp b/tests/src/observability/metrics_endpoint_test.cpp new file mode 100644 index 0000000..60a3291 --- /dev/null +++ b/tests/src/observability/metrics_endpoint_test.cpp @@ -0,0 +1,203 @@ +// Metrics pillar, mirroring GET /metrics: instruments, cardinality caps, +// series caps, Prometheus exposition shape, framework metrics, render cost. + +#include +#include +#include + +#include "observability/metrics/counter.h" +#include "observability/metrics/gauge.h" +#include "observability/metrics/histogram.h" +#include "observability/metrics/labels.h" +#include "observability/metrics/prometheus.h" +#include "observability/metrics/registry.h" +#include "observability/metrics/timer.h" +#include "test_helper.h" + +namespace { + +bool contains(const std::string& haystack, const std::string& needle) { + return haystack.find(needle) != std::string::npos; +} + +} // namespace + +int main() { + TestCtx ctx; + + // Counter basics and automatic _total suffix. + { + molo::metrics::Registry registry; + auto& counter = registry.counter("molo_http_requests", "Requests"); + CHECK(ctx, counter.name() == "molo_http_requests_total"); + counter.inc(); + counter.inc(2.0); + counter.inc(-5.0); + CHECK(ctx, counter.value() == 3.0); + auto& same = registry.counter("molo_http_requests_total", "Requests"); + CHECK(ctx, &same == &counter); + bool threw = false; + try { + registry.counter("bad name!", "x"); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(ctx, threw); + } + + // Gauge up and down. + { + molo::metrics::Gauge gauge("g", "h"); + gauge.set(2.0); + gauge.inc(); + gauge.dec(0.5); + CHECK(ctx, gauge.value() == 2.5); + } + + // Labels: caps and truncation. + { + molo::metrics::Labels labels; + labels.add("method", "GET").add("route", "/items"); + CHECK(ctx, labels.size() == 2); + molo::metrics::Labels fat; + for (int i = 0; i < 16; ++i) fat.add("k" + std::to_string(i), "v"); + bool threw = false; + try { + fat.add("overflow", "v"); + } catch (const std::invalid_argument&) { + threw = true; + } + CHECK(ctx, threw); + molo::metrics::Labels trunc; + trunc.add("k", std::string(200, 'a')); + CHECK(ctx, trunc.items()[0].second.size() == molo::obs::Caps::kMaxLabelValueLen); + } + + // Cardinality cap: 65th distinct value maps to "other" and counts. + { + molo::metrics::Registry registry; + for (int i = 0; i < 70; ++i) { + molo::metrics::Labels labels; + labels.add("route", "/r" + std::to_string(i)); + registry.counter("molo_card", "card", labels).inc(); + } + CHECK(ctx, registry.cardinality_overflows() == 6); + std::string body = molo::metrics::render(registry); + CHECK(ctx, contains(body, "molo_card_total{route=\"other\"} 6")); + CHECK(ctx, contains(body, "molo_metrics_cardinality_overflows_total 6")); + } + + // Series cap: 65536 series max, excess hits the blackhole and counts. + { + molo::metrics::Registry registry; + for (int i = 0; i < 65537; ++i) { + registry.gauge("fill_" + std::to_string(i), "filler").set(1.0); + } + CHECK(ctx, registry.series_count() == 65536); + CHECK(ctx, registry.cardinality_overflows() >= 1); + std::string body = molo::metrics::render(registry); + CHECK(ctx, contains(body, "fill_0 1")); + CHECK(ctx, !contains(body, "fill_65536")); + } + + // Histogram cumulative buckets, sum and count. + { + molo::metrics::Registry registry; + molo::metrics::Labels labels; + labels.add("method", "GET"); + auto& hist = registry.histogram("molo_http_request_duration_seconds", "Latency", labels, + {0.1, 0.5, 1.0}); + hist.observe(0.05); + hist.observe(0.3); + hist.observe(5.0); + auto snap = hist.snapshot(); + CHECK(ctx, snap.count == 3); + CHECK(ctx, snap.cumulative.size() == 3); + CHECK(ctx, snap.cumulative[0] == 1); + CHECK(ctx, snap.cumulative[1] == 2); + CHECK(ctx, snap.cumulative[2] == 2); + CHECK(ctx, snap.sum > 5.3 && snap.sum < 5.4); + std::string body = molo::metrics::render(registry); + CHECK(ctx, contains(body, "# TYPE molo_http_request_duration_seconds histogram")); + CHECK(ctx, contains(body, "molo_http_request_duration_seconds_bucket{method=\"GET\",le=\"0.1\"} 1")); + CHECK(ctx, contains(body, "molo_http_request_duration_seconds_bucket{method=\"GET\",le=\"+Inf\"} 3")); + CHECK(ctx, contains(body, "molo_http_request_duration_seconds_sum{method=\"GET\"}")); + CHECK(ctx, contains(body, "molo_http_request_duration_seconds_count{method=\"GET\"} 3")); + } + + // Render shape and escaping. + { + molo::metrics::Registry registry; + molo::metrics::Labels labels; + labels.add("route", "/a\"b\\c\nd"); + registry.counter("molo_esc", "help\nline\\x", labels).inc(3.0); + std::string body = molo::metrics::render(registry); + CHECK(ctx, contains(body, "# HELP molo_esc_total help\\nline\\\\x")); + CHECK(ctx, contains(body, "# TYPE molo_esc_total counter")); + CHECK(ctx, contains(body, "molo_esc_total{route=\"/a\\\"b\\\\c\\nd\"} 3")); + CHECK(ctx, molo::metrics::format_sample_double(0.5) == "0.5"); + CHECK(ctx, molo::metrics::format_sample_double(3.0) == "3"); + CHECK(ctx, molo::metrics::format_sample_double(1.0 / 0.0) == "+Inf"); + } + + // Framework metrics shipped by default. + { + molo::metrics::Registry registry; + registry.ensure_framework_metrics(); + std::string body = molo::metrics::render(registry); + CHECK(ctx, contains(body, "molo_http_requests_in_flight 0")); + CHECK(ctx, contains(body, "molo_log_records_total{level=\"info\"} 0")); + CHECK(ctx, contains(body, "molo_log_records_total{level=\"critical\"} 0")); + CHECK(ctx, contains(body, "molo_spans_total{kind=\"server\"} 0")); + CHECK(ctx, contains(body, "molo_log_dropped_total 0")); + CHECK(ctx, contains(body, "molo_spans_dropped_total 0")); + CHECK(ctx, contains(body, "molo_trace_propagator_errors_total 0")); + CHECK(ctx, contains(body, "molo_metrics_cardinality_overflows_total 0")); + } + + // RAII timer records, cancel skips. + { + molo::metrics::Histogram hist("t", "h", molo::metrics::Labels{}, {1.0, 10.0}); + { + molo::metrics::Timer timer(hist); + } + CHECK(ctx, hist.snapshot().count == 1); + { + molo::metrics::Timer timer(hist); + timer.cancel(); + } + CHECK(ctx, hist.snapshot().count == 1); + { + MOLO_TIMER(hist); + } + CHECK(ctx, hist.snapshot().count == 2); + } + + // Runtime toggle disables updates with one branch. + { + molo::metrics::Counter counter("rt_total", "h"); + counter.inc(); + molo::obs::RuntimeToggles::instance().metrics.store(false); + counter.inc(); + molo::obs::RuntimeToggles::instance().metrics.store(true); + counter.inc(); + CHECK(ctx, counter.value() == 2.0); + } + + // Render cost: full exposition of 10k series well under the 100 ms budget. + { + molo::metrics::Registry registry; + for (int i = 0; i < 10000; ++i) { + registry.gauge("perf_gauge_" + std::to_string(i), "perf").set(static_cast(i)); + } + auto begin = std::chrono::steady_clock::now(); + std::string body = molo::metrics::render(registry); + auto end = std::chrono::steady_clock::now(); + double ms = std::chrono::duration(end - begin).count(); + std::printf("info: render 10k series %.1f ms (%zu bytes)\n", ms, body.size()); + CHECK(ctx, !body.empty()); + CHECK(ctx, ms < 1000.0); + } + + return test_summary(ctx, "metrics-endpoint"); +} diff --git a/tests/src/observability/otlp_export_test.cpp b/tests/src/observability/otlp_export_test.cpp new file mode 100644 index 0000000..45aa06a --- /dev/null +++ b/tests/src/observability/otlp_export_test.cpp @@ -0,0 +1,217 @@ +// Export path: OTLP/HTTP JSON shape, retry with backoff, batch processor +// behavior and bounds. + +#include +#include +#include +#include +#include + +#include "observability/tracing/batch_processor.h" +#include "observability/tracing/exporter.h" +#include "observability/tracing/http_sender.h" +#include "observability/tracing/ids.h" +#include "observability/tracing/memory_exporter.h" +#include "observability/tracing/otlp_http.h" +#include "observability/tracing/span.h" +#include "test_helper.h" + +namespace { + +bool contains(const std::string& haystack, const std::string& needle) { + return haystack.find(needle) != std::string::npos; +} + +molo::trace::FinishedSpan make_span(const std::string& name, bool with_parent) { + molo::trace::FinishedSpan span; + span.name = name; + span.kind = molo::trace::SpanKind::Server; + span.context.trace_id = molo::trace::TraceId::generate(); + span.context.span_id = molo::trace::SpanId::generate(); + span.context.flags.value = molo::trace::TraceFlags::kSampled; + if (with_parent) span.parent_span_id = molo::trace::SpanId::generate(); + span.start = std::chrono::system_clock::now(); + span.end = span.start + std::chrono::milliseconds(5); + span.attributes.push_back({"http.method", std::string("GET")}); + span.attributes.push_back({"http.status", std::int64_t(200)}); + span.attributes.push_back({"ratio", 0.5}); + span.attributes.push_back({"cached", true}); + molo::trace::SpanEvent event; + event.timestamp = span.start; + event.name = "db.start"; + span.events.push_back(event); + span.status = molo::trace::StatusCode::Ok; + return span; +} + +} // namespace + +int main() { + TestCtx ctx; + + // OTLP JSON document shape. + { + std::vector batch; + batch.push_back(make_span("GET /root", false)); + batch.push_back(make_span("db.query", true)); + std::string body = molo::trace::OtlpHttpExporter::build_request_json(batch, "molo"); + CHECK(ctx, contains(body, "\"resourceSpans\":[{")); + CHECK(ctx, contains(body, "\"scope\":{\"name\":\"molo\"}")); + CHECK(ctx, contains(body, "\"traceId\":\"" + batch[0].context.trace_id.to_hex() + "\"")); + CHECK(ctx, contains(body, "\"spanId\":\"" + batch[0].context.span_id.to_hex() + "\"")); + CHECK(ctx, contains(body, "\"parentSpanId\":\"" + batch[1].parent_span_id.to_hex() + "\"")); + CHECK(ctx, contains(body, "\"name\":\"GET /root\"")); + CHECK(ctx, contains(body, "\"kind\":2")); + CHECK(ctx, contains(body, "\"startTimeUnixNano\":\"")); + CHECK(ctx, contains(body, "\"key\":\"http.method\",\"value\":{\"stringValue\":\"GET\"}")); + CHECK(ctx, contains(body, "\"key\":\"http.status\",\"value\":{\"intValue\":\"200\"}")); + CHECK(ctx, contains(body, "\"doubleValue\":0.5")); + CHECK(ctx, contains(body, "\"boolValue\":true")); + CHECK(ctx, contains(body, "\"status\":{\"code\":1}")); + CHECK(ctx, contains(body, "\"name\":\"db.start\"")); + // Root span carries no parentSpanId field. + std::size_t first_span = body.find("\"name\":\"GET /root\""); + std::size_t parent_key = body.find("\"parentSpanId\""); + CHECK(ctx, first_span != std::string::npos && parent_key != std::string::npos); + CHECK(ctx, parent_key > first_span); + } + + // Successful export posts JSON with the right headers. + { + std::string seen_url; + std::string seen_body; + std::string seen_content_type; + auto sender = std::make_shared( + [&](const std::string& url, const molo::trace::HttpSender::HeaderList& headers, + const std::string& body, std::chrono::seconds timeout) { + seen_url = url; + seen_body = body; + for (const auto& [key, value] : headers) { + if (key == "Content-Type") seen_content_type = value; + } + CHECK(ctx, timeout == std::chrono::seconds(10)); + return true; + }); + molo::trace::OtlpHttpExporter exporter("http://collector:4318/v1/traces", sender); + std::vector batch; + batch.push_back(make_span("GET /ok", false)); + CHECK(ctx, exporter.export_spans(batch)); + CHECK(ctx, seen_url == "http://collector:4318/v1/traces"); + CHECK(ctx, seen_content_type == "application/json"); + CHECK(ctx, contains(seen_body, "\"resourceSpans\"")); + CHECK(ctx, exporter.requests_total() == 1); + CHECK(ctx, exporter.requests_failed() == 0); + CHECK(ctx, exporter.name() == "otlp-http"); + } + + // Retry: two failures then success. + { + int calls = 0; + auto sender = std::make_shared( + [&](const std::string&, const molo::trace::HttpSender::HeaderList&, + const std::string&, std::chrono::seconds) { return ++calls >= 3; }); + molo::trace::OtlpHttpExporter exporter( + "http://collector:4318/v1/traces", sender, + {std::chrono::milliseconds(0), std::chrono::milliseconds(0), + std::chrono::milliseconds(0)}); + std::vector batch; + batch.push_back(make_span("GET /retry", false)); + CHECK(ctx, exporter.export_spans(batch)); + CHECK(ctx, calls == 3); + CHECK(ctx, exporter.requests_total() == 3); + CHECK(ctx, exporter.requests_failed() == 2); + } + + // Persistent failure: first attempt plus 3 retries, then give up. + { + int calls = 0; + auto sender = std::make_shared( + [&](const std::string&, const molo::trace::HttpSender::HeaderList&, + const std::string&, std::chrono::seconds) { + ++calls; + return false; + }); + molo::trace::OtlpHttpExporter exporter( + "http://collector:4318/v1/traces", sender, + {std::chrono::milliseconds(0), std::chrono::milliseconds(0), + std::chrono::milliseconds(0)}); + std::vector batch; + batch.push_back(make_span("GET /fail", false)); + CHECK(ctx, !exporter.export_spans(batch)); + CHECK(ctx, calls == 4); + } + + // Empty batches are free. + { + auto sender = std::make_shared( + [](const std::string&, const molo::trace::HttpSender::HeaderList&, + const std::string&, std::chrono::seconds) { return false; }); + molo::trace::OtlpHttpExporter exporter("http://x", sender); + CHECK(ctx, exporter.export_spans({})); + CHECK(ctx, exporter.requests_total() == 0); + } + + // The marked stub drops and reports failure. + { + molo::trace::NullHttpSender stub; + molo::trace::OtlpHttpExporter exporter( + "http://x", std::make_shared()); + std::vector batch; + batch.push_back(make_span("GET /stub", false)); + CHECK(ctx, !exporter.export_spans(batch)); + CHECK(ctx, !stub.post("http://x", {}, "{}", std::chrono::seconds(10))); + CHECK(ctx, stub.attempts() == 1); + } + + // gzip without an injected codec sends identity (documented fallback). + { + std::string encoding; + auto sender = std::make_shared( + [&](const std::string&, const molo::trace::HttpSender::HeaderList& headers, + const std::string&, std::chrono::seconds) { + for (const auto& [key, value] : headers) { + if (key == "Content-Encoding") encoding = value; + } + return true; + }); + molo::trace::OtlpHttpExporter exporter("http://x", sender, + {std::chrono::milliseconds(0)}, std::chrono::seconds(10), + true); + std::vector batch; + batch.push_back(make_span("GET /gzip", false)); + CHECK(ctx, exporter.export_spans(batch)); + CHECK(ctx, encoding.empty()); + exporter.set_compressor([](const std::string& input, std::string& output) { + output = "COMPRESSED:" + input; + return true; + }); + CHECK(ctx, exporter.export_spans(batch)); + CHECK(ctx, encoding == "gzip"); + } + + // Batch processor: batches of 512 or 5 s, bounded buffer with drops. + { + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter); + for (int i = 0; i < 600; ++i) processor.add(make_span("s" + std::to_string(i), false)); + processor.flush(); + CHECK(ctx, exporter->spans().size() == 600); + CHECK(ctx, processor.dropped() == 0); + CHECK(ctx, processor.exported_total() >= 1); + CHECK(ctx, processor.failed_batches() == 0); + } + { + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter, 8, 1000000, std::chrono::seconds(3600)); + for (int i = 0; i < 20; ++i) processor.add(make_span("o" + std::to_string(i), false)); + CHECK(ctx, processor.dropped() == 12); + } + { + auto exporter = std::make_shared(); + CHECK(ctx, exporter->name() == "memory"); + exporter->clear(); + CHECK(ctx, exporter->spans().empty()); + } + + return test_summary(ctx, "otlp-export"); +} diff --git a/tests/src/observability/propagation_test.cpp b/tests/src/observability/propagation_test.cpp new file mode 100644 index 0000000..ddc30d4 --- /dev/null +++ b/tests/src/observability/propagation_test.cpp @@ -0,0 +1,108 @@ +// W3C propagation: spec vectors round trip, malformed headers start a +// new trace and count an error, outbound injection always emits context. + +#include + +#include "observability/config.h" +#include "observability/tracing/ids.h" +#include "observability/tracing/propagator.h" +#include "observability/tracing/span_context.h" +#include "test_helper.h" + +int main() { + TestCtx ctx; + + // Spec-style vector parses and round trips exactly. + { + molo::trace::Headers headers = { + {"traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}, + {"tracestate", "rojo=00f067aa0ba902b7,congo=t61rcWkgMzE"}}; + int errors = 0; + auto parsed = + molo::trace::W3CPropagator::extract(headers, [&errors] { ++errors; }); + CHECK(ctx, parsed.has_value()); + CHECK(ctx, errors == 0); + CHECK(ctx, parsed->trace_id.to_hex() == "4bf92f3577b34da6a3ce929d0e0e4736"); + CHECK(ctx, parsed->span_id.to_hex() == "00f067aa0ba902b7"); + CHECK(ctx, parsed->is_sampled()); + CHECK(ctx, parsed->tracestate == "rojo=00f067aa0ba902b7,congo=t61rcWkgMzE"); + + molo::trace::Headers outbound; + molo::trace::W3CPropagator::inject(*parsed, outbound); + CHECK(ctx, outbound["traceparent"] == + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"); + CHECK(ctx, outbound["tracestate"] == "rojo=00f067aa0ba902b7,congo=t61rcWkgMzE"); + } + + // Header names are case-insensitive. + { + molo::trace::Headers headers = { + {"TraceParent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00"}}; + auto parsed = molo::trace::W3CPropagator::extract(headers); + CHECK(ctx, parsed.has_value()); + CHECK(ctx, !parsed->is_sampled()); + } + + // Missing header means no context, and no error. + { + molo::trace::Headers headers; + int errors = 0; + auto parsed = molo::trace::W3CPropagator::extract(headers, [&errors] { ++errors; }); + CHECK(ctx, !parsed.has_value()); + CHECK(ctx, errors == 0); + } + + // Malformed headers fail and count exactly one error each. + const char* malformed[] = { + "bogus", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7", + "ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "00-00000000000000000000000000000000-00f067aa0ba902b7-01", + "00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-01", + "00-zzzzzf3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-zz", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-0100", + }; + for (const char* value : malformed) { + molo::trace::Headers headers = {{"traceparent", value}}; + int errors = 0; + auto parsed = molo::trace::W3CPropagator::extract(headers, [&errors] { ++errors; }); + CHECK(ctx, !parsed.has_value()); + CHECK(ctx, errors == 1); + } + + // Overlong tracestate is truncated to the cap. + { + molo::trace::Headers headers = { + {"traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}, + {"tracestate", std::string(2000, 's')}}; + auto parsed = molo::trace::W3CPropagator::extract(headers); + CHECK(ctx, parsed.has_value()); + CHECK(ctx, parsed->tracestate.size() == molo::obs::Caps::kMaxTracestate); + } + + // Invalid contexts are never injected. + { + molo::trace::Headers headers; + molo::trace::W3CPropagator::inject(molo::trace::SpanContext::invalid(), headers); + CHECK(ctx, headers.empty()); + } + + // Fresh contexts render valid version 00 headers. + { + molo::trace::SpanContext context; + context.trace_id = molo::trace::TraceId::generate(); + context.span_id = molo::trace::SpanId::generate(); + context.flags.value = 0; + std::string rendered = molo::trace::W3CPropagator::render_traceparent(context); + CHECK(ctx, rendered.size() == 55); + CHECK(ctx, rendered.substr(0, 3) == "00-"); + molo::trace::Headers headers = {{"traceparent", rendered}}; + auto back = molo::trace::W3CPropagator::extract(headers); + CHECK(ctx, back.has_value()); + CHECK(ctx, back->trace_id == context.trace_id); + CHECK(ctx, back->span_id == context.span_id); + } + + return test_summary(ctx, "propagation"); +} diff --git a/tests/src/observability/test_helper.h b/tests/src/observability/test_helper.h new file mode 100644 index 0000000..b13b148 --- /dev/null +++ b/tests/src/observability/test_helper.h @@ -0,0 +1,29 @@ +#pragma once + +// Minimal test helper. Tests print FAIL lines per broken check and a PASS +// line with counts, and return nonzero when any check fails. + +#include + +struct TestCtx { + int checks = 0; + int failures = 0; +}; + +#define CHECK(ctx, cond) \ + do { \ + ++(ctx).checks; \ + if (!(cond)) { \ + ++(ctx).failures; \ + std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +inline int test_summary(TestCtx& ctx, const char* name) { + if (ctx.failures == 0) { + std::printf("PASS %s (%d checks)\n", name, ctx.checks); + return 0; + } + std::printf("FAIL %s (%d of %d checks failed)\n", name, ctx.failures, ctx.checks); + return 1; +} diff --git a/tests/src/observability/tracing_test.cpp b/tests/src/observability/tracing_test.cpp new file mode 100644 index 0000000..2400118 --- /dev/null +++ b/tests/src/observability/tracing_test.cpp @@ -0,0 +1,234 @@ +// Tracing core: IDs, samplers, span lifecycle, attribute caps, +// null-span fast path, redaction. + +#include +#include +#include + +#include "observability/config.h" +#include "observability/tracing/batch_processor.h" +#include "observability/tracing/ids.h" +#include "observability/tracing/memory_exporter.h" +#include "observability/tracing/sampler.h" +#include "observability/tracing/span.h" +#include "observability/tracing/span_context.h" +#include "observability/tracing/tracer.h" +#include "test_helper.h" + +int main() { + TestCtx ctx; + + // IDs: generation, hex round trip, strict parsing. + { + auto trace = molo::trace::TraceId::generate(); + auto span = molo::trace::SpanId::generate(); + CHECK(ctx, trace.valid()); + CHECK(ctx, span.valid()); + CHECK(ctx, trace.to_hex().size() == 32); + CHECK(ctx, span.to_hex().size() == 16); + auto back = molo::trace::TraceId::from_hex(trace.to_hex()); + CHECK(ctx, back.has_value() && *back == trace); + auto back_span = molo::trace::SpanId::from_hex(span.to_hex()); + CHECK(ctx, back_span.has_value() && *back_span == span); + CHECK(ctx, !molo::trace::TraceId::from_hex("short").has_value()); + CHECK(ctx, !molo::trace::TraceId::from_hex(std::string(32, 'z')).has_value()); + CHECK(ctx, !molo::trace::SpanId::from_hex("short").has_value()); + auto zero = molo::trace::TraceId::from_hex("00000000000000000000000000000000"); + CHECK(ctx, zero.has_value() && !zero->valid()); + + auto flags = molo::trace::TraceFlags::from_hex("01"); + CHECK(ctx, flags.has_value() && flags->is_sampled()); + auto flags0 = molo::trace::TraceFlags::from_hex("00"); + CHECK(ctx, flags0.has_value() && !flags0->is_sampled()); + CHECK(ctx, !molo::trace::TraceFlags::from_hex("0").has_value()); + CHECK(ctx, !molo::trace::TraceFlags::from_hex("zz").has_value()); + + molo::trace::SpanContext invalid = molo::trace::SpanContext::invalid(); + CHECK(ctx, !invalid.is_valid()); + CHECK(ctx, !invalid.is_sampled()); + } + + // Samplers. + { + molo::trace::AlwaysOnSampler on; + molo::trace::AlwaysOffSampler off; + auto id = molo::trace::TraceId::generate(); + CHECK(ctx, on.should_sample(id)); + CHECK(ctx, !off.should_sample(id)); + CHECK(ctx, on.name() == "always_on"); + + molo::trace::RatioSampler half(0.5); + const int total = 100000; + int kept = 0; + for (int i = 0; i < total; ++i) { + if (half.should_sample(molo::trace::TraceId::generate())) ++kept; + } + double rate = static_cast(kept) / static_cast(total); + CHECK(ctx, rate > 0.45 && rate < 0.55); + auto fixed = molo::trace::TraceId::generate(); + CHECK(ctx, half.should_sample(fixed) == half.should_sample(fixed)); + molo::trace::RatioSampler none(0.0); + molo::trace::RatioSampler all(1.0); + CHECK(ctx, !none.should_sample(fixed)); + CHECK(ctx, all.should_sample(fixed)); + } + + // Span lifecycle with parent and child linkage. + { + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter); + molo::trace::Tracer tracer("test", std::make_shared(), + &processor); + { + molo::trace::Span parent = + tracer.start_span("GET /items/{id}", molo::trace::SpanKind::Server); + CHECK(ctx, parent.valid()); + parent.set_attribute("http.request.method", std::string("GET")); + parent.set_attribute("http.response.status_code", std::int64_t(200)); + parent.set_attribute("ratio", 0.25); + parent.set_attribute("cached", true); + parent.add_event("db.start"); + { + molo::trace::Span child = + tracer.start_span("db.query", molo::trace::SpanKind::Client, parent.context()); + CHECK(ctx, child.valid()); + CHECK(ctx, child.context().trace_id == parent.context().trace_id); + CHECK(ctx, child.context().span_id != parent.context().span_id); + child.set_status(molo::trace::StatusCode::Ok); + } + parent.set_status(molo::trace::StatusCode::Ok); + parent.end(); + parent.end(); + } + processor.flush(); + auto spans = exporter->spans(); + CHECK(ctx, spans.size() == 2); + const molo::trace::FinishedSpan* server = nullptr; + const molo::trace::FinishedSpan* client = nullptr; + for (const auto& span : spans) { + if (span.kind == molo::trace::SpanKind::Server) server = &span; + if (span.kind == molo::trace::SpanKind::Client) client = &span; + } + CHECK(ctx, server != nullptr && client != nullptr); + CHECK(ctx, server->name == "GET /items/{id}"); + CHECK(ctx, client->parent_span_id == server->context.span_id); + CHECK(ctx, !server->parent_span_id.valid()); + CHECK(ctx, server->attributes.size() == 4); + CHECK(ctx, server->events.size() == 1); + CHECK(ctx, server->status == molo::trace::StatusCode::Ok); + CHECK(ctx, server->end >= server->start); + CHECK(ctx, !server->attributes_truncated); + CHECK(ctx, tracer.spans_started() == 2); + } + + // Attribute and event caps with truncation flags. + { + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter); + molo::trace::Tracer tracer("caps", std::make_shared(), + &processor); + molo::trace::Span span = tracer.start_span("caps"); + for (int i = 0; i < 70; ++i) { + span.set_attribute("k" + std::to_string(i), std::int64_t(i)); + } + span.set_attribute("big", std::string(2000, 'v')); + for (int i = 0; i < 40; ++i) span.add_event("e" + std::to_string(i)); + span.end(); + processor.flush(); + auto spans = exporter->spans(); + CHECK(ctx, spans.size() == 1); + CHECK(ctx, spans[0].attributes.size() == 64); + CHECK(ctx, spans[0].attributes_truncated); + CHECK(ctx, spans[0].events.size() == 32); + CHECK(ctx, spans[0].events_truncated); + } + + // Attribute value truncation at 1024 bytes. + { + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter); + molo::trace::Tracer tracer("trunc", std::make_shared(), + &processor); + molo::trace::Span span = tracer.start_span("trunc"); + span.set_attribute("long", std::string(2000, 'w')); + span.end(); + processor.flush(); + auto spans = exporter->spans(); + CHECK(ctx, spans.size() == 1); + auto* text = std::get_if(&spans[0].attributes[0].value); + CHECK(ctx, text != nullptr && text->size() == 1024); + CHECK(ctx, spans[0].attributes_truncated); + } + + // Unsampled spans are null but carry a propagatable context. + { + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter); + molo::trace::Tracer tracer("unsampled", std::make_shared(), + &processor); + molo::trace::Span span = tracer.start_span("dropped"); + CHECK(ctx, !span.valid()); + CHECK(ctx, span.context().is_valid()); + CHECK(ctx, !span.context().is_sampled()); + span.set_attribute("k", std::int64_t(1)); + span.end(); + processor.flush(); + CHECK(ctx, exporter->spans().empty()); + } + + // Sampling decision is inherited from the parent (immutable per trace). + { + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter); + molo::trace::Tracer tracer("inherit", std::make_shared(), + &processor); + molo::trace::SpanContext sampled_parent; + sampled_parent.trace_id = molo::trace::TraceId::generate(); + sampled_parent.span_id = molo::trace::SpanId::generate(); + sampled_parent.flags.value = molo::trace::TraceFlags::kSampled; + molo::trace::Span child = + tracer.start_span("child", molo::trace::SpanKind::Internal, sampled_parent); + CHECK(ctx, child.valid()); + CHECK(ctx, child.context().is_sampled()); + child.end(); + } + + // Secrets in attributes are redacted. + { + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter); + molo::trace::Tracer tracer("redact", std::make_shared(), + &processor); + molo::trace::Span span = tracer.start_span("redact"); + span.set_attribute("auth_token", std::string("abc")); + span.end(); + processor.flush(); + auto spans = exporter->spans(); + CHECK(ctx, spans.size() == 1); + auto* text = std::get_if(&spans[0].attributes[0].value); + CHECK(ctx, text != nullptr && *text == "***"); + } + + // Disabled runtime and missing processor yield null spans and count drops. + { + molo::trace::Tracer tracer("off"); + molo::trace::Span span = tracer.start_span("x"); + CHECK(ctx, !span.valid()); + CHECK(ctx, tracer.spans_dropped() == 1); + + auto exporter = std::make_shared(); + molo::trace::BatchProcessor processor(exporter); + tracer.set_processor(&processor); + molo::obs::RuntimeToggles::instance().tracing.store(false); + molo::trace::Span span2 = tracer.start_span("y"); + molo::obs::RuntimeToggles::instance().tracing.store(true); + CHECK(ctx, !span2.valid()); + molo::trace::Span span3 = tracer.start_span("z"); + CHECK(ctx, span3.valid()); + span3.end(); + } + + CHECK(ctx, molo::trace::span_kind_name(molo::trace::SpanKind::Server) == "server"); + + return test_summary(ctx, "tracing"); +} diff --git a/tests/src/observability/zero_cost_probe.cpp b/tests/src/observability/zero_cost_probe.cpp new file mode 100644 index 0000000..85b0d7c --- /dev/null +++ b/tests/src/observability/zero_cost_probe.cpp @@ -0,0 +1,47 @@ +// Zero-cost probe: one TU touching every pillar. Must compile both with +// full instrumentation and with MOLO_OBSERVABILITY=0 plus all ENABLE +// flags at 0, where every call below must vanish (macros expand to +// ((void)0), instruments guard on if constexpr). + +#include + +#include "observability/events/dispatcher.h" +#include "observability/events/event.h" +#include "observability/events/lifecycle_events.h" +#include "observability/logging/logger.h" +#include "observability/logging/macros.h" +#include "observability/logging/registry.h" +#include "observability/metrics/counter.h" +#include "observability/metrics/histogram.h" +#include "observability/metrics/registry.h" +#include "observability/metrics/timer.h" +#include "observability/tracing/span.h" +#include "observability/tracing/tracer.h" + +void probe_hot_path(molo::log::Logger& logger, molo::metrics::Counter& counter, + molo::metrics::Histogram& histogram, molo::trace::Tracer& tracer, + molo::events::Dispatcher& dispatcher) { + MOLO_LOG_DEBUG(logger, "hot.path.tick"); + MOLO_LOG_INFO(logger, "hot.path.tick", {{"n", std::int64_t(1)}}); + counter.inc(); + { + MOLO_TIMER(histogram); + molo::trace::Span span = + tracer.start_span("hot", molo::trace::SpanKind::Internal); + span.set_attribute("k", std::int64_t(1)); + } + molo::events::Event event("hot.tick"); + dispatcher.emit(event); +} + +int main() { + molo::log::Registry logs; + auto logger = logs.get("probe"); + molo::metrics::Registry metrics; + auto& counter = metrics.counter("probe_ticks", "probe"); + auto& histogram = metrics.histogram("probe_latency", "probe"); + molo::trace::Tracer tracer("probe"); + molo::events::Dispatcher dispatcher; + probe_hot_path(*logger, counter, histogram, tracer, dispatcher); + return 0; +} diff --git a/tests/src/operability/main.cpp b/tests/src/operability/main.cpp new file mode 100644 index 0000000..be7a45e --- /dev/null +++ b/tests/src/operability/main.cpp @@ -0,0 +1,30 @@ +// Standalone operability test runner. No framework, stdlib only: each +// suite returns its failure count and main sums them. +#include + +int run_coord_tests(); +int run_lifecycle_tests(); +int run_breaker_bulkhead_tests(); +int run_shed_deadline_tests(); +int run_deploy_tests(); +int run_slo_tests(); +int run_chaos_tests(); +int run_debug_tests(); + +int main() { + int fail = 0; + fail += run_coord_tests(); + fail += run_lifecycle_tests(); + fail += run_breaker_bulkhead_tests(); + fail += run_shed_deadline_tests(); + fail += run_deploy_tests(); + fail += run_slo_tests(); + fail += run_chaos_tests(); + fail += run_debug_tests(); + if (fail == 0) { + std::printf("operability: all suites passed\n"); + return 0; + } + std::printf("operability: %d check(s) failed\n", fail); + return 1; +} diff --git a/tests/src/operability/test_breaker_bulkhead.cpp b/tests/src/operability/test_breaker_bulkhead.cpp new file mode 100644 index 0000000..02b1309 --- /dev/null +++ b/tests/src/operability/test_breaker_bulkhead.cpp @@ -0,0 +1,154 @@ +// Breaker and bulkhead tests: trip thresholds, half open probes, bulkhead +// admission and promotion. Frozen clock throughout. +#include +#include + +#include "operability/resilience/breaker.h" +#include "operability/resilience/bulkhead.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++g_fail; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +using State = molo::resilience::Breaker::State; + +void test_breaker_consecutive_trip() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Breaker b(clock, obs, "svc"); + CHECK(b.state() == State::kClosed); + CHECK(b.allow()); + // Pad the window with successes so the rate rule stays quiet and only + // the 20 consecutive failures rule can trip. + for (int i = 0; i < 100; ++i) b.on_success(); + for (int i = 0; i < 19; ++i) b.on_failure(); + CHECK(b.state() == State::kClosed); + b.on_failure(); + CHECK(b.state() == State::kOpen); + CHECK(!b.allow()); +} + +void test_breaker_half_open_close() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Breaker b(clock, obs, "svc"); + for (int i = 0; i < 20; ++i) b.on_failure(); + CHECK(!b.allow()); + clock.advance(31000); // past the 30 s open hold + CHECK(b.allow()); + CHECK(b.state() == State::kHalfOpen); + b.on_success(); + // Only 3 probes may be inflight; use them one at a time. + CHECK(b.allow()); + b.on_success(); + CHECK(b.allow()); + b.on_success(); + CHECK(b.state() == State::kClosed); + CHECK(b.allow()); +} + +void test_breaker_half_open_reopen() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Breaker b(clock, obs, "svc"); + for (int i = 0; i < 20; ++i) b.on_failure(); + clock.advance(31000); + CHECK(b.allow()); + b.on_failure(); + CHECK(b.state() == State::kOpen); + CHECK(!b.allow()); +} + +void test_breaker_rate_trip() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Breaker b(clock, obs, "svc"); + for (int i = 0; i < 5; ++i) b.on_success(); + for (int i = 0; i < 4; ++i) b.on_failure(); + CHECK(b.state() == State::kClosed); // 4/9, below 50 percent + b.on_failure(); // 5/10, hits 50 percent + CHECK(b.state() == State::kOpen); +} + +void test_breaker_window_prunes_old_failures() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Breaker b(clock, obs, "svc"); + for (int i = 0; i < 5; ++i) b.on_failure(); + clock.advance(11000); // past the 10 s window + b.on_success(); // resets consecutive count and prunes the old samples + for (int i = 0; i < 4; ++i) b.on_failure(); + for (int i = 0; i < 5; ++i) b.on_success(); + // Window holds 10 samples with 4 failures: 40 percent, stays closed. If + // the 5 expired failures were still counted it would read 9/15 and trip. + CHECK(b.state() == State::kClosed); + CHECK(b.allow()); +} + +void test_breaker_success_resets_consecutive() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Breaker b(clock, obs, "svc"); + for (int i = 0; i < 200; ++i) b.on_success(); // keep the rate rule quiet + for (int i = 0; i < 19; ++i) b.on_failure(); + b.on_success(); // breaks the streak: 19 + 1 + 19 must not trip 20 + for (int i = 0; i < 19; ++i) b.on_failure(); + CHECK(b.state() == State::kClosed); + CHECK(b.allow()); +} + +void test_bulkhead_admit_queue_reject() { + molo::common::RecordingObserver obs; + molo::resilience::Bulkhead bh(obs, 1, 2, 1); + using A = molo::resilience::Bulkhead::Admission; + CHECK(bh.try_acquire("p") == A::kAdmitted); + CHECK(bh.try_acquire("p") == A::kAdmitted); + CHECK(bh.inflight("p") == 2); + CHECK(bh.try_acquire("p") == A::kQueued); + CHECK(bh.queued("p") == 1); + CHECK(bh.try_acquire("p") == A::kRejected); + bh.release("p"); // promotes the queued request + CHECK(bh.queued("p") == 0); + CHECK(bh.inflight("p") == 2); + bh.release("p"); + CHECK(bh.inflight("p") == 1); + bh.release("p"); + CHECK(bh.inflight("p") == 0); +} + +void test_bulkhead_caps_partitions() { + molo::common::RecordingObserver obs; + molo::resilience::Bulkhead bh(obs, 100, 1000, 1000); + CHECK(bh.partition_count() == 32); + // Clamped to 64 concurrent: 64 admitted, then queued, never more inflight. + int admitted = 0; + for (int i = 0; i < 64; ++i) { + if (bh.try_acquire("solo") == molo::resilience::Bulkhead::Admission::kAdmitted) ++admitted; + } + CHECK(admitted == 64); + CHECK(bh.inflight("solo") == 64); +} + +} // namespace + +int run_breaker_bulkhead_tests() { + test_breaker_consecutive_trip(); + test_breaker_half_open_close(); + test_breaker_half_open_reopen(); + test_breaker_rate_trip(); + test_breaker_window_prunes_old_failures(); + test_breaker_success_resets_consecutive(); + test_bulkhead_admit_queue_reject(); + test_bulkhead_caps_partitions(); + return g_fail; +} diff --git a/tests/src/operability/test_chaos.cpp b/tests/src/operability/test_chaos.cpp new file mode 100644 index 0000000..4379a2d --- /dev/null +++ b/tests/src/operability/test_chaos.cpp @@ -0,0 +1,122 @@ +// Chaos tests: deterministic hooks, injector decisions and caps, +// experiment blast radius, duration and burn abort. +#include +#include + +#include "operability/chaos/experiment.h" +#include "operability/chaos/hook.h" +#include "operability/chaos/injector.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++g_fail; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +using Action = molo::chaos::Hook::Action; +using Decision = molo::chaos::Injector::Decision; + +void test_hook_sampling() { + molo::chaos::Hook always("a", Action::kFail, 100); + molo::chaos::Hook never("n", Action::kFail, 0); + molo::chaos::Hook none("x", Action::kNone, 100); + for (std::uint64_t s = 0; s < 16; ++s) { + CHECK(always.fires_for(s)); + CHECK(!never.fires_for(s)); + CHECK(!none.fires_for(s)); + } + // Partial rate fires for some but not all of a large sweep. + molo::chaos::Hook half("h", Action::kFail, 50); + int fired = 0; + for (std::uint64_t s = 0; s < 1000; ++s) { + if (half.fires_for(s)) ++fired; + } + CHECK(fired > 300 && fired < 700); +} + +void test_injector_decisions() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::chaos::Injector inj(clock, obs); + CHECK(inj.evaluate(1) == Decision::kProceed); + CHECK(inj.add_hook(molo::chaos::Hook("slow", Action::kDelay, 100, 250))); + CHECK(inj.evaluate(2) == Decision::kDelayed); + CHECK(clock.steady_ms() == 250); + CHECK(inj.add_hook(molo::chaos::Hook("boom", Action::kFail, 100))); + CHECK(inj.evaluate(3) == Decision::kFailed); + CHECK(inj.add_hook(molo::chaos::Hook("stop", Action::kAbort, 100))); + CHECK(inj.evaluate(4) == Decision::kAborted); + CHECK(inj.size() == 3); +} + +void test_injector_caps_and_dupes() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::chaos::Injector inj(clock, obs); + for (int i = 0; i < 64; ++i) { + CHECK(inj.add_hook(molo::chaos::Hook("h" + std::to_string(i), Action::kFail, 0))); + } + CHECK(!inj.add_hook(molo::chaos::Hook("one-more", Action::kFail, 0))); + CHECK(!inj.add_hook(molo::chaos::Hook("h0", Action::kFail, 0))); + CHECK(!inj.add_hook(molo::chaos::Hook("", Action::kFail, 0))); + CHECK(inj.remove_hook("h0")); + CHECK(!inj.remove_hook("h0")); + CHECK(inj.size() == 63); + inj.clear(); + CHECK(inj.size() == 0); +} + +void test_experiment_blast_and_duration() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::chaos::Experiment ex(clock, obs, "latency", 100, 1000, 2.0); + CHECK(!ex.should_apply(1)); // idle: never applies + CHECK(ex.start()); + CHECK(!ex.start()); + CHECK(ex.should_apply(1)); + clock.advance(1001); + CHECK(!ex.should_apply(2)); // duration elapsed, auto finished + CHECK(ex.state() == molo::chaos::Experiment::State::kFinished); +} + +void test_experiment_zero_blast() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::chaos::Experiment ex(clock, obs, "off", 0, 0, 2.0); + CHECK(ex.start()); + CHECK(!ex.should_apply(1)); + ex.finish(); + CHECK(ex.state() == molo::chaos::Experiment::State::kFinished); +} + +void test_experiment_burn_abort() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::chaos::Experiment ex(clock, obs, "risky", 10, 0, 1.0); + CHECK(ex.start()); + CHECK(!ex.abort_if_breached([] { return 0.5; })); + CHECK(ex.state() == molo::chaos::Experiment::State::kRunning); + CHECK(ex.abort_if_breached([] { return 4.0; })); + CHECK(ex.state() == molo::chaos::Experiment::State::kAborted); + CHECK(!ex.should_apply(1)); +} + +} // namespace + +int run_chaos_tests() { + test_hook_sampling(); + test_injector_decisions(); + test_injector_caps_and_dupes(); + test_experiment_blast_and_duration(); + test_experiment_zero_blast(); + test_experiment_burn_abort(); + return g_fail; +} diff --git a/tests/src/operability/test_coord.cpp b/tests/src/operability/test_coord.cpp new file mode 100644 index 0000000..70013a4 --- /dev/null +++ b/tests/src/operability/test_coord.cpp @@ -0,0 +1,155 @@ +// Coordination tests: lease, distributed lock, fence, leader election. +// Frozen clock throughout; no real waiting. +#include +#include +#include + +#include "operability/coord/dist_lock.h" +#include "operability/coord/election.h" +#include "operability/coord/fence.h" +#include "operability/coord/lease.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/kv_store.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++g_fail; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +void test_lease_basic() { + molo::common::FrozenClock clock(1000); + molo::common::InMemoryKvStore kv(clock); + molo::common::RecordingObserver obs; + molo::coord::Lease a(kv, clock, obs, "k/a", "owner-a"); + molo::coord::Lease b(kv, clock, obs, "k/a", "owner-b"); + CHECK(a.acquire(5000)); + CHECK(a.holds()); + CHECK(!b.acquire(5000)); + CHECK(a.renew(5000)); + a.release(); + CHECK(!a.holds()); + CHECK(b.acquire(5000)); + CHECK(b.holds()); +} + +void test_lease_expiry() { + molo::common::FrozenClock clock(0); + molo::common::InMemoryKvStore kv(clock); + molo::common::RecordingObserver obs; + molo::coord::Lease a(kv, clock, obs, "k/exp", "a"); + molo::coord::Lease b(kv, clock, obs, "k/exp", "b"); + CHECK(a.acquire(1000)); + clock.advance(1001); + CHECK(b.acquire(1000)); + CHECK(!a.renew(1000)); + CHECK(!a.holds()); +} + +void test_lease_rejects_bad_args() { + molo::common::FrozenClock clock(0); + molo::common::InMemoryKvStore kv(clock); + molo::common::RecordingObserver obs; + molo::coord::Lease bad(kv, clock, obs, "", ""); + CHECK(!bad.acquire(1000)); + molo::coord::Lease c(kv, clock, obs, "k/c", "c"); + CHECK(!c.acquire(0)); + CHECK(!c.renew(1000)); + CHECK(molo::coord::Lease::live_count() >= 2); +} + +void test_dist_lock() { + molo::common::FrozenClock clock(0); + molo::common::InMemoryKvStore kv(clock); + molo::common::RecordingObserver obs; + molo::coord::DistLock a(kv, clock, obs, "k/lock", "a"); + molo::coord::DistLock b(kv, clock, obs, "k/lock", "b"); + const std::uint64_t t1 = a.try_lock(60000, 0); + CHECK(t1 != 0); + CHECK(a.locked()); + CHECK(b.try_lock(60000, 0) == 0); + a.unlock(); + CHECK(!a.locked()); + const std::uint64_t t2 = b.try_lock(60000, 100); + CHECK(t2 != 0); + CHECK(t2 > t1); + b.unlock(); +} + +void test_fence() { + molo::common::FrozenClock clock(0); + molo::common::InMemoryKvStore kv(clock); + molo::coord::Fence f1(kv, "k/f"); + molo::coord::Fence f2(kv, "k/f"); + const std::uint64_t t1 = f1.next(); + const std::uint64_t t2 = f2.next(); + CHECK(t1 != 0 && t2 != 0 && t2 > t1); + CHECK(f1.check(t2)); + CHECK(!f1.check(t1)); +} + +void test_election() { + molo::common::FrozenClock clock(0); + molo::common::InMemoryKvStore kv(clock); + molo::common::RecordingObserver obs; + molo::coord::LeaderElection n1(kv, clock, obs, "molo/scheduler/leader", "n1"); + molo::coord::LeaderElection n2(kv, clock, obs, "molo/scheduler/leader", "n2"); + CHECK(n1.tick()); + CHECK(n1.is_leader()); + CHECK(!n2.tick()); + CHECK(!n2.is_leader()); + CHECK(n2.leader() == "n1"); + n1.resign(); + CHECK(!n1.is_leader()); + CHECK(n2.tick()); + CHECK(n2.is_leader()); +} + +void test_election_failover_on_expiry() { + molo::common::FrozenClock clock(0); + molo::common::InMemoryKvStore kv(clock); + molo::common::RecordingObserver obs; + molo::coord::LeaderElection n1(kv, clock, obs, "e/fail", "n1", 10000, 3000); + molo::coord::LeaderElection n2(kv, clock, obs, "e/fail", "n2", 10000, 3000); + CHECK(n1.tick()); + clock.advance(11000); // n1 never renews + CHECK(n2.tick()); + CHECK(n2.is_leader()); + CHECK(!n1.tick()); + CHECK(!n1.is_leader()); +} + +void test_election_renew_keeps_leadership() { + molo::common::FrozenClock clock(0); + molo::common::InMemoryKvStore kv(clock); + molo::common::RecordingObserver obs; + molo::coord::LeaderElection n1(kv, clock, obs, "e/keep", "n1", 10000, 3000); + CHECK(n1.tick()); + for (int i = 0; i < 10; ++i) { + clock.advance(3000); + CHECK(n1.tick()); + } + CHECK(n1.is_leader()); + CHECK(n1.leader() == "n1"); +} + +} // namespace + +int run_coord_tests() { + test_lease_basic(); + test_lease_expiry(); + test_lease_rejects_bad_args(); + test_dist_lock(); + test_fence(); + test_election(); + test_election_failover_on_expiry(); + test_election_renew_keeps_leadership(); + return g_fail; +} diff --git a/tests/src/operability/test_debug.cpp b/tests/src/operability/test_debug.cpp new file mode 100644 index 0000000..4d89f5b --- /dev/null +++ b/tests/src/operability/test_debug.cpp @@ -0,0 +1,103 @@ +// Debug tests: endpoint routing, auth gate, response caps and snapshots. +// When MOLO_ENABLE_DEBUG is off the endpoint must 404 everything; the +// standalone build runs this suite twice, once with the flag and once +// without, to prove both sides. +#include +#include + +#include "operability/debug/endpoint.h" +#include "operability/debug/snapshot.h" +#include "supervisor/common/clock.hpp" + +namespace { + +int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++g_fail; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +void test_endpoint_disabled() { + molo::debug::Endpoint ep([] { return std::string("{}"); }); + const auto r = ep.handle("/debug/ping", ""); + CHECK(r.status == 404); + CHECK(ep.handle("/debug/snapshot", "").status == 404); +} + +void test_endpoint_routing() { + molo::debug::Endpoint ep([] { return std::string("{\"ok\":true}"); }); + const auto ping = ep.handle("/debug/ping", ""); + CHECK(ping.status == 200); + CHECK(ping.body == "pong"); + const auto snap = ep.handle("/debug/snapshot", ""); + CHECK(snap.status == 200); + CHECK(snap.content_type == "application/json"); + CHECK(snap.body == "{\"ok\":true}"); + CHECK(ep.handle("/debug/nope", "").status == 404); + CHECK(ep.handle("/metrics", "").status == 404); + CHECK(ep.handle(std::string(600, 'a'), "").status == 414); +} + +void test_endpoint_auth() { + molo::debug::Endpoint ep([] { return std::string("{}"); }, + [](const std::string& t) { return t == "secret"; }); + CHECK(ep.handle("/debug/ping", "wrong").status == 403); + CHECK(ep.handle("/debug/ping", "secret").status == 200); +} + +void test_endpoint_truncates() { + molo::debug::Endpoint ep([] { return std::string(2 * 1024 * 1024, 'x'); }); + const auto r = ep.handle("/debug/snapshot", ""); + CHECK(r.status == 200); + CHECK(r.body.size() == 1024 * 1024); +} + +void test_snapshot_capture() { + molo::common::FrozenClock clock(5000); + molo::debug::Snapshot snap(clock, "1.2.3"); + CHECK(snap.add_section("breaker", "{\"open\":0}")); + CHECK(!snap.add_section("breaker", "{}")); + clock.advance(100); + const std::string out = snap.capture(); + CHECK(out.find("\"version\":\"1.2.3\"") != std::string::npos); + CHECK(out.find("\"wall_ms\":5100") != std::string::npos); + CHECK(out.find("\"uptime_ms\":100") != std::string::npos); + CHECK(out.find("\"breaker\":{\"open\":0}") != std::string::npos); + CHECK(snap.section_count() == 1); + CHECK(snap.remove_section("breaker")); + CHECK(!snap.remove_section("breaker")); + CHECK(snap.section_count() == 0); +} + +void test_snapshot_caps() { + molo::common::FrozenClock clock(0); + molo::debug::Snapshot snap(clock); + for (int i = 0; i < 64; ++i) { + CHECK(snap.add_section("s" + std::to_string(i), "1")); + } + CHECK(!snap.add_section("overflow", "1")); + CHECK(!snap.add_section("", "1")); + molo::common::FrozenClock clock2(0); + molo::debug::Snapshot big(clock2); + CHECK(big.add_section("huge", "\"" + std::string(2 * 1024 * 1024, 'y') + "\"")); + CHECK(big.capture().size() == 1024 * 1024); +} + +} // namespace + +int run_debug_tests() { + if (!molo::debug::Endpoint::enabled()) { + test_endpoint_disabled(); + } else { + test_endpoint_routing(); + test_endpoint_auth(); + test_endpoint_truncates(); + } + test_snapshot_capture(); + test_snapshot_caps(); + return g_fail; +} diff --git a/tests/src/operability/test_deploy.cpp b/tests/src/operability/test_deploy.cpp new file mode 100644 index 0000000..e34cb5e --- /dev/null +++ b/tests/src/operability/test_deploy.cpp @@ -0,0 +1,176 @@ +// Deploy strategy tests: rolling batches with readiness gates and the +// blue/green stage machine with rollback. Frozen clock throughout. +#include + +#include "operability/deploy/blue_green.h" +#include "operability/deploy/rolling.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++g_fail; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +using State = molo::deploy::Strategy::State; + +void test_rolling_happy_path() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Rolling::Config cfg; + cfg.total_instances = 4; + cfg.batch_size = 2; + cfg.pause_between_batches_ms = 100; + int upgraded = 0; + molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t) { return true; }, + [&](std::size_t) { ++upgraded; }); + CHECK(r.start()); + CHECK(!r.start()); // second start refuses + CHECK(r.step()); + CHECK(r.step()); + CHECK(r.state() == State::kSucceeded); + CHECK(r.upgraded() == 4); + CHECK(r.failed() == 0); + CHECK(r.pending() == 0); + CHECK(upgraded == 4); + CHECK(clock.steady_ms() >= 100); // batch pause observed +} + +void test_rolling_gate_failure() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Rolling::Config cfg; + cfg.total_instances = 3; + cfg.batch_size = 1; + cfg.ready_timeout_ms = 200; + molo::deploy::Rolling r(cfg, clock, obs, + [](std::size_t i) { return i != 1; }); // index 1 never ready + CHECK(r.start()); + CHECK(r.step()); // index 0 upgrades + CHECK(!r.step()); // index 1 times out, max_unavailable is 0 + CHECK(r.state() == State::kFailed); + CHECK(r.upgraded() == 1); + CHECK(r.failed() == 1); +} + +void test_rolling_tolerates_limited_outage() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Rolling::Config cfg; + cfg.total_instances = 3; + cfg.batch_size = 1; + cfg.ready_timeout_ms = 100; + cfg.max_unavailable = 1; + molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t i) { return i != 1; }); + CHECK(r.start()); + CHECK(r.step()); + CHECK(r.step()); // failure tolerated, run continues + CHECK(!r.step()); // ends failed because one instance failed + CHECK(r.upgraded() == 2); + CHECK(r.failed() == 1); +} + +void test_rolling_rejects_empty() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Rolling::Config cfg; + molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t) { return true; }); + CHECK(!r.start()); +} + +void test_blue_green_happy_path() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::BlueGreen::Config cfg; + cfg.green_instances = 2; + cfg.rollback_window_ms = 1000; + bool switched = false; + bool retired_green = false; + molo::deploy::BlueGreen bg( + cfg, clock, obs, [](std::size_t) { return true; }, + [&] { + switched = true; + return true; + }, + [&](bool green) { retired_green = green; }); + CHECK(bg.start()); + CHECK(bg.step()); // deploy green + CHECK(bg.step()); // verify green + CHECK(bg.step()); // switch + CHECK(switched); + CHECK(bg.serving_green()); + CHECK(bg.step()); // still inside the rollback window + CHECK(bg.state() == State::kRunning); + clock.advance(1001); + CHECK(bg.step()); // window elapsed, move to retire + CHECK(bg.step()); // retire blue + CHECK(bg.state() == State::kSucceeded); + CHECK(retired_green); +} + +void test_blue_green_rollback() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::BlueGreen::Config cfg; + cfg.green_instances = 1; + cfg.rollback_window_ms = 60000; + molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return true; }); + CHECK(bg.start()); + CHECK(bg.step()); + CHECK(bg.step()); + CHECK(bg.step()); + CHECK(bg.serving_green()); + CHECK(bg.rollback()); + CHECK(!bg.serving_green()); + CHECK(bg.state() == State::kRolledBack); + CHECK(!bg.rollback()); // second rollback refuses +} + +void test_blue_green_verify_timeout() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::BlueGreen::Config cfg; + cfg.green_instances = 1; + cfg.ready_timeout_ms = 100; + molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return false; }); + CHECK(bg.start()); + CHECK(bg.step()); + CHECK(!bg.step()); // green never ready + CHECK(bg.state() == State::kFailed); + CHECK(!bg.serving_green()); +} + +void test_blue_green_switch_refused() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::BlueGreen::Config cfg; + cfg.green_instances = 1; + molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return true; }, + [] { return false; }); + CHECK(bg.start()); + CHECK(bg.step()); + CHECK(bg.step()); + CHECK(!bg.step()); // switch hook refuses + CHECK(bg.state() == State::kFailed); +} + +} // namespace + +int run_deploy_tests() { + test_rolling_happy_path(); + test_rolling_gate_failure(); + test_rolling_tolerates_limited_outage(); + test_rolling_rejects_empty(); + test_blue_green_happy_path(); + test_blue_green_rollback(); + test_blue_green_verify_timeout(); + test_blue_green_switch_refused(); + return g_fail; +} diff --git a/tests/src/operability/test_lifecycle.cpp b/tests/src/operability/test_lifecycle.cpp new file mode 100644 index 0000000..5cdf1d0 --- /dev/null +++ b/tests/src/operability/test_lifecycle.cpp @@ -0,0 +1,94 @@ +// Lifecycle tests: boot completion and the four phase drain. +#include +#include + +#include "operability/deploy/lifecycle.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++g_fail; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +void test_mark_serving() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Lifecycle lc(clock, obs); + bool called = false; + lc.on_serving([&] { called = true; }); + lc.mark_serving(); + CHECK(called); + CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kServing); +} + +void test_clean_drain() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Lifecycle lc(clock, obs); + bool stopped = false; + bool done = false; + lc.on_stop_listeners([&] { stopped = true; }); + lc.on_connections_drained([] { return true; }); + lc.on_background_drained([] { return true; }); + lc.on_complete([&] { done = true; }); + CHECK(lc.run_drain(1000)); + CHECK(stopped && done); + CHECK(lc.clean()); + CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kDone); +} + +void test_delayed_drain() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Lifecycle lc(clock, obs); + // Probes flip true once the frozen clock passes their marks; the drain + // loop advances the clock via sleep_ms, so no real waiting happens. + lc.on_connections_drained([&] { return clock.steady_ms() >= 200; }); + lc.on_background_drained([&] { return clock.steady_ms() >= 400; }); + CHECK(lc.run_drain(5000, 50)); + CHECK(lc.clean()); + CHECK(clock.steady_ms() >= 400); +} + +void test_drain_timeout_drops_background() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Lifecycle lc(clock, obs); + bool dropped = false; + lc.on_connections_drained([] { return true; }); + lc.on_background_drained([] { return false; }); + lc.on_drop_background([&](const std::string&) { dropped = true; }); + CHECK(!lc.run_drain(200, 25)); + CHECK(!lc.clean()); + CHECK(dropped); + CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kDone); +} + +void test_drain_timeout_connections() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::deploy::Lifecycle lc(clock, obs); + lc.on_connections_drained([] { return false; }); + lc.on_background_drained([] { return true; }); + CHECK(!lc.run_drain(100, 25)); + CHECK(!lc.clean()); +} + +} // namespace + +int run_lifecycle_tests() { + test_mark_serving(); + test_clean_drain(); + test_delayed_drain(); + test_drain_timeout_drops_background(); + test_drain_timeout_connections(); + return g_fail; +} diff --git a/tests/src/operability/test_shed_deadline.cpp b/tests/src/operability/test_shed_deadline.cpp new file mode 100644 index 0000000..e230a1f --- /dev/null +++ b/tests/src/operability/test_shed_deadline.cpp @@ -0,0 +1,78 @@ +// Shedder and deadline tests: overload shedding with Retry-After plus +// X-Deadline-Ms parsing and remaining budget math. +#include +#include + +#include "operability/resilience/deadline.h" +#include "operability/resilience/shedder.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++g_fail; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +void test_shedderadmits_when_cool() { + molo::common::RecordingObserver obs; + molo::resilience::Shedder s(obs, [] { return false; }); + CHECK(!s.should_shed()); +} + +void test_shedder_sheds_with_retry_after() { + molo::common::RecordingObserver obs; + molo::resilience::Shedder s(obs, [] { return true; }, 7); + CHECK(s.should_shed()); + const auto r = s.reject(); + CHECK(r.status == 503); + CHECK(r.retry_after == "7"); + CHECK(s.retry_after_s() == 7); +} + +void test_shedder_clamps_retry_after() { + molo::common::RecordingObserver obs; + molo::resilience::Shedder hi(obs, [] { return true; }, 99999); + CHECK(hi.retry_after_s() == 120); + molo::resilience::Shedder lo(obs, [] { return true; }, -5); + CHECK(lo.retry_after_s() == 0); +} + +void test_deadline_parse() { + const auto ok = molo::resilience::Deadline::parse_header("12345"); + CHECK(ok.has_value() && *ok == 12345); + CHECK(!molo::resilience::Deadline::parse_header("").has_value()); + CHECK(!molo::resilience::Deadline::parse_header("abc").has_value()); + CHECK(!molo::resilience::Deadline::parse_header("12x").has_value()); + CHECK(!molo::resilience::Deadline::parse_header("99999999999999999999999").has_value()); + CHECK(molo::resilience::Deadline::format_header(42) == "42"); + CHECK(molo::resilience::Deadline::format_header(-1) == "0"); +} + +void test_deadline_remaining() { + molo::common::FrozenClock clock(100000); + molo::resilience::Deadline d(clock); + CHECK(d.remaining_ms(100500) == 450); // 500 left minus the 50 ms margin + CHECK(d.remaining_ms(100050) == 0); // exactly the margin: unusable + CHECK(d.remaining_ms(100040) == 0); // inside the margin + CHECK(d.remaining_ms(90000) == 0); // already past + CHECK(!d.expired(100500)); + CHECK(d.expired(100040)); +} + +} // namespace + +int run_shed_deadline_tests() { + test_shedderadmits_when_cool(); + test_shedder_sheds_with_retry_after(); + test_shedder_clamps_retry_after(); + test_deadline_parse(); + test_deadline_remaining(); + return g_fail; +} diff --git a/tests/src/operability/test_slo.cpp b/tests/src/operability/test_slo.cpp new file mode 100644 index 0000000..5efdac1 --- /dev/null +++ b/tests/src/operability/test_slo.cpp @@ -0,0 +1,101 @@ +// SLO tests: budgets, burn rates, bad event counters, prometheus render. +#include +#include + +#include "operability/resilience/slo.h" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++g_fail; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +bool near(double a, double b) { + const double d = a > b ? a - b : b - a; + return d < 0.0001; +} + +void test_budget_and_burn() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Slo::Config cfg; + cfg.target_ratio = 0.9; + cfg.window_ms = 3600000; + molo::resilience::Slo slo(clock, obs, cfg); + for (int i = 0; i < 19; ++i) slo.record("/api/items", true); + slo.record("/api/items", false, "boom"); + // 20 samples, 1 error, 10 percent budget: half the budget remains. + CHECK(near(slo.budget_remaining("/api/items"), 0.5)); + // Observed 5 percent over budgeted 10 percent: burn 0.5. + CHECK(near(slo.burn_rate("/api/items", 3600000), 0.5)); + CHECK(slo.bad_events("/api/items", "boom") == 1); + CHECK(slo.bad_events("/api/items", "other") == 0); +} + +void test_unknown_route_is_healthy() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Slo slo(clock, obs); + CHECK(near(slo.budget_remaining("/nope"), 1.0)); + CHECK(near(slo.burn_rate("/nope", 60000), 0.0)); +} + +void test_budget_exhausted() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Slo::Config cfg; + cfg.target_ratio = 0.99; + molo::resilience::Slo slo(clock, obs, cfg); + for (int i = 0; i < 10; ++i) slo.record("/bad", false, "down"); + CHECK(near(slo.budget_remaining("/bad"), 0.0)); + CHECK(slo.burn_rate("/bad", 3600000) > 1.0); +} + +void test_window_pruning_restores_budget() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Slo::Config cfg; + cfg.target_ratio = 0.9; + cfg.window_ms = 1000; + molo::resilience::Slo slo(clock, obs, cfg); + slo.record("/flap", false, "x"); + CHECK(near(slo.budget_remaining("/flap"), 0.0)); + clock.advance(2000); + CHECK(near(slo.budget_remaining("/flap"), 1.0)); + CHECK(near(slo.burn_rate("/flap", 1000), 0.0)); +} + +void test_prometheus_render() { + molo::common::FrozenClock clock(0); + molo::common::RecordingObserver obs; + molo::resilience::Slo slo(clock, obs); + slo.record("/api/items", true); + slo.record("/api/items", false, "boom"); + const std::string out = slo.render_prometheus(); + CHECK(out.find("molo_slo_budget_remaining_ratio") != std::string::npos); + CHECK(out.find("molo_slo_burn_rate") != std::string::npos); + CHECK(out.find("molo_slo_bad_events_total") != std::string::npos); + CHECK(out.find("route=\"/api/items\"") != std::string::npos); + CHECK(out.find("reason=\"boom\"") != std::string::npos); + CHECK(out.find("window=\"5m\"") != std::string::npos); + CHECK(out.find("window=\"1h\"") != std::string::npos); +} + +} // namespace + +int run_slo_tests() { + test_budget_and_burn(); + test_unknown_route_is_healthy(); + test_budget_exhausted(); + test_window_pruning_restores_budget(); + test_prometheus_render(); + return g_fail; +} diff --git a/tests/src/realtime/dav/files_test.cpp b/tests/src/realtime/dav/files_test.cpp new file mode 100644 index 0000000..0ebaa20 --- /dev/null +++ b/tests/src/realtime/dav/files_test.cpp @@ -0,0 +1,185 @@ +// Route: /dav/*. Full litmus-style sequence: MKCOL, PUT, GET, +// PROPFIND depths, PROPPATCH, COPY/MOVE, LOCK/UNLOCK, quota 507, +// client streaming GET with ETag cache. + +#include +#include + +#include "test_main.h" +#include "realtime/shared/http_seam.h" +#include "realtime/webdav/client/webdav_client.h" +#include "realtime/webdav/server/dav_view.h" + +using namespace molo::realtime; + +namespace { + +shared::HttpRequest dav_req(const std::string& method, const std::string& path, + const std::string& body = "") { + shared::HttpRequest req; + req.method = method; + req.path = path; + req.body = body; + return req; +} + +} // namespace + +int main() { + realtime_test::Results r; + webdav::server::DavView dav("/dav"); + + // OPTIONS advertises DAV: 1. + { + auto res = dav.dispatch(dav_req("OPTIONS", "/dav")); + T_CHECK(r, res.status == 200); + T_CHECK(r, shared::header_first(res, "DAV") == "1"); + } + + // MKCOL then PUT then GET. + T_CHECK(r, dav.dispatch(dav_req("MKCOL", "/dav/docs")).status == 201); + T_CHECK(r, dav.dispatch(dav_req("MKCOL", "/dav/docs")).status == 405); + T_CHECK(r, dav.dispatch(dav_req("MKCOL", "/dav/no/file")).status == 409); + { + auto res = dav.dispatch(dav_req("PUT", "/dav/docs/a.txt", "hello dav")); + T_CHECK(r, res.status == 201); + auto get = dav.dispatch(dav_req("GET", "/dav/docs/a.txt")); + T_CHECK(r, get.status == 200 && get.body == "hello dav"); + T_CHECK(r, !shared::header_first(get, "ETag").empty()); + // Conditional GET: 304. + auto cond = dav_req("GET", "/dav/docs/a.txt"); + cond.headers.emplace("If-None-Match", + shared::header_first(get, "ETag")); + T_CHECK(r, dav.dispatch(cond).status == 304); + // Range GET: 206 with Content-Range. + auto rng = dav_req("GET", "/dav/docs/a.txt"); + rng.headers.emplace("Range", "bytes=0-4"); + auto r206 = dav.dispatch(rng); + T_CHECK(r, r206.status == 206 && r206.body == "hello"); + auto bad = dav_req("GET", "/dav/docs/a.txt"); + bad.headers.emplace("Range", "bytes=99-"); + T_CHECK(r, dav.dispatch(bad).status == 416); + } + + // PROPFIND depth 1 lists the collection plus child. + { + auto req = dav_req("PROPFIND", "/dav/docs"); + req.headers.emplace("Depth", "1"); + auto res = dav.dispatch(req); + T_CHECK(r, res.status == 207); + T_CHECK(r, res.body.find("/dav/docs/a.txt") != std::string::npos); + auto entries = webdav::client::WebDavClient::parse_propfind(res.body); + T_CHECK(r, entries.size() == 2); + } + // Depth infinity is capped, still 207. + { + auto req = dav_req("PROPFIND", "/dav"); + req.headers.emplace("Depth", "infinity"); + T_CHECK(r, dav.dispatch(req).status == 207); + auto bad = dav_req("PROPFIND", "/dav"); + bad.headers.emplace("Depth", "2"); + T_CHECK(r, dav.dispatch(bad).status == 400); + } + + // PROPPATCH sets a dead property, readable via prop selection. + { + auto req = dav_req("PROPPATCH", "/dav/docs/a.txt", + "" + "ada" + ""); + T_CHECK(r, dav.dispatch(req).status == 207); + } + + // COPY then MOVE with Overwrite handling. + { + auto copy = dav_req("COPY", "/dav/docs/a.txt"); + copy.headers.emplace("Destination", "/dav/docs/b.txt"); + T_CHECK(r, dav.dispatch(copy).status == 201); + auto copy2 = dav_req("COPY", "/dav/docs/a.txt"); + copy2.headers.emplace("Destination", "/dav/docs/b.txt"); + copy2.headers.emplace("Overwrite", "F"); + T_CHECK(r, dav.dispatch(copy2).status == 412); + auto move = dav_req("MOVE", "/dav/docs/b.txt"); + move.headers.emplace("Destination", "/dav/docs/c.txt"); + T_CHECK(r, dav.dispatch(move).status == 201); + T_CHECK(r, dav.dispatch(dav_req("GET", "/dav/docs/b.txt")).status == 404); + T_CHECK(r, dav.dispatch(dav_req("GET", "/dav/docs/c.txt")).status == 200); + } + + // LOCK writes, COPY without token is 423, UNLOCK releases. + { + auto lock = dav_req("LOCK", "/dav/docs/c.txt", + "" + "" + "" + "tester"); + lock.headers.emplace("Timeout", "Second-60"); + auto locked = dav.dispatch(lock); + T_CHECK(r, locked.status == 200); + std::string token = shared::header_first(locked, "Lock-Token"); + T_CHECK(r, !token.empty()); + auto copy = dav_req("COPY", "/dav/docs/c.txt"); + copy.headers.emplace("Destination", "/dav/docs/d.txt"); + T_CHECK(r, dav.dispatch(copy).status == 423); + auto unlock = dav_req("UNLOCK", "/dav/docs/c.txt"); + unlock.headers.emplace("Lock-Token", token); + T_CHECK(r, dav.dispatch(unlock).status == 204); + T_CHECK(r, dav.dispatch(copy).status == 201); + } + + // DELETE removes; non-empty collection delete fails. + { + T_CHECK(r, + dav.dispatch(dav_req("DELETE", "/dav/docs/c.txt")).status == 204); + T_CHECK(r, dav.dispatch(dav_req("DELETE", "/dav/docs")).status == 409); + } + + // Path escape fails closed. + T_CHECK(r, dav.dispatch(dav_req("GET", "/dav/../etc/passwd")).status == 404); + + // Quota exceeded is 507. + { + webdav::server::DavConfig cfg; + cfg.quota_bytes = 8; + webdav::server::DavView small("/dav", cfg); + T_CHECK(r, small.dispatch(dav_req("MKCOL", "/dav/d")).status == 201); + auto res = small.dispatch(dav_req("PUT", "/dav/d/big", "0123456789")); + T_CHECK(r, res.status == 507); + } + + // Client: streaming GET with ETag cache and quota hook. + { + webdav::client::WebDavClient client; + std::size_t quota_down = 0; + client.set_quota_hook( + [&](const std::string& dir, std::size_t n) { + if (dir == "down") { + quota_down += n; + } + return true; + }); + auto req = client.get("/dav/docs/a.txt"); + auto res = dav.dispatch(req); + std::string streamed; + auto got = client.on_get_response(req, res, + [&](const uint8_t* data, std::size_t n) { + streamed.append( + reinterpret_cast(data), + n); + return true; + }); + T_CHECK(r, got.ok && !got.cached && streamed == "hello dav"); + T_CHECK(r, quota_down == streamed.size()); + T_CHECK(r, client.cache_entries() == 1); + // Second GET is conditional and hits the cache. + auto req2 = client.get("/dav/docs/a.txt"); + T_CHECK(r, + !shared::header_first(req2, "If-None-Match").empty()); + auto res2 = dav.dispatch(req2); + auto got2 = client.on_get_response( + req2, res2, [&](const uint8_t*, std::size_t) { return true; }); + T_CHECK(r, got2.ok && got2.cached); + } + + return r.report("dav/files"); +} diff --git a/tests/src/realtime/engineio/handshake_test.cpp b/tests/src/realtime/engineio/handshake_test.cpp new file mode 100644 index 0000000..3060da1 --- /dev/null +++ b/tests/src/realtime/engineio/handshake_test.cpp @@ -0,0 +1,160 @@ +// Route: /engine.io/. Open handshake shape, polling framing, +// upgrade probe sequence, duplicate GET rejection, unknown sid. + +#include +#include + +#include "test_main.h" +#include "realtime/engineio/heartbeat_scheduler.h" +#include "realtime/engineio/packet_codec.h" +#include "realtime/engineio/polling_transport.h" +#include "realtime/engineio/upgrade_manager.h" +#include "realtime/engineio/websocket_transport.h" +#include "realtime/shared/mini_json.h" + +using namespace molo::realtime; + +int main() { + realtime_test::Results r; + using engineio::EnginePacket; + using engineio::EnginePacketType; + using engineio::EngineSession; + using engineio::PacketCodec; + + // Single packet round-trips for all 7 types. + for (int t = 0; t <= 6; ++t) { + EnginePacket p; + p.type = static_cast(t); + p.data = t == 4 ? "hello" : ""; + auto wire = PacketCodec::encode(p); + T_CHECK(r, wire[0] == '0' + t); + auto back = PacketCodec::decode(wire); + T_CHECK(r, back.has_value() && back->type == p.type && + back->data == p.data); + } + T_CHECK(r, !PacketCodec::decode("").has_value()); + T_CHECK(r, !PacketCodec::decode("7x").has_value()); + + // Polling payload framing round-trips; errors fail clean. + { + EnginePacket a, b; + a.type = EnginePacketType::kMessage; + a.data = "hi"; + b.type = EnginePacketType::kNoop; + std::string payload = PacketCodec::encode_polling({a, b}); + T_CHECK(r, payload == "3:4hi1:6"); + auto back = PacketCodec::decode_polling(payload); + T_CHECK(r, back.has_value() && back->size() == 2); + T_CHECK(r, (*back)[0].data == "hi"); + T_CHECK(r, !PacketCodec::decode_polling("9:4hi").has_value()); + T_CHECK(r, !PacketCodec::decode_polling("xx").has_value()); + } + + // Open packet JSON has exactly the spec keys. + { + shared::ConnectionContext ctx; + EngineSession session(ctx); + T_CHECK(r, !session.sid().empty()); + EnginePacket open = session.open_packet(); + T_CHECK(r, open.type == EnginePacketType::kOpen); + auto json = shared::Json::parse(open.data); + T_CHECK(r, json.has_value()); + T_CHECK(r, json->find("sid") != nullptr); + T_CHECK(r, json->find("upgrades") != nullptr); + T_CHECK(r, json->find("pingInterval") != nullptr); + T_CHECK(r, json->find("pingTimeout") != nullptr); + T_CHECK(r, json->find("maxPayload") != nullptr); + } + + // Polling GET drains queued packets; empty holds; dup GET is 400. + { + shared::ConnectionContext ctx; + EngineSession session(ctx); + engineio::PollingTransport polling; + bool hold = false; + auto held = polling.answer_get(session, hold); + (void)held; + T_CHECK(r, hold); + bool hold2 = false; + auto dup = polling.answer_get(session, hold2); + T_CHECK(r, dup.status == 400); + EnginePacket m; + m.type = EnginePacketType::kMessage; + m.data = "4hi"; + session.enqueue(m); + auto flushed = polling.flush_held(session); + T_CHECK(r, flushed.status == 200); + T_CHECK(r, flushed.body == "4:44hi"); + T_CHECK(r, engineio::PollingTransport::unknown_sid().status == 400); + } + + // POST bodies decode; oversized is 400. + { + shared::ConnectionContext ctx; + EngineSession session(ctx); + engineio::PollingTransport polling; + shared::HttpRequest req; + req.method = "POST"; + req.body = "3:4hi"; + auto res = polling.accept_post(session, req); + T_CHECK(r, res.ok && res.packets.size() == 1); + req.body = "bogus"; + T_CHECK(r, !polling.accept_post(session, req).ok); + } + + // WebSocket transport admit checks EIO/transport/sid. + { + shared::HttpRequest req; + req.query = "EIO=4&transport=websocket&sid=abc"; + T_CHECK(r, engineio::WebSocketTransport::admit(req, true).ok); + T_CHECK(r, !engineio::WebSocketTransport::admit(req, false).ok); + req.query = "EIO=3&transport=websocket"; + T_CHECK(r, !engineio::WebSocketTransport::admit(req, true).ok); + } + + // Upgrade probe: 2probe -> 3probe, 5 upgrades. + { + shared::ConnectionContext ctx; + EngineSession session(ctx); + engineio::UpgradeManager up; + EnginePacket probe; + probe.type = EnginePacketType::kPing; + probe.data = "probe"; + auto answer = up.on_packet(session, probe); + T_CHECK(r, answer.has_value() && *answer == "3probe"); + T_CHECK(r, !up.upgraded()); + EnginePacket fin; + fin.type = EnginePacketType::kUpgrade; + auto consumed = up.on_packet(session, fin); + T_CHECK(r, consumed.has_value() && consumed->empty()); + T_CHECK(r, up.upgraded() && session.upgraded()); + EnginePacket other; + other.type = EnginePacketType::kMessage; + other.data = "x"; + T_CHECK(r, !up.on_packet(session, other).has_value()); + } + + // Heartbeat pings on interval, closes on silence. + { + shared::ConnectionContext ctx; + engineio::EngineConfig cfg; + cfg.ping_interval = std::chrono::milliseconds(25); + cfg.ping_timeout = std::chrono::milliseconds(100); + EngineSession session(ctx, cfg); + engineio::HeartbeatScheduler hb(&session); + int pings = 0; + bool dead = false; + hb.on_send([&](const EnginePacket&) { ++pings; }); + hb.on_timeout([&] { dead = true; }); + auto t0 = std::chrono::steady_clock::now(); + hb.start(t0); + session.note_activity(t0); + hb.tick(t0 + session.config().ping_interval); + T_CHECK(r, pings == 1 && !dead); + hb.tick(t0 + session.config().ping_interval + + session.config().ping_timeout); + T_CHECK(r, dead && session.closed()); + } + + return r.report("engineio/handshake"); +} diff --git a/tests/src/realtime/shared/foundation_test.cpp b/tests/src/realtime/shared/foundation_test.cpp new file mode 100644 index 0000000..498bfae --- /dev/null +++ b/tests/src/realtime/shared/foundation_test.cpp @@ -0,0 +1,149 @@ +// Shared foundations: base64/SHA-1 vectors, UTF-8, send-queue shed +// policies, jittered backoff bounds, valve watermarks, JSON codec, +// metrics names, auth headers. + +#include +#include + +#include "test_main.h" +#include "realtime/shared/backpressure_valve.h" +#include "realtime/shared/base64.h" +#include "realtime/shared/base_realtime_client.h" +#include "realtime/shared/mini_json.h" +#include "realtime/shared/realtime_metrics.h" +#include "realtime/shared/reconnect_policy.h" +#include "realtime/shared/send_queue.h" +#include "realtime/shared/sha1.h" +#include "realtime/shared/utf8.h" + +using namespace molo::realtime; + +int main() { + realtime_test::Results r; + + // SHA-1 golden vector ("abc"). + { + auto d = shared::Sha1::hash("abc"); + std::string hex; + for (uint8_t b : d) { + char buf[3]; + std::snprintf(buf, sizeof(buf), "%02x", b); + hex += buf; + } + T_CHECK(r, hex == "a9993e364706816aba3e25717850c26c9cd0d89d"); + } + // Base64 round-trip plus rejection. + { + T_CHECK(r, shared::base64_encode("Man") == "TWFu"); + T_CHECK(r, shared::base64_encode("Ma") == "TWE="); + auto dec = shared::base64_decode("TWFu"); + T_CHECK(r, dec.has_value() && + std::string(dec->begin(), dec->end()) == "Man"); + T_CHECK(r, !shared::base64_decode("!!!").has_value()); + T_CHECK(r, !shared::base64_decode("abc").has_value()); + } + // UTF-8 strictness. + { + T_CHECK(r, shared::is_valid_utf8("hello")); + T_CHECK(r, shared::is_valid_utf8("\xc3\xa9")); // e-acute. + T_CHECK(r, !shared::is_valid_utf8("\xc3\x28")); + T_CHECK(r, !shared::is_valid_utf8("\xed\xa0\x80")); // Surrogate. + T_CHECK(r, !shared::is_valid_utf8("\xf0\x82\x82\xac")); // Overlong. + } + + // Send queue shed policies. + { + shared::SendQueue q(2, shared::ShedPolicy::kDropOldest); + shared::SendItem a, b, c; + a.bytes = {'a'}; + b.bytes = {'b'}; + c.bytes = {'c'}; + T_CHECK(r, q.push(a) && q.push(b) && q.push(c)); + T_CHECK(r, q.shed_count() == 1); + shared::SendItem out; + T_CHECK(r, q.pop(out) && out.bytes[0] == 'b'); + } + { + shared::SendQueue q(1, shared::ShedPolicy::kDropNewest); + shared::SendItem a, b; + a.bytes = {'a'}; + b.bytes = {'b'}; + T_CHECK(r, q.push(a)); + T_CHECK(r, !q.push(b)); + T_CHECK(r, q.shed_count() == 1); + } + { + shared::SendQueue q(1, shared::ShedPolicy::kClose); + shared::SendItem a, b; + a.bytes = {'a'}; + b.bytes = {'b'}; + T_CHECK(r, q.push(a)); + T_CHECK(r, !q.push(b)); + T_CHECK(r, q.must_close()); + } + + // Backoff bounds double then cap; jitter stays inside. + { + shared::ReconnectPolicy p(std::chrono::milliseconds(100), + std::chrono::milliseconds(1000), 10); + T_CHECK(r, p.bound_for(0) == std::chrono::milliseconds(100)); + T_CHECK(r, p.bound_for(1) == std::chrono::milliseconds(200)); + T_CHECK(r, p.bound_for(9) == std::chrono::milliseconds(1000)); + for (int i = 0; i < 20; ++i) { + auto d = p.delay_for(2); + T_CHECK(r, d >= std::chrono::milliseconds(0) && + d <= std::chrono::milliseconds(400)); + } + } + + // Valve pauses at high, resumes at low. + { + shared::BackpressureValve v(10, 100); + v.note_buffered(50); + T_CHECK(r, !v.paused()); + v.note_buffered(60); + T_CHECK(r, v.paused()); + v.note_flushed(50); + T_CHECK(r, v.paused()); // Still above low. + v.note_flushed(55); + T_CHECK(r, !v.paused()); + } + + // JSON round-trip and rejection. + { + auto v = shared::Json::parse("{\"a\":[1,true,\"x\"],\"b\":null}"); + T_CHECK(r, v.has_value()); + T_CHECK(r, v->find("a")->array()[1].boolean()); + T_CHECK(r, v->dump() == "{\"a\":[1,true,\"x\"],\"b\":null}"); + T_CHECK(r, !shared::Json::parse("{oops").has_value()); + T_CHECK(r, !shared::Json::parse("[1,]").has_value()); + } + + // Metrics count and reset. + { + auto& m = shared::RealtimeMetrics::instance(); + m.reset_all(); + m.inc(shared::RealtimeMetric::kMessagesSent, 3); + T_CHECK(r, m.get(shared::RealtimeMetric::kMessagesSent) == 3); + T_CHECK(r, std::string(shared::metric_name( + shared::RealtimeMetric::kMessagesSent)) == + "molo_realtime_messages_sent"); + } + + // Client auth headers: exactly one scheme at a time. + { + shared::ClientAuth anon; + T_CHECK(r, anon.anonymous() && anon.headers().empty()); + shared::ClientAuth bearer; + bearer.bearer_token = "tok"; + auto h = bearer.headers(); + T_CHECK(r, h.size() == 1 && h[0].second == "Bearer tok"); + shared::ClientAuth basic; + basic.basic_user = "u"; + basic.basic_password = "p"; + auto h2 = basic.headers(); + T_CHECK(r, h2.size() == 1 && h2[0].second == "Basic dTpw"); + } + + return r.report("shared/foundation"); +} diff --git a/tests/src/realtime/shared/robustness_test.cpp b/tests/src/realtime/shared/robustness_test.cpp new file mode 100644 index 0000000..8f62683 --- /dev/null +++ b/tests/src/realtime/shared/robustness_test.cpp @@ -0,0 +1,95 @@ +// Parser robustness: deterministic garbage into every byte-level +// parser must fail clean, never crash or hang. Fixed seed, bounded +// iterations, small inputs. + +#include +#include +#include +#include + +#include "test_main.h" +#include "realtime/engineio/packet_codec.h" +#include "realtime/shared/base64.h" +#include "realtime/shared/mini_json.h" +#include "realtime/shared/utf8.h" +#include "realtime/socketio/packet_codec.h" +#include "realtime/sse/sse_client.h" +#include "realtime/webdav/server/dav_xml.h" +#include "realtime/websocket/server/frame_parser.h" + +using namespace molo::realtime; + +namespace { + +std::vector garbage(std::mt19937_64& rng, std::size_t max_len) { + std::uniform_int_distribution len(0, max_len); + std::uniform_int_distribution byte(0, 255); + std::uniform_int_distribution pick(0, 3); + std::vector out(len(rng)); + for (auto& b : out) { + // Mix raw bytes with protocol-ish alphabet to hit deep paths. + int p = pick(rng); + if (p == 0) { + static const char k[] = "0123456-:,/[]{}<>\" _placeholder"; + b = static_cast(k[byte(rng) % (sizeof(k) - 1)]); + } else { + b = static_cast(byte(rng)); + } + } + return out; +} + +} // namespace + +int main() { + realtime_test::Results r; + std::mt19937_64 rng(0xC10C4E7E); + constexpr int kIters = 4000; + + int ws_frames = 0; + for (int i = 0; i < kIters; ++i) { + auto g = garbage(rng, 96); + websocket::server::FrameParser parser(true); + // Feed one byte at a time to stress incremental states. + for (uint8_t b : g) { + std::size_t consumed = 0; + auto step = parser.feed(&b, 1, consumed); + if (step.outcome == websocket::server::ParseOutcome::kFrame) { + ++ws_frames; + } + if (step.outcome == websocket::server::ParseOutcome::kFailed) { + break; + } + } + std::string s(g.begin(), g.end()); + (void)engineio::PacketCodec::decode(s); + (void)engineio::PacketCodec::decode_polling(s); + (void)socketio::PacketCodec::decode_text(s); + (void)shared::Json::parse(s); + (void)shared::base64_decode(s); + (void)shared::is_valid_utf8(g.data(), g.size()); + (void)webdav::server::parse_dav_xml(s, 65536); + } + T_CHECK(r, ws_frames >= 0); // Ran to completion without crashing. + + // SSE client survives garbage streams and stays responsive. + { + sse::SseClient c; + shared::HttpResponse head; + head.status = 200; + shared::set_header(head, "Content-Type", "text/event-stream"); + c.on_head(head); + int events = 0; + c.on_event([&](const sse::SseEvent&) { ++events; }); + for (int i = 0; i < 1000; ++i) { + auto g = garbage(rng, 128); + c.on_bytes(g.data(), g.size()); + } + // Leading newline terminates any partial garbage line first: + // without it the next bytes would correctly merge into that line. + c.on_bytes("\ndata: alive\n\n"); + T_CHECK(r, events >= 1); + } + + return r.report("shared/robustness"); +} diff --git a/tests/src/realtime/socketio/chat_test.cpp b/tests/src/realtime/socketio/chat_test.cpp new file mode 100644 index 0000000..1f92e8b --- /dev/null +++ b/tests/src/realtime/socketio/chat_test.cpp @@ -0,0 +1,221 @@ +// Route: /socket.io/ chat. Packet vectors for all 7 types, +// namespace connect/refuse, rooms, ack round-trip, client parity +// over an in-process server plus long-polling-only mode. + +#include +#include + +#include "test_main.h" +#include "realtime/shared/mini_json.h" +#include "realtime/socketio/ack_tracker.h" +#include "realtime/socketio/client/socketio_client.h" +#include "realtime/socketio/packet_codec.h" +#include "realtime/socketio/socketio_server.h" + +using namespace molo::realtime; + +int main() { + realtime_test::Results r; + using socketio::PacketCodec; + using socketio::SocketPacket; + using socketio::SocketPacketType; + + // Official-style vectors for text packets. + { + auto ev = PacketCodec::decode_text("2[\"hi\",\"a\"]"); + T_CHECK(r, ev.has_value() && ev->type == SocketPacketType::kEvent); + T_CHECK(r, ev->nsp == "/" && ev->ack_id < 0); + T_CHECK(r, ev->data.array()[0].string() == "hi"); + auto back = PacketCodec::encode_text(*ev); + T_CHECK(r, back.has_value() && *back == "2[\"hi\",\"a\"]"); + } + { + auto ev = PacketCodec::decode_text("2/chat,13[\"msg\",1]"); + T_CHECK(r, ev.has_value() && ev->nsp == "/chat" && ev->ack_id == 13); + auto back = PacketCodec::encode_text(*ev); + T_CHECK(r, back.has_value() && *back == "2/chat,13[\"msg\",1]"); + } + { + auto ack = PacketCodec::decode_text("336[\"ok\"]"); + T_CHECK(r, ack.has_value() && ack->type == SocketPacketType::kAck && + ack->ack_id == 36); + } + { + auto disc = PacketCodec::decode_text("1/chat,"); + T_CHECK(r, disc.has_value() && + disc->type == SocketPacketType::kDisconnect && + disc->nsp == "/chat"); + } + { + auto conn = PacketCodec::decode_text("0{\"token\":\"t\"}"); + T_CHECK(r, conn.has_value() && + conn->type == SocketPacketType::kConnect); + } + { + auto err = PacketCodec::decode_text("4{\"message\":\"no\"}"); + T_CHECK(r, err.has_value() && + err->type == SocketPacketType::kConnectError); + } + // Binary with placeholders. + { + auto bin = PacketCodec::decode_text( + "51-[\"frame\",{\"_placeholder\":true,\"num\":0}]"); + T_CHECK(r, bin.has_value() && + bin->type == SocketPacketType::kBinaryEvent); + T_CHECK(r, bin->attachments.size() == 1); + std::vector> frames = {{1, 2, 3}}; + T_CHECK(r, PacketCodec::attach(*bin, frames)); + T_CHECK(r, bin->attachments[0] == frames[0]); + auto back = PacketCodec::encode_text(*bin); + T_CHECK(r, back.has_value()); + } + T_CHECK(r, !PacketCodec::decode_text("").has_value()); + T_CHECK(r, !PacketCodec::decode_text("9[]").has_value()); + T_CHECK(r, !PacketCodec::valid_nsp("chat")); + T_CHECK(r, !PacketCodec::valid_nsp("/a,b")); + + // Ack tracker expiry. + { + socketio::AckTracker acks(std::chrono::milliseconds(10)); + auto t0 = std::chrono::steady_clock::now(); + long long id = acks.expect(t0); + T_CHECK(r, acks.pending() == 1); + T_CHECK(r, acks.resolve(id)); + T_CHECK(r, !acks.resolve(id)); + acks.expect(t0); + T_CHECK(r, acks.tick(t0 + std::chrono::milliseconds(50)) == 1); + T_CHECK(r, acks.pending() == 0); + } + + // In-process client/server session with rooms and acks. + socketio::SocketIoServer server; + struct Out { + std::string sid; + std::string text; + }; + std::vector server_out; + server.set_sink([&](const std::string& sid, const std::string& text, + const std::vector>&) { + server_out.push_back({sid, text}); + }); + server.dispatcher().on("/", "join", [&](const std::string& sid, + const shared::Json& args, auto ack) { + (void)ack; + std::string room = args.array()[1].string(); + server.rooms().join("/", room, sid); + }); + server.dispatcher().on("/", "say", [&](const std::string& sid, + const shared::Json& args, auto ack) { + std::string room = args.array()[1].string(); + std::string msg = args.array()[2].string(); + shared::Json fwd = shared::Json::make_array(); + fwd.push(shared::Json::str(msg)); + server.emit_to_room("/", room, "said", fwd, sid, false); + if (ack) { + shared::Json reply = shared::Json::make_array(); + reply.push(shared::Json::str("ok")); + ack(std::move(reply)); + } + }); + + socketio::client::SocketIoClient alice; + std::vector alice_out; + alice.on_emit([&](const std::string& text, + const std::vector>&) { + alice_out.push_back(text); + auto p = PacketCodec::decode_text(text); + if (p.has_value()) { + server.on_packet("alice", *p); + } + }); + // Long-polling-only client exposes the same API. + socketio::client::SocketIoClient poller(true); + T_CHECK(r, poller.polling_only()); + poller.on_emit([&](const std::string& text, + const std::vector>&) { + auto p = PacketCodec::decode_text(text); + if (p.has_value()) { + server.on_packet("poller", *p); + } + }); + // Pump helper: server replies back into each client. + auto pump_server = [&] { + for (const auto& o : server_out) { + auto p = PacketCodec::decode_text(o.text); + if (!p.has_value()) { + continue; + } + if (o.sid == "alice") { + alice.on_packet(*p); + } else if (o.sid == "poller") { + poller.on_packet(*p); + } + // "bob" speaks the raw codec directly; nothing to feed. + } + server_out.clear(); + }; + + bool alice_joined = false; + alice.on_connect([&](const std::string& nsp) { + (void)nsp; + alice_joined = true; + }); + std::vector heard; + alice.on_event("/", "said", + [&](const std::string&, const shared::Json& args, auto) { + heard.push_back(args.array()[1].string()); + }); + bool acked = false; + alice.connect_nsp("/"); + pump_server(); + T_CHECK(r, alice_joined && alice.connected_nsp("/")); + + shared::Json join_args = shared::Json::make_array(); + join_args.push(shared::Json::str("lobby")); + alice.emit("/", "join", join_args); + pump_server(); + T_CHECK(r, server.rooms().room_size("/", "lobby") == 1); + + // Bob joins through the raw codec path (third-party shape). + auto raw_join = PacketCodec::decode_text("0{}"); + server.on_packet("bob", *raw_join); + pump_server(); + auto bob_join = PacketCodec::decode_text("2[\"join\",\"lobby\"]"); + server.on_packet("bob", *bob_join); + T_CHECK(r, server.rooms().room_size("/", "lobby") == 2); + + shared::Json say_args = shared::Json::make_array(); + say_args.push(shared::Json::str("lobby")); + say_args.push(shared::Json::str("hello")); + alice.emit("/", "say", say_args, [&](shared::Json reply) { + acked = reply.array()[0].string() == "ok"; + }); + pump_server(); // Server emits broadcast plus ack. + pump_server(); // Ack text flows back to alice. + T_CHECK(r, acked); + // Alice excluded herself from the broadcast; bob got one packet. + T_CHECK(r, heard.empty()); + + // Long-polling-only client connects through the same path. + poller.connect_nsp("/"); + pump_server(); + T_CHECK(r, poller.connected_nsp("/")); + + // Namespace refusal carries CONNECT_ERROR. + server.get_or_create_namespace("/vip"); + server.get_or_create_namespace("/vip") = + socketio::Namespace("/vip", [](const std::string&, + const shared::Json&) { + return std::string("members only"); + }); + std::string refusal; + alice.on_connect_error([&](const std::string&, const std::string& msg) { + refusal = msg; + }); + alice.connect_nsp("/vip"); + pump_server(); + T_CHECK(r, refusal == "members only"); + T_CHECK(r, !alice.connected_nsp("/vip")); + + return r.report("socketio/chat"); +} diff --git a/tests/src/realtime/sse/client_test.cpp b/tests/src/realtime/sse/client_test.cpp new file mode 100644 index 0000000..8402d5b --- /dev/null +++ b/tests/src/realtime/sse/client_test.cpp @@ -0,0 +1,82 @@ +// SSE client: head handling, stream parsing, reconnect state. + +#include "test_main.h" + +#include + +#include "realtime/sse/sse_client.h" + +using namespace molo::realtime; + +int main() { + realtime_test::Results r; + + // Opens on 200 plus event-stream. + { + sse::SseClient c; + bool opened = false; + c.on_open([&] { opened = true; }); + shared::HttpResponse head; + head.status = 200; + shared::set_header(head, "Content-Type", "text/event-stream"); + T_CHECK(r, c.on_head(head)); + T_CHECK(r, opened); + T_CHECK(r, c.state() == shared::ClientState::kOpen); + } + + // Wrong content type fails cleanly. + { + sse::SseClient c; + shared::HttpResponse head; + head.status = 200; + shared::set_header(head, "Content-Type", "text/html"); + T_CHECK(r, !c.on_head(head)); + T_CHECK(r, !c.failure_reason().empty()); + } + + // Redirect and stop. + { + sse::SseClient c; + shared::HttpResponse head; + head.status = 307; + shared::set_header(head, "Location", "/sse/other"); + T_CHECK(r, !c.on_head(head)); + T_CHECK(r, c.redirect_url() == "/sse/other"); + } + { + sse::SseClient c; + shared::HttpResponse head; + head.status = 204; + T_CHECK(r, !c.on_head(head)); + T_CHECK(r, c.stopped()); + T_CHECK(r, c.state() == shared::ClientState::kClosed); + } + + // Stream parsing: multiline data joins, ids tracked, retry honored. + { + sse::SseClient c; + shared::HttpResponse head; + head.status = 200; + shared::set_header(head, "Content-Type", "text/event-stream"); + c.on_head(head); + std::vector got; + c.on_event([&](const sse::SseEvent& e) { got.push_back(e); }); + c.on_bytes(": keep-alive\n\n"); + c.on_bytes("event: tick\ndata: one\ndata: two\nid: 7\nretry: 5000\n\n"); + T_CHECK(r, got.size() == 1); + T_CHECK(r, got[0].event == "tick"); + T_CHECK(r, got[0].data == "one\ntwo"); + T_CHECK(r, got[0].id == "7"); + T_CHECK(r, c.last_event_id() == "7"); + T_CHECK(r, c.retry_delay() == std::chrono::milliseconds(5000)); + // Partial delivery across feeds still dispatches. + c.on_bytes("data: half"); + T_CHECK(r, got.size() == 1); + c.on_bytes("-done\n\n"); + T_CHECK(r, got.size() == 2); + T_CHECK(r, got[1].data == "half-done"); + T_CHECK(r, got[1].event.empty()); // Default message type. + } + + return r.report("sse/client"); +} diff --git a/tests/src/realtime/sse/feed_test.cpp b/tests/src/realtime/sse/feed_test.cpp new file mode 100644 index 0000000..4424ca6 --- /dev/null +++ b/tests/src/realtime/sse/feed_test.cpp @@ -0,0 +1,102 @@ +// Route: /sse/feed. Byte-exact encoding, headers, replay, stop. + +#include "test_main.h" + +#include +#include + +#include "realtime/sse/event_encoder.h" +#include "realtime/sse/event_store.h" +#include "realtime/sse/sse_response.h" + +using namespace molo::realtime; + +int main() { + realtime_test::Results r; + + // Byte-exact encoding with all fields. + { + sse::SseEvent e; + e.event = "price"; + e.data = "line1\nline2"; + e.id = "42"; + T_CHECK(r, sse::EventEncoder::encode(e) == + "event: price\ndata: line1\ndata: line2\nid: 42\n\n"); + } + // Default type omits the event line; CRLF normalizes. + { + sse::SseEvent e; + e.data = "a\r\nb\rc"; + T_CHECK(r, sse::EventEncoder::encode(e) == + "data: a\ndata: b\ndata: c\n\n"); + } + // Retry clamps to 1 s..300 s. + T_CHECK(r, sse::EventEncoder::clamp_retry(std::chrono::milliseconds(5)) == + std::chrono::milliseconds(1000)); + T_CHECK(r, + sse::EventEncoder::clamp_retry(std::chrono::milliseconds(9999999)) == + std::chrono::milliseconds(300000)); + + // Response headers per rule 6. + { + sse::SseResponse res; + auto head = res.start(); + T_CHECK(r, head.status == 200 && head.stream); + T_CHECK(r, shared::header_first(head, "Content-Type") == + "text/event-stream"); + T_CHECK(r, shared::header_first(head, "Cache-Control") == "no-store"); + } + + // Store assigns ids and replays after Last-Event-ID. + { + sse::EventStore store(10, std::chrono::minutes(5)); + sse::SseEvent a, b, c; + a.data = "a"; + b.data = "b"; + c.data = "c"; + auto sa = store.append(a); + auto sb = store.append(b); + store.append(c); + T_CHECK(r, sa.id == "1" && sb.id == "2"); + auto replay = store.replay_after("1"); + T_CHECK(r, replay.size() == 2); + T_CHECK(r, replay[0].data == "b" && replay[1].data == "c"); + T_CHECK(r, store.replay_after("3").empty()); + // Unknown id replays the whole window. + T_CHECK(r, store.replay_after("zzz").size() == 3); + } + + // Cap eviction: count wins over age here. + { + sse::EventStore store(2, std::chrono::minutes(5)); + sse::SseEvent e; + e.data = "x"; + store.append(e); + store.append(e); + store.append(e); + T_CHECK(r, store.size() == 2); + } + + // SseResponse wires store plus encoding plus 204 stop. + { + sse::SseResponse res; + res.set_retry(std::chrono::milliseconds(2000)); + sse::SseEvent e; + e.event = "msg"; + e.data = "hi"; + std::string wire = res.send(e); + T_CHECK(r, wire.find("retry: 2000\n") != std::string::npos); + T_CHECK(r, wire.find("id: 1\n") != std::string::npos); + sse::SseEvent e2; + e2.data = "again"; + std::string wire2 = res.send(e2); + T_CHECK(r, wire2.find("retry:") == std::string::npos); // Once only. + std::string replay = res.replay_bytes("1"); + T_CHECK(r, replay.find("data: again") != std::string::npos); + T_CHECK(r, sse::SseResponse::stop().status == 204); + } + + T_CHECK(r, sse::SseResponse::keep_alive() == ": ping\n\n"); + + return r.report("sse/feed"); +} diff --git a/tests/src/realtime/stream/passthrough_test.cpp b/tests/src/realtime/stream/passthrough_test.cpp new file mode 100644 index 0000000..5419cc4 --- /dev/null +++ b/tests/src/realtime/stream/passthrough_test.cpp @@ -0,0 +1,107 @@ +// Route: /stream/download. Chunked framing, range parsing, +// passthrough pump with bounded memory and disconnect cleanup. + +#include "test_main.h" + +#include +#include + +#include "realtime/streaming/chunked_body.h" +#include "realtime/streaming/passthrough_body.h" + +using namespace molo::realtime; + +int main() { + realtime_test::Results r; + + // Chunked framing is byte-exact. + { + streaming::ChunkedBody body; + std::string out = body.write("hello"); + out += body.flush(); + out += body.finish(); + T_CHECK(r, out == "5\r\nhello\r\n0\r\n\r\n"); + T_CHECK(r, body.finish().empty()); // Idempotent. + T_CHECK(r, body.total_bytes() == 5); + } + + // Empty writes produce no chunks. + { + streaming::ChunkedBody body; + T_CHECK(r, body.write("").empty()); + T_CHECK(r, body.finish() == "0\r\n\r\n"); + } + + // Range parsing: single ranges only. + { + auto full = streaming::parse_range("bytes=0-99", 100); + T_CHECK(r, full.satisfiable && full.first == 0 && full.last == 99); + auto tail = streaming::parse_range("bytes=-10", 100); + T_CHECK(r, tail.satisfiable && tail.first == 90 && tail.last == 99); + auto open = streaming::parse_range("bytes=50-", 100); + T_CHECK(r, open.satisfiable && open.first == 50 && open.last == 99); + T_CHECK(r, !streaming::parse_range("bytes=0-10,20-30", 100).satisfiable); + T_CHECK(r, !streaming::parse_range("bytes=200-", 100).satisfiable); + T_CHECK(r, !streaming::parse_range("items=0-1", 100).satisfiable); + } + + // Passthrough pumps 8 MiB through a 64 KiB ring with flat memory. + { + const std::size_t kTotal = 8u * 1024u * 1024u; + std::size_t produced = 0; + std::size_t consumed = 0; + uint64_t checksum = 0; + streaming::PassthroughBody pump(64u * 1024u); + std::size_t moved = pump.pump( + [&](uint8_t* out, std::size_t max) { + std::size_t left = kTotal - produced; + std::size_t n = left < max ? left : max; + for (std::size_t i = 0; i < n; ++i) { + out[i] = static_cast((produced + i) & 0xFFu); + } + produced += n; + return n; + }, + [&](const uint8_t* data, std::size_t len) { + for (std::size_t i = 0; i < len; ++i) { + checksum += data[i]; + } + consumed += len; + return true; + }); + T_CHECK(r, moved == kTotal && consumed == kTotal); + T_CHECK(r, !pump.aborted()); + // Sum of 0..255 repeating over 8 MiB. + uint64_t expect = (kTotal / 256) * (255u * 256u / 2); + T_CHECK(r, checksum == expect); + } + + // Client disconnect aborts the pump promptly. + { + streaming::PassthroughBody pump(4096); + std::size_t calls = 0; + std::size_t moved = pump.pump( + [&](uint8_t* out, std::size_t max) { + (void)out; + ++calls; + return max; // Endless producer. + }, + [&](const uint8_t*, std::size_t) { return false; }); + T_CHECK(r, pump.aborted()); + T_CHECK(r, moved == 0 && calls == 1); + } + + // Max-bytes cap truncates. + { + streaming::PassthroughBody pump(4096); + std::size_t moved = pump.pump( + [&](uint8_t* out, std::size_t max) { + (void)out; + return max; + }, + [&](const uint8_t*, std::size_t) { return true; }, 100); + T_CHECK(r, moved == 100); + } + + return r.report("stream/passthrough"); +} diff --git a/tests/src/realtime/test_main.h b/tests/src/realtime/test_main.h new file mode 100644 index 0000000..8365854 --- /dev/null +++ b/tests/src/realtime/test_main.h @@ -0,0 +1,45 @@ +#pragma once + +// Minimal test helper for realtime integration tests. Tests run +// through real objects and real sockets (socketpair loopback), never +// mocks. CHECK records failures; main returns nonzero when any fail. + +#include +#include +#include + +namespace realtime_test { + +class Results { + public: + void check(bool ok, const char* expr, const char* file, int line) { + ++total_; + if (ok) { + ++passed_; + } else { + ++failed_; + failures_.push_back(std::string(file) + ":" + std::to_string(line) + + ": " + expr); + } + } + + int report(const char* name) { + std::printf("%s: passed=%d failed=%d total=%d\n", name, passed_, failed_, + total_); + for (const auto& f : failures_) { + std::printf(" FAIL %s\n", f.c_str()); + } + return failed_ == 0 ? 0 : 1; + } + + private: + int total_ = 0; + int passed_ = 0; + int failed_ = 0; + std::vector failures_; +}; + +} // namespace realtime_test + +#define T_CHECK(results, expr) \ + (results).check((expr), #expr, __FILE__, __LINE__) diff --git a/tests/src/realtime/ws/echo_test.cpp b/tests/src/realtime/ws/echo_test.cpp new file mode 100644 index 0000000..492d477 --- /dev/null +++ b/tests/src/realtime/ws/echo_test.cpp @@ -0,0 +1,139 @@ +// Route: /ws/echo. Full round-trip over a real socketpair: client +// handshake bytes, server view admit/accept, text plus binary echo, +// close handshake. Exercises the real FD path, not memory doubles. + +#include +#include + +#include +#include +#include + +#include "test_main.h" +#include "realtime/shared/transport.h" +#include "realtime/websocket/client/client_handshake.h" +#include "realtime/websocket/client/websocket_client.h" +#include "realtime/websocket/server/websocket_handler.h" +#include "realtime/websocket/server/websocket_view.h" + +using namespace molo::realtime; + +namespace { + +class EchoHandler : public websocket::server::WebSocketHandler { + public: + void on_text(const std::string& message) override { + send_text("echo:" + message); + } + void on_binary(const std::vector& payload) override { + send_binary(payload); + } +}; + +std::string read_available(int fd) { + char buf[65536]; + ssize_t n = ::read(fd, buf, sizeof(buf)); + return n <= 0 ? std::string{} : std::string(buf, buf + n); +} + +} // namespace + +int main() { + realtime_test::Results r; + + int sv[2]; + T_CHECK(r, ::socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0); + + websocket::server::WebSocketView view( + [] { return std::make_unique(); }); + + // Client builds the handshake, sends it over the real socket. + websocket::client::WebSocketClient client; + shared::FdTransport client_fd(sv[0]); + client.bind(&client_fd, "localhost", "/ws/echo"); + std::string key; + shared::HttpRequest hs_req = + websocket::client::ClientHandshake::build_request( + "localhost", "/ws/echo", {}, key); + std::string wire = websocket::client::ClientHandshake::render(hs_req); + T_CHECK(r, client_fd.send(wire)); + + // Server reads raw bytes and admits from the parsed head. + std::string got = read_available(sv[1]); + T_CHECK(r, got.find("Sec-WebSocket-Key: ") != std::string::npos); + auto admit = view.admit(hs_req); + T_CHECK(r, admit.ok); + + // Server 101 travels back; client validates the accept hash. + T_CHECK(r, websocket::client::ClientHandshake::check_response( + admit.response, key) + .ok); + client.note_open(); + + // Bind the server connection to the other FD. + shared::FdTransport server_fd(sv[1]); + shared::ConnectionContext ctx; + ctx.route = "/ws/echo"; + ctx.correlation_id = "echo-1"; + websocket::server::WebSocketConnection* conn = + view.accept(&server_fd, ctx); + T_CHECK(r, conn != nullptr); + + // Client text round-trips through the echo handler. + std::string echoed; + client.on_message([&](bool is_text, const std::vector& p) { + (void)is_text; + echoed = std::string(p.begin(), p.end()); + }); + T_CHECK(r, client.send_text("hello")); + std::string c2s = read_available(sv[1]); + T_CHECK(r, !c2s.empty()); + T_CHECK(r, conn->on_bytes(reinterpret_cast(c2s.data()), + c2s.size())); + T_CHECK(r, conn->flush()); + std::string s2c = read_available(sv[0]); + T_CHECK(r, !s2c.empty()); + T_CHECK(r, client.on_bytes(reinterpret_cast(s2c.data()), + s2c.size())); + T_CHECK(r, echoed == "echo:hello"); + + // Binary round-trips byte-exact. + std::vector bin_in = {0x00, 0xFF, 0x01, 0xFE}; + std::vector bin_out; + client.on_message([&](bool is_text, const std::vector& p) { + (void)is_text; + bin_out = p; + }); + T_CHECK(r, client.send_binary(bin_in.data(), bin_in.size())); + c2s = read_available(sv[1]); + T_CHECK(r, conn->on_bytes(reinterpret_cast(c2s.data()), + c2s.size())); + T_CHECK(r, conn->flush()); + s2c = read_available(sv[0]); + T_CHECK(r, client.on_bytes(reinterpret_cast(s2c.data()), + s2c.size())); + T_CHECK(r, bin_out == bin_in); + + // Ping from the client is answered with an identical pong. + client.send_ping({'p', 'i'}); + c2s = read_available(sv[1]); + T_CHECK(r, conn->on_bytes(reinterpret_cast(c2s.data()), + c2s.size())); + s2c = read_available(sv[0]); + T_CHECK(r, s2c.size() == 4 && s2c[0] == (char)0x8A && s2c[2] == 'p'); + + // Clean close both directions. + bool client_closed = false; + client.on_close([&](uint16_t, const std::string&) { client_closed = true; }); + client.close(1000, "done"); + c2s = read_available(sv[1]); + conn->on_bytes(reinterpret_cast(c2s.data()), c2s.size()); + T_CHECK(r, !conn->open()); + s2c = read_available(sv[0]); + client.on_bytes(reinterpret_cast(s2c.data()), s2c.size()); + T_CHECK(r, client_closed); + + ::close(sv[0]); + ::close(sv[1]); + return r.report("ws/echo"); +} diff --git a/tests/src/realtime/ws/frames_test.cpp b/tests/src/realtime/ws/frames_test.cpp new file mode 100644 index 0000000..b0190fa --- /dev/null +++ b/tests/src/realtime/ws/frames_test.cpp @@ -0,0 +1,169 @@ +// Route: /ws/* framing. Byte-level RFC 6455 vectors: masked parse, +// fragmentation, control limits, UTF-8 and mask rejection. + +#include "test_main.h" + +#include "realtime/shared/transport.h" +#include "realtime/websocket/server/frame_parser.h" +#include "realtime/websocket/server/frame_serializer.h" +#include "realtime/websocket/server/websocket_response.h" + +using namespace molo::realtime; + +namespace { + +// RFC 6455 section 5.7 masked "Hello" with mask 37 fa 21 3d. +std::vector rfc_hello() { + return {0x81, 0x85, 0x37, 0xfa, 0x21, 0x3d, 0x7f, 0x9f, 0x4d, 0x51, 0x58}; +} + +} // namespace + +int main() { + realtime_test::Results r; + using server = websocket::server::FrameParser; + using serializer = websocket::server::FrameSerializer; + using websocket::server::CloseCode; + using websocket::server::Opcode; + using websocket::server::ParseOutcome; + + // Golden masked frame decodes to "Hello". + { + server p(true); + auto wire = rfc_hello(); + std::size_t consumed = 0; + auto step = p.feed(wire.data(), wire.size(), consumed); + T_CHECK(r, step.outcome == ParseOutcome::kFrame); + T_CHECK(r, step.frame.opcode == Opcode::kText); + T_CHECK(r, std::string(step.frame.payload.begin(), + step.frame.payload.end()) == "Hello"); + } + + // Unmasked client frame fails with 1002 (rule 2). + { + server p(true); + auto wire = serializer::text("hi"); + std::size_t consumed = 0; + auto step = p.feed(wire.data(), wire.size(), consumed); + T_CHECK(r, step.outcome == ParseOutcome::kFailed); + T_CHECK(r, step.fail_code == CloseCode::kProtocolError); + } + + // Server frames are never masked: round-trip unmasked parse. + { + server p(false); + auto wire = serializer::text("hello"); + std::size_t consumed = 0; + auto step = p.feed(wire.data(), wire.size(), consumed); + T_CHECK(r, step.outcome == ParseOutcome::kFrame); + } + { + server p(false); + uint8_t mask[4] = {1, 2, 3, 4}; + auto wire = serializer::masked_frame(true, Opcode::kText, + reinterpret_cast("hi"), + 2, mask); + std::size_t consumed = 0; + auto step = p.feed(wire.data(), wire.size(), consumed); + T_CHECK(r, step.outcome == ParseOutcome::kFailed); + } + + // Fragmented control frame is a protocol error (rule 4). + { + server p(true); + uint8_t mask[4] = {0, 0, 0, 0}; + uint8_t payload[1] = {'x'}; + auto wire = + serializer::masked_frame(false, Opcode::kPing, payload, 1, mask); + std::size_t consumed = 0; + auto step = p.feed(wire.data(), wire.size(), consumed); + T_CHECK(r, step.outcome == ParseOutcome::kFailed); + } + + // Control payload over 125 bytes fails. + { + server p(true); + std::vector big(126, 'a'); + uint8_t mask[4] = {0, 0, 0, 0}; + auto wire = + serializer::masked_frame(true, Opcode::kPing, big.data(), big.size(), + mask); + std::size_t consumed = 0; + auto step = p.feed(wire.data(), wire.size(), consumed); + T_CHECK(r, step.outcome == ParseOutcome::kFailed); + } + + // 16-bit and 64-bit lengths round-trip. + for (std::size_t n : {126u, 1000u, 65536u}) { + std::vector payload(n, 'b'); + auto wire = serializer::frame(true, Opcode::kBinary, payload); + server p(false); + std::size_t consumed = 0; + auto step = p.feed(wire.data(), wire.size(), consumed); + T_CHECK(r, step.outcome == ParseOutcome::kFrame); + T_CHECK(r, step.frame.payload.size() == n); + } + + // Invalid UTF-8 text fails the connection with 1007 (rule 3). + { + shared::MemoryTransport t; + shared::ConnectionContext ctx; + websocket::server::WebSocketConnection conn(&t, ctx); + bool closed = false; + conn.on_close([&](uint16_t code, const std::string&) { + closed = code == 1007; + }); + uint8_t mask[4] = {9, 9, 9, 9}; + uint8_t bad[2] = {0xC3, 0x28}; // Invalid 2-byte sequence. + auto wire = + serializer::masked_frame(true, Opcode::kText, bad, 2, mask); + T_CHECK(r, !conn.on_bytes(wire.data(), wire.size())); + T_CHECK(r, closed); + } + + // Fragmented message reassembles; ping is answered with pong. + { + shared::MemoryTransport t; + shared::ConnectionContext ctx; + websocket::server::WebSocketConnection conn(&t, ctx); + std::string got; + conn.on_message([&](bool is_text, const std::vector& p) { + got = std::string(p.begin(), p.end()); + (void)is_text; + }); + uint8_t mask[4] = {0, 0, 0, 0}; + auto f1 = serializer::masked_frame( + false, Opcode::kText, reinterpret_cast("he"), 2, mask); + auto f2 = serializer::masked_frame( + true, Opcode::kContinuation, + reinterpret_cast("llo"), 3, mask); + T_CHECK(r, conn.on_bytes(f1.data(), f1.size())); + T_CHECK(r, conn.on_bytes(f2.data(), f2.size())); + T_CHECK(r, got == "hello"); + auto ping = serializer::masked_frame(true, Opcode::kPing, + reinterpret_cast("p"), + 1, mask); + T_CHECK(r, conn.on_bytes(ping.data(), ping.size())); + // Server answered 0x8a pong with payload "p". + T_CHECK(r, t.sent().size() == 3 && t.sent()[0] == 0x8A && + t.sent()[2] == 'p'); + } + + // Close handshake echoes the code. + { + shared::MemoryTransport t; + shared::ConnectionContext ctx; + websocket::server::WebSocketConnection conn(&t, ctx); + uint16_t code = 0; + conn.on_close([&](uint16_t c, const std::string&) { code = c; }); + uint8_t mask[4] = {0, 0, 0, 0}; + uint8_t body[2] = {0x03, 0xE8}; // 1000. + auto wire = + serializer::masked_frame(true, Opcode::kClose, body, 2, mask); + conn.on_bytes(wire.data(), wire.size()); + T_CHECK(r, code == 1000); + T_CHECK(r, !conn.open()); + } + + return r.report("ws/frames"); +} diff --git a/tests/src/realtime/ws/handshake_test.cpp b/tests/src/realtime/ws/handshake_test.cpp new file mode 100644 index 0000000..b311dbf --- /dev/null +++ b/tests/src/realtime/ws/handshake_test.cpp @@ -0,0 +1,86 @@ +// Route: /ws/* handshake. Golden RFC 6455 vectors plus failure modes. + +#include "test_main.h" + +#include "realtime/shared/http_seam.h" +#include "realtime/websocket/server/handshake_validator.h" + +using namespace molo::realtime; + +namespace { + +websocket::server::HandshakeValidator validator() { + websocket::server::HandshakeConfig cfg; + cfg.subprotocols = {"chat", "super"}; + return websocket::server::HandshakeValidator(cfg); +} + +shared::HttpRequest base_req() { + shared::HttpRequest req; + req.method = "GET"; + req.path = "/ws/chat"; + req.headers.emplace("Upgrade", "websocket"); + req.headers.emplace("Connection", "Upgrade"); + req.headers.emplace("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ=="); + req.headers.emplace("Sec-WebSocket-Version", "13"); + return req; +} + +} // namespace + +int main() { + realtime_test::Results r; + + // RFC 6455 section 1.3 golden vector. + T_CHECK(r, websocket::server::HandshakeValidator::accept_for( + "dGhlIHNhbXBsZSBub25jZQ==") == "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="); + + auto ok = validator().validate(base_req()); + T_CHECK(r, ok.ok); + T_CHECK(r, ok.response.status == 101); + T_CHECK(r, shared::header_first(ok.response, "Sec-WebSocket-Accept") == + "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="); + T_CHECK(r, shared::header_first(ok.response, "Upgrade") == "websocket"); + + // Subprotocol negotiation picks the server's first match. + auto req = base_req(); + req.headers.emplace("Sec-WebSocket-Protocol", "super, chat"); + auto sub = validator().validate(req); + T_CHECK(r, sub.ok && sub.subprotocol == "chat"); + + // Unknown subprotocol fails when the server offers any. + req = base_req(); + req.headers.emplace("Sec-WebSocket-Protocol", "nope"); + T_CHECK(r, !validator().validate(req).ok); + + // Failure modes. + req = base_req(); + req.method = "POST"; + T_CHECK(r, !validator().validate(req).ok); + req = base_req(); + req.headers.erase("Upgrade"); + T_CHECK(r, !validator().validate(req).ok); + req = base_req(); + req.headers.erase("Connection"); + T_CHECK(r, !validator().validate(req).ok); + req = base_req(); + req.headers.erase("Sec-WebSocket-Key"); + T_CHECK(r, !validator().validate(req).ok); + req = base_req(); + req.headers.emplace("Sec-WebSocket-Key", "!!!!"); + req.headers.erase("Sec-WebSocket-Key"); + req.headers.emplace("Sec-WebSocket-Key", "!!!!"); + T_CHECK(r, !validator().validate(req).ok); + req = base_req(); + req.headers.erase("Sec-WebSocket-Key"); + req.headers.emplace("Sec-WebSocket-Key", "aGk="); // Decodes to 2 bytes. + T_CHECK(r, !validator().validate(req).ok); + + // Case-insensitive token match on Connection with extras. + req = base_req(); + req.headers.erase("Connection"); + req.headers.emplace("Connection", "keep-alive, Upgrade"); + T_CHECK(r, validator().validate(req).ok); + + return r.report("ws/handshake"); +} diff --git a/tests/src/realtime/ws/heartbeat_test.cpp b/tests/src/realtime/ws/heartbeat_test.cpp new file mode 100644 index 0000000..ac09e47 --- /dev/null +++ b/tests/src/realtime/ws/heartbeat_test.cpp @@ -0,0 +1,51 @@ +// Heartbeat manager: ping schedule plus pong-timeout close. + +#include "test_main.h" + +#include "realtime/websocket/server/heartbeat_manager.h" + +using namespace molo::realtime; + +int main() { + realtime_test::Results r; + using clock = std::chrono::steady_clock; + using websocket::server::HeartbeatManager; + + // Pings fire on interval while activity flows. + { + HeartbeatManager hb(std::chrono::milliseconds(25), + std::chrono::milliseconds(100)); + int pings = 0; + bool dead = false; + hb.on_ping([&] { ++pings; }); + hb.on_timeout([&] { dead = true; }); + auto t0 = clock::now(); + hb.start(t0); + hb.note_activity(t0); + T_CHECK(r, hb.tick(t0 + std::chrono::milliseconds(10))); + T_CHECK(r, pings == 0); + hb.note_activity(t0 + std::chrono::milliseconds(10)); + T_CHECK(r, hb.tick(t0 + std::chrono::milliseconds(30))); + T_CHECK(r, pings == 1); + T_CHECK(r, !dead); + } + + // Silence past the timeout fires once and sticks. + { + HeartbeatManager hb(std::chrono::milliseconds(25), + std::chrono::milliseconds(50)); + int pings = 0; + int timeouts = 0; + hb.on_ping([&] { ++pings; }); + hb.on_timeout([&] { ++timeouts; }); + auto t0 = clock::now(); + hb.start(t0); + T_CHECK(r, !hb.tick(t0 + std::chrono::milliseconds(60))); + T_CHECK(r, timeouts == 1); + T_CHECK(r, !hb.tick(t0 + std::chrono::milliseconds(70))); + T_CHECK(r, timeouts == 1); + T_CHECK(r, hb.timed_out()); + } + + return r.report("ws/heartbeat"); +} diff --git a/tests/src/runtime/compress/test_brotli.cpp b/tests/src/runtime/compress/test_brotli.cpp new file mode 100644 index 0000000..d01a5f7 --- /dev/null +++ b/tests/src/runtime/compress/test_brotli.cpp @@ -0,0 +1,119 @@ +#include "test_framework.h" + +#include +#include + +#include "runtime/compress/brotli.h" + +using namespace molo::runtime::compress; + +namespace { +std::string Hex(const std::string& hex) { + std::string out; + for (std::size_t i = 0; i + 1 < hex.size(); i += 2) { + auto nyb = [](char c) -> unsigned { + if (c >= '0' && c <= '9') return (unsigned)(c - '0'); + return (unsigned)(c - 'a' + 10); + }; + out.push_back((char)((nyb(hex[i]) << 4) | nyb(hex[i + 1]))); + } + return out; +} + +std::string EncodeAll(const std::string& in, CodecStatus& st) { + BrotliEncoder e; + std::string out; + std::vector buf(1000); + std::size_t pos = 0; + CodecResult r; + do { + std::size_t n = in.size() - pos; + if (n > 70000) n = 70000; + bool finish = (pos + n >= in.size()); + r = e.feed(in.data() + pos, n, buf.data(), buf.size(), finish); + pos += r.in_used; + out.append(buf.data(), r.out_used); + } while (r.status == CodecStatus::Ok || + r.status == CodecStatus::NeedOutput); + st = r.status; + return out; +} + +std::string DecodeAll(const std::string& in, CodecStatus& st, + std::size_t chunk = 100000) { + BrotliDecoder d; + std::string out; + std::vector buf(500); + std::size_t pos = 0; + CodecResult r; + do { + std::size_t n = in.size() - pos; + if (n > chunk) n = chunk; + bool finish = (pos + n >= in.size()); + r = d.feed(in.data() + pos, n, buf.data(), buf.size(), finish); + pos += r.in_used; + out.append(buf.data(), r.out_used); + } while (r.status == CodecStatus::Ok || + r.status == CodecStatus::NeedOutput); + st = r.status; + return out; +} +} // namespace + +TEST(BrotliRoundTrip) { + CodecStatus st; + std::string in; + for (int i = 0; i < 3000; ++i) in += "row " + std::to_string(i) + "\n"; + std::string enc = EncodeAll(in, st); + ASSERT_EQ(st, CodecStatus::Done); + ASSERT_EQ(DecodeAll(enc, st), in); + ASSERT_EQ(st, CodecStatus::Done); + ASSERT_EQ(DecodeAll(enc, st, 7), in); + ASSERT_EQ(st, CodecStatus::Done); +} + +TEST(BrotliMultiBlock) { + CodecStatus st; + std::string in(200000, 'x'); + for (std::size_t i = 0; i < in.size(); i += 7) in[i] = (char)('a' + i % 26); + std::string enc = EncodeAll(in, st); + ASSERT_EQ(st, CodecStatus::Done); + ASSERT_EQ(DecodeAll(enc, st, 13), in); + ASSERT_EQ(st, CodecStatus::Done); +} + +TEST(BrotliEmptyIsOneByte) { + CodecStatus st; + std::string enc = EncodeAll("", st); + ASSERT_EQ(st, CodecStatus::Done); + ASSERT_EQ(enc, std::string("\x06", 1)); // WBITS + ISLAST + ISLASTEMPTY + ASSERT_EQ(DecodeAll(enc, st), ""); + ASSERT_EQ(st, CodecStatus::Done); +} + +TEST(BrotliReferenceCompressedIsUnsupported) { + // Real entropy-coded Brotli (CPython brotli, quality 4): valid framing we + // report as Unsupported per the documented v1 gap, never garbage. + std::string ref = Hex( + "1bf9000044e796ea43a106601f14af9b18840d38114c2cf66c211f2672824120edf7" + "ee45b4c6179544fce13c539e2cc38e4fe3aec1052a50"); + CodecStatus st; + DecodeAll(ref, st); + ASSERT_EQ(st, CodecStatus::Unsupported); +} + +TEST(BrotliCorrupt) { + CodecStatus st; + DecodeAll("\x1b\xff\x00 somewhere invalid brotli bytes....", st); + ASSERT_TRUE(st == CodecStatus::Error || st == CodecStatus::Unsupported); + // Truncated mid-block. + CodecStatus st2; + std::string in(5000, 'q'); + std::string enc = EncodeAll(in, st2); + ASSERT_EQ(st2, CodecStatus::Done); + DecodeAll(enc.substr(0, 100), st); + ASSERT_EQ(st, CodecStatus::Error); + // Trailing garbage after a valid stream. + DecodeAll(enc + "trailing", st); + ASSERT_EQ(st, CodecStatus::Error); +} diff --git a/tests/src/runtime/compress/test_gzip.cpp b/tests/src/runtime/compress/test_gzip.cpp new file mode 100644 index 0000000..89903f2 --- /dev/null +++ b/tests/src/runtime/compress/test_gzip.cpp @@ -0,0 +1,139 @@ +#include "test_framework.h" + +#include +#include + +#include "runtime/compress/gzip.h" + +using namespace molo::runtime::compress; + +namespace { +std::string Hex(const std::string& hex) { + std::string out; + for (std::size_t i = 0; i + 1 < hex.size(); i += 2) { + auto nyb = [](char c) -> unsigned { + if (c >= '0' && c <= '9') return (unsigned)(c - '0'); + return (unsigned)(c - 'a' + 10); + }; + out.push_back((char)((nyb(hex[i]) << 4) | nyb(hex[i + 1]))); + } + return out; +} + +std::string EncodeAll(const std::string& in, CodecStatus& st) { + GzipEncoder e; + std::string out; + std::vector buf(1000); + std::size_t pos = 0; + CodecResult r; + do { + std::size_t n = in.size() - pos; + if (n > 7000) n = 7000; + bool finish = (pos + n >= in.size()); + r = e.feed(in.data() + pos, n, buf.data(), buf.size(), finish); + pos += r.in_used; + out.append(buf.data(), r.out_used); + } while (r.status == CodecStatus::Ok || + r.status == CodecStatus::NeedOutput); + st = r.status; + return out; +} + +std::string DecodeAll(const std::string& in, CodecStatus& st, + std::size_t chunk = 100000, std::size_t cap = 65536) { + GzipDecoder d; + std::string out; + std::vector buf(cap); + std::size_t pos = 0; + CodecResult r; + do { + std::size_t n = in.size() - pos; + if (n > chunk) n = chunk; + bool finish = (pos + n >= in.size()); + r = d.feed(in.data() + pos, n, buf.data(), buf.size(), finish); + pos += r.in_used; + out.append(buf.data(), r.out_used); + } while (r.status == CodecStatus::Ok || + r.status == CodecStatus::NeedOutput); + st = r.status; + return out; +} +} // namespace + +TEST(GzipReferenceVector) { + // Produced by CPython gzip.compress(mtime=0); any gzip decodes it. + std::string ref = Hex( + "1f8b08000000000002ffcb48cdc9c957284a4d4b2d4acd4b4e5528cf2fca495154c818" + "151e7ac200bb4c4e2acc010000"); + CodecStatus st; + std::string plain; + for (int i = 0; i < 20; ++i) plain += "hello reference world! "; + ASSERT_EQ(DecodeAll(ref, st), plain); + ASSERT_EQ(st, CodecStatus::Done); + // Same vector in 7-byte input chunks and 11-byte output caps. + ASSERT_EQ(DecodeAll(ref, st, 7, 11), plain); + ASSERT_EQ(st, CodecStatus::Done); +} + +TEST(GzipRoundTrip) { + CodecStatus st; + std::string in; + for (int i = 0; i < 2000; ++i) in += "line " + std::to_string(i % 97) + "\n"; + // Irregular tail keeps the ratio realistic (under the 10x bomb guard). + uint32_t x = 0x12345678; + for (int i = 0; i < 20000; ++i) { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + in.push_back((char)(x >> 24)); + } + std::string enc = EncodeAll(in, st); + ASSERT_EQ(st, CodecStatus::Done); + ASSERT_TRUE(enc.size() < in.size()); // real compression + ASSERT_EQ(enc.substr(0, 2), std::string("\x1f\x8b", 2)); + ASSERT_EQ(DecodeAll(enc, st), in); + ASSERT_EQ(st, CodecStatus::Done); +} + +TEST(GzipEmpty) { + CodecStatus st; + std::string enc = EncodeAll("", st); + ASSERT_EQ(st, CodecStatus::Done); + ASSERT_EQ(DecodeAll(enc, st), ""); + ASSERT_EQ(st, CodecStatus::Done); +} + +TEST(GzipBombGuard) { + // 217x ratio input: the 10x guard must stop it. + std::string bomb_in(10000, 'A'); + CodecStatus st; + std::string enc = EncodeAll(bomb_in, st); + ASSERT_EQ(st, CodecStatus::Done); + ASSERT_TRUE(enc.size() * 10 < bomb_in.size()); + std::string out = DecodeAll(enc, st); + ASSERT_EQ(st, CodecStatus::Error); + (void)out; +} + +TEST(GzipCorrupt) { + std::string ref = Hex( + "1f8b08000000000002ffcb48cdc9c957284a4d4b2d4acd4b4e5528cf2fca495154c818" + "151e7ac200bb4c4e2acc010000"); + CodecStatus st; + std::string bad = ref; + bad[20] ^= 0x40; + DecodeAll(bad, st); + ASSERT_EQ(st, CodecStatus::Error); + DecodeAll(ref.substr(0, ref.size() - 1), st); // truncated + ASSERT_EQ(st, CodecStatus::Error); + std::string badcrc = ref; // broken CRC + badcrc[badcrc.size() - 8] ^= 0x01; + DecodeAll(badcrc, st); + ASSERT_EQ(st, CodecStatus::Error); + DecodeAll("not gzip at all, clearly...........", st); + ASSERT_EQ(st, CodecStatus::Error); +} + +TEST(GzipCrc) { + ASSERT_EQ(Crc32("123456789", 9), 0xcbf43926u); +} diff --git a/tests/src/runtime/compress/test_negotiate.cpp b/tests/src/runtime/compress/test_negotiate.cpp new file mode 100644 index 0000000..96ae348 --- /dev/null +++ b/tests/src/runtime/compress/test_negotiate.cpp @@ -0,0 +1,61 @@ +#include "test_framework.h" + +#include "runtime/compress/negotiate.h" + +using namespace molo::runtime::compress; + +namespace { +std::string Neg(const std::string& ae, const std::string& ct, + std::size_t len) { + return Negotiator::Negotiate(ae, ct, len, false, false, 200).encoding; +} +} // namespace + +TEST(NegotiateMatrix) { + ASSERT_EQ(Neg("br, gzip", "text/html", 5000), "br"); + ASSERT_EQ(Neg("gzip, br", "text/html", 5000), "br"); // tie goes br + ASSERT_EQ(Neg("gzip", "text/html", 5000), "gzip"); + ASSERT_EQ(Neg("br;q=0.5, gzip;q=0.9", "text/html", 5000), "gzip"); + ASSERT_EQ(Neg("br;q=0.9, gzip;q=0.5", "text/html", 5000), "br"); + ASSERT_EQ(Neg("br;q=0, gzip;q=0", "text/html", 5000), "identity"); + ASSERT_EQ(Neg("", "text/html", 5000), "identity"); + ASSERT_EQ(Neg("*", "text/html", 5000), "br"); + ASSERT_EQ(Neg("x-gzip", "text/html", 5000), "gzip"); + ASSERT_EQ(Neg("compress, gzip;q=0.1", "text/html", 5000), "gzip"); +} + +TEST(NegotiateThresholds) { + ASSERT_EQ(Neg("br, gzip", "text/html", 1023), "identity"); + ASSERT_EQ(Neg("br, gzip", "text/html", 1024), "br"); + ASSERT_EQ(Neg("br, gzip", "image/png", 5000), "identity"); + ASSERT_EQ(Neg("br, gzip", "video/mp4", 5000), "identity"); + ASSERT_EQ(Neg("br, gzip", "application/pdf", 5000), "identity"); + ASSERT_EQ(Neg("br, gzip", "application/json; charset=utf-8", 5000), "br"); + ASSERT_EQ(Neg("br, gzip", "image/svg+xml", 5000), "br"); + ASSERT_EQ(Neg("br, gzip", "application/hal+json", 5000), "br"); + Decision d = + Negotiator::Negotiate("br", "text/html", 5000, true, false, 200); + ASSERT_EQ(d.encoding, "identity"); // already encoded + d = Negotiator::Negotiate("br", "text/html", 5000, false, true, 200); + ASSERT_EQ(d.encoding, "identity"); // HEAD + d = Negotiator::Negotiate("br", "text/html", 5000, false, false, 204); + ASSERT_EQ(d.encoding, "identity"); // 204 + d = Negotiator::Negotiate("br", "text/html", 5000, false, false, 304); + ASSERT_EQ(d.encoding, "identity"); // 304 + ASSERT_EQ(std::string(Negotiator::VaryValue()), "Accept-Encoding"); +} + +TEST(NegotiatePrecompressed) { + Precompressed p = Negotiator::SelectPrecompressed("br, gzip", true, true); + ASSERT_TRUE(p.found); + ASSERT_EQ(p.suffix, ".br"); + p = Negotiator::SelectPrecompressed("gzip", true, true); + ASSERT_TRUE(p.found); + ASSERT_EQ(p.suffix, ".gz"); + p = Negotiator::SelectPrecompressed("br", false, false); + ASSERT_FALSE(p.found); + p = Negotiator::SelectPrecompressed("", true, true); + ASSERT_FALSE(p.found); + p = Negotiator::SelectPrecompressed("br", true, false); + ASSERT_FALSE(p.found); // only .gz on disk, client wants br +} diff --git a/tests/src/runtime/cookie/test_cookie.cpp b/tests/src/runtime/cookie/test_cookie.cpp new file mode 100644 index 0000000..f17b9b9 --- /dev/null +++ b/tests/src/runtime/cookie/test_cookie.cpp @@ -0,0 +1,118 @@ +#include "test_framework.h" + +#include "runtime/cookie/cookie.h" +#include "runtime/cookie/jar.h" +#include "runtime/cookie/signer.h" + +using namespace molo::runtime::cookie; + +TEST(CookieParseRfc) { + std::vector> out; + ASSERT_EQ(ParseCookieHeader("SID=31d4d96e407aad42; lang=en-US", out), + CookieError::Ok); + ASSERT_EQ(out.size(), 2u); + ASSERT_EQ(out[0].first, "SID"); + ASSERT_EQ(out[0].second, "31d4d96e407aad42"); + ASSERT_EQ(out[1].first, "lang"); + ASSERT_EQ(out[1].second, "en-US"); + out.clear(); + ASSERT_EQ(ParseCookieHeader("a=1;;b=2; ; c = 3 ", out), CookieError::Ok); + ASSERT_EQ(out.size(), 3u); + out.clear(); + ASSERT_EQ(ParseCookieHeader("q=\"quoted value\"; e=", out), CookieError::Ok); + ASSERT_EQ(out[0].second, "quoted value"); + ASSERT_EQ(out[1].second, ""); + out.clear(); + ASSERT_EQ(ParseCookieHeader("bad;name=x", out), CookieError::Ok); + ASSERT_EQ(out[0].first, "bad"); + out.clear(); + ASSERT_EQ(ParseCookieHeader("a=b; bad\x01=x", out), CookieError::BadName); + out.clear(); + ASSERT_EQ(ParseCookieHeader("a=b; good=bad\x01x", out), CookieError::BadValue); +} + +TEST(CookieSerialize) { + Cookie c; + c.name = "sid"; + c.value = "abc123"; + c.path = "/"; + c.max_age = 3600; + std::string out; + ASSERT_EQ(SerializeSetCookie(c, out), CookieError::Ok); + ASSERT_EQ(out, "sid=abc123; Path=/; Max-Age=3600; HttpOnly; SameSite=Lax"); + c.secure = true; + c.same_site = SameSite::Strict; + c.domain = "example.com"; + c.partitioned = true; + ASSERT_EQ(SerializeSetCookie(c, out), CookieError::Ok); + ASSERT_TRUE(out.find("Domain=example.com") != std::string::npos); + ASSERT_TRUE(out.find("Secure") != std::string::npos); + ASSERT_TRUE(out.find("SameSite=Strict") != std::string::npos); + ASSERT_TRUE(out.find("Partitioned") != std::string::npos); + // None without Secure is rejected. + c.secure = false; + c.same_site = SameSite::None; + ASSERT_EQ(SerializeSetCookie(c, out), CookieError::BadValue); + // Oversized value is an error, never truncated. + c.value = std::string(5000, 'x'); + c.secure = true; + c.same_site = SameSite::Lax; + ASSERT_EQ(SerializeSetCookie(c, out), CookieError::BadValue); + // Oversized full line (valid value plus long attrs) is TooLarge. + c.value = std::string(4000, 'x'); + c.path = "/" + std::string(200, 'p'); + ASSERT_EQ(SerializeSetCookie(c, out), CookieError::TooLarge); + // Bad name rejected. + c.name = "has space"; + c.value = "v"; + ASSERT_EQ(SerializeSetCookie(c, out), CookieError::BadName); +} + +TEST(JarJoinAndCaps) { + CookieJar jar; + // H2 split fields join with "; ". + ASSERT_EQ(jar.ParseRequest({"a=1", "b=2"}), JarError::Ok); + ASSERT_EQ(jar.Get("a"), "1"); + ASSERT_EQ(jar.Get("b"), "2"); + ASSERT_EQ(jar.Get("nope"), ""); + // Count cap. + std::string many; + for (int i = 0; i < 60; ++i) many += "c" + std::to_string(i) + "=v;"; + ASSERT_EQ(jar.ParseRequest({many}), JarError::TooMany); + // Size cap. + ASSERT_EQ(jar.ParseRequest({std::string(9000, 'x')}), JarError::TooLarge); +} + +TEST(JarSetAndSecure) { + CookieJar plain(false); + Cookie c; + c.name = "a"; + c.value = "b"; + ASSERT_EQ(plain.Set(c), JarError::Ok); + ASSERT_EQ(plain.set_cookie_headers().size(), 1u); + // Never joined: two cookies give two headers. + ASSERT_EQ(plain.Set(c), JarError::Ok); + ASSERT_EQ(plain.set_cookie_headers().size(), 2u); + CookieJar secure(true); + Cookie d; + d.name = "s"; + d.value = "v"; + ASSERT_EQ(secure.Set(d), JarError::Ok); + ASSERT_TRUE(secure.set_cookie_headers()[0].find("Secure") != + std::string::npos); +} + +TEST(SignerNullRoundTrip) { + NullSigner s; + std::string signed_v = s.Sign("payload-here"); + ASSERT_TRUE(signed_v.compare(0, 3, "v1.") == 0); + std::string payload; + ASSERT_TRUE(s.Verify(signed_v, payload)); + ASSERT_EQ(payload, "payload-here"); + ASSERT_FALSE(s.Verify(signed_v + "tamper", payload)); + ASSERT_FALSE(s.Verify("v1.bm8tampered.tag", payload)); + ASSERT_FALSE(s.Verify("nope", payload)); + CookieJar jar; + ASSERT_EQ(jar.SetSigned("sess", "user=7", s), JarError::Ok); + ASSERT_TRUE(jar.ParseRequest({"other=1"}) == JarError::Ok); +} diff --git a/tests/src/runtime/fixtures/www/.hidden b/tests/src/runtime/fixtures/www/.hidden new file mode 100644 index 0000000..57350a6 --- /dev/null +++ b/tests/src/runtime/fixtures/www/.hidden @@ -0,0 +1 @@ +top secret diff --git a/tests/src/runtime/fixtures/www/hello.txt b/tests/src/runtime/fixtures/www/hello.txt new file mode 100644 index 0000000..b90f6dd --- /dev/null +++ b/tests/src/runtime/fixtures/www/hello.txt @@ -0,0 +1 @@ +Hello, static world! This is a test file. diff --git a/tests/src/runtime/fixtures/www/index.html b/tests/src/runtime/fixtures/www/index.html new file mode 100644 index 0000000..c281c6e --- /dev/null +++ b/tests/src/runtime/fixtures/www/index.html @@ -0,0 +1 @@ +

    hi

    diff --git a/tests/src/runtime/fixtures/www/sub/file.txt b/tests/src/runtime/fixtures/www/sub/file.txt new file mode 100644 index 0000000..63dba76 --- /dev/null +++ b/tests/src/runtime/fixtures/www/sub/file.txt @@ -0,0 +1 @@ +nested file contents here diff --git a/tests/src/runtime/http/test_integration.cpp b/tests/src/runtime/http/test_integration.cpp new file mode 100644 index 0000000..a78fdda --- /dev/null +++ b/tests/src/runtime/http/test_integration.cpp @@ -0,0 +1,649 @@ +// End-to-end integration through real TCP: listener, H1 keep-alive/chunked/ +// pipelining/errors, static files, timeouts, budgets, h2c upgrade, drain. +#include "test_framework.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "runtime/event_loop.h" +#include "runtime/http/connection.h" +#include "runtime/http/listener.h" +#include "runtime/static/disk_provider.h" +#include "runtime/static/file_server.h" +#include "runtime/task.h" +#include "runtime/tls_handle.h" +#include "runtime/transport_tcp.h" + +using namespace molo::runtime; +using namespace molo::runtime::http; +using namespace std::chrono_literals; + +namespace { +bool WriteAll(int fd, const std::string& s) { + std::size_t off = 0; + while (off < s.size()) { + ssize_t n = write(fd, s.data() + off, s.size() - off); + if (n <= 0) return false; + off += (std::size_t)n; + } + return true; +} + +// Reads one H1 response head plus body (content-length or close-delimited). +struct HttpResp { + int status = 0; + std::string head; + std::string body; + bool eof = false; +}; + +HttpResp ReadHttpResp(int fd) { + HttpResp r; + std::string buf; + char tmp[4096]; + for (;;) { + std::size_t end = buf.find("\r\n\r\n"); + if (end != std::string::npos) break; + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n <= 0) { + r.eof = true; + return r; + } + buf.append(tmp, (std::size_t)n); + } + std::size_t hend = buf.find("\r\n\r\n"); + r.head = buf.substr(0, hend + 4); + buf.erase(0, hend + 4); + { + std::size_t sp = r.head.find(' '); + r.status = std::stoi(r.head.substr(sp + 1, 3)); + } + std::size_t len = 0; + bool has_len = false; + { + std::string low = r.head; + for (char& c : low) c = (char)tolower((unsigned char)c); + std::size_t p = low.find("content-length:"); + if (p != std::string::npos) { + has_len = true; + len = (std::size_t)std::stoul(low.substr(p + 15)); + } + } + while (buf.size() < len) { + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n <= 0) { + r.eof = true; + break; + } + buf.append(tmp, (std::size_t)n); + } + r.body = buf.substr(0, len); + if (!has_len && len == 0) { + // Close-delimited or empty: nothing more to read here. + } + return r; +} + +void SetTimeout(int fd, int secs = 5) { + struct timeval tv; + tv.tv_sec = secs; + tv.tv_usec = 0; + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); +} + +int Connect(uint16_t port) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + inet_pton(AF_INET, "127.0.0.1", &addr.sin_addr); + if (connect(fd, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + close(fd); + return -1; + } + SetTimeout(fd); + return fd; +} + +class Server { + public: + Server() : listener(loop) {} + + void Start(ConnectionConfig cfg = ConnectionConfig{}) { + provider = + std::make_unique("tests/src/runtime/fixtures/www"); + files = std::make_unique(*provider); + ASSERT_TRUE(provider->valid()); + ASSERT_TRUE(listener.bind("127.0.0.1", 0)); + port = listener.port(); + ASSERT_TRUE(port != 0); + listener.on_accept([this, cfg](int fd, const std::string& peer) { + (void)peer; + Handler h = [this](const FullRequest& req) -> Task { + if (req.head.target == "/slow") { + co_await loop.sleep(2s); // budget tests rely on this stall + } + co_return Handle(req); + }; + auto c = Connection::Create(loop, std::make_unique(fd), + std::make_unique(), h, + cfg); + { + std::lock_guard lock(mutex); + conns.push_back(c); + } + listener.track(c); + c->start(); + }); + thread = std::thread([this] { loop.run(); }); + std::this_thread::sleep_for(30ms); + } + + FullResponse Handle(const FullRequest& req) { + if (req.head.target == "/slow") { + FullResponse r; + r.head.status = 200; + r.body = "slow"; + return r; + } + if (req.head.target.compare(0, 8, "/static/") == 0) { + RequestHead sub = req.head; + sub.target = req.head.target.substr(7); // keep leading slash + static_::FileOutcome o = files->Handle(sub); + FullResponse r; + r.head = o.head; + if (o.asset.has_value() && !o.head_only && o.length > 0) { + static_::Asset& a = *o.asset; + if (a.fd() >= 0) { + r.file = FileBody{a.release(), o.offset, o.length}; + } else { + r.body.assign(a.data() + o.offset, (std::size_t)o.length); + } + } + if (o.status == 500 && !o.asset.has_value()) { + r.body = "static error\n"; + } + return r; + } + if (req.head.target == "/echo") { + FullResponse r; + r.head.status = 200; + r.head.headers.set("content-type", "text/plain"); + r.body = "echo:" + req.head.method + ":" + req.body; + return r; + } + if (req.head.target == "/bigfile" || req.head.target == "/bigfile-lie") { + FullResponse r; + int fd = open(big_path.c_str(), O_RDONLY); + if (fd < 0) { + r.head.status = 500; + r.body = "open failed\n"; + return r; + } + r.head.status = 200; + r.head.headers.set("content-type", "application/octet-stream"); + uint64_t len = big_size; + if (req.head.target == "/bigfile-lie") len += 1000000; // must clamp + r.file = FileBody{fd, 0, len}; + return r; + } + FullResponse r; + r.head.status = 404; + r.body = "nope\n"; + return r; + } + + ~Server() { + try { + Stop(); + } catch (...) { + } + } + + void Stop() { + listener.stop(); + loop.stop(); + if (thread.joinable()) thread.join(); + std::lock_guard lock(mutex); + conns.clear(); + } + + void DrainAll() { + std::lock_guard lock(mutex); + for (auto& c : conns) c->drain(); + } + + EventLoop loop; + Listener listener; + std::unique_ptr provider; + std::unique_ptr files; + uint16_t port = 0; + std::thread thread; + std::mutex mutex; + std::vector> conns; + // File-body routes. Set before Start; the handler opens per request. + std::string big_path; + std::size_t big_size = 0; +}; + +char FileByte(std::size_t i) { + return (char)(((i * 31u + 7u) ^ (i >> 8)) & 0xffu); +} + +std::string ExpectedFileBytes(std::size_t n) { + std::string out(n, '\0'); + for (std::size_t i = 0; i < n; ++i) out[i] = FileByte(i); + return out; +} + +// Writes n pattern bytes to a temp file; returns the path (unlink after). +std::string MakeBigTempFile(std::size_t n) { + std::string tpl = "/tmp/molo_big_test_XXXXXX"; + int fd = mkstemp(tpl.data()); + if (fd < 0) return ""; + std::string chunk = ExpectedFileBytes(n > 65536 ? 65536 : n); + std::size_t pos = 0; + bool ok = true; + while (pos < n) { + std::size_t w = n - pos; + if (w > chunk.size()) w = chunk.size(); + for (std::size_t i = 0; i < w; ++i) chunk[i] = FileByte(pos + i); + ssize_t r = write(fd, chunk.data(), w); + if (r <= 0) { + ok = false; + break; + } + pos += (std::size_t)r; + } + close(fd); + if (!ok) { + unlink(tpl.c_str()); + return ""; + } + return tpl; +} + +std::string HeaderValue(const std::string& head, const std::string& name) { + std::string low = head; + for (char& c : low) c = (char)tolower((unsigned char)c); + std::string key = name; + for (char& c : key) c = (char)tolower((unsigned char)c); + std::size_t p = low.find(key + ":"); + if (p == std::string::npos) return ""; + std::size_t eol = head.find("\r\n", p); + std::string v = head.substr(p + key.size() + 1, eol - p - key.size() - 1); + while (!v.empty() && (v.front() == ' ' || v.front() == '\t')) { + v.erase(v.begin()); + } + return v; +} +} // namespace + +TEST(IntegEchoKeepAlive) { + Server s; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(c >= 0); + ASSERT_TRUE(WriteAll(c, "GET /echo HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r1 = ReadHttpResp(c); + ASSERT_EQ(r1.status, 200); + ASSERT_EQ(r1.body, "echo:GET:"); + // Same connection, second request (keep-alive). + ASSERT_TRUE(WriteAll(c, "POST /echo HTTP/1.1\r\nHost: x\r\n" + "Content-Length: 4\r\n\r\nabcd")); + HttpResp r2 = ReadHttpResp(c); + ASSERT_EQ(r2.status, 200); + ASSERT_EQ(r2.body, "echo:POST:abcd"); + close(c); + s.Stop(); +} + +TEST(IntegChunkedAndPipelined) { + Server s; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(c >= 0); + ASSERT_TRUE(WriteAll(c, "POST /echo HTTP/1.1\r\nHost: x\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "3\r\nabc\r\n2\r\nde\r\n0\r\n\r\n" + "GET /echo HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r1 = ReadHttpResp(c); + ASSERT_EQ(r1.status, 200); + ASSERT_EQ(r1.body, "echo:POST:abcde"); + HttpResp r2 = ReadHttpResp(c); + ASSERT_EQ(r2.status, 200); + ASSERT_EQ(r2.body, "echo:GET:"); + close(c); + s.Stop(); +} + +TEST(IntegSmuggling400) { + Server s; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "POST /echo HTTP/1.1\r\nHost: x\r\n" + "Content-Length: 4\r\n" + "Transfer-Encoding: chunked\r\n\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 400); + // Connection must be closed after a framing error. + char tmp[8]; + ssize_t n = read(c, tmp, sizeof(tmp)); + ASSERT_EQ(n, 0); + close(c); + s.Stop(); +} + +TEST(IntegUnknownTE501) { + Server s; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "POST /echo HTTP/1.1\r\nHost: x\r\n" + "Transfer-Encoding: compress\r\n\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 501); + close(c); + s.Stop(); +} + +TEST(IntegNotFound) { + Server s; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "GET /missing HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 404); + close(c); + s.Stop(); +} + +TEST(IntegIdleTimeout) { + Server s; + ConnectionConfig cfg; + cfg.idle_timeout = 200ms; + s.Start(cfg); + int c = Connect(s.port); + ASSERT_TRUE(c >= 0); + char tmp[8]; + ssize_t n = read(c, tmp, sizeof(tmp)); + ASSERT_EQ(n, 0); // silent close on idle expiry + close(c); + s.Stop(); +} + +TEST(IntegHeaderTimeout408) { + Server s; + ConnectionConfig cfg; + cfg.header_timeout = 200ms; + cfg.idle_timeout = 10s; + s.Start(cfg); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "GET /echo HTTP/1.1\r\nHost: x\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 408); + close(c); + s.Stop(); +} + +TEST(IntegBodyTimeout408) { + Server s; + ConnectionConfig cfg; + cfg.body_timeout = 200ms; + cfg.idle_timeout = 10s; + s.Start(cfg); + int c = Connect(s.port); + // Complete head, then a stalled body. + ASSERT_TRUE(WriteAll(c, "POST /echo HTTP/1.1\r\nHost: x\r\n" + "Content-Length: 100\r\n\r\n12345")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 408); + close(c); + s.Stop(); +} + +TEST(IntegExpectContinue) { + Server s; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "POST /echo HTTP/1.1\r\nHost: x\r\n" + "Content-Length: 3\r\n" + "Expect: 100-continue\r\n\r\n")); + // Interim 100 arrives before the client sends the body. + std::string interim; + char tmp[256]; + for (;;) { + ssize_t n = read(c, tmp, sizeof(tmp)); + ASSERT_TRUE(n > 0); + interim.append(tmp, (std::size_t)n); + if (interim.find("\r\n\r\n") != std::string::npos) break; + } + ASSERT_TRUE(interim.find("100 Continue") != std::string::npos); + ASSERT_TRUE(WriteAll(c, "xyz")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 200); + ASSERT_EQ(r.body, "echo:POST:xyz"); + close(c); + s.Stop(); +} + +TEST(IntegHandlerBudget) { + Server s; + ConnectionConfig cfg; + cfg.handler_budget = 200ms; + s.Start(cfg); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "GET /slow HTTP/1.1\r\nHost: x\r\n\r\n")); + // Over-budget handler: connection dies with no 200. + char tmp[4096]; + ssize_t n = read(c, tmp, sizeof(tmp)); + ASSERT_TRUE(n <= 0); + close(c); + s.Stop(); +} + +TEST(IntegStaticFile) { + Server s; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "GET /static/hello.txt HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 200); + ASSERT_EQ(r.body, "Hello, static world! This is a test file.\n"); + ASSERT_EQ(HeaderValue(r.head, "content-type"), "text/plain; charset=utf-8"); + std::string etag = HeaderValue(r.head, "etag"); + ASSERT_FALSE(etag.empty()); + ASSERT_FALSE(HeaderValue(r.head, "last-modified").empty()); + ASSERT_EQ(HeaderValue(r.head, "accept-ranges"), "bytes"); + // Conditional: match wins 304. + ASSERT_TRUE(WriteAll(c, "GET /static/hello.txt HTTP/1.1\r\nHost: x\r\n" + + std::string("If-None-Match: ") + etag + "\r\n\r\n")); + HttpResp r2 = ReadHttpResp(c); + ASSERT_EQ(r2.status, 304); + ASSERT_TRUE(r2.body.empty()); + // Range. + ASSERT_TRUE(WriteAll(c, "GET /static/hello.txt HTTP/1.1\r\nHost: x\r\n" + + std::string("Range: bytes=0-4\r\n\r\n"))); + HttpResp r3 = ReadHttpResp(c); + ASSERT_EQ(r3.status, 206); + ASSERT_EQ(r3.body, "Hello"); + // Bad range. + ASSERT_TRUE(WriteAll(c, "GET /static/hello.txt HTTP/1.1\r\nHost: x\r\n" + + std::string("Range: bytes=99999-\r\n\r\n"))); + HttpResp r4 = ReadHttpResp(c); + ASSERT_EQ(r4.status, 416); + // Traversal fails closed. + ASSERT_TRUE(WriteAll(c, "GET /static/../hello.txt HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r5 = ReadHttpResp(c); + ASSERT_EQ(r5.status, 404); + // HEAD has no body. + ASSERT_TRUE(WriteAll(c, "HEAD /static/hello.txt HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r6 = ReadHttpResp(c); + ASSERT_EQ(r6.status, 200); + ASSERT_EQ(HeaderValue(r6.head, "content-length"), "42"); + close(c); + s.Stop(); +} + +TEST(IntegUpgradeH2c) { + Server s; + s.Start(); + int c = Connect(s.port); + // H1 request asking for h2c with empty client SETTINGS. + ASSERT_TRUE(WriteAll(c, "GET /echo HTTP/1.1\r\nHost: x\r\n" + "Connection: Upgrade, HTTP2-Settings\r\n" + "Upgrade: h2c\r\nHTTP2-Settings: \r\n\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_TRUE(r.status == 101 || r.status == 400); + if (r.status == 400) { + // Empty HTTP2-Settings value is benign (server strips space); any 400 + // here would be a server bug, fail loudly below. + ASSERT_EQ(r.status, 101); + } + // Read server SETTINGS frame, then the stream-1 response (HEADERS+DATA). + auto read_frame = [&]() { + char h[9]; + if (![&] { + std::size_t off = 0; + while (off < 9) { + ssize_t n = read(c, h + off, 9 - off); + if (n <= 0) return false; + off += (std::size_t)n; + } + return true; + }()) { + return std::string(); + } + std::size_t len = ((std::size_t)(unsigned char)h[0] << 16) | + ((std::size_t)(unsigned char)h[1] << 8) | + (std::size_t)(unsigned char)h[2]; + std::string p(len, '\0'); + std::size_t off = 0; + while (off < len) { + ssize_t n = read(c, p.data() + off, len - off); + if (n <= 0) return std::string(); + off += (std::size_t)n; + } + return std::string(h, 9) + p; + }; + std::string f1 = read_frame(); + ASSERT_EQ(f1.size(), 9u + 30u); // empty-SETTINGS server preface is 30 bytes + ASSERT_EQ((unsigned char)f1[3], 0x04u); + // Send our SETTINGS + ACK to be polite, then read stream 1 response. + std::string cli_settings(9, '\0'); + cli_settings[2] = 0; + cli_settings[3] = 0x04; + ASSERT_TRUE(WriteAll(c, cli_settings)); + bool saw_data_end = false; + std::string body; + for (int i = 0; i < 16 && !saw_data_end; ++i) { + std::string fr = read_frame(); + ASSERT_FALSE(fr.empty()); + uint8_t type = (unsigned char)fr[3]; + uint8_t flags = (unsigned char)fr[4]; + uint32_t sid = ((uint32_t)((unsigned char)fr[5] & 0x7f) << 24) | + ((uint32_t)(unsigned char)fr[6] << 16) | + ((uint32_t)(unsigned char)fr[7] << 8) | + (uint32_t)(unsigned char)fr[8]; + if (type == 0x0 && sid == 1) { + body += fr.substr(9); + if (flags & 0x1) saw_data_end = true; + } + } + ASSERT_TRUE(saw_data_end); + ASSERT_EQ(body, "echo:GET:"); + close(c); + s.Stop(); +} + +TEST(IntegDrain) { + Server s; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "GET /echo HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 200); + s.listener.stop(); + s.DrainAll(); + // Drained connection closes after the in-flight request. + char tmp[8]; + ssize_t n = read(c, tmp, sizeof(tmp)); + ASSERT_EQ(n, 0); + close(c); + // Listener no longer accepts. + int c2 = socket(AF_INET, SOCK_STREAM, 0); + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(s.port); + inet_pton(AF_INET, "127.0.0.1", &addr.sin_addr); + SetTimeout(c2, 2); + int rc = connect(c2, (struct sockaddr*)&addr, sizeof(addr)); + close(c2); + ASSERT_TRUE(rc != 0); + s.Stop(); +} + +TEST(IntegFileBodySendfile) { + constexpr std::size_t kSize = 400 * 1024; // past the 256 KiB watermark + std::string path = MakeBigTempFile(kSize); + ASSERT_FALSE(path.empty()); + std::string want = ExpectedFileBytes(kSize); + Server s; + s.big_path = path; + s.big_size = kSize; + s.Start(); + int c = Connect(s.port); + ASSERT_TRUE(WriteAll(c, "GET /bigfile HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 200); + ASSERT_EQ(HeaderValue(r.head, "content-length"), std::to_string(kSize)); + ASSERT_EQ(r.body.size(), kSize); + ASSERT_TRUE(r.body == want); + // HEAD reports the file length with no body. + ASSERT_TRUE(WriteAll(c, "HEAD /bigfile HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp h = ReadHttpResp(c); + ASSERT_EQ(h.status, 200); + ASSERT_EQ(HeaderValue(h.head, "content-length"), std::to_string(kSize)); + ASSERT_TRUE(h.body.empty()); + // Connection still healthy for keep-alive. + ASSERT_TRUE(WriteAll(c, "GET /echo HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r2 = ReadHttpResp(c); + ASSERT_EQ(r2.status, 200); + ASSERT_EQ(r2.body, "echo:GET:"); + close(c); + s.Stop(); + unlink(path.c_str()); +} + +TEST(IntegFileBodyClamp) { + constexpr std::size_t kSize = 100 * 1024; + std::string path = MakeBigTempFile(kSize); + ASSERT_FALSE(path.empty()); + std::string want = ExpectedFileBytes(kSize); + Server s; + s.big_path = path; + s.big_size = kSize; + s.Start(); + int c = Connect(s.port); + // Handler claims a million extra bytes; the server clamps to the file. + ASSERT_TRUE(WriteAll(c, "GET /bigfile-lie HTTP/1.1\r\nHost: x\r\n\r\n")); + HttpResp r = ReadHttpResp(c); + ASSERT_EQ(r.status, 200); + ASSERT_EQ(HeaderValue(r.head, "content-length"), std::to_string(kSize)); + ASSERT_TRUE(r.body == want); + close(c); + s.Stop(); + unlink(path.c_str()); +} diff --git a/tests/src/runtime/http/test_message.cpp b/tests/src/runtime/http/test_message.cpp new file mode 100644 index 0000000..304757d --- /dev/null +++ b/tests/src/runtime/http/test_message.cpp @@ -0,0 +1,87 @@ +#include "test_framework.h" + +#include +#include + +#include + +#include "runtime/http/message.h" + +using namespace molo::runtime::http; + +TEST(HeadersCaseInsensitive) { + Headers h; + h.add("Content-Type", "text/html"); + h.add("X-Custom", "1"); + h.add("X-Custom", "2"); + ASSERT_EQ(h.get("content-type"), "text/html"); + ASSERT_EQ(h.get("CONTENT-TYPE"), "text/html"); + ASSERT_EQ(h.get("missing"), ""); + ASSERT_TRUE(h.has("x-custom")); + ASSERT_EQ(h.get_all("X-CUSTOM").size(), 2u); + h.set("X-Custom", "3"); + ASSERT_EQ(h.get_all("x-custom").size(), 1u); + h.remove("x-custom"); + ASSERT_FALSE(h.has("X-Custom")); +} + +TEST(ContentLengthParse) { + Headers h; + bool conflict = true; + ASSERT_FALSE(ParseContentLength(h, conflict).has_value()); + h.add("Content-Length", "42"); + auto v = ParseContentLength(h, conflict); + ASSERT_TRUE(v.has_value()); + ASSERT_EQ(*v, 42u); + ASSERT_FALSE(conflict); + h.add("content-length", "42"); + v = ParseContentLength(h, conflict); + ASSERT_TRUE(v.has_value()); + ASSERT_FALSE(conflict); + h.add("Content-Length", "43"); + v = ParseContentLength(h, conflict); + ASSERT_TRUE(conflict); + Headers bad; + bad.add("content-length", "4x2"); + ASSERT_FALSE(ParseContentLength(bad, conflict).has_value()); +} + +TEST(StatusReasons) { + ASSERT_EQ(StatusReason(200), "OK"); + ASSERT_EQ(StatusReason(404), "Not Found"); + ASSERT_EQ(StatusReason(431), "Request Header Fields Too Large"); + ASSERT_EQ(StatusReason(501), "Not Implemented"); +} + +TEST(TrimAndLower) { + ASSERT_EQ(Trim(" a b\t"), "a b"); + ASSERT_EQ(ToLower("AbC-1"), "abc-1"); + ASSERT_TRUE(EqCaseInsensitive("HoSt", "host")); + ASSERT_FALSE(EqCaseInsensitive("host", "hosts")); +} + +TEST(ClampFileBody) { + std::string tpl = "/tmp/molo_clamp_test_XXXXXX"; + int fd = mkstemp(tpl.data()); + ASSERT_TRUE(fd >= 0); + std::string data(100, 'x'); + ASSERT_EQ(write(fd, data.data(), data.size()), (ssize_t)data.size()); + FileBody a{fd, 0, 1000}; // over-long: clamps to the file size + ASSERT_TRUE(ClampFileBody(a)); + ASSERT_EQ(a.offset, 0u); + ASSERT_EQ(a.length, 100u); + FileBody b{fd, 50, 100}; // tail range clamps + ASSERT_TRUE(ClampFileBody(b)); + ASSERT_EQ(b.offset, 50u); + ASSERT_EQ(b.length, 50u); + FileBody c{fd, 500, 10}; // past EOF: empty range at the end + ASSERT_TRUE(ClampFileBody(c)); + ASSERT_EQ(c.offset, 100u); + ASSERT_EQ(c.length, 0u); + close(fd); + unlink(tpl.c_str()); + FileBody d{-1, 0, 10}; + ASSERT_FALSE(ClampFileBody(d)); + FileBody e{fd, 0, 10}; // closed fd is not stat-able + ASSERT_FALSE(ClampFileBody(e)); +} diff --git a/tests/src/runtime/http/test_protocol_detect.cpp b/tests/src/runtime/http/test_protocol_detect.cpp new file mode 100644 index 0000000..8f65e46 --- /dev/null +++ b/tests/src/runtime/http/test_protocol_detect.cpp @@ -0,0 +1,34 @@ +#include "test_framework.h" + +#include "runtime/http/protocol_detect.h" + +using namespace molo::runtime::http; + +TEST(DetectPriorKnowledge) { + ProtocolDetector d; + ASSERT_EQ(d.feed("PRI * HT", 8), Protocol::NeedMore); + ASSERT_EQ(d.feed("TP/2.0\r\n\r\nSM\r\n\r\n", 16), Protocol::H2PriorKnowledge); + ASSERT_EQ(d.feed("more", 4), Protocol::H2PriorKnowledge); +} + +TEST(DetectHttp1) { + ProtocolDetector d; + ASSERT_EQ(d.feed("GET / HTTP/1.1\r\n", 16), Protocol::Http1); + d.reset(); + ASSERT_EQ(d.feed("P", 1), Protocol::NeedMore); + ASSERT_EQ(d.feed("OST", 3), Protocol::Http1); +} + +TEST(DetectMismatchLate) { + ProtocolDetector d; + ASSERT_EQ(d.feed("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\nX", 25), + Protocol::H2PriorKnowledge); + d.reset(); + ASSERT_EQ(d.feed("PRI * HTTP/2.0\r\n\r\nSM\r\n\rX\n", 24), Protocol::Http1); +} + +TEST(DetectAlpn) { + ASSERT_EQ(ProtocolDetector::from_alpn("h2"), Protocol::H2PriorKnowledge); + ASSERT_EQ(ProtocolDetector::from_alpn("http/1.1"), Protocol::Http1); + ASSERT_EQ(ProtocolDetector::from_alpn(""), Protocol::Http1); +} diff --git a/tests/src/runtime/http/v1/test_parser.cpp b/tests/src/runtime/http/v1/test_parser.cpp new file mode 100644 index 0000000..abfbcb1 --- /dev/null +++ b/tests/src/runtime/http/v1/test_parser.cpp @@ -0,0 +1,156 @@ +#include "test_framework.h" + +#include "runtime/http/v1/parser.h" + +using namespace molo::runtime::http::v1; + +namespace { +ParseResult FeedAll(Parser& p, const std::string& s, std::size_t chunk = 0) { + if (chunk == 0) return p.feed(s.data(), s.size()); + ParseResult r = ParseResult::NeedMore; + for (std::size_t i = 0; i < s.size(); i += chunk) { + std::size_t n = s.size() - i; + if (n > chunk) n = chunk; + r = p.feed(s.data() + i, n); + if (r != ParseResult::NeedMore) break; + } + return r; +} +} // namespace + +TEST(ParseSimpleGet) { + Parser p; + std::string req = "GET /hello?name=x HTTP/1.1\r\nHost: a.com\r\n\r\n"; + ASSERT_EQ(FeedAll(p, req), ParseResult::Complete); + ASSERT_EQ(p.head().method, "GET"); + ASSERT_EQ(p.head().target, "/hello?name=x"); + ASSERT_EQ(p.head().headers.get("host"), "a.com"); + ASSERT_TRUE(p.keep_alive()); + ASSERT_TRUE(p.body().empty()); +} + +TEST(ParseByteByByte) { + Parser p; + std::string req = + "POST /p HTTP/1.1\r\nHost: h\r\nContent-Length: 5\r\n\r\nabcde"; + ASSERT_EQ(FeedAll(p, req, 1), ParseResult::Complete); + ASSERT_EQ(p.body(), "abcde"); +} + +TEST(ParseChunked) { + Parser p; + std::string req = + "POST /c HTTP/1.1\r\nHost: h\r\nTransfer-Encoding: chunked\r\n\r\n" + "5; ext=1\r\nhello\r\n6\r\n world\r\n0\r\nX-T: v\r\n\r\n"; + ASSERT_EQ(FeedAll(p, req, 3), ParseResult::Complete); + ASSERT_EQ(p.body(), "hello world"); + ASSERT_EQ(p.trailers().get("x-t"), "v"); +} + +TEST(ParseSmugglingRejected) { + Parser p; + std::string req = + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 5\r\n" + "Transfer-Encoding: chunked\r\n\r\n"; + ASSERT_EQ(FeedAll(p, req), ParseResult::Error400); +} + +TEST(ParseUnknownTERejected) { + Parser p; + std::string req = + "POST / HTTP/1.1\r\nHost: h\r\nTransfer-Encoding: rot13\r\n\r\n"; + ASSERT_EQ(FeedAll(p, req), ParseResult::Error501); +} + +TEST(ParseHostRequired) { + Parser p; + ASSERT_EQ(FeedAll(p, "GET / HTTP/1.1\r\nX-A: b\r\n\r\n"), + ParseResult::Error400); +} + +TEST(ParseObsFoldRejected) { + Parser p; + ASSERT_EQ(FeedAll(p, "GET / HTTP/1.1\r\nHost: h\r\n X: folded\r\n\r\n"), + ParseResult::Error400); +} + +TEST(ParseBadVersion) { + Parser p; + ASSERT_EQ(FeedAll(p, "GET / HTTP/2.0\r\nHost: h\r\n\r\n"), + ParseResult::Error400); +} + +TEST(ParseBadChunk) { + Parser p; + std::string req = + "POST / HTTP/1.1\r\nHost: h\r\nTransfer-Encoding: chunked\r\n\r\n" + "zz\r\n"; + ASSERT_EQ(FeedAll(p, req), ParseResult::Error400); +} + +TEST(ParseHeaderFlood431) { + Parser p; + std::string req = "GET / HTTP/1.1\r\nHost: h\r\n"; + for (int i = 0; i < 200; ++i) req += "X-Flood: abcdef\r\n"; + req += "\r\n"; + ASSERT_EQ(FeedAll(p, req), ParseResult::Error431); +} + +TEST(ParseBodyCap413) { + Parser p(10); + std::string req = + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 100\r\n\r\n"; + ASSERT_EQ(FeedAll(p, req), ParseResult::Error413); +} + +TEST(ParseChunkedCap413) { + Parser p(10); + std::string req = + "POST / HTTP/1.1\r\nHost: h\r\nTransfer-Encoding: chunked\r\n\r\n" + "64\r\n"; + ASSERT_EQ(FeedAll(p, req), ParseResult::Error413); +} + +TEST(ParseKeepAlive10) { + Parser p; + ASSERT_EQ(FeedAll(p, "GET / HTTP/1.0\r\nHost: h\r\n\r\n"), + ParseResult::Complete); + ASSERT_FALSE(p.keep_alive()); + p.reset(); + ASSERT_EQ(FeedAll(p, "GET / HTTP/1.0\r\nHost: h\r\nConnection: keep-alive\r\n\r\n"), + ParseResult::Complete); + ASSERT_TRUE(p.keep_alive()); + p.reset(); + ASSERT_EQ(FeedAll(p, "GET / HTTP/1.1\r\nHost: h\r\nConnection: close\r\n\r\n"), + ParseResult::Complete); + ASSERT_FALSE(p.keep_alive()); +} + +TEST(ParseExpectContinue) { + Parser p; + ASSERT_EQ(FeedAll(p, "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 1\r\n" + "Expect: 100-continue\r\n\r\na"), + ParseResult::Complete); + ASSERT_TRUE(p.expect_continue()); +} + +TEST(ParsePipelinedLeavesRemainder) { + Parser p; + std::string two = + "GET /1 HTTP/1.1\r\nHost: h\r\n\r\nGET /2 HTTP/1.1\r\nHost: h\r\n\r\n"; + ASSERT_EQ(FeedAll(p, two), ParseResult::Complete); + ASSERT_EQ(p.head().target, "/1"); + std::string rest = p.take_pending(); + ASSERT_EQ(rest, "GET /2 HTTP/1.1\r\nHost: h\r\n\r\n"); + p.reset(); + ASSERT_EQ(FeedAll(p, rest), ParseResult::Complete); + ASSERT_EQ(p.head().target, "/2"); + ASSERT_TRUE(p.consumed() > 0); +} + +TEST(ParseConflictingLengths) { + Parser p; + ASSERT_EQ(FeedAll(p, "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 1\r\n" + "Content-Length: 2\r\n\r\n"), + ParseResult::Error400); +} diff --git a/tests/src/runtime/http/v1/test_serializer.cpp b/tests/src/runtime/http/v1/test_serializer.cpp new file mode 100644 index 0000000..8167c27 --- /dev/null +++ b/tests/src/runtime/http/v1/test_serializer.cpp @@ -0,0 +1,58 @@ +#include "test_framework.h" + +#include "runtime/http/v1/serializer.h" + +using namespace molo::runtime::http; +using namespace molo::runtime::http::v1; + +TEST(SerializeBasic) { + ResponseHead h; + h.status = 200; + h.headers.set("content-type", "text/plain"); + std::string s = + Serializer::SerializeHead(h, "HTTP/1.1", false, 5, false, false); + ASSERT_TRUE(s.find("HTTP/1.1 200 OK\r\n") == 0); + ASSERT_TRUE(s.find("content-length: 5\r\n") != std::string::npos); + ASSERT_TRUE(s.find("date: ") != std::string::npos); + ASSERT_TRUE(s.compare(s.size() - 4, 4, "\r\n\r\n") == 0 || + s.find("\r\n\r\n") != std::string::npos); +} + +TEST(SerializeChunked) { + ResponseHead h; + h.status = 200; + std::string s = + Serializer::SerializeHead(h, "HTTP/1.1", true, 0, false, false); + ASSERT_TRUE(s.find("transfer-encoding: chunked\r\n") != std::string::npos); + ASSERT_TRUE(s.find("content-length") == std::string::npos); +} + +TEST(SerializeNoBody) { + ResponseHead h; + h.status = 304; + std::string s = + Serializer::SerializeHead(h, "HTTP/1.1", false, 10, false, false); + ASSERT_TRUE(s.find("content-length") == std::string::npos); + h.status = 204; + s = Serializer::SerializeHead(h, "HTTP/1.1", false, 10, false, false); + ASSERT_TRUE(s.find("content-length") == std::string::npos); + // HEAD carries the length the matching GET would send (RFC 9110 8.6). + h.status = 200; + s = Serializer::SerializeHead(h, "HTTP/1.1", false, 10, true, false); + ASSERT_TRUE(s.find("content-length: 10\r\n") != std::string::npos); +} + +TEST(SerializeClose) { + ResponseHead h; + h.status = 200; + std::string s = + Serializer::SerializeHead(h, "HTTP/1.1", false, 1, false, true); + ASSERT_TRUE(s.find("connection: close\r\n") != std::string::npos); +} + +TEST(SerializeChunks) { + ASSERT_EQ(Serializer::Chunk("hello", 5), "5\r\nhello\r\n"); + ASSERT_EQ(Serializer::Chunk(nullptr, 0), ""); + ASSERT_EQ(Serializer::LastChunk(), "0\r\n\r\n"); + ASSERT_FALSE(Serializer::HttpDateNow().empty()); +} diff --git a/tests/src/runtime/http/v2/test_flow.cpp b/tests/src/runtime/http/v2/test_flow.cpp new file mode 100644 index 0000000..00443c4 --- /dev/null +++ b/tests/src/runtime/http/v2/test_flow.cpp @@ -0,0 +1,45 @@ +#include "test_framework.h" + +#include "runtime/http/v2/flow_control.h" + +using namespace molo::runtime::http::v2; + +TEST(FlowBasic) { + FlowController f; + ASSERT_EQ(f.ConnectionAvailable(), 65535); + ASSERT_TRUE(f.TakeConnection(1000)); + ASSERT_EQ(f.ConnectionAvailable(), 64535); + ASSERT_FALSE(f.TakeConnection(100000)); + f.OnConnectionWindowUpdate(1000); + ASSERT_EQ(f.ConnectionAvailable(), 65535); + f.AddStream(1, 100); + ASSERT_EQ(f.StreamAvailable(1), 100); + ASSERT_TRUE(f.TakeStream(1, 60)); + ASSERT_FALSE(f.TakeStream(1, 60)); + f.OnStreamWindowUpdate(1, 60); + ASSERT_TRUE(f.TakeStream(1, 60)); + f.RemoveStream(1); + ASSERT_EQ(f.StreamAvailable(1), 0); +} + +TEST(FlowAdjustStreams) { + FlowController f; + f.AddStream(1, 100); + f.AddStream(3, 100); + ASSERT_TRUE(f.AdjustAllStreams(-150, 0)); + ASSERT_EQ(f.StreamAvailable(1), -50); + ASSERT_FALSE(f.TakeStream(1, 1)); + ASSERT_TRUE(f.AdjustAllStreams(150, 0)); + ASSERT_TRUE(f.TakeStream(1, 100)); + ASSERT_FALSE(f.AdjustAllStreams(0x7fffffffLL, 0)); +} + +TEST(FlowRefunds) { + FlowController f; + f.ConsumedConnection(500); + f.ConsumedStream(1, 300); + ASSERT_EQ(f.TakeConnectionRefund(), 500u); + ASSERT_EQ(f.TakeConnectionRefund(), 0u); + ASSERT_EQ(f.TakeStreamRefund(1), 300u); + ASSERT_EQ(f.TakeStreamRefund(9), 0u); +} diff --git a/tests/src/runtime/http/v2/test_h2connection.cpp b/tests/src/runtime/http/v2/test_h2connection.cpp new file mode 100644 index 0000000..97bcdcd --- /dev/null +++ b/tests/src/runtime/http/v2/test_h2connection.cpp @@ -0,0 +1,526 @@ +// H2 over a real socketpair: hand-rolled client speaking prior-knowledge +// h2c against Connection + H2Connection on a background loop thread. +#include "test_framework.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "runtime/event_loop.h" +#include "runtime/http/connection.h" +#include "runtime/http/v2/hpack.h" +#include "runtime/task.h" +#include "runtime/tls_handle.h" +#include "runtime/transport_tcp.h" + +using namespace molo::runtime; +using namespace molo::runtime::http; +using namespace std::chrono_literals; + +namespace { +std::string Frame(uint8_t type, uint8_t flags, uint32_t id, + const std::string& payload) { + std::string out; + out.resize(9 + payload.size()); + out[0] = (char)((payload.size() >> 16) & 0xff); + out[1] = (char)((payload.size() >> 8) & 0xff); + out[2] = (char)(payload.size() & 0xff); + out[3] = (char)type; + out[4] = (char)flags; + out[5] = (char)((id >> 24) & 0x7f); + out[6] = (char)((id >> 16) & 0xff); + out[7] = (char)((id >> 8) & 0xff); + out[8] = (char)(id & 0xff); + out.replace(9, payload.size(), payload); + return out; +} + +bool WriteAll(int fd, const std::string& s) { + std::size_t off = 0; + while (off < s.size()) { + ssize_t n = write(fd, s.data() + off, s.size() - off); + if (n <= 0) return false; + off += (std::size_t)n; + } + return true; +} + +bool ReadExact(int fd, char* buf, std::size_t n) { + std::size_t off = 0; + while (off < n) { + ssize_t r = read(fd, buf + off, n - off); + if (r <= 0) return false; + off += (std::size_t)r; + } + return true; +} + +struct GotFrame { + uint8_t type = 0; + uint8_t flags = 0; + uint32_t id = 0; + std::string payload; +}; + +bool ReadFrame(int fd, GotFrame& f) { + char h[9]; + if (!ReadExact(fd, h, 9)) return false; + std::size_t len = ((std::size_t)(unsigned char)h[0] << 16) | + ((std::size_t)(unsigned char)h[1] << 8) | + (std::size_t)(unsigned char)h[2]; + f.type = (unsigned char)h[3]; + f.flags = (unsigned char)h[4]; + f.id = ((uint32_t)((unsigned char)h[5] & 0x7f) << 24) | + ((uint32_t)(unsigned char)h[6] << 16) | + ((uint32_t)(unsigned char)h[7] << 8) | (uint32_t)(unsigned char)h[8]; + f.payload.resize(len); + if (len > 0 && !ReadExact(fd, f.payload.data(), len)) return false; + return true; +} + +void SetTimeout(int fd) { + struct timeval tv; + tv.tv_sec = 5; + tv.tv_usec = 0; + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); +} + +struct Harness { + int sv[2] = {-1, -1}; + EventLoop loop; + std::shared_ptr conn; + std::thread thread; + std::mutex mutex; + std::vector seen; + + void Start(Handler h, const std::string& alpn = "http/1.1", + bool is_tls = false) { + socketpair(AF_UNIX, SOCK_STREAM, 0, sv); + SetTimeout(sv[1]); + int flags = fcntl(sv[0], F_GETFL, 0); + fcntl(sv[0], F_SETFL, flags | O_NONBLOCK); + auto transport = std::make_unique(sv[0], "test"); + auto tls = std::make_unique(alpn, is_tls); + conn = Connection::Create(loop, std::move(transport), std::move(tls), h); + conn->start(); + thread = std::thread([this] { loop.run(); }); + // Give the loop a moment to arm. + std::this_thread::sleep_for(20ms); + } + + ~Harness() { + try { + Stop(); + } catch (...) { + } + } + + void Stop() { + if (sv[1] >= 0) { + shutdown(sv[1], SHUT_RDWR); + close(sv[1]); + sv[1] = -1; + } + loop.stop(); + if (thread.joinable()) thread.join(); + conn.reset(); + } +}; +} // namespace + +TEST(H2GetRoundTrip) { + Harness t; + Handler h = [&](const FullRequest& req) -> Task { + std::lock_guard lock(t.mutex); + t.seen.push_back(req); + FullResponse resp; + resp.head.status = 200; + resp.head.headers.set("content-type", "text/plain"); + resp.body = "echo:" + req.body; + co_return resp; + }; + t.Start(h); + int c = t.sv[1]; + // Client preface: magic + empty SETTINGS. + ASSERT_TRUE(WriteAll(c, std::string(kH2Preface))); + ASSERT_TRUE(WriteAll(c, Frame(0x4, 0, 0, ""))); + // Server preface: SETTINGS. + GotFrame f; + ASSERT_TRUE(ReadFrame(c, f)); + ASSERT_EQ(f.type, 0x4u); + ASSERT_TRUE(WriteAll(c, Frame(0x4, 0x1, 0, ""))); // our ACK + // GET / on stream 1 (RFC-style block, non-Huffman authority). + std::string block("\x82\x86\x84\x41\x0f", 5); + block += "www.example.com"; + ASSERT_TRUE(WriteAll(c, Frame(0x1, 0x4 | 0x1, 1, block))); + // Expect HEADERS + DATA with END_STREAM. + std::string hblock; + std::string body; + bool got_data_end = false; + for (int i = 0; i < 16 && !got_data_end; ++i) { + ASSERT_TRUE(ReadFrame(c, f)); + if (f.type == 0x4 && (f.flags & 0x1)) continue; // settings ack + if (f.type == 0x8) continue; // window update + if (f.type == 0x1) { + ASSERT_EQ(f.id, 1u); + hblock += f.payload; + if (f.flags & 0x1) break; // END_STREAM on headers (empty body?) + } else if (f.type == 0x0) { + ASSERT_EQ(f.id, 1u); + body += f.payload; + if (f.flags & 0x1) got_data_end = true; + } + } + ASSERT_TRUE(got_data_end); + ASSERT_EQ(body, "echo:"); + v2::HpackDecoder dec; + std::vector fields; + ASSERT_EQ(dec.Decode(hblock.data(), hblock.size(), fields), + v2::HpackError::Ok); + ASSERT_TRUE(fields.size() >= 1); + ASSERT_EQ(fields[0].name, ":status"); + ASSERT_EQ(fields[0].value, "200"); + { + std::lock_guard lock(t.mutex); + ASSERT_EQ(t.seen.size(), 1u); + ASSERT_EQ(t.seen[0].head.method, "GET"); + ASSERT_EQ(t.seen[0].head.target, "/"); + ASSERT_EQ(t.seen[0].protocol, "HTTP/2"); + } + // Ping round trip. + ASSERT_TRUE(WriteAll(c, Frame(0x6, 0, 0, "12345678"))); + bool pong = false; + for (int i = 0; i < 8 && !pong; ++i) { + ASSERT_TRUE(ReadFrame(c, f)); + if (f.type == 0x6 && (f.flags & 0x1)) { + pong = true; + ASSERT_EQ(f.payload, "12345678"); + } + } + ASSERT_TRUE(pong); + // Clean shutdown from the client. + ASSERT_TRUE(WriteAll(c, Frame(0x7, 0, 0, std::string(8, '\0')))); + t.Stop(); + ASSERT_EQ(t.conn, nullptr); +} + +TEST(H2PostAndRst) { + Harness t; + Handler h = [&](const FullRequest& req) -> Task { + std::lock_guard lock(t.mutex); + t.seen.push_back(req); + FullResponse resp; + resp.head.status = 200; + resp.body = "got " + std::to_string(req.body.size()); + co_return resp; + }; + t.Start(h); + int c = t.sv[1]; + ASSERT_TRUE(WriteAll(c, std::string(kH2Preface))); + ASSERT_TRUE(WriteAll(c, Frame(0x4, 0, 0, ""))); + GotFrame f; + ASSERT_TRUE(ReadFrame(c, f)); // server settings + // POST with two DATA frames on stream 1. + v2::HpackEncoder enc; + std::string block = enc.Encode({{":method", "POST"}, + {":scheme", "http"}, + {":authority", "h"}, + {":path", "/up"}, + {"content-length", "10"}}); + ASSERT_TRUE(WriteAll(c, Frame(0x1, 0x4, 1, block))); + ASSERT_TRUE(WriteAll(c, Frame(0x0, 0, 1, "01234"))); + ASSERT_TRUE(WriteAll(c, Frame(0x0, 0x1, 1, "56789"))); + std::string body; + bool done = false; + for (int i = 0; i < 16 && !done; ++i) { + ASSERT_TRUE(ReadFrame(c, f)); + if (f.type == 0x0 && f.id == 1) { + body += f.payload; + if (f.flags & 0x1) done = true; + } + } + ASSERT_TRUE(done); + ASSERT_EQ(body, "got 10"); + // Open stream 3 then reset it; stream 5 must still work. + std::string b3 = enc.Encode( + {{":method", "GET"}, {":scheme", "http"}, {":authority", "h"}, + {":path", "/x"}}); + ASSERT_TRUE(WriteAll(c, Frame(0x1, 0x4 | 0x1, 3, b3))); + char rst[4] = {0, 0, 0, 8}; + ASSERT_TRUE(WriteAll(c, Frame(0x3, 0, 3, std::string(rst, 4)))); + ASSERT_TRUE(WriteAll(c, Frame(0x1, 0x4 | 0x1, 5, b3))); + bool s5done = false; + for (int i = 0; i < 16 && !s5done; ++i) { + ASSERT_TRUE(ReadFrame(c, f)); + if (f.type == 0x0 && f.id == 5 && (f.flags & 0x1)) s5done = true; + if (f.type == 0x1 && f.id == 5 && (f.flags & 0x1)) s5done = true; + } + ASSERT_TRUE(s5done); + t.Stop(); +} + +TEST(H2BadPreface) { + Harness t; + Handler h = [&](const FullRequest&) -> Task { + FullResponse resp; + co_return resp; + }; + t.Start(h); + int c = t.sv[1]; + ASSERT_TRUE(WriteAll(c, "GARBAGE GARBAGE GARBAGE\r\n\r\n")); + // Not the H2 magic: the server treats it as H1 and 400s, then closes. + char buf[256]; + ssize_t n = read(c, buf, sizeof(buf)); + ASSERT_TRUE(n > 0); + std::string resp(buf, (std::size_t)n); + ASSERT_TRUE(resp.find("400") != std::string::npos); + t.Stop(); +} + +TEST(H2AlpnNoMagic) { + // TLS with ALPN h2: straight to SETTINGS, no PRI magic. + Harness t; + Handler h = [&](const FullRequest& req) -> Task { + FullResponse resp; + resp.head.status = 200; + resp.body = std::string(req.is_tls ? "tls:" : "plain:") + req.head.target; + co_return resp; + }; + t.Start(h, "h2", true); + int c = t.sv[1]; + ASSERT_TRUE(WriteAll(c, Frame(0x4, 0, 0, ""))); + GotFrame f; + ASSERT_TRUE(ReadFrame(c, f)); + ASSERT_EQ(f.type, 0x4u); + v2::HpackEncoder enc; + std::string block = enc.Encode({{":method", "GET"}, + {":scheme", "https"}, + {":authority", "h"}, + {":path", "/a"}}); + ASSERT_TRUE(WriteAll(c, Frame(0x1, 0x4 | 0x1, 1, block))); + std::string body; + bool done = false; + for (int i = 0; i < 16 && !done; ++i) { + ASSERT_TRUE(ReadFrame(c, f)); + if (f.type == 0x0 && f.id == 1) { + body += f.payload; + if (f.flags & 0x1) done = true; + } + } + ASSERT_TRUE(done); + ASSERT_EQ(body, "tls:/a"); + t.Stop(); +} + +TEST(H2ContinuationAndPriority) { + Harness t; + Handler h = [&](const FullRequest& req) -> Task { + FullResponse resp; + resp.head.status = 200; + resp.body = req.head.headers.get("x-big"); + co_return resp; + }; + t.Start(h); + int c = t.sv[1]; + ASSERT_TRUE(WriteAll(c, std::string(kH2Preface))); + ASSERT_TRUE(WriteAll(c, Frame(0x4, 0, 0, ""))); + GotFrame f; + ASSERT_TRUE(ReadFrame(c, f)); + // PRIORITY is accepted and ignored. + ASSERT_TRUE(WriteAll(c, Frame(0x2, 0, 1, std::string("\0\0\0\0\0", 5)))); + // Split header block across HEADERS + CONTINUATION. + v2::HpackEncoder enc; + std::string big(500, 'z'); + std::string block = enc.Encode({{":method", "GET"}, + {":scheme", "http"}, + {":authority", "h"}, + {":path", "/"}, + {"x-big", big}}); + std::size_t half = block.size() / 2; + ASSERT_TRUE( + WriteAll(c, Frame(0x1, 0x1, 1, block.substr(0, half)))); // END_STREAM + ASSERT_TRUE( + WriteAll(c, Frame(0x9, 0x4, 1, block.substr(half)))); // END_HEADERS + std::string body; + bool done = false; + for (int i = 0; i < 16 && !done; ++i) { + ASSERT_TRUE(ReadFrame(c, f)); + if (f.type == 0x0 && f.id == 1) { + body += f.payload; + if (f.flags & 0x1) done = true; + } + } + ASSERT_TRUE(done); + ASSERT_EQ(body, big); + t.Stop(); +} + +TEST(H2DrainGoaway) { + Harness t; + Handler h = [&](const FullRequest&) -> Task { + FullResponse resp; + resp.head.status = 200; + resp.body = "ok"; + co_return resp; + }; + t.Start(h); + int c = t.sv[1]; + ASSERT_TRUE(WriteAll(c, std::string(kH2Preface))); + ASSERT_TRUE(WriteAll(c, Frame(0x4, 0, 0, ""))); + GotFrame f; + ASSERT_TRUE(ReadFrame(c, f)); + v2::HpackEncoder enc; + std::string block = enc.Encode({{":method", "GET"}, + {":scheme", "http"}, + {":authority", "h"}, + {":path", "/"}}); + ASSERT_TRUE(WriteAll(c, Frame(0x1, 0x4 | 0x1, 1, block))); + bool done = false; + for (int i = 0; i < 16 && !done; ++i) { + ASSERT_TRUE(ReadFrame(c, f)); + if (f.type == 0x0 && f.id == 1 && (f.flags & 0x1)) done = true; + } + ASSERT_TRUE(done); + // Drain from the test thread: GOAWAY then close. + t.conn->drain(); + bool goaway = false; + for (int i = 0; i < 16 && !goaway; ++i) { + if (!ReadFrame(c, f)) break; + if (f.type == 0x7) { + goaway = true; + ASSERT_TRUE(f.payload.size() >= 8); + uint32_t code = ((uint32_t)(unsigned char)f.payload[4] << 24) | + ((uint32_t)(unsigned char)f.payload[5] << 16) | + ((uint32_t)(unsigned char)f.payload[6] << 8) | + (uint32_t)(unsigned char)f.payload[7]; + ASSERT_EQ(code, 0u); + } + } + ASSERT_TRUE(goaway); + char tmp[8]; + ASSERT_EQ(read(c, tmp, sizeof(tmp)), 0); // EOF after drain + t.Stop(); +} + +TEST(H2FirstFrameMustBeSettings) { + Harness t; + Handler h = [&](const FullRequest&) -> Task { + FullResponse resp; + co_return resp; + }; + t.Start(h); + int c = t.sv[1]; + ASSERT_TRUE(WriteAll(c, std::string(kH2Preface))); + ASSERT_TRUE(WriteAll(c, Frame(0x6, 0, 0, "12345678"))); // PING, not SETTINGS + GotFrame f; + ASSERT_TRUE(ReadFrame(c, f)); // server settings first + ASSERT_EQ(f.type, 0x4u); + ASSERT_TRUE(ReadFrame(c, f)); // then GOAWAY PROTOCOL_ERROR + ASSERT_EQ(f.type, 0x7u); + ASSERT_TRUE(f.payload.size() >= 8); + uint32_t code = ((uint32_t)(unsigned char)f.payload[4] << 24) | + ((uint32_t)(unsigned char)f.payload[5] << 16) | + ((uint32_t)(unsigned char)f.payload[6] << 8) | + (uint32_t)(unsigned char)f.payload[7]; + ASSERT_EQ(code, 0x1u); + t.Stop(); +} + +TEST(H2FileBodyStream) { + // 400 KiB: past the 64 KiB flow windows and the 256 KiB queue watermark, + // so this exercises WINDOW_UPDATE interplay plus queue backpressure. + constexpr std::size_t kSize = 400 * 1024; + std::string tpl = "/tmp/molo_h2file_test_XXXXXX"; + int tfd = mkstemp(tpl.data()); + ASSERT_TRUE(tfd >= 0); + { + std::string chunk(65536, '\0'); + std::size_t pos = 0; + while (pos < kSize) { + std::size_t w = kSize - pos; + if (w > chunk.size()) w = chunk.size(); + for (std::size_t i = 0; i < w; ++i) { + std::size_t j = pos + i; + chunk[i] = (char)(((j * 31u + 7u) ^ (j >> 8)) & 0xffu); + } + ASSERT_TRUE(write(tfd, chunk.data(), w) == (ssize_t)w); + pos += w; + } + } + close(tfd); + std::string want(kSize, '\0'); + for (std::size_t i = 0; i < kSize; ++i) { + want[i] = (char)(((i * 31u + 7u) ^ (i >> 8)) & 0xffu); + } + Harness t; + Handler h = [&](const FullRequest&) -> Task { + FullResponse resp; + int fd = open(tpl.c_str(), O_RDONLY); + resp.head.status = 200; + resp.head.headers.set("content-type", "application/octet-stream"); + resp.file = FileBody{fd, 0, kSize}; + co_return resp; + }; + t.Start(h); + int c = t.sv[1]; + ASSERT_TRUE(WriteAll(c, std::string(kH2Preface))); + ASSERT_TRUE(WriteAll(c, Frame(0x4, 0, 0, ""))); + GotFrame f; + ASSERT_TRUE(ReadFrame(c, f)); + ASSERT_EQ(f.type, 0x4u); + ASSERT_TRUE(WriteAll(c, Frame(0x4, 0x1, 0, ""))); + std::string block("\x82\x86\x84\x41\x0f", 5); + block += "www.example.com"; + ASSERT_TRUE(WriteAll(c, Frame(0x1, 0x4 | 0x1, 1, block))); + auto window_update = [&](uint32_t sid, std::size_t n) { + std::string p(4, '\0'); + p[0] = (char)((n >> 24) & 0x7f); + p[1] = (char)((n >> 16) & 0xff); + p[2] = (char)((n >> 8) & 0xff); + p[3] = (char)(n & 0xff); + return WriteAll(c, Frame(0x8, 0, sid, p)); + }; + std::string hblock; + std::string body; + bool got_data_end = false; + bool head_end_stream = false; + for (int i = 0; i < 128 && !got_data_end; ++i) { + ASSERT_TRUE(ReadFrame(c, f)); + if (f.type == 0x4 && (f.flags & 0x1)) continue; + if (f.type == 0x8) continue; + if (f.type == 0x7) break; // GOAWAY: fail below via got_data_end + if (f.type == 0x1) { + ASSERT_EQ(f.id, 1u); + hblock += f.payload; + if (f.flags & 0x1) head_end_stream = true; + } else if (f.type == 0x0) { + ASSERT_EQ(f.id, 1u); + body += f.payload; + ASSERT_TRUE(window_update(1, f.payload.size())); + ASSERT_TRUE(window_update(0, f.payload.size())); + if (f.flags & 0x1) got_data_end = true; + } + } + ASSERT_FALSE(head_end_stream); // HEADERS left the stream open for DATA + ASSERT_TRUE(got_data_end); + ASSERT_EQ(body.size(), kSize); + ASSERT_TRUE(body == want); + v2::HpackDecoder dec; + std::vector fields; + ASSERT_EQ(dec.Decode(hblock.data(), hblock.size(), fields), + v2::HpackError::Ok); + ASSERT_EQ(fields[0].name, ":status"); + ASSERT_EQ(fields[0].value, "200"); + ASSERT_TRUE(WriteAll(c, Frame(0x7, 0, 0, std::string(8, '\0')))); + t.Stop(); + unlink(tpl.c_str()); +} diff --git a/tests/src/runtime/http/v2/test_hpack.cpp b/tests/src/runtime/http/v2/test_hpack.cpp new file mode 100644 index 0000000..3c2ae87 --- /dev/null +++ b/tests/src/runtime/http/v2/test_hpack.cpp @@ -0,0 +1,174 @@ +#include "test_framework.h" + +#include + +#include "runtime/http/v2/hpack.h" + +using namespace molo::runtime::http::v2; + +namespace { +std::string Hex(const std::string& hex) { + std::string out; + for (std::size_t i = 0; i + 1 < hex.size(); i += 2) { + auto nyb = [](char c) -> unsigned { + if (c >= '0' && c <= '9') return (unsigned)(c - '0'); + if (c >= 'a' && c <= 'f') return (unsigned)(c - 'a' + 10); + return (unsigned)(c - 'A' + 10); + }; + out.push_back((char)((nyb(hex[i]) << 4) | nyb(hex[i + 1]))); + } + return out; +} +} // namespace + +TEST(HuffmanRfcVector) { + // RFC 7541 C.4.1: "www.example.com" Huffman-coded. + std::string enc = Hex("f1e3c2e5f23a6ba0ab90f4ff"); + auto dec = HuffmanCodec::Decode(enc.data(), enc.size()); + ASSERT_TRUE(dec.has_value()); + ASSERT_EQ(*dec, "www.example.com"); + ASSERT_EQ(HuffmanCodec::Encode("www.example.com"), enc); +} + +TEST(HuffmanMoreVectors) { + // Reference-verified against python-hpack. + const char* pairs[][2] = { + {"custom-key", "25a849e95ba97d7f"}, + {"custom-value", "25a849e95bb8e8b4bf"}, + {"Mon, 21 Oct 2013 20:13:21 GMT", + "d07abe941054d444a8200595040b8166e082a62d1bff"}, + {"private", "aec3771a4b"}, + }; + for (auto& pr : pairs) { + std::string enc = Hex(pr[1]); + ASSERT_EQ(HuffmanCodec::Encode(pr[0]), enc); + auto dec = HuffmanCodec::Decode(enc.data(), enc.size()); + ASSERT_TRUE(dec.has_value()); + ASSERT_EQ(*dec, pr[0]); + } +} + +TEST(HuffmanAllBytesRoundTrip) { + std::string all; + for (int i = 0; i < 256; ++i) all.push_back((char)i); + std::string enc = HuffmanCodec::Encode(all); + auto dec = HuffmanCodec::Decode(enc.data(), enc.size()); + ASSERT_TRUE(dec.has_value()); + ASSERT_EQ(*dec, all); + // Empty string. + ASSERT_EQ(HuffmanCodec::Encode(""), ""); + auto empty = HuffmanCodec::Decode("", 0); + ASSERT_TRUE(empty.has_value()); + ASSERT_TRUE(empty->empty()); +} + +TEST(HuffmanRejects) { + // All three rejected by the python-hpack reference as well. + for (const char* hx : {"ffffffff", "00", "ff"}) { + std::string v = Hex(hx); + ASSERT_FALSE(HuffmanCodec::Decode(v.data(), v.size()).has_value()); + } + // Single-symbol plus padding (reference agrees: b'0'). + std::string zero = Hex("07"); + auto zdec = HuffmanCodec::Decode(zero.data(), zero.size()); + ASSERT_TRUE(zdec.has_value()); + ASSERT_EQ(*zdec, "0"); + // A mutated tail that still parses (reference decodes it to a longer + // string) must be accepted, not mis-rejected. + std::string longer = Hex("f1e3c2e5f23a6ba0ab90f4f0"); + auto dec = HuffmanCodec::Decode(longer.data(), longer.size()); + ASSERT_TRUE(dec.has_value()); + ASSERT_EQ(*dec, "www.example.comU"); +} + +TEST(HpackRfcC41) { + // RFC 7541 C.4.1 first request, literal without Huffman for authority. + HpackDecoder d; + std::vector out; + std::string block = Hex("828684410f7777772e6578616d706c652e636f6d"); + ASSERT_EQ(d.Decode(block.data(), block.size(), out), HpackError::Ok); + ASSERT_EQ(out.size(), 4u); + ASSERT_EQ(out[0].name, ":method"); + ASSERT_EQ(out[0].value, "GET"); + ASSERT_EQ(out[1].name, ":scheme"); + ASSERT_EQ(out[1].value, "http"); + ASSERT_EQ(out[2].name, ":path"); + ASSERT_EQ(out[2].value, "/"); + ASSERT_EQ(out[3].name, ":authority"); + ASSERT_EQ(out[3].value, "www.example.com"); +} + +TEST(HpackRfcC41Huffman) { + HpackDecoder d; + std::vector out; + std::string block = Hex("828684418cf1e3c2e5f23a6ba0ab90f4ff"); + ASSERT_EQ(d.Decode(block.data(), block.size(), out), HpackError::Ok); + ASSERT_EQ(out.size(), 4u); + ASSERT_EQ(out[3].name, ":authority"); + ASSERT_EQ(out[3].value, "www.example.com"); + // The literal used incremental indexing: dynamic table now holds it. + ASSERT_TRUE(d.table_size() > 0); + // Index 62 addresses the new dynamic entry. + std::vector out2; + std::string idx("\xbe", 1); // indexed 62 + ASSERT_EQ(d.Decode(idx.data(), idx.size(), out2), HpackError::Ok); + ASSERT_EQ(out2.size(), 1u); + ASSERT_EQ(out2[0].name, ":authority"); +} + +TEST(HpackEncoderRoundTrip) { + HpackEncoder e; + HpackDecoder d; + std::vector fields = { + {":method", "POST"}, + {":scheme", "https"}, + {":path", "/submit?q=1"}, + {":authority", "x.test"}, + {"content-type", "application/json"}, + {"x-custom", "v"}, + }; + std::string block = e.Encode(fields); + std::vector out; + ASSERT_EQ(d.Decode(block.data(), block.size(), out), HpackError::Ok); + ASSERT_EQ(out.size(), fields.size()); + for (std::size_t i = 0; i < fields.size(); ++i) { + ASSERT_EQ(out[i].name, fields[i].name); + ASSERT_EQ(out[i].value, fields[i].value); + } +} + +TEST(HpackErrors) { + HpackDecoder d; + std::vector out; + std::string bad_index("\x80", 1); // indexed 0: invalid + ASSERT_EQ(d.Decode(bad_index.data(), bad_index.size(), out), + HpackError::BadIndex); + std::string huge_index("\xff\x9a\x0a", 3); // indexed 1337 + out.clear(); + ASSERT_EQ(d.Decode(huge_index.data(), huge_index.size(), out), + HpackError::BadIndex); + std::string trunc("\x00\x05", 2); // literal name len 5, missing bytes + out.clear(); + ASSERT_EQ(d.Decode(trunc.data(), trunc.size(), out), HpackError::Truncated); +} + +TEST(HpackTableSizeUpdate) { + HpackDecoder d(4096); + std::vector out; + std::string upd("\x3f\x21", 2); // size update to 32+... 0x3f=31, +0x21 + ASSERT_EQ(d.Decode(upd.data(), upd.size(), out), HpackError::Ok); + std::string too_big("\x3f\xe2\x1f", 3); // 4097: over the 4096 max + out.clear(); + ASSERT_EQ(d.Decode(too_big.data(), too_big.size(), out), + HpackError::TableSizeChanged); +} + +TEST(HpackHeaderListCap) { + HpackDecoder d(4096, 100); + std::vector out; + // One literal field larger than the 100-byte list cap. + std::string block("\x00\x01\x61\x64", 4); + block.append(100, 'b'); + ASSERT_EQ(d.Decode(block.data(), block.size(), out), + HpackError::HeaderListTooLarge); +} diff --git a/tests/src/runtime/http/v2/test_settings.cpp b/tests/src/runtime/http/v2/test_settings.cpp new file mode 100644 index 0000000..b800430 --- /dev/null +++ b/tests/src/runtime/http/v2/test_settings.cpp @@ -0,0 +1,42 @@ +#include "test_framework.h" + +#include "runtime/http/v2/settings.h" + +using namespace molo::runtime::http::v2; + +TEST(SettingsRoundTrip) { + Settings s; + s.max_concurrent_streams = 50; + std::string p = EncodeSettingsPayload(s); + ASSERT_EQ(p.size(), 30u); + Settings out; + ASSERT_EQ(DecodeSettingsPayload(p.data(), p.size(), out), + SettingsError::Ok); + ASSERT_EQ(out.max_concurrent_streams, 50u); + ASSERT_EQ(out.initial_window_size, 65535u); +} + +TEST(SettingsRejects) { + Settings out; + // Odd length. + ASSERT_EQ(DecodeSettingsPayload("abc", 3, out), SettingsError::ProtocolError); + // ENABLE_PUSH = 1 is accepted and ignored (deprecated knob real + // clients still send; we never push regardless). + char push[6] = {0, 2, 0, 0, 0, 1}; + ASSERT_EQ(DecodeSettingsPayload(push, 6, out), SettingsError::Ok); + ASSERT_FALSE(out.enable_push); + // Bad window. + char win[6] = {0, 4, (char)0xff, (char)0xff, (char)0xff, (char)0xff}; + ASSERT_EQ(DecodeSettingsPayload(win, 6, out), + SettingsError::FlowControlError); + // Bad frame size. + char frame[6] = {0, 5, 0, 0, 0, 1}; + ASSERT_EQ(DecodeSettingsPayload(frame, 6, out), SettingsError::ProtocolError); + // Oversized header table. + char tab[6] = {0, 1, 0, 1, 0, 0}; + ASSERT_EQ(DecodeSettingsPayload(tab, 6, out), + SettingsError::CompressionError); + // Unknown id ignored. + char unk[6] = {(char)0xde, (char)0xad, 0, 0, 0, 0}; + ASSERT_EQ(DecodeSettingsPayload(unk, 6, out), SettingsError::Ok); +} diff --git a/tests/src/runtime/http/v2/test_stream.cpp b/tests/src/runtime/http/v2/test_stream.cpp new file mode 100644 index 0000000..6349cd6 --- /dev/null +++ b/tests/src/runtime/http/v2/test_stream.cpp @@ -0,0 +1,103 @@ +#include "test_framework.h" + +#include "runtime/http/v2/stream.h" + +using namespace molo::runtime::http::v2; + +namespace { +std::vector GetFields() { + return {{":method", "GET"}, + {":scheme", "http"}, + {":authority", "h"}, + {":path", "/"}}; +} +} // namespace + +TEST(StreamGetNoBody) { + H2Stream s(1, 1024); + ASSERT_EQ(s.state(), StreamState::Idle); + StreamOutcome o = s.OnHeaders(GetFields(), true); + ASSERT_EQ(o.event, StreamEvent::MessageComplete); + ASSERT_EQ(s.state(), StreamState::HalfClosedRemote); + ASSERT_EQ(s.head().method, "GET"); + ASSERT_EQ(s.head().target, "/"); + ASSERT_EQ(s.head().headers.get("host"), "h"); +} + +TEST(StreamPostBody) { + H2Stream s(1, 1024); + StreamOutcome o = s.OnHeaders(GetFields(), false); + ASSERT_EQ(o.event, StreamEvent::HeadersComplete); + ASSERT_EQ(s.state(), StreamState::Open); + o = s.OnData("ab", 2, false); + ASSERT_EQ(o.event, StreamEvent::None); + o = s.OnData("c", 1, true); + ASSERT_EQ(o.event, StreamEvent::MessageComplete); + ASSERT_EQ(s.body(), "abc"); + ASSERT_EQ(s.state(), StreamState::HalfClosedRemote); +} + +TEST(StreamBodyCap) { + H2Stream s(1, 4); + s.OnHeaders(GetFields(), false); + StreamOutcome o = s.OnData("abcde", 5, true); + ASSERT_EQ(o.event, StreamEvent::LocalReset); + ASSERT_EQ(o.rst_code, 0x8u); +} + +TEST(StreamTrailers) { + H2Stream s(1, 1024); + s.OnHeaders(GetFields(), false); + std::vector tr = {{"x-sum", "1"}}; + StreamOutcome o = s.OnHeaders(tr, true); + ASSERT_EQ(o.event, StreamEvent::MessageComplete); + ASSERT_EQ(s.trailers().get("x-sum"), "1"); +} + +TEST(StreamBadPseudo) { + H2Stream s(1, 1024); + std::vector bad = {{":method", "GET"}}; + StreamOutcome o = s.OnHeaders(bad, true); + ASSERT_EQ(o.event, StreamEvent::LocalReset); + ASSERT_EQ(o.rst_code, 0x1u); + H2Stream c(3, 1024); + std::vector connect = { + {":method", "CONNECT"}, {":scheme", "http"}, {":path", "/"}}; + ASSERT_EQ(c.OnHeaders(connect, true).event, StreamEvent::LocalReset); +} + +TEST(StreamDataBeforeHeaders) { + H2Stream s(1, 1024); + StreamOutcome o = s.OnData("x", 1, false); + ASSERT_EQ(o.event, StreamEvent::LocalReset); +} + +TEST(StreamRst) { + H2Stream s(1, 1024); + s.OnHeaders(GetFields(), false); + StreamOutcome o = s.OnRst(0x8); + ASSERT_EQ(o.event, StreamEvent::ResetByPeer); + ASSERT_EQ(s.state(), StreamState::Closed); +} + +TEST(StreamWindowZero) { + H2Stream s(1, 1024); + s.OnHeaders(GetFields(), false); + StreamOutcome o = s.OnWindowUpdate(0); + ASSERT_EQ(o.event, StreamEvent::LocalReset); + H2Stream idle(3, 1024); + ASSERT_EQ(idle.OnWindowUpdate(100).event, StreamEvent::LocalReset); +} + +TEST(StreamResponseStates) { + H2Stream s(1, 1024); + s.OnHeaders(GetFields(), true); + s.MarkResponseHeadersSent(false); + ASSERT_EQ(s.state(), StreamState::HalfClosedLocal); + s.MarkResponseDataSent(true); + ASSERT_EQ(s.state(), StreamState::Closed); + H2Stream e(3, 1024); + e.OnHeaders(GetFields(), true); + e.MarkResponseHeadersSent(true); + ASSERT_EQ(e.state(), StreamState::Closed); +} diff --git a/tests/src/runtime/interop/interop.py b/tests/src/runtime/interop/interop.py new file mode 100644 index 0000000..5bc60a6 --- /dev/null +++ b/tests/src/runtime/interop/interop.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""Real-client interop for the runtime part. Spawns the C++ test server and +drives it with stdlib http.client (H1) and python-h2 (h2c prior knowledge). +Skips the H2 half when the h2 package is unavailable.""" +import gzip +import hashlib +import http.client +import socket +import subprocess +import sys +import time + +FAILS = [] + + +def check(name, cond, extra=""): + print(("PASS " if cond else "FAIL ") + name + (" " + extra if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def main(): + server_bin = sys.argv[1] + www = "tests/src/runtime/interop/www" + proc = subprocess.Popen( + [server_bin, "0", www], stdout=subprocess.PIPE, text=True + ) + try: + line = proc.stdout.readline().strip() + assert line.startswith("READY "), line + port = int(line.split()[1]) + time.sleep(0.2) + run_h1(port) + run_h2(port) + finally: + proc.terminate() + proc.wait(timeout=10) + print("FAILURES: %d" % len(FAILS)) + return 1 if FAILS else 0 + + +def run_h1(port): + c = http.client.HTTPConnection("127.0.0.1", port, timeout=10) + c.request("GET", "/echo") + r = c.getresponse() + check("h1-echo-status", r.status == 200, str(r.status)) + check("h1-echo-body", r.read() == b"GET:", "") + # keep-alive reuse on the same connection + c.request("POST", "/echo", body=b"abc") + r = c.getresponse() + check("h1-echo-post", r.read() == b"POST:abc") + # static file with validators + c.request("GET", "/static/hello.txt") + r = c.getresponse() + body = r.read() + etag = r.getheader("ETag") + lastmod = r.getheader("Last-Modified") + check("h1-static", r.status == 200 and body.startswith(b"Hello, static")) + check("h1-static-etag", bool(etag), repr(etag)) + c.request("GET", "/static/hello.txt", headers={"If-None-Match": etag}) + r = c.getresponse() + r.read() + check("h1-static-304", r.status == 304, str(r.status)) + c.request("GET", "/static/hello.txt", headers={"Range": "bytes=0-4"}) + r = c.getresponse() + check("h1-static-206", r.status == 206 and r.read() == b"Hello", str(r.status)) + c.request("GET", "/static/hello.txt", headers={"If-Modified-Since": lastmod}) + r = c.getresponse() + r.read() + check("h1-static-ims", r.status == 304, str(r.status)) + # gzip negotiation + c.request("GET", "/gzip", headers={"Accept-Encoding": "gzip"}) + r = c.getresponse() + gz = r.read() + check("h1-gzip-enc", r.getheader("Content-Encoding") == "gzip") + check("h1-gzip-vary", r.getheader("Vary") == "Accept-Encoding") + try: + plain = gzip.decompress(gz) + check("h1-gzip-body", plain.startswith(b"interop line 0"), str(len(plain))) + except Exception as e: + check("h1-gzip-body", False, repr(e)) + c.request("GET", "/gzip", headers={"Accept-Encoding": "identity"}) + r = c.getresponse() + check("h1-gzip-identity", r.getheader("Content-Encoding") in (None, "identity") + and r.read().startswith(b"interop line 0")) + # brotli negotiation + try: + import brotli # noqa + + c.request("GET", "/br", headers={"Accept-Encoding": "br, gzip"}) + r = c.getresponse() + br = r.read() + check("h1-br-enc", r.getheader("Content-Encoding") == "br") + try: + plain = brotli.decompress(br) + check("h1-br-body", plain.startswith(b"interop line 0")) + except Exception as e: + check("h1-br-body", False, repr(e)) + except ImportError: + print("SKIP h1-br (no brotli package)") + # cookies + c.request("GET", "/cookie-set") + r = c.getresponse() + r.read() + sets = [v for k, v in r.getheaders() if k.lower() == "set-cookie"] + check("h1-cookie-set-count", len(sets) == 2, repr(sets)) + check("h1-cookie-defaults", all("HttpOnly" in s and "SameSite=Lax" in s for s in sets), + repr(sets)) + jar = "; ".join(s.split(";", 1)[0] for s in sets) + c.request("GET", "/cookie-get", headers={"Cookie": jar}) + r = c.getresponse() + check("h1-cookie-get", r.read() == b"plain=v1 sess_ok=1 sess=user=42") + # big body over H1 + c.request("GET", "/big") + r = c.getresponse() + big = r.read() + check("h1-big", len(big) > 200000, str(len(big))) + # file-backed body over H1 (sendfile path) + c.request("GET", "/file") + r = c.getresponse() + fbody = r.read() + check("h1-file", r.status == 200 and fbody == file_pattern(), + "%s len=%d" % (r.status, len(fbody))) + c.request("HEAD", "/file") + r = c.getresponse() + r.read() + check("h1-file-head", r.status == 200 and + r.getheader("Content-Length") == str(300 * 1024), + "%s %r" % (r.status, r.getheader("Content-Length"))) + c.close() + + +def file_pattern(): + return bytes((((i * 31 + 7) ^ (i >> 8)) & 0xff) for i in range(300 * 1024)) + + +def run_h2(port): + try: + from h2.connection import H2Connection + from h2.events import ResponseReceived, DataReceived, StreamEnded + except ImportError: + print("SKIP h2 (no h2 package)") + return + sock = socket.create_connection(("127.0.0.1", port), timeout=10) + h2 = H2Connection() + h2.initiate_connection() + sock.sendall(h2.data_to_send()) + try: + # single GET + h2.send_headers(1, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/echo")], + end_stream=True) + sock.sendall(h2.data_to_send()) + status, body = read_stream(sock, h2, 1) + check("h2-get-status", status == "200", repr(status)) + check("h2-get-body", body == b"GET:", repr(body[:20])) + # POST with request body + h2.send_headers(3, [(":method", "POST"), (":scheme", "http"), + (":authority", "x"), (":path", "/echo")], + end_stream=False) + h2.send_data(3, b"stream-body", end_stream=True) + sock.sendall(h2.data_to_send()) + status, body = read_stream(sock, h2, 3) + check("h2-post", status == "200" and body == b"POST:stream-body", + repr((status, body[:30]))) + # two concurrent streams + h2.send_headers(5, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/echo")], + end_stream=True) + h2.send_headers(7, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/big")], + end_stream=True) + sock.sendall(h2.data_to_send()) + s5, b5 = read_stream(sock, h2, 5) + s7, b7 = read_stream(sock, h2, 7) + check("h2-concurrent", (s5, b5) == ("200", b"GET:") and s7 == "200" + and len(b7) > 200000, repr((s5, len(b5), s7, len(b7)))) + # gzip over H2 (request with accept-encoding) + h2.send_headers(9, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/gzip"), + ("accept-encoding", "gzip")], + end_stream=True) + sock.sendall(h2.data_to_send()) + status, body, headers = read_stream_full(sock, h2, 9) + enc = dict(headers).get("content-encoding", "") + check("h2-gzip-enc", enc == "gzip", repr(enc)) + try: + check("h2-gzip-body", gzip.decompress(body).startswith(b"interop line 0")) + except Exception as e: + check("h2-gzip-body", False, repr(e)) + # reset one stream, keep the connection usable + h2.send_headers(11, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/big")], + end_stream=True) + sock.sendall(h2.data_to_send()) + h2.reset_stream(11) + sock.sendall(h2.data_to_send()) + h2.send_headers(13, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/echo")], + end_stream=True) + sock.sendall(h2.data_to_send()) + status, body = read_stream(sock, h2, 13) + check("h2-after-rst", (status, body) == ("200", b"GET:"), + repr((status, body[:20]))) + # cookies over H2 (split cookie header join) + h2.send_headers(15, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/cookie-set")], + end_stream=True) + sock.sendall(h2.data_to_send()) + _, _, hdrs = read_stream_full(sock, h2, 15) + sets = [v for k, v in hdrs if k == "set-cookie"] + check("h2-cookie-set", len(sets) == 2, repr(sets)) + jar = "; ".join(s.split(";", 1)[0] for s in sets) + h2.send_headers(17, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/cookie-get"), + ("cookie", jar)], + end_stream=True) + sock.sendall(h2.data_to_send()) + status, body = read_stream(sock, h2, 17) + check("h2-cookie-get", body == b"plain=v1 sess_ok=1 sess=user=42", + repr(body[:40])) + # file-backed body over H2 (streamed DATA with flow control) + h2.send_headers(19, [(":method", "GET"), (":scheme", "http"), + (":authority", "x"), (":path", "/file")], + end_stream=True) + sock.sendall(h2.data_to_send()) + status, body = read_stream(sock, h2, 19) + check("h2-file", status == "200" and + hashlib.sha256(body).hexdigest() == + hashlib.sha256(file_pattern()).hexdigest(), + repr((status, len(body)))) + h2.close_connection() + sock.sendall(h2.data_to_send()) + sock.close() + except Eof as e: + check("h2-eof", False, str(e)) + sock.close() + + +class Eof(Exception): + pass + + +def pump(sock, h2, timeout=10): + sock.settimeout(timeout) + try: + data = sock.recv(65536) + except socket.timeout: + raise Eof("timeout waiting for server bytes") + if not data: + raise Eof("server closed the connection") + return h2.receive_data(data) + + +def read_stream(sock, h2, sid): + status, body, _ = read_stream_full(sock, h2, sid) + return status, body + + +def _s(x): + return x.decode("utf-8") if isinstance(x, bytes) else x + + +def read_stream_full(sock, h2, sid): + from h2.events import ResponseReceived, DataReceived, StreamEnded + status = "" + headers = [] + chunks = [] + ended = False + while not ended: + for ev in pump(sock, h2): + if isinstance(ev, ResponseReceived) and ev.stream_id == sid: + headers = [(_s(k), _s(v)) for k, v in ev.headers] + for k, v in headers: + if k == ":status": + status = v + elif isinstance(ev, DataReceived) and ev.stream_id == sid: + chunks.append(ev.data) + h2.acknowledge_received_data(len(ev.data), sid) + elif isinstance(ev, StreamEnded) and ev.stream_id == sid: + ended = True + data = h2.data_to_send() + if data: + sock.sendall(data) + return status, b"".join(chunks), headers + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/src/runtime/interop/test_server.cpp b/tests/src/runtime/interop/test_server.cpp new file mode 100644 index 0000000..337f892 --- /dev/null +++ b/tests/src/runtime/interop/test_server.cpp @@ -0,0 +1,255 @@ +// Interop test server (test-only): exercises the runtime over real sockets +// for tests/src/runtime/interop/interop.py. Routes: +// /echo method + body echo +// /static/* files from the www dir argument +// /gzip compressible body, explicit gzip when accepted +// /br compressible body, explicit brotli when accepted +// /cookie-set sets two cookies (one signed with NullSigner) +// /cookie-get echoes parsed cookies as JSON-ish text +// /big 200 KiB body (multi-frame H2 DATA, flow control) +// /file 300 KiB file-backed body (sendfile on H1, stream on H2) +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "runtime/compress/brotli.h" +#include "runtime/compress/gzip.h" +#include "runtime/compress/negotiate.h" +#include "runtime/cookie/jar.h" +#include "runtime/event_loop.h" +#include "runtime/http/connection.h" +#include "runtime/http/listener.h" +#include "runtime/static/disk_provider.h" +#include "runtime/static/file_server.h" +#include "runtime/task.h" +#include "runtime/tls_handle.h" +#include "runtime/transport_tcp.h" + +using namespace molo::runtime; +using namespace molo::runtime::http; + +namespace { +std::string CompressGzip(const std::string& in) { + compress::GzipEncoder e; + std::string out; + std::vector buf(65536); + compress::CodecResult r; + std::size_t pos = 0; + do { + std::size_t n = in.size() - pos; + bool finish = true; + r = e.feed(in.data() + pos, n, buf.data(), buf.size(), finish); + pos += r.in_used; + out.append(buf.data(), r.out_used); + } while (r.status == compress::CodecStatus::Ok || + r.status == compress::CodecStatus::NeedOutput); + return out; +} + +std::string CompressBrotli(const std::string& in) { + compress::BrotliEncoder e; + std::string out; + std::vector buf(65536); + compress::CodecResult r; + std::size_t pos = 0; + do { + std::size_t n = in.size() - pos; + bool finish = true; + r = e.feed(in.data() + pos, n, buf.data(), buf.size(), finish); + pos += r.in_used; + out.append(buf.data(), r.out_used); + } while (r.status == compress::CodecStatus::Ok || + r.status == compress::CodecStatus::NeedOutput); + return out; +} + +std::string BigBody() { + std::string out; + for (int i = 0; i < 4000; ++i) { + out += "interop line " + std::to_string(i) + + " with some padding text to add up bytes.\n"; + } + return out; +} + +// Deterministic file bytes; interop.py regenerates the same pattern. +constexpr std::size_t kFileBytes = 300 * 1024; +char FileByte(std::size_t i) { + return (char)(((i * 31u + 7u) ^ (i >> 8)) & 0xffu); +} + +std::string MakeFileBody() { + std::string tpl = "/tmp/molo_interop_file_XXXXXX"; + int fd = mkstemp(tpl.data()); + if (fd < 0) return ""; + std::string chunk(65536, '\0'); + std::size_t pos = 0; + while (pos < kFileBytes) { + std::size_t n = kFileBytes - pos; + if (n > chunk.size()) n = chunk.size(); + for (std::size_t i = 0; i < n; ++i) chunk[i] = FileByte(pos + i); + ssize_t w = write(fd, chunk.data(), n); + if (w <= 0) { + close(fd); + unlink(tpl.c_str()); + return ""; + } + pos += (std::size_t)w; + } + close(fd); + return tpl; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 3) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + uint16_t port = (uint16_t)atoi(argv[1]); + static_::DiskProvider disk(argv[2]); + if (!disk.valid()) { + fprintf(stderr, "bad www dir\n"); + return 2; + } + static_::FileServer files(disk); + cookie::NullSigner signer; + std::string big = BigBody(); + std::string file_path = MakeFileBody(); + if (file_path.empty()) { + fprintf(stderr, "file body setup failed\n"); + return 2; + } + + EventLoop loop; + Listener listener(loop); + if (!listener.bind("127.0.0.1", port)) { + fprintf(stderr, "bind failed\n"); + return 2; + } + printf("READY %d\n", listener.port()); + fflush(stdout); + + Handler handler = [&](const FullRequest& req) -> Task { + const std::string& t = req.head.target; + if (t == "/echo") { + FullResponse r; + r.head.status = 200; + r.head.headers.set("content-type", "text/plain"); + r.body = req.head.method + ":" + req.body; + co_return r; + } + if (t.compare(0, 8, "/static/") == 0) { + RequestHead sub = req.head; + sub.target = t.substr(7); + static_::FileOutcome o = files.Handle(sub); + FullResponse r; + r.head = o.head; + if (o.asset.has_value() && !o.head_only && o.length > 0) { + static_::Asset& a = *o.asset; + if (a.fd() >= 0) { + r.file = FileBody{a.release(), o.offset, o.length}; + } else { + r.body.assign(a.data() + o.offset, (std::size_t)o.length); + } + } + co_return r; + } + if (t == "/gzip" || t == "/br") { + std::string ae = req.head.headers.get("accept-encoding"); + compress::Decision d = compress::Negotiator::Negotiate( + ae, "text/plain", big.size(), false, req.head.method == "HEAD", + 200); + FullResponse r; + r.head.status = 200; + r.head.headers.set("content-type", "text/plain"); + if ((t == "/gzip" && d.encoding == "gzip") || + (t == "/br" && d.encoding == "br")) { + r.body = (t == "/gzip") ? CompressGzip(big) : CompressBrotli(big); + r.head.headers.set("content-encoding", d.encoding); + r.head.headers.set("vary", compress::Negotiator::VaryValue()); + } else { + r.body = big; + } + co_return r; + } + if (t == "/cookie-set") { + cookie::CookieJar jar(req.is_tls); + cookie::Cookie c; + c.name = "plain"; + c.value = "v1"; + c.path = "/"; + jar.Set(c); + jar.SetSigned("sess", "user=42", signer, "/"); + FullResponse r; + r.head.status = 200; + for (const std::string& h : jar.set_cookie_headers()) { + r.head.headers.add("set-cookie", h); + } + r.body = "cookies set"; + co_return r; + } + if (t == "/cookie-get") { + cookie::CookieJar jar(req.is_tls); + std::vector fields = + req.head.headers.get_all("cookie"); + FullResponse r; + r.head.status = 200; + r.head.headers.set("content-type", "text/plain"); + if (jar.ParseRequest(fields) != cookie::JarError::Ok) { + r.head.status = 431; + r.body = "cookie overflow"; + co_return r; + } + std::string payload; + bool verified = + jar.VerifySigned("sess", const_cast(signer), + payload); + r.body = "plain=" + jar.Get("plain") + " sess_ok=" + + (verified ? "1" : "0") + " sess=" + payload; + co_return r; + } + if (t == "/big") { + FullResponse r; + r.head.status = 200; + r.head.headers.set("content-type", "text/plain"); + r.body = big; + co_return r; + } + if (t == "/file") { + FullResponse r; + int fd = open(file_path.c_str(), O_RDONLY); + if (fd < 0) { + r.head.status = 500; + r.body = "file open failed\n"; + co_return r; + } + r.head.status = 200; + r.head.headers.set("content-type", "application/octet-stream"); + r.file = FileBody{fd, 0, kFileBytes}; + co_return r; + } + FullResponse r; + r.head.status = 404; + r.body = "nope\n"; + co_return r; + }; + + listener.on_accept([&](int fd, const std::string& peer) { + (void)peer; + auto c = Connection::Create(loop, std::make_unique(fd), + std::make_unique(), + handler); + listener.track(c); + c->start(); + }); + loop.run(); + return 0; +} diff --git a/tests/src/runtime/interop/www/hello.txt b/tests/src/runtime/interop/www/hello.txt new file mode 100644 index 0000000..b90f6dd --- /dev/null +++ b/tests/src/runtime/interop/www/hello.txt @@ -0,0 +1 @@ +Hello, static world! This is a test file. diff --git a/tests/src/runtime/main.cpp b/tests/src/runtime/main.cpp new file mode 100644 index 0000000..a1ddd7d --- /dev/null +++ b/tests/src/runtime/main.cpp @@ -0,0 +1,3 @@ +#include "test_framework.h" + +int main() { return ::molo::test::RunAll(); } diff --git a/tests/src/runtime/static/test_file_server.cpp b/tests/src/runtime/static/test_file_server.cpp new file mode 100644 index 0000000..792f3f2 --- /dev/null +++ b/tests/src/runtime/static/test_file_server.cpp @@ -0,0 +1,266 @@ +#include "test_framework.h" + +#include +#include +#include + +#include +#include + +#include "runtime/compress/brotli.h" +#include "runtime/compress/gzip.h" +#include "runtime/http/message.h" +#include "runtime/static/disk_provider.h" +#include "runtime/static/embedded_provider.h" +#include "runtime/static/etag.h" +#include "runtime/static/file_server.h" + +using namespace molo::runtime; +using namespace molo::runtime::http; +using namespace molo::runtime::static_; + +namespace { +RequestHead Get(const std::string& target) { + RequestHead h; + h.method = "GET"; + h.target = target; + return h; +} + +std::string BodyOf(const FileOutcome& o) { + if (!o.asset.has_value() || o.head_only) return ""; + const Asset& a = *o.asset; + if (a.fd() >= 0) { + std::string out((std::size_t)o.length, '\0'); + std::size_t off = 0; + while (off < o.length) { + ssize_t n = pread(a.fd(), out.data() + off, + (std::size_t)(o.length - off), + (off_t)(o.offset + off)); + if (n <= 0) break; + off += (std::size_t)n; + } + out.resize(off); + return out; + } + return std::string(a.data() + o.offset, (std::size_t)o.length); +} + +std::string GzipBytes(const std::string& in) { + compress::GzipEncoder e; + std::string out; + std::vector buf(4096); + compress::CodecResult r; + std::size_t pos = 0; + do { + std::size_t n = in.size() - pos; + bool finish = true; + r = e.feed(in.data() + pos, n, buf.data(), buf.size(), finish); + pos += r.in_used; + out.append(buf.data(), r.out_used); + } while (r.status == compress::CodecStatus::Ok || + r.status == compress::CodecStatus::NeedOutput); + return out; +} + +std::string BrotliBytes(const std::string& in) { + compress::BrotliEncoder e; + std::string out; + std::vector buf(4096); + compress::CodecResult r; + std::size_t pos = 0; + do { + std::size_t n = in.size() - pos; + bool finish = true; + r = e.feed(in.data() + pos, n, buf.data(), buf.size(), finish); + pos += r.in_used; + out.append(buf.data(), r.out_used); + } while (r.status == compress::CodecStatus::Ok || + r.status == compress::CodecStatus::NeedOutput); + return out; +} +} // namespace + +TEST(FileServerDiskBasics) { + DiskProvider disk("tests/src/runtime/fixtures/www"); + ASSERT_TRUE(disk.valid()); + FileServer fs(disk); + FileOutcome o = fs.Handle(Get("/hello.txt")); + ASSERT_EQ(o.status, 200); + ASSERT_EQ(BodyOf(o), "Hello, static world! This is a test file.\n"); + ASSERT_EQ(o.head.headers.get("content-type"), "text/plain; charset=utf-8"); + ASSERT_FALSE(o.head.headers.get("etag").empty()); + ASSERT_FALSE(o.head.headers.get("last-modified").empty()); + ASSERT_EQ(o.head.headers.get("accept-ranges"), "bytes"); + ASSERT_EQ(o.head.headers.get("cache-control"), "public, max-age=3600"); + FileOutcome html = fs.Handle(Get("/index.html")); + ASSERT_EQ(html.head.headers.get("cache-control"), "no-cache"); + ASSERT_EQ(html.head.headers.get("content-type"), "text/html; charset=utf-8"); +} + +TEST(FileServerNotFoundAndMethods) { + DiskProvider disk("tests/src/runtime/fixtures/www"); + FileServer fs(disk); + ASSERT_EQ(fs.Handle(Get("/nope.txt")).status, 404); + ASSERT_EQ(fs.Handle(Get("/../hello.txt")).status, 404); + ASSERT_EQ(fs.Handle(Get("/.hidden")).status, 404); + ASSERT_EQ(fs.Handle(Get("/sub")).status, 403); // index off + RequestHead post = Get("/hello.txt"); + post.method = "POST"; + FileOutcome m = fs.Handle(post); + ASSERT_EQ(m.status, 405); + ASSERT_EQ(m.head.headers.get("allow"), "GET, HEAD, OPTIONS"); + RequestHead opt = Get("/hello.txt"); + opt.method = "OPTIONS"; + ASSERT_EQ(fs.Handle(opt).status, 204); + RequestHead head = Get("/hello.txt"); + head.method = "HEAD"; + FileOutcome h = fs.Handle(head); + ASSERT_EQ(h.status, 200); + ASSERT_TRUE(h.head_only); + ASSERT_EQ(BodyOf(h), ""); + ASSERT_EQ(h.head.headers.get("content-length"), "42"); +} + +TEST(FileServerConditional) { + DiskProvider disk("tests/src/runtime/fixtures/www"); + FileServer fs(disk); + FileOutcome o = fs.Handle(Get("/hello.txt")); + std::string etag = o.head.headers.get("etag"); + std::string lm = o.head.headers.get("last-modified"); + RequestHead inm = Get("/hello.txt"); + inm.headers.set("if-none-match", etag); + ASSERT_EQ(fs.Handle(inm).status, 304); + RequestHead inm_list = Get("/hello.txt"); + inm_list.headers.set("if-none-match", "\"other\", " + etag); + ASSERT_EQ(fs.Handle(inm_list).status, 304); + RequestHead star = Get("/hello.txt"); + star.headers.set("if-none-match", "*"); + ASSERT_EQ(fs.Handle(star).status, 304); + RequestHead ims = Get("/hello.txt"); + ims.headers.set("if-modified-since", lm); + ASSERT_EQ(fs.Handle(ims).status, 304); + // INM precedence: wrong INM plus fresh IMS still serves 200. + RequestHead both = Get("/hello.txt"); + both.headers.set("if-none-match", "\"wrong\""); + both.headers.set("if-modified-since", lm); + ASSERT_EQ(fs.Handle(both).status, 200); + FileOutcome c = fs.Handle(inm); + ASSERT_EQ(c.head.headers.get("etag"), etag); +} + +TEST(FileServerRanges) { + DiskProvider disk("tests/src/runtime/fixtures/www"); + FileServer fs(disk); + RequestHead r = Get("/hello.txt"); + r.headers.set("range", "bytes=0-4"); + FileOutcome o = fs.Handle(r); + ASSERT_EQ(o.status, 206); + ASSERT_EQ(BodyOf(o), "Hello"); + ASSERT_EQ(o.head.headers.get("content-range"), "bytes 0-4/42"); + RequestHead suf = Get("/hello.txt"); + suf.headers.set("range", "bytes=-6"); + FileOutcome s = fs.Handle(suf); + ASSERT_EQ(s.status, 206); + ASSERT_EQ(BodyOf(s), "file.\n"); + RequestHead open = Get("/hello.txt"); + open.headers.set("range", "bytes=7-"); + FileOutcome p = fs.Handle(open); + ASSERT_EQ(p.status, 206); + ASSERT_EQ(BodyOf(p), "static world! This is a test file.\n"); + RequestHead bad = Get("/hello.txt"); + bad.headers.set("range", "bytes=99-"); + FileOutcome b = fs.Handle(bad); + ASSERT_EQ(b.status, 416); + ASSERT_EQ(b.head.headers.get("content-range"), "bytes */42"); + RequestHead multi = Get("/hello.txt"); + multi.headers.set("range", "bytes=0-1,3-4"); + ASSERT_EQ(fs.Handle(multi).status, 416); + // If-Range mismatch serves the whole file. + RequestHead ir = Get("/hello.txt"); + ir.headers.set("range", "bytes=0-4"); + ir.headers.set("if-range", "\"mismatch\""); + FileOutcome w = fs.Handle(ir); + ASSERT_EQ(w.status, 200); + ASSERT_EQ(BodyOf(w).size(), 42u); +} + +TEST(FileServerEmbeddedIdentical) { + DiskProvider disk("tests/src/runtime/fixtures/www"); + FileServer fs_disk(disk); + EmbeddedProvider emb; + FileOutcome from_disk = fs_disk.Handle(Get("/hello.txt")); + std::string bytes = BodyOf(from_disk); + emb.Register("hello.txt", std::vector(bytes.begin(), bytes.end()), + 1700000000); + FileServer fs_emb(emb); + FileOutcome o = fs_emb.Handle(Get("/hello.txt")); + ASSERT_EQ(o.status, 200); + ASSERT_EQ(BodyOf(o), bytes); // byte-identical bodies + ASSERT_EQ(fs_emb.Handle(Get("/missing")).status, 404); +} + +TEST(FileServerPrecompressed) { + EmbeddedProvider emb; + std::string body(3000, 'z'); + emb.Register("f.txt", std::vector(body.begin(), body.end()), + 1700000000); + std::string gz = GzipBytes(body); + emb.Register("f.txt.gz", std::vector(gz.begin(), gz.end()), + 1700000000); + std::string br = BrotliBytes(body); + emb.Register("f.txt.br", std::vector(br.begin(), br.end()), + 1700000000); + FileServer fs(emb); + RequestHead plain = Get("/f.txt"); + FileOutcome o = fs.Handle(plain); + ASSERT_EQ(o.status, 200); + ASSERT_EQ(o.head.headers.get("content-encoding"), ""); + RequestHead want_br = Get("/f.txt"); + want_br.headers.set("accept-encoding", "gzip, br"); + FileOutcome b = fs.Handle(want_br); + ASSERT_EQ(b.status, 200); + ASSERT_EQ(b.head.headers.get("content-encoding"), "br"); + ASSERT_EQ(BodyOf(b), br); + ASSERT_EQ(b.head.headers.get("vary"), "Accept-Encoding"); + ASSERT_TRUE(b.head.headers.get("etag").find("-br\"") != std::string::npos); + RequestHead want_gz = Get("/f.txt"); + want_gz.headers.set("accept-encoding", "gzip"); + FileOutcome g = fs.Handle(want_gz); + ASSERT_EQ(g.head.headers.get("content-encoding"), "gzip"); + ASSERT_EQ(BodyOf(g), gz); +} + +TEST(FileServerSymlinkEscape) { + // Build scratch only: link inside root pointing outside must fail closed. + mkdir("build/symlink-test", 0755); + mkdir("build/symlink-test/root", 0755); + int fd = ::open("build/symlink-test/outside.txt", O_WRONLY | O_CREAT | O_TRUNC, + 0644); + ASSERT_TRUE(fd >= 0); + const char* secret = "outside secret"; + ASSERT_TRUE(write(fd, secret, 14) == 14); + close(fd); + unlink("build/symlink-test/root/evil"); + ASSERT_EQ(symlink("../outside.txt", "build/symlink-test/root/evil"), 0); + DiskProvider disk("build/symlink-test/root"); + ASSERT_TRUE(disk.valid()); + FileServer fs(disk); + ASSERT_EQ(fs.Handle(Get("/evil")).status, 404); + // Following is opt-in and still contained. + DiskProvider strict("build/symlink-test/root", true); + FileServer fs2(strict); + ASSERT_EQ(fs2.Handle(Get("/evil")).status, 404); // escapes root: denied +} + +TEST(EtagDateRoundTrip) { + std::string d = Etag::HttpDate(1700000000); + ASSERT_EQ(d, "Tue, 14 Nov 2023 22:13:20 GMT"); + ASSERT_EQ(Etag::ParseHttpDate(d), 1700000000); + ASSERT_EQ(Etag::ParseHttpDate("garbage"), -1); + ASSERT_TRUE(Etag::Matches("\"a\", \"b\"", "\"b\"")); + ASSERT_TRUE(Etag::Matches("W/\"b\"", "\"b\"")); + ASSERT_FALSE(Etag::Matches("\"a\"", "\"b\"")); + ASSERT_TRUE(Etag::RangeMatches("\"b\"", "\"b\"")); + ASSERT_FALSE(Etag::RangeMatches("W/\"b\"", "\"b\"")); +} diff --git a/tests/src/runtime/static/test_path_guard.cpp b/tests/src/runtime/static/test_path_guard.cpp new file mode 100644 index 0000000..fc46dd8 --- /dev/null +++ b/tests/src/runtime/static/test_path_guard.cpp @@ -0,0 +1,39 @@ +#include "test_framework.h" + +#include "runtime/static/path_guard.h" + +using namespace molo::runtime::static_; + +TEST(GuardNormalizes) { + ASSERT_EQ(NormalizeUrlPath("/a/b/c").value(), "a/b/c"); + ASSERT_EQ(NormalizeUrlPath("/a/./b//c").value(), "a/b/c"); + ASSERT_EQ(NormalizeUrlPath("/a/b/../c").value(), "a/c"); + ASSERT_EQ(NormalizeUrlPath("/a/b/../../c").value(), "c"); + ASSERT_EQ(NormalizeUrlPath("/hello.txt?x=1#frag").value(), "hello.txt"); + ASSERT_EQ(NormalizeUrlPath("/%68%65%6c%6c%6f").value(), "hello"); + ASSERT_EQ(NormalizeUrlPath("/").value(), ""); +} + +TEST(GuardRejectsEscape) { + ASSERT_FALSE(NormalizeUrlPath("/..").has_value()); + ASSERT_FALSE(NormalizeUrlPath("/a/../../..").has_value()); + ASSERT_FALSE(NormalizeUrlPath("/%2e%2e/%2e%2e/x").has_value()); + ASSERT_FALSE(NormalizeUrlPath("/%2E%2E/x").has_value()); + ASSERT_FALSE(NormalizeUrlPath("relative/path").has_value()); + ASSERT_FALSE(NormalizeUrlPath("").has_value()); +} + +TEST(GuardRejectsNasty) { + ASSERT_FALSE(NormalizeUrlPath("/%00").has_value()); // NUL + ASSERT_FALSE(NormalizeUrlPath("/a%00b").has_value()); + ASSERT_FALSE(NormalizeUrlPath("/a\\b").has_value()); // backslash + ASSERT_FALSE(NormalizeUrlPath("/%5cb").has_value()); // encoded backslash + ASSERT_FALSE(NormalizeUrlPath("/%zz").has_value()); // bad hex + ASSERT_FALSE(NormalizeUrlPath("/abc%").has_value()); + // Double encoding stays literal (no second decode pass). + auto v = NormalizeUrlPath("/%252e%252e/x"); + ASSERT_TRUE(v.has_value()); + ASSERT_EQ(*v, "%2e%2e/x"); + // Overlong path. + ASSERT_FALSE(NormalizeUrlPath("/" + std::string(5000, 'a')).has_value()); +} diff --git a/tests/src/runtime/test_base64.cpp b/tests/src/runtime/test_base64.cpp new file mode 100644 index 0000000..e936f1a --- /dev/null +++ b/tests/src/runtime/test_base64.cpp @@ -0,0 +1,38 @@ +#include "test_framework.h" + +#include "runtime/base64.h" + +using namespace molo::runtime; + +TEST(Base64UrlVectors) { + ASSERT_EQ(Base64Url::Encode(""), ""); + ASSERT_EQ(Base64Url::Encode("f"), "Zg"); + ASSERT_EQ(Base64Url::Encode("fo"), "Zm8"); + ASSERT_EQ(Base64Url::Encode("foo"), "Zm9v"); + ASSERT_EQ(Base64Url::Encode("foob"), "Zm9vYg"); + ASSERT_EQ(Base64Url::Encode("fooba"), "Zm9vYmE"); + ASSERT_EQ(Base64Url::Encode("foobar"), "Zm9vYmFy"); + // URL-safe alphabet: bytes that give +/ in std base64. + std::string raw("\xfb\xff\xfe", 3); + ASSERT_EQ(Base64Url::Encode(raw), "-__-"); +} + +TEST(Base64UrlRoundTrip) { + for (int len = 0; len < 64; ++len) { + std::string in; + for (int i = 0; i < len; ++i) in.push_back((char)(i * 37 + 11)); + std::string enc = Base64Url::Encode(in); + auto dec = Base64Url::Decode(enc); + ASSERT_TRUE(dec.has_value()); + ASSERT_EQ(*dec, in); + } +} + +TEST(Base64UrlRejects) { + ASSERT_FALSE(Base64Url::Decode("a").has_value()); // bad length + ASSERT_FALSE(Base64Url::Decode("****").has_value()); + // Padding tolerated. + auto d = Base64Url::Decode("Zg=="); + ASSERT_TRUE(d.has_value()); + ASSERT_EQ(*d, "f"); +} diff --git a/tests/src/runtime/test_channel_strand.cpp b/tests/src/runtime/test_channel_strand.cpp new file mode 100644 index 0000000..28198b2 --- /dev/null +++ b/tests/src/runtime/test_channel_strand.cpp @@ -0,0 +1,70 @@ +#include "test_framework.h" + +#include +#include +#include +#include + +#include "runtime/channel.h" +#include "runtime/event_loop.h" +#include "runtime/strand.h" + +using namespace molo::runtime; +using namespace std::chrono_literals; + +TEST(StrandSerializes) { + EventLoop loop; + Strand s(loop); + std::vector order; + for (int i = 0; i < 5; ++i) { + s.post([&, i] { order.push_back(i); }); + } + loop.run_once(50); + loop.run_once(50); + ASSERT_EQ(order.size(), 5u); + for (int i = 0; i < 5; ++i) ASSERT_EQ(order[(std::size_t)i], i); + loop.stop(); +} + +TEST(ChannelPushPop) { + Channel ch(4); + ASSERT_TRUE(ch.valid()); + ASSERT_TRUE(ch.try_push(1)); + ASSERT_TRUE(ch.try_push(2)); + int v = 0; + ASSERT_TRUE(ch.try_pop(v)); + ASSERT_EQ(v, 1); + ASSERT_TRUE(ch.try_pop(v)); + ASSERT_EQ(v, 2); + ASSERT_FALSE(ch.try_pop(v)); +} + +TEST(ChannelCapAndClose) { + Channel ch(2); + ASSERT_TRUE(ch.try_push(1)); + ASSERT_TRUE(ch.try_push(2)); + ASSERT_FALSE(ch.try_push(3)); + ch.close(); + ASSERT_TRUE(ch.closed()); + ASSERT_FALSE(ch.try_push(4)); +} + +TEST(ChannelWakesLoop) { + EventLoop loop; + Channel ch(16); + std::vector got; + ASSERT_TRUE(ch.attach(loop, [&] { + std::string s; + while (ch.try_pop(s)) got.push_back(s); + loop.stop(); + })); + std::thread t([&] { + std::this_thread::sleep_for(20ms); + ch.try_push("hello"); + }); + loop.run(); + t.join(); + ASSERT_EQ(got.size(), 1u); + ASSERT_EQ(got[0], "hello"); + ch.detach(); +} diff --git a/tests/src/runtime/test_event_loop.cpp b/tests/src/runtime/test_event_loop.cpp new file mode 100644 index 0000000..ba5110c --- /dev/null +++ b/tests/src/runtime/test_event_loop.cpp @@ -0,0 +1,126 @@ +#include "test_framework.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "runtime/event_loop.h" +#include "runtime/signal_handler.h" + +using namespace molo::runtime; +using namespace std::chrono_literals; + +TEST(LoopDeferRuns) { + EventLoop loop; + bool ran = false; + loop.defer([&] { ran = true; }); + ASSERT_TRUE(loop.run_once(10)); + ASSERT_TRUE(ran); + loop.stop(); + ASSERT_FALSE(loop.run_once(10)); +} + +TEST(LoopSleepAndYield) { + EventLoop loop; + std::vector order; + // Named closure: a coroutine frame references (not copies) the lambda + // object, so an immediately-invoked temporary would dangle. + auto body = [&](EventLoop& l) -> Task { + order.push_back(1); + co_await l.yield(); + order.push_back(2); + co_await l.sleep(30ms); + order.push_back(3); + l.stop(); + co_return; + }; + Task t = body(loop); + loop.post(std::move(t)); + loop.run(); + ASSERT_EQ(order.size(), 3u); + ASSERT_EQ(order[0], 1); + ASSERT_EQ(order[1], 2); + ASSERT_EQ(order[2], 3); +} + +TEST(LoopTimers) { + EventLoop loop; + int n = 0; + loop.schedule_after(20ms, [&] { ++n; }); + auto id = loop.schedule_after(20ms, [&] { n += 100; }); + ASSERT_TRUE(loop.cancel_timer(id)); + auto start = std::chrono::steady_clock::now(); + while (n == 0) { + if (!loop.run_once(50)) break; + if (std::chrono::steady_clock::now() - start > 2s) break; + } + ASSERT_EQ(n, 1); + loop.stop(); +} + +TEST(LoopPollerEcho) { + int sv[2]; + ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0, sv), 0); + EventLoop loop; + std::string got; + bool ok = loop.poller().add_fd(sv[0], IPoller::kRead, + [&](int fd, uint32_t ev) { + (void)ev; + char buf[64]; + ssize_t n = read(fd, buf, sizeof(buf)); + if (n > 0) got.append(buf, (std::size_t)n); + }); + ASSERT_TRUE(ok); + const char* msg = "ping"; + ASSERT_EQ(write(sv[1], msg, 4), 4); + auto start = std::chrono::steady_clock::now(); + while (got.empty()) { + loop.run_once(50); + if (std::chrono::steady_clock::now() - start > 2s) break; + } + ASSERT_EQ(got, "ping"); + loop.poller().remove_fd(sv[0]); + close(sv[0]); + close(sv[1]); + loop.stop(); +} + +TEST(LoopStopFromOtherThread) { + EventLoop loop; + std::thread t([&] { + std::this_thread::sleep_for(30ms); + loop.stop(); + }); + loop.run(); + t.join(); + ASSERT_TRUE(loop.stopped()); +} + +TEST(LoopSignals) { + EventLoop loop; + SignalHandler sig(loop); + int got = 0; + ASSERT_TRUE(sig.install([&](int signo) { got = signo; })); + raise(SIGTERM); + auto start = std::chrono::steady_clock::now(); + while (got == 0) { + loop.run_once(50); + if (std::chrono::steady_clock::now() - start > 2s) break; + } + ASSERT_EQ(got, SIGTERM); + sig.uninstall(); + loop.stop(); +} + +TEST(LoopOffLoopPool) { + EventLoop loop; + bool ran = false; + ASSERT_TRUE(loop.pool().try_submit([&] { ran = true; })); + loop.pool().shutdown(); + ASSERT_TRUE(ran); +} diff --git a/tests/src/runtime/test_executor.cpp b/tests/src/runtime/test_executor.cpp new file mode 100644 index 0000000..20c1ce3 --- /dev/null +++ b/tests/src/runtime/test_executor.cpp @@ -0,0 +1,47 @@ +#include "test_framework.h" + +#include +#include +#include + +#include "runtime/executor.h" + +using namespace molo::runtime; + +TEST(PoolRunsJobs) { + ThreadPool pool(2, 16); + std::atomic n{0}; + for (int i = 0; i < 10; ++i) { + ASSERT_TRUE(pool.try_submit([&] { n.fetch_add(1); })); + } + pool.shutdown(); + ASSERT_EQ(n.load(), 10); +} + +TEST(PoolOverflowReturnsFalse) { + ThreadPool pool(1, 2); + // Block the single worker, fill the queue, then overflow. + std::atomic release{false}; + struct Guard { + std::atomic& r; + ~Guard() { r.store(true); } + } guard{release}; + std::atomic entered{false}; + ASSERT_TRUE(pool.try_submit([&] { + entered.store(true); + while (!release.load()) std::this_thread::yield(); + })); + while (!entered.load()) std::this_thread::yield(); + ASSERT_TRUE(pool.try_submit([] {})); + ASSERT_TRUE(pool.try_submit([] {})); + ASSERT_FALSE(pool.try_submit([] {})); + ASSERT_TRUE(pool.pending() >= 2u); + // No explicit shutdown: the guard releases the worker first, then the + // pool destructor joins. (Explicit shutdown here would deadlock.) +} + +TEST(PoolClampsThreads) { + ThreadPool pool(1000000, 4); + ASSERT_TRUE(pool.threads() <= 64u); + pool.shutdown(); +} diff --git a/tests/src/runtime/test_framework.h b/tests/src/runtime/test_framework.h new file mode 100644 index 0000000..ba9ad49 --- /dev/null +++ b/tests/src/runtime/test_framework.h @@ -0,0 +1,114 @@ +// Minimal test framework for the runtime part. No third-party runners: +// TEST(name) registers a case, ASSERT_* throws on failure, main runs all. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace molo::test { + +struct Failure : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +struct Case { + std::string name; + std::function fn; +}; + +inline std::vector& Registry() { + static std::vector r; + return r; +} + +inline void Add(std::string name, std::function fn) { + Registry().push_back({std::move(name), std::move(fn)}); +} + +inline int RunAll() { + std::cout.setf(std::ios::unitbuf); // unbuffered: hangs show progress + int pass = 0, fail = 0; + for (auto& c : Registry()) { + try { + c.fn(); + ++pass; + std::cout << "PASS " << c.name << "\n"; + } catch (const Failure& e) { + ++fail; + std::cout << "FAIL " << c.name << ": " << e.what() << "\n"; + } catch (const std::exception& e) { + ++fail; + std::cout << "FAIL " << c.name << ": std::exception: " << e.what() + << "\n"; + } catch (...) { + ++fail; + std::cout << "FAIL " << c.name << ": unknown exception\n"; + } + } + std::cout << pass << " passed, " << fail << " failed\n"; + return fail ? 1 : 0; +} + +template +void PrintInto(std::ostringstream& o, const T& v) { + if constexpr (std::is_arithmetic_v) { + o << v; + } else if constexpr (std::is_same_v || + std::is_same_v) { + o << "\"" << v << "\""; + } else { + o << "[unprintable]"; + } +} + +} // namespace molo::test + +#define TEST(name) \ + static void name##_body(); \ + static bool name##_reg = \ + ([] { ::molo::test::Add(#name, name##_body); return true; })(); \ + static void name##_body() + +#define ASSERT_TRUE(x) \ + do { \ + if (!(x)) { \ + std::ostringstream _o; \ + _o << __FILE__ << ":" << __LINE__ << ": ASSERT_TRUE(" #x ") failed"; \ + throw ::molo::test::Failure(_o.str()); \ + } \ + } while (0) + +#define ASSERT_FALSE(x) ASSERT_TRUE(!(x)) + +#define ASSERT_EQ(a, b) \ + do { \ + auto _a = (a); \ + auto _b = (b); \ + if (!(_a == _b)) { \ + std::ostringstream _o; \ + _o << __FILE__ << ":" << __LINE__ << ": ASSERT_EQ(" #a ", " #b \ + << ") failed: "; \ + ::molo::test::PrintInto(_o, _a); \ + _o << " != "; \ + ::molo::test::PrintInto(_o, _b); \ + throw ::molo::test::Failure(_o.str()); \ + } \ + } while (0) + +#define ASSERT_NE(a, b) \ + do { \ + auto _a = (a); \ + auto _b = (b); \ + if (!(_a != _b)) { \ + std::ostringstream _o; \ + _o << __FILE__ << ":" << __LINE__ << ": ASSERT_NE(" #a ", " #b \ + << ") failed"; \ + throw ::molo::test::Failure(_o.str()); \ + } \ + } while (0) diff --git a/tests/src/runtime/test_task.cpp b/tests/src/runtime/test_task.cpp new file mode 100644 index 0000000..100d4e6 --- /dev/null +++ b/tests/src/runtime/test_task.cpp @@ -0,0 +1,92 @@ +#include "test_framework.h" + +#include + +#include "runtime/task.h" + +using namespace molo::runtime; + +namespace { +Task AddOne(int x) { co_return x + 1; } + +Task Chain(int x) { + int a = co_await AddOne(x); + int b = co_await AddOne(a); + co_return b; +} + +Task Failer() { + throw std::runtime_error("boom"); + co_return; +} + +Task Catcher() { + try { + co_await Failer(); + } catch (const std::runtime_error&) { + co_return 7; + } + co_return -1; +} + +Generator Range(int n) { + for (int i = 0; i < n; ++i) co_yield i; +} +} // namespace + +TEST(TaskLazyStart) { + bool ran = false; + auto body = [&]() -> Task { + ran = true; + co_return 1; + }; + Task t = body(); + ASSERT_FALSE(ran); + t.start(); + ASSERT_TRUE(ran); + ASSERT_TRUE(t.done()); +} + +TEST(TaskChain) { + Task t = Chain(41); + t.start(); + ASSERT_TRUE(t.done()); + int v = std::move(t).await_resume(); + ASSERT_EQ(v, 43); +} + +TEST(TaskException) { + Task t = Catcher(); + t.start(); + ASSERT_EQ(std::move(t).await_resume(), 7); +} + +TEST(TaskDetachRuns) { + bool ran = false; + auto body = [&]() -> Task { + ran = true; + co_return; + }; + Detach(body()); + ASSERT_TRUE(ran); +} + +TEST(GeneratorRange) { + int sum = 0; + for (int v : Range(5)) sum += v; + ASSERT_EQ(sum, 10); +} + +TEST(CancelToken) { + CancellationToken tok; + ASSERT_FALSE(tok.is_cancelled()); + tok.cancel(); + ASSERT_TRUE(tok.is_cancelled()); + bool threw = false; + try { + tok.throw_if_cancelled(); + } catch (const Cancelled&) { + threw = true; + } + ASSERT_TRUE(threw); +} diff --git a/tests/src/runtime/test_timer.cpp b/tests/src/runtime/test_timer.cpp new file mode 100644 index 0000000..bae2f82 --- /dev/null +++ b/tests/src/runtime/test_timer.cpp @@ -0,0 +1,57 @@ +#include "test_framework.h" + +#include +#include +#include + +#include "runtime/timer_wheel.h" + +using namespace molo::runtime; +using namespace std::chrono_literals; + +TEST(TimerFiresInOrder) { + TimerWheel w(10, 64); + std::vector fired; + w.add(50ms, [&] { fired.push_back(2); }); + w.add(20ms, [&] { fired.push_back(1); }); + ASSERT_EQ(w.size(), 2u); + std::this_thread::sleep_for(120ms); + w.run_due(); + ASSERT_EQ(fired.size(), 2u); + ASSERT_EQ(fired[0], 1); + ASSERT_EQ(fired[1], 2); + ASSERT_EQ(w.size(), 0u); +} + +TEST(TimerCancel) { + TimerWheel w(10, 64); + bool fired = false; + auto id = w.add(10ms, [&] { fired = true; }); + ASSERT_TRUE(id != 0); + ASSERT_TRUE(w.cancel(id)); + ASSERT_FALSE(w.cancel(id)); + std::this_thread::sleep_for(40ms); + w.run_due(); + ASSERT_FALSE(fired); + ASSERT_EQ(w.size(), 0u); +} + +TEST(TimerNextExpiry) { + TimerWheel w(10, 64); + ASSERT_EQ(w.next_expiry_ms(), -1); + w.add(100ms, [] {}); + int ms = w.next_expiry_ms(); + ASSERT_TRUE(ms >= 0 && ms <= 100); +} + +TEST(TimerFarFuture) { + TimerWheel w(10, 16); // 160 ms span; overflow path beyond it + bool fired = false; + w.add(500ms, [&] { fired = true; }); + std::this_thread::sleep_for(100ms); + w.run_due(); + ASSERT_FALSE(fired); + std::this_thread::sleep_for(500ms); + w.run_due(); + ASSERT_TRUE(fired); +} diff --git a/tests/src/security/auth/auth_test.cpp b/tests/src/security/auth/auth_test.cpp new file mode 100644 index 0000000..a8bf90a --- /dev/null +++ b/tests/src/security/auth/auth_test.cpp @@ -0,0 +1,158 @@ +// Auth test: Basic parsing, login paths, re-auth gate, route guards. +#include +#include +#include +#include + +#include "Authenticator.h" +#include "DenialSink.h" +#include "Guard.h" +#include "HttpMessageView.h" +#include "PasswordHasher.h" + +using namespace molo::security::auth; +using molo::security::guard::HttpRequestView; +using molo::security::guard::NullSink; +using molo::security::session::Session; + +static int failures = 0; +static void check(bool ok, const char* name) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", name); + if (!ok) ++failures; +} + +struct TestUsers : IUserStore { + Account acct; + bool has = false; + bool find(const std::string& username, Account& out) override { + if (!has || username != "alice") return false; + out = acct; + return true; + } + bool store_rehash(const std::string& username, const std::string& encoded) override { + if (username != "alice") return false; + acct.argon2id_hash = encoded; + return true; + } +}; + +int main() { + NullSink sink; + + // Basic parsing. + { + HttpRequestView r; + r.headers.emplace_back("authorization", "Basic YWxpY2U6czNjcml0"); // alice:s3crit + auto c = Authenticator::parse_basic(r); + check(c.present && c.username == "alice" && c.password == "s3crit", "basic parse"); + HttpRequestView r2; + check(!Authenticator::parse_basic(r2).present, "basic missing"); + HttpRequestView r3; + r3.headers.emplace_back("authorization", "Basic !!!"); + check(!Authenticator::parse_basic(r3).present, "basic malformed"); + HttpRequestView r4; + r4.headers.emplace_back("authorization", "Basic bm9jb2xvbg=="); // "nocolon" + check(!Authenticator::parse_basic(r4).present, "basic no colon"); + } + + // Login ok / bad / unknown at small test params. + auto users = std::make_shared(); + { + molo::security::crypto::PasswordHasher::Params p; + p.memory_kib = 8; + p.passes = 1; + p.lanes = 1; + std::string enc; + check(molo::security::crypto::PasswordHasher::hash_password("s3crit", p, enc), "mk hash"); + users->acct.user_id = "u-1"; + users->acct.argon2id_hash = enc; + users->acct.roles = {{"reader", {{{"read"}}}}}; + users->has = true; + } + { + Authenticator auth(users, sink); + std::uint64_t now = 5000; + auth.set_clock([&] { return now; }); + HttpRequestView r; + r.now_unix_sec = now; + r.path = "/login"; + BasicCredentials good{"alice", "s3crit", true}; + BasicCredentials bad{"alice", "wrong", true}; + BasicCredentials unknown{"mallory", "x", true}; + Session s; + check(auth.login(good, "10.0.0.1", s, r) == Authenticator::LoginResult::Ok && + s.user_id == "u-1", + "login ok"); + // Rehash to OWASP params happened on login; restore small hash for speed. + { + molo::security::crypto::PasswordHasher::Params p; + p.memory_kib = 8; + p.passes = 1; + p.lanes = 1; + std::string enc; + molo::security::crypto::PasswordHasher::hash_password("s3crit", p, enc); + users->acct.argon2id_hash = enc; + } + Session s2; + check(auth.login(bad, "10.0.0.1", s2, r) == Authenticator::LoginResult::BadCredentials, + "login bad password"); + Session s3; + check(auth.login(unknown, "10.0.0.2", s3, r) == + Authenticator::LoginResult::BadCredentials, + "login unknown user"); + // Rate limit: 10/min budget, 11th attempt trips. + Authenticator::LoginResult last = Authenticator::LoginResult::Ok; + for (int i = 0; i < 12; ++i) { + Session tmp; + BasicCredentials fresh{"mallory", "nope", true}; + last = auth.login(fresh, "10.7.7.7", tmp, r); + } + check(last == Authenticator::LoginResult::RateLimited, "login rate limit trips"); + } + + // Locked account. + { + auto locked = std::make_shared(); + locked->has = true; + locked->acct.user_id = "u-2"; + locked->acct.argon2id_hash = ""; + Authenticator auth(locked, sink); + HttpRequestView r; + Session s; + BasicCredentials c{"alice", "anything", true}; + check(auth.login(c, "10.0.0.9", s, r) == Authenticator::LoginResult::Locked, "locked"); + } + + // Re-auth gate. + { + Session s; + s.privilege = Session::Privilege::User; + s.last_reauth_at = 1000; + check(Authenticator::reauth_fresh(s, 1000 + 9 * 60), "reauth fresh"); + check(!Authenticator::reauth_fresh(s, 1000 + 11 * 60), "reauth stale"); + Authenticator::mark_reauth(s, 2000); + check(Authenticator::reauth_fresh(s, 2000), "reauth marked"); + Session anon; + check(!Authenticator::reauth_fresh(anon, 2000), "reauth anon denied"); + } + + // Route guards: deny by default. + { + Session anon; + HttpRequestView r; + auto v = Guard::check_logged(anon, {}, {{{"read"}}}, r, sink); + check(!v.allowed && v.http_status == 401, "guard anon 401"); + Session user; + user.privilege = Session::Privilege::User; + std::vector roles = {{"reader", {{{"read"}}}}}; + auto v2 = Guard::check(user, roles, {{{"read"}}}); + check(v2.allowed, "guard granted"); + auto v3 = Guard::check(user, roles, {{{"admin"}}}); + check(!v3.allowed && v3.http_status == 403, "guard forbidden 403"); + auto v4 = Guard::check(user, {}, {}); + check(v4.allowed, "guard no requirements passes authed"); + } + + std::printf(failures == 0 ? "ALL AUTH TESTS PASSED\n" : "FAILURES: %d\n", failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/src/security/crypto/crypto_test.cpp b/tests/src/security/crypto/crypto_test.cpp new file mode 100644 index 0000000..51a7335 --- /dev/null +++ b/tests/src/security/crypto/crypto_test.cpp @@ -0,0 +1,308 @@ +// Security crypto test: primitives, key hierarchy, password hashing. +// Standalone: links libsecurity only. Exit 0 on full pass, 1 otherwise. +#include +#include +#include +#include + +#include "AesGcm.h" +#include "Base64.h" +#include "Blake2b.h" +#include "ChaCha20Poly1305.h" +#include "CryptoBox.h" +#include "Hmac.h" +#include "KeyHierarchy.h" +#include "PasswordHasher.h" +#include "SecureRandom.h" +#include "Sha256.h" + +using namespace molo::security::crypto; + +static int failures = 0; + +static void check(bool ok, const char* name) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", name); + if (!ok) ++failures; +} + +static std::string hex(const std::uint8_t* d, std::size_t n) { + static const char* k = "0123456789abcdef"; + std::string o; + for (std::size_t i = 0; i < n; ++i) { + o.push_back(k[d[i] >> 4]); + o.push_back(k[d[i] & 15]); + } + return o; +} + +static std::string hex(const std::vector& v) { + return hex(v.data(), v.size()); +} + +int main() { + // Base64 RFC 4648 vectors. + check(Base64::encode(std::string()) == "", "b64 empty"); + check(Base64::encode("f") == "Zg==", "b64 f"); + check(Base64::encode("foobar") == "Zm9vYmFy", "b64 foobar"); + { + std::string out; + check(Base64::decode("Zm9vYmFy", out) && out == "foobar", "b64 decode"); + check(!Base64::decode("Zm9vYmF!", out), "b64 reject bad char"); + check(!Base64::decode("abc", out), "b64 reject bad length"); + check(Base64::encode_url("foobar") == "Zm9vYmFy", "b64url foobar"); + check(Base64::decode_url("Zm9vYmFy", out) && out == "foobar", "b64url decode"); + check(!Base64::decode_url("a", out), "b64url reject len 1 mod 4"); + } + + // SHA-256 FIPS vectors. + check(Sha256::hex(Sha256::hash("")) == + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "sha256 empty"); + check(Sha256::hex(Sha256::hash("abc")) == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "sha256 abc"); + { + Sha256 s; + std::string a(1000, 'a'); + for (int i = 0; i < 1000; ++i) s.update(a); + check(Sha256::hex(s.final()) == + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0", + "sha256 1M a"); + } + + // HMAC-SHA-256 RFC 4231 cases 1..3. + { + std::uint8_t k1[20]; + for (int i = 0; i < 20; ++i) k1[i] = 0x0b; + auto t1 = Hmac::sha256(k1, 20, (const std::uint8_t*)"Hi There", 8); + check(Hmac::hex(t1) == "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", + "hmac case1"); + auto t2 = Hmac::sha256("Jefe", "what do ya want for nothing?"); + check(Hmac::hex(t2) == "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", + "hmac case2"); + std::uint8_t k3[20], d3[50]; + for (int i = 0; i < 20; ++i) k3[i] = 0xaa; + for (int i = 0; i < 50; ++i) d3[i] = 0xdd; + auto t3 = Hmac::sha256(k3, 20, d3, 50); + check(Hmac::hex(t3) == "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", + "hmac case3"); + } + + // HKDF RFC 5869 case 1 (cross-checked with openssl). + { + std::uint8_t ikm[22], salt[13], info[10]; + for (int i = 0; i < 22; ++i) ikm[i] = 0x0b; + for (int i = 0; i < 13; ++i) salt[i] = (std::uint8_t)i; + for (int i = 0; i < 10; ++i) info[i] = (std::uint8_t)(0xf0 + i); + std::uint8_t okm[42]; + check(Hkdf::derive(ikm, 22, salt, 13, info, 10, okm, 42) && + hex(okm, 42) == + "3cb25f25faacd57a90434f64d0362f2a2d2d0a90cf1a5a4c5db02d56ecc4c5bf34007208d5b8" + "87185865", + "hkdf case1"); + } + + // BLAKE2b (cross-checked with python hashlib). + { + auto h = Blake2b::hash(nullptr, 0, 64); + check(hex(h) == + "786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419d25e1031afee58" + "5313896444934eb04b903a685b1448b755d56f701afe9be2ce", + "blake2b empty"); + auto h2 = Blake2b::hash((const std::uint8_t*)"abc", 3, 64); + check(hex(h2) == + "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d17d87c5392aab79" + "2dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923", + "blake2b abc"); + } + + // AES-256-GCM (cross-checked with python cryptography). + { + std::uint8_t key[32] = {0}, nonce[12] = {0}, tag[16], dummy = 0; + check(AesGcm::seal(key, nonce, nullptr, 0, nullptr, 0, &dummy, tag) && + hex(tag, 16) == "530f8afbc74536b9a963b4f1c4cb738b", + "aesgcm zero tag"); + std::uint8_t pt[16] = {0}, ct[16], back[16]; + check(AesGcm::seal(key, nonce, nullptr, 0, pt, 16, ct, tag) && + hex(ct, 16) == "cea7403d4d606b6e074ec5d3baf39d18" && + hex(tag, 16) == "d0d1c8a799996bf0265b98b5d48ab919", + "aesgcm zero block"); + check(AesGcm::open(key, nonce, nullptr, 0, ct, 16, tag, back), "aesgcm roundtrip"); + tag[0] ^= 1; + std::uint8_t untouched[16]; + for (int i = 0; i < 16; ++i) untouched[i] = 0xAA; + check(!AesGcm::open(key, nonce, nullptr, 0, ct, 16, tag, untouched) && + untouched[0] == 0xAA, + "aesgcm tamper fails closed"); + } + + // ChaCha20-Poly1305 RFC 8439 A.5 (cross-checked with python cryptography). + { + std::uint8_t key[32], nonce[12] = {0, 0, 0, 9, 0, 0, 0, 74, 0, 0, 0, 0}; + for (int i = 0; i < 32; ++i) key[i] = (std::uint8_t)i; + std::uint8_t aad[12] = {0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, + 0xc3, 0xc4, 0xc5, 0xc6, 0xc7}; + std::string pt = + "Ladies and Gentlemen of the class of '99: If I could offer you only one tip for the " + "future, sunscreen would be it."; + std::vector ct(pt.size()), back(pt.size()); + std::uint8_t tag[16]; + check(ChaCha20Poly1305::seal(key, nonce, aad, 12, (const std::uint8_t*)pt.data(), pt.size(), + ct.data(), tag) && + hex(tag, 16) == "a0dcd40f06b40fe14d7d51791c187b9b" && + hex(ct).substr(0, 32) == "5c90838db44879743e6bfd58c64e05a8", + "chacha A.5 vector"); + check(ChaCha20Poly1305::open(key, nonce, aad, 12, ct.data(), ct.size(), tag, back.data()) && + std::string((char*)back.data(), back.size()) == pt, + "chacha roundtrip"); + ct[0] ^= 1; + check(!ChaCha20Poly1305::open(key, nonce, aad, 12, ct.data(), ct.size(), tag, back.data()), + "chacha tamper fails closed"); + } + + // SecureRandom basics. + { + auto a = SecureRandom::bytes(32); + auto b = SecureRandom::bytes(32); + check(a.size() == 32 && b.size() == 32 && a != b, "csprng unique"); + check(SecureRandom::constant_time_equal("abc", "abc") && + !SecureRandom::constant_time_equal("abc", "abd") && + !SecureRandom::constant_time_equal("abc", "abcd"), + "ct equal"); + check(SecureRandom::token(16).size() == 22, "token length"); + check(SecureRandom::bytes(0).empty(), "zero bytes ok"); + check(!SecureRandom::bytes(nullptr, 4), "null fails closed"); + } + + // KeyHierarchy: seed, derive, rotate, overlap, expiry. + { + KeyHierarchy kh; + std::uint8_t out[32]; + check(!kh.subkey(1, KeyHierarchy::Purpose::Data, out), "no key before seed"); + check(kh.seed_random(), "seed random"); + check(kh.current_epoch() == 1, "epoch 1"); + std::uint8_t d1[32], c1[32]; + check(kh.subkey(1, KeyHierarchy::Purpose::Data, d1), "derive data e1"); + check(kh.subkey(1, KeyHierarchy::Purpose::Cookies, c1), "derive cookies e1"); + check(std::string((char*)d1, 32) != std::string((char*)c1, 32), "purpose separation"); + check(kh.rotate(1000), "rotate"); + check(kh.current_epoch() == 2, "epoch 2"); + check(kh.subkey(1, KeyHierarchy::Purpose::Data, out), "old epoch readable in window"); + check(kh.rotate(1000 + 25 * 3600), "rotate after window"); + check(!kh.subkey(1, KeyHierarchy::Purpose::Data, out), "old epoch expired"); + KeyHierarchy::KeyId id{7, KeyHierarchy::Purpose::Csrf}, back; + check(KeyHierarchy::KeyId::decode(id.encode(), back) && back.epoch == 7 && + back.purpose == KeyHierarchy::Purpose::Csrf, + "keyid codec"); + check(!KeyHierarchy::KeyId::decode("bogus", back), "keyid reject"); + } + + // CryptoBox envelopes, both algorithms. + for (auto alg : {CryptoBox::Alg::AesGcm, CryptoBox::Alg::ChaCha20Poly1305}) { + KeyHierarchy kh; + check(kh.seed_random(), "box seed"); + CryptoBox box(kh, alg); + std::string env, plain; + const char* tag = alg == CryptoBox::Alg::AesGcm ? "aes" : "chacha"; + char name[64]; + std::snprintf(name, sizeof name, "box %s roundtrip", tag); + check(box.seal("hello envelope", KeyHierarchy::Purpose::Data, env) && + box.open(env, KeyHierarchy::Purpose::Data, plain) && plain == "hello envelope", + name); + std::snprintf(name, sizeof name, "box %s wrong purpose", tag); + check(!box.open(env, KeyHierarchy::Purpose::Cookies, plain), name); + std::snprintf(name, sizeof name, "box %s tamper", tag); + std::string bad = env; + bad[bad.size() / 2] ^= 1; + check(!box.open(bad, KeyHierarchy::Purpose::Data, plain), name); + std::snprintf(name, sizeof name, "box %s rotation overlap", tag); + check(kh.rotate(5000) && box.open(env, KeyHierarchy::Purpose::Data, plain) && + plain == "hello envelope", + name); + } + + // Argon2id oracle vectors (python argon2-cffi, Type.ID). + struct ArgonCase { + const char* pwd; + std::size_t pwd_len; // 0 means use full literal below via marker + const char* salt; + std::uint32_t t, m, p, len; + const char* hex; + }; + const ArgonCase cases[] = { + {"password", 8, "somesalt", 2, 8, 1, 32, + "fdb4ddb6d5887131b66f0b2a3740c077dd05b755845861f6b5a1dde8b1071646"}, + {"password", 8, "somesalt", 3, 64, 1, 32, + "b9e3bfda58fb95f1e1e479af4e01bfade5a0dba34d400a254f770462674169e6"}, + {"hello world", 11, "12345678", 1, 256, 1, 16, "134c882631b4af2720ffb6f4d1c54ac8"}, + {"", 0, "emptypass", 1, 8, 1, 24, "0c520f6d084e137c8163c2811be4505efaa8d6058083adc4"}, + {"password", 8, "somesalt", 2, 16, 2, 32, + "747d7631b182faf749d7efc31aec31df4ecfe3b57c792f53800ac2c9978b4888"}, + }; + for (std::size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); ++i) { + const auto& c = cases[i]; + PasswordHasher::Params prm; + prm.passes = c.t; + prm.memory_kib = c.m; + prm.lanes = c.p; + prm.hash_len = c.len; + prm.salt_len = std::uint32_t(std::string(c.salt).size()); + std::vector tag; + std::string pwd(c.pwd, c.pwd_len); + bool ok = PasswordHasher::derive(pwd, c.salt, "", "", prm, tag); + char name[64]; + std::snprintf(name, sizeof name, "argon2id oracle case %zu", i); + check(ok && hex(tag) == c.hex, name); + } + { // 100-byte password, 1 MiB memory, 64-byte tag + PasswordHasher::Params prm; + prm.passes = 2; + prm.memory_kib = 1024; + prm.lanes = 1; + prm.hash_len = 64; + prm.salt_len = 16; + std::vector tag; + check(PasswordHasher::derive(std::string(100, 'p'), std::string(16, 's'), "", "", prm, + tag) && + hex(tag) == + "f27d9a7175df6d14e8ebef7318b5a546df0a04102752abcd1dcd288eb00c547ae84f4ffe7fc" + "b1115282b7aa6e0dd013ec2ffac6298469155254a84efdde0cb04", + "argon2id oracle case 5"); + } + + // Password hash/verify round trip at small test params. + { + PasswordHasher::Params prm; + prm.memory_kib = 8; + prm.passes = 1; + prm.lanes = 1; + std::string enc; + check(PasswordHasher::hash_password("s3cret!", prm, enc), "hash small"); + check(PasswordHasher::verify("s3cret!", enc), "verify good"); + check(!PasswordHasher::verify("s3cret?", enc), "verify bad"); + check(!PasswordHasher::verify("s3cret!", "$argon2id$v=19$m=8,t=1,p=1$broken"), + "verify malformed"); + PasswordHasher::Params weak = prm; + weak.memory_kib = 8; + PasswordHasher::Params strong = prm; + strong.memory_kib = 64; + check(!PasswordHasher::needs_rehash(enc, weak), "no rehash same params"); + check(PasswordHasher::needs_rehash(enc, strong), "rehash stronger params"); + std::string encp; + check(PasswordHasher::hash_password_peppered("pw", "pepper", prm, encp) && + PasswordHasher::verify_peppered("pw", "pepper", encp) && + !PasswordHasher::verify_peppered("pw", "wrong", encp) && + !PasswordHasher::verify("pw", encp), + "pepper roundtrip"); + } + + // OWASP defaults sanity. + { + auto d = PasswordHasher::owasp_defaults(); + check(d.memory_kib == 19456 && d.passes == 2 && d.lanes == 1 && d.salt_len == 16, + "owasp params"); + } + + std::printf(failures == 0 ? "ALL CRYPTO TESTS PASSED\n" : "FAILURES: %d\n", failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/src/security/guard/guard_test.cpp b/tests/src/security/guard/guard_test.cpp new file mode 100644 index 0000000..c0e5bfd --- /dev/null +++ b/tests/src/security/guard/guard_test.cpp @@ -0,0 +1,256 @@ +// Guard test: CSRF, CORS, headers, rate limit, body limits, paths, SQL. +#include +#include +#include +#include +#include + +#include "BodyLimits.h" +#include "CorsPolicy.h" +#include "CsrfGuard.h" +#include "DenialSink.h" +#include "HttpMessageView.h" +#include "PathGuard.h" +#include "RateLimiter.h" +#include "SecurityHeaders.h" +#include "Session.h" +#include "SqlSafety.h" + +using namespace molo::security::guard; +using molo::security::session::Session; + +static int failures = 0; +static void check(bool ok, const char* name) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", name); + if (!ok) ++failures; +} + +static HttpRequestView post(const std::string& path) { + HttpRequestView r; + r.method = "POST"; + r.path = path; + r.peer_ip = "10.0.0.1"; + r.now_unix_sec = 1000; + return r; +} + +static Session mk_session() { + Session s; + s.id = "AAAAAAAAAAAAAAAAAAAAAA"; + s.csrf_token = "tok-123"; + return s; +} + +int main() { + NullSink sink; + + // CSRF: synchronizer via header and form field. + { + CsrfGuard g(CsrfGuard::Policy(), "hmac-key-for-tests", sink); + Session s = mk_session(); + auto r = post("/submit"); + HttpResponseView res; + check(!g.enforce(r, res, s) && res.status == 403, "csrf missing token 403"); + auto r2 = post("/submit"); + r2.headers.emplace_back("x-csrf-token", "tok-123"); + check(g.enforce(r2, res, s), "csrf header token passes"); + auto r3 = post("/submit"); + check(g.check(r3, s, "tok-123") == CsrfGuard::Verdict::Allow, "csrf form field passes"); + Session other = mk_session(); + other.id = "BBBBBBBBBBBBBBBBBBBBBB"; + other.csrf_token = "tok-999"; + auto r4 = post("/submit"); + r4.headers.emplace_back("x-csrf-token", "tok-123"); + check(g.check(r4, other) == CsrfGuard::Verdict::Deny, "csrf foreign token fails"); + HttpRequestView safe; + safe.method = "GET"; + check(g.check(safe, s) == CsrfGuard::Verdict::Allow, "csrf safe method passes"); + } + + // CSRF: signed double-submit fallback. + { + CsrfGuard g(CsrfGuard::Policy(), "hmac-key-for-tests", sink); + Session s = mk_session(); + std::string nonce = "n-1"; + std::string cookie = nonce + "." + g.fallback_token(s.id, nonce); + auto r = post("/submit"); + r.headers.emplace_back("cookie", "molo_csrf=" + cookie); + r.headers.emplace_back("x-csrf-token", cookie); + check(g.check(r, s) == CsrfGuard::Verdict::Allow, "csrf fallback passes"); + auto r2 = post("/submit"); + r2.headers.emplace_back("cookie", "molo_csrf=" + cookie); + r2.headers.emplace_back("x-csrf-token", "tampered"); + check(g.check(r2, s) == CsrfGuard::Verdict::Deny, "csrf fallback echo fails"); + } + + // CSRF: Fetch Metadata block and exemptions. + { + CsrfGuard::Policy p; + p.exempt_paths = {"/hooks/stripe"}; + CsrfGuard g(p, "k", sink); + Session s = mk_session(); + auto r = post("/submit"); + r.headers.emplace_back("sec-fetch-site", "cross-site"); + r.headers.emplace_back("x-csrf-token", "tok-123"); + check(g.check(r, s) == CsrfGuard::Verdict::Deny, "csrf cross-site blocked"); + auto r2 = post("/hooks/stripe"); + check(g.check(r2, s) == CsrfGuard::Verdict::Exempt, "csrf exempt path"); + } + + // CORS: default deny, allow-list, preflight, wildcard guard. + { + CorsPolicy::Policy p; + check(CorsPolicy::validate(p).empty(), "cors empty valid"); + CorsPolicy::Policy bad; + bad.allow_origins = {"*"}; + bad.allow_credentials = true; + check(!CorsPolicy::validate(bad).empty(), "cors wildcard+creds rejected"); + CorsPolicy closed(p, sink); + auto r = post("/api"); + r.headers.emplace_back("origin", "https://evil.test"); + HttpResponseView res; + closed.apply(r, res); + check(res.header("Access-Control-Allow-Origin").empty(), "cors default deny"); + CorsPolicy::Policy open; + open.allow_origins = {"https://app.test"}; + open.allow_credentials = true; + CorsPolicy c(open, sink); + auto r2 = post("/api"); + r2.headers.emplace_back("origin", "https://app.test"); + HttpResponseView res2; + c.apply(r2, res2); + check(res2.header("Access-Control-Allow-Origin") == "https://app.test" && + res2.header("Access-Control-Allow-Credentials") == "true", + "cors exact origin with creds"); + HttpRequestView pre; + pre.method = "OPTIONS"; + pre.path = "/api"; + pre.headers.emplace_back("origin", "https://app.test"); + pre.headers.emplace_back("access-control-request-method", "POST"); + HttpResponseView res3; + check(c.preflight(pre, res3) && res3.status == 204 && + !res3.header("Access-Control-Allow-Methods").empty(), + "cors preflight"); + HttpRequestView pre2 = pre; + pre2.headers.clear(); + pre2.headers.emplace_back("origin", "https://evil.test"); + pre2.headers.emplace_back("access-control-request-method", "POST"); + HttpResponseView res4; + check(c.preflight(pre2, res4) && res4.status == 403, "cors preflight deny"); + } + + // Security headers: all six families, TLS-gated HSTS, opt-out. + { + SecurityHeaders h; + HttpRequestView r; + r.is_tls = true; + r.path = "/"; + HttpResponseView res; + h.apply(r, res); + check(!res.header("Strict-Transport-Security").empty() && + res.header("X-Frame-Options") == "DENY" && + res.header("X-Content-Type-Options") == "nosniff" && + !res.header("Referrer-Policy").empty() && + !res.header("Permissions-Policy").empty() && + !res.header("Content-Security-Policy").empty(), + "headers six families"); + HttpRequestView plain; + plain.path = "/"; + HttpResponseView res2; + h.apply(plain, res2); + check(res2.header("Strict-Transport-Security").empty() && + !res2.header("X-Frame-Options").empty(), + "hsts tls only"); + SecurityHeaders::Policy p; + p.opt_out_paths = {"/legacy"}; + SecurityHeaders h2(p); + HttpRequestView leg; + leg.is_tls = true; + leg.path = "/legacy"; + HttpResponseView res3; + h2.apply(leg, res3); + check(res3.header("X-Frame-Options").empty(), "headers opt-out"); + } + + // Rate limiter: burst then 429 with Retry-After. + { + RateLimiter::Policy p; + p.per_ip = {5, 1, 10}; // 5/min, no burst headroom + p.per_user = {1000, 2, 10}; + RateLimiter lim(p, &sink); + std::uint64_t now = 9000; + lim.set_clock([&] { return now; }); + HttpRequestView r; + r.peer_ip = "10.1.1.1"; + r.path = "/t"; + int allowed = 0; + for (int i = 0; i < 7; ++i) { + if (lim.check(r, "").allowed) ++allowed; + } + check(allowed == 5, "ratelimit burst cap"); + HttpResponseView res; + check(!lim.enforce(r, res, "") && res.status == 429 && + !res.header("Retry-After").empty(), + "ratelimit 429 with retry-after"); + now += 61; + check(lim.check(r, "").allowed, "ratelimit refills"); + } + + // Body limits: 431/413 mapping, JSON depth, form fields. + { + BodyLimits lim(BodyLimits::Policy(), sink); + HttpRequestView r; + HttpResponseView res; + check(!lim.enforce(lim.check_headers(17000, 100, r), r, res) && res.status == 431, + "limits headers total 431"); + check(!lim.enforce(lim.check_headers(100, 9000, r), r, res), "limits single 431"); + check(!lim.enforce(lim.check_body(2 * 1024 * 1024, 0, r), r, res) && res.status == 413, + "limits body 413"); + check(lim.enforce(lim.check_body(2 * 1024 * 1024, 5 * 1024 * 1024, r), r, res), + "limits route override"); + check(!lim.enforce(lim.check_body(11 * 1024 * 1024, 100 * 1024 * 1024, r), r, res), + "limits override capped at 10MB"); + std::string deep(33, '['); + check(!lim.check_json_depth(deep, r), "limits json depth 33"); + check(lim.check_json_depth("{\"a\":[1,2]}", r), "limits json ok"); + check(!lim.check_form_fields(1001, r) && lim.check_form_fields(1000, r), + "limits form fields"); + } + + // Path guard: traversal, abs, drive, symlink, jail. + { + std::string out; + check(PathGuard::canonicalize("/a/b/../c", out) && out == "/a/c", "path canonical"); + check(!PathGuard::canonicalize("/../etc", out), "path climb rejected"); + check(!PathGuard::canonicalize("/a/../../x", out), "path deep climb"); + check(!PathGuard::canonicalize("C:\\win", out), "path drive rejected"); + ::mkdir("/tmp/molo_path_root", 0755); + ::mkdir("/tmp/molo_path_root/sub", 0755); + ::unlink("/tmp/molo_path_root/link"); + check(::symlink("/etc", "/tmp/molo_path_root/link") == 0, "path fixture symlink"); + check(PathGuard::check("/tmp/molo_path_root", "/sub", out, &sink, nullptr) && + out == "/tmp/molo_path_root/sub", + "path jail ok"); + check(!PathGuard::check("/tmp/molo_path_root", "/link/passwd", out, &sink, nullptr), + "path symlink rejected"); + check(PathGuard::check("/tmp/molo_path_root", "/link", out, &sink, nullptr, true), + "path symlink allowed when opted in"); + ::unlink("/tmp/molo_path_root/link"); + } + + // SQL safety contract. + { + BoundQuery q{"SELECT * FROM t WHERE a = ? AND b = ?", + {{BoundValue::Type::Text, "x"}, {BoundValue::Type::Int, "1"}}}; + std::string reason; + check(SqlSafety::check(q, reason), "sql bound ok"); + BoundQuery bad{"SELECT * FROM t WHERE a = ?", + {{BoundValue::Type::Text, "x"}, {BoundValue::Type::Int, "1"}}}; + check(!SqlSafety::check(bad, reason), "sql arity mismatch"); + check(SqlSafety::quote_identifier("users") == "\"users\"", "sql quote ok"); + check(SqlSafety::quote_identifier("a;b").empty(), "sql quote rejects"); + } + + std::printf(failures == 0 ? "ALL GUARD TESTS PASSED\n" : "FAILURES: %d\n", failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/src/security/secret/secrets_test.cpp b/tests/src/security/secret/secrets_test.cpp new file mode 100644 index 0000000..f63eb46 --- /dev/null +++ b/tests/src/security/secret/secrets_test.cpp @@ -0,0 +1,88 @@ +// Secrets test: JSON parsing, file plus env override, rotation, redaction. +#include +#include +#include +#include + +#include "SecretsProvider.h" + +using namespace molo::security::secret; + +static int failures = 0; +static void check(bool ok, const char* name) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", name); + if (!ok) ++failures; +} + +int main() { + // JSON parser. + { + std::unordered_map m; + check(SecretsProvider::parse_json_object("{\"a\":\"b\",\"n\":42,\"t\":true}", m) && + m["a"] == "b" && m["n"] == "42" && m["t"] == "true", + "json flat"); + check(SecretsProvider::parse_json_object("{\"o\":{\"k\":\"v\"}}", m) && m["o.k"] == "v", + "json nested"); + check(!SecretsProvider::parse_json_object("{bad", m), "json malformed"); + check(!SecretsProvider::parse_json_object("{\"a\":\"b\"} trailing", m), "json trailing"); + } + + // File load with env override winning. + const char* path = "/tmp/molo_test_env.json"; + { + FILE* fp = ::fopen(path, "w"); + ::fputs("{\"master_key\":\"file-value-0123456789abcdef\"," + "\"workers\":4,\"debug\":false}", + fp); + ::fclose(fp); + } + { + SecretsProvider sp; + sp.declare({"master_key", true, 16, 128}); + sp.declare({"workers", false, 0, 16}); + sp.declare({"debug", false, 0, 16}); + ::setenv("workers", "8", 1); + std::string reason; + check(sp.load(path, reason), "secrets load"); + std::string v; + check(sp.get("master_key", v) && v == "file-value-0123456789abcdef", "secrets file value"); + std::uint64_t n = 0; + check(sp.get_u64("workers", n) && n == 8, "secrets env override wins"); + bool b = true; + check(sp.get_bool("debug", b) && !b, "secrets typed bool"); + ::unsetenv("workers"); + auto red = sp.redacted(); + bool leak = false; + for (const auto& line : red) { + if (line.find("file-value") != std::string::npos) leak = true; + } + check(!leak && red.size() == 3, "secrets redacted dump"); + check(reason.empty(), "secrets no reason on success"); + } + + // Missing required fails with a value-free reason. + { + SecretsProvider sp; + sp.declare({"must_have", true, 1, 64}); + std::string reason; + check(!sp.load("/tmp/molo_test_env.json", reason) && + reason.find("must_have") != std::string::npos, + "secrets missing required"); + } + + // Rotation swaps values, rejects unknown names. + { + SecretsProvider sp; + sp.declare({"k", false, 1, 64}); + std::string reason; + check(sp.load("", reason), "secrets empty path ok"); + check(sp.rotate("k", "v1"), "rotate set"); + std::string v; + check(sp.get("k", v) && v == "v1", "rotate read"); + check(sp.rotate("k", "v2") && sp.get("k", v) && v == "v2", "rotate swap"); + check(!sp.rotate("nope", "x"), "rotate unknown rejected"); + } + + std::printf(failures == 0 ? "ALL SECRETS TESTS PASSED\n" : "FAILURES: %d\n", failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/src/security/session/session_test.cpp b/tests/src/security/session/session_test.cpp new file mode 100644 index 0000000..a2a80c2 --- /dev/null +++ b/tests/src/security/session/session_test.cpp @@ -0,0 +1,155 @@ +// Session test: manager lifecycle, timeouts, rotation, both stores. +#include +#include +#include +#include + +#include "DbStore.h" +#include "DenialSink.h" +#include "HttpMessageView.h" +#include "MemoryStore.h" +#include "SessionManager.h" + +using namespace molo::security::session; +using molo::security::guard::HttpRequestView; +using molo::security::guard::HttpResponseView; +using molo::security::guard::NullSink; + +static int failures = 0; +static void check(bool ok, const char* name) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", name); + if (!ok) ++failures; +} + +struct FakeRows : IDbSessionRows { + MemoryStore mem; + bool select(const std::string& id, Session& out) override { return mem.get(id, 0, out); } + bool upsert(const Session& s) override { return mem.put(s); } + bool erase(const std::string& id) override { return mem.destroy(id); } + std::size_t erase_expired(std::uint64_t now) override { return mem.sweep(now); } +}; + +static HttpRequestView req_with_cookie(const std::string& id, std::uint64_t now) { + HttpRequestView r; + r.now_unix_sec = now; + r.peer_ip = "10.0.0.1"; + r.path = "/app"; + if (!id.empty()) r.headers.emplace_back("cookie", "__Host-molo_sid=" + id); + return r; +} + +int main() { + NullSink sink; + + // Fresh request mints an anonymous session. + { + auto store = std::make_shared(); + SessionManager mgr(store, SessionPolicy(), sink); + Session s; + auto r = mgr.load(req_with_cookie("", 1000), s); + check(r == SessionManager::LoadResult::Fresh && s.id.size() == 22, "fresh mints id"); + HttpResponseView res; + mgr.save(s, res); + check(res.header("Set-Cookie").find("__Host-molo_sid=") != std::string::npos && + res.header("Set-Cookie").find("Secure") != std::string::npos && + res.header("Set-Cookie").find("HttpOnly") != std::string::npos && + res.header("Set-Cookie").find("SameSite=Strict") != std::string::npos, + "cookie flags"); + } + + // Resume, idle timeout, absolute cap. + { + auto store = std::make_shared(); + SessionManager mgr(store, SessionPolicy(), sink); + Session s; + mgr.load(req_with_cookie("", 1000), s); + HttpResponseView res; + mgr.save(s, res); + Session s2; + check(mgr.load(req_with_cookie(s.id, 1100), s2) == SessionManager::LoadResult::Resumed && + s2.id == s.id, + "resume"); + Session s3; + check(mgr.load(req_with_cookie(s.id, 1100 + 31 * 60), s3) == + SessionManager::LoadResult::ExpiredIdle, + "idle expiry at 31 min"); + Session s4; + mgr.load(req_with_cookie("", 2000), s4); + mgr.save(s4, res); + Session s5; + check(mgr.load(req_with_cookie(s4.id, 2000 + 12 * 3600 + 1), s5) != + SessionManager::LoadResult::Resumed, + "absolute expiry enforced"); + } + + // Rotation on privilege change, old ID dies. + { + auto store = std::make_shared(); + SessionManager mgr(store, SessionPolicy(), sink); + Session s; + mgr.load(req_with_cookie("", 1000), s); + HttpResponseView res; + mgr.save(s, res); + std::string old = s.id; + check(mgr.set_privilege(s, Session::Privilege::User, res, 1001) && s.id != old, + "privilege rotates id"); + Session probe; + check(!store->get(old, 1001, probe), "old id dead"); + check(store->get(s.id, 1001, probe) && probe.privilege == Session::Privilege::User, + "new id live"); + mgr.logout(s, res); + check(!store->get(s.id, 1002, probe) && + res.header_all("Set-Cookie").back().find("Max-Age=0") != std::string::npos, + "logout destroys and clears"); + } + + // Unknown cookie fails closed with a fresh ID. + { + auto store = std::make_shared(); + SessionManager mgr(store, SessionPolicy(), sink); + Session s; + check(mgr.load(req_with_cookie("AAAAAAAAAAAAAAAAAAAAAA", 1000), s) == + SessionManager::LoadResult::Invalid && + s.id != "AAAAAAAAAAAAAAAAAAAAAA", + "unknown cookie invalid"); + } + + // MemoryStore caps and sweep. + { + MemoryStore tiny(2); + Session a, b, c; + a.id = "AAAAAAAAAAAAAAAAAAAAAA"; + b.id = "BBBBBBBBBBBBBBBBBBBBBB"; + c.id = "CCCCCCCCCCCCCCCCCCCCCC"; + a.absolute_expiry = 100; + b.absolute_expiry = 200; + c.absolute_expiry = 300; + check(tiny.put(a) && tiny.put(b) && tiny.put(c) && tiny.size() == 2, "cap evicts oldest"); + check(!tiny.get("AAAAAAAAAAAAAAAAAAAAAA", 50, a), "evicted entry gone"); + check(tiny.sweep(250) == 1 && tiny.size() == 1, "sweep expires"); + Session bad; + bad.id = "short"; + check(!tiny.put(bad), "bad shape rejected"); + } + + // DbStore over fake rows, plus null-rows fail-closed. + { + auto rows = std::make_shared(); + DbStore db(rows); + Session s; + s.id = "DDDDDDDDDDDDDDDDDDDDDD"; + s.absolute_expiry = 5000; + check(db.put(s), "db put"); + Session out; + check(db.get(s.id, 100, out), "db get"); + s.id = "EEEEEEEEEEEEEEEEEEEEEE"; + check(db.rotate("DDDDDDDDDDDDDDDDDDDDDD", s), "db rotate"); + check(!db.get("DDDDDDDDDDDDDDDDDDDDDD", 100, out), "db old gone"); + check(db.destroy(s.id), "db destroy"); + DbStore nulldb(nullptr); + check(!nulldb.put(s) && !nulldb.get("x", 0, out), "null rows fail closed"); + } + + std::printf(failures == 0 ? "ALL SESSION TESTS PASSED\n" : "FAILURES: %d\n", failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/src/security/tls/tls_test.cpp b/tests/src/security/tls/tls_test.cpp new file mode 100644 index 0000000..05396ee --- /dev/null +++ b/tests/src/security/tls/tls_test.cpp @@ -0,0 +1,198 @@ +// TLS test: config validation, stub fail-closed, acceptor driver with a +// fake backend (handshake ok, SNI reject, timeout, backoff, unavailable). +#include +#include +#include +#include +#include + +#include "TlsAcceptor.h" +#include "TlsBackend.h" +#include "TlsConfig.h" +#include "TlsContext.h" +#include "WolfSslBackend.h" + +using namespace molo::security::tls; + +static int failures = 0; +static void check(bool ok, const char* name) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", name); + if (!ok) ++failures; +} + +// Fake backend: scripted handshake (WantRead once, then Done). +struct FakeSession : ITlsSession { + int steps = 0; + std::string sni; + bool fail = false; + explicit FakeSession(std::string s, bool f = false) : sni(std::move(s)), fail(f) {} + TlsStep handshake_step(const std::uint8_t*, std::size_t, std::vector& out) override { + if (fail) return TlsStep::Failed; + if (steps == 0) { + ++steps; + out = {0x16, 0x03}; + return TlsStep::WantRead; + } + return TlsStep::Done; + } + bool decrypt(const std::uint8_t*, std::size_t, std::vector&, std::size_t) override { + return true; + } + bool encrypt(const std::uint8_t*, std::size_t, std::vector&) override { + return true; + } + std::string alpn_selected() const override { return "h2"; } + std::string sni_hostname() const override { return sni; } + std::string cipher_name() const override { return "TLS_AES_128_GCM_SHA256"; } + std::string last_error() const override { return fail ? "scripted fail" : ""; } + bool shutdown(std::vector&) override { return true; } +}; + +struct FakeBackend : ITlsBackend { + std::string sni = "example.com"; + bool fail = false; + std::string name() const override { return "fake"; } + bool available() const override { return true; } + bool configure(const TlsConfig&, std::string&) override { return true; } + bool reload_certs(std::string&) override { return true; } + std::unique_ptr new_session(TlsRole) override { + return std::unique_ptr(new FakeSession(sni, fail)); + } +}; + +struct FakeHooks : ITlsConnHooks { + std::vector closed; + std::vector promoted; + std::string last_alpn; + bool send_ok = true; + bool send_bytes(std::uint64_t, const std::vector&) override { return send_ok; } + void close_conn(std::uint64_t id, const std::string&) override { closed.push_back(id); } + void promote_conn(std::uint64_t id, std::unique_ptr, const std::string& alpn) override { + promoted.push_back(id); + last_alpn = alpn; + } +}; + +int main() { + // Config validation. + TlsConfig good; + good.cert_path = "/etc/molo/cert.pem"; + good.key_path = "/etc/molo/key.pem"; + check(good.validate().empty(), "tls config valid"); + TlsConfig bad12 = good; + bad12.min_version_minor = 3; // TLS 1.2 attempt + check(!bad12.validate().empty(), "tls 1.2 rejected"); + TlsConfig badcert = good; + badcert.key_path.clear(); + check(!badcert.validate().empty(), "tls missing key rejected"); + TlsConfig badto = good; + badto.handshake_timeout_ms = 100; + check(!badto.validate().empty(), "tls tiny timeout rejected"); + + // Stub backend fails closed with a marked reason. + { + WolfSslBackend stub; + check(!WolfSslBackend::kNativeBuild, "stub build marked"); + check(!stub.available(), "stub unavailable"); + std::string reason; + check(!stub.configure(good, reason) && reason.find("STUB") != std::string::npos, + "stub configure fails closed"); + auto s = stub.new_session(TlsRole::Server); + std::vector out; + check(s && s->handshake_step(nullptr, 0, out) == TlsStep::Unavailable, + "stub session unavailable"); + } + + // Context boot + reload with fake backend. + TlsContext ctx; + { + std::string reason; + check(ctx.boot(std::unique_ptr(new FakeBackend), good, reason), "ctx boot"); + check(ctx.booted(), "ctx booted"); + check(ctx.reload_certs(reason), "ctx reload"); + auto s = ctx.new_session(TlsRole::Server); + check(s && ctx.stats().sessions_opened == 1, "ctx session"); + } + { + TlsContext ctx2; + std::string reason; + check(!ctx2.boot(std::unique_ptr(new FakeBackend), bad12, reason), + "ctx boot rejects bad config"); + } + + // Acceptor: happy path handshake. + { + FakeHooks hooks; + TlsAcceptor acc(ctx, hooks); + std::uint64_t now = 100000; + acc.set_clock([&] { return now; }); + check(acc.on_accept(1, "10.0.0.1"), "accept ok"); + std::uint8_t hello[4] = {0x16, 0x03, 0x01, 0x00}; + acc.on_bytes(1, hello, sizeof hello); + check(hooks.promoted.size() == 1 && hooks.last_alpn == "h2", "handshake promotes"); + check(acc.counters().completed == 1, "counter completed"); + } + + // Acceptor: SNI reject with allow-list. + { + FakeHooks hooks; + TlsAcceptorPolicy pol; + pol.sni_allow = {"allowed.test"}; + TlsAcceptor acc(ctx, hooks, pol); + acc.set_clock([] { return std::uint64_t(100000); }); + bool logged = false; + acc.set_sni_log([&](const SniDecision& d) { + logged = !d.allowed && d.hostname == "example.com"; + }); + check(acc.on_accept(2, "10.0.0.2"), "accept sni case"); + std::uint8_t hello[4] = {0x16, 0x03, 0x01, 0x00}; + acc.on_bytes(2, hello, sizeof hello); + check(hooks.closed.size() == 1 && acc.counters().sni_rejected == 1 && logged, + "sni rejected and logged"); + } + + // Acceptor: handshake timeout sweep. + { + FakeHooks hooks; + TlsAcceptor acc(ctx, hooks); + std::uint64_t now = 200000; + acc.set_clock([&] { return now; }); + check(acc.on_accept(3, "10.0.0.3"), "accept timeout case"); + now += 6000; + check(acc.on_tick() == 1 && acc.counters().timed_out == 1 && hooks.closed.size() == 1, + "timeout sweep closes"); + } + + // Acceptor: failure backoff per IP. + { + ctx.new_session(TlsRole::Server); // keep stats moving + TlsContext fctx; + std::string reason; + auto fb = new FakeBackend; + fb->fail = true; + fctx.boot(std::unique_ptr(fb), good, reason); + FakeHooks hooks; + TlsAcceptor acc(fctx, hooks); + acc.set_clock([] { return std::uint64_t(300000); }); + std::uint8_t hello[1] = {0}; + for (int i = 0; i < 5; ++i) { + std::uint64_t id = 10 + std::uint64_t(i); + if (acc.on_accept(id, "10.9.9.9")) acc.on_bytes(id, hello, 1); + } + check(acc.counters().failed == 5, "five failures counted"); + check(!acc.on_accept(99, "10.9.9.9") && acc.counters().backoff_rejected == 1, + "backoff rejects sixth accept"); + } + + // Acceptor: unavailable backend (stub context) fails closed. + { + TlsContext stub_ctx; // never booted + FakeHooks hooks; + TlsAcceptor acc(stub_ctx, hooks); + check(!acc.on_accept(5, "10.0.0.5") && acc.counters().unavailable == 1, + "unbooted context fails closed"); + } + + std::printf(failures == 0 ? "ALL TLS TESTS PASSED\n" : "FAILURES: %d\n", failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/src/security/upload/upload_test.cpp b/tests/src/security/upload/upload_test.cpp new file mode 100644 index 0000000..d3570fd --- /dev/null +++ b/tests/src/security/upload/upload_test.cpp @@ -0,0 +1,117 @@ +// Upload test: validator traps and store flow with stub AV. +#include +#include +#include +#include +#include +#include + +#include "DenialSink.h" +#include "HttpMessageView.h" +#include "UploadStore.h" +#include "UploadValidator.h" + +using namespace molo::security::upload; +using molo::security::guard::HttpRequestView; +using molo::security::guard::NullSink; + +static int failures = 0; +static void check(bool ok, const char* name) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", name); + if (!ok) ++failures; +} + +static UploadFile mk(const std::string& name, std::vector bytes) { + UploadFile f; + f.field_name = "file"; + f.client_filename = name; + f.content_type = "application/octet-stream"; + f.bytes = std::move(bytes); + return f; +} + +static std::vector png_bytes() { + std::vector b = {0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A, 0, 1, 2, 3}; + return b; +} + +struct InfectedScanner : IAvScanner { + ScanResult scan(const std::string&) override { return ScanResult::Infected; } +}; + +int main() { + NullSink sink; + HttpRequestView req; + req.path = "/upload"; + + // Validator traps. + { + UploadValidator v(UploadValidator::Policy(), sink); + check(v.check(mk("a.png", png_bytes()), req) == UploadValidator::Verdict::Allow, + "upload valid png"); + check(v.check(mk("x.jpg.php", png_bytes()), req) != UploadValidator::Verdict::Allow, + "upload double extension"); + check(v.check(mk("x.pHp", png_bytes()), req) != UploadValidator::Verdict::Allow, + "upload case trick"); + std::string nulled = std::string("a.png\0.php", 10); + check(v.check(mk(nulled, png_bytes()), req) != UploadValidator::Verdict::Allow, + "upload null byte"); + std::vector elf = {'E', 'L', 'F', '!', 0, 0, 0, 0}; + UploadFile spoof = mk("run.png", elf); + spoof.content_type = "image/png"; + check(v.check(spoof, req) == UploadValidator::Verdict::MagicMismatch, + "upload spoofed content-type"); + UploadFile big = mk("big.png", std::vector(11 * 1024 * 1024, 0)); + check(v.check(big, req) == UploadValidator::Verdict::TooLarge, "upload too large"); + check(!v.check_count(5, req) && v.check_count(4, req), "upload count cap"); + std::vector zip = {'P', 'K', 0x03, 0x04, 0, 0}; + check(v.check(mk("doc.docx", zip), req) == UploadValidator::Verdict::Allow, + "upload docx zip-family"); + } + + // Store flow with clean and infected scanners. + const char* tmpd = "/tmp/molo_up_tmp"; + const char* stdir = "/tmp/molo_up_store"; + const char* qdir = "/tmp/molo_up_quar"; + ::mkdir(tmpd, 0700); + ::mkdir(stdir, 0700); + ::mkdir(qdir, 0700); + { + UploadStore::Policy p{tmpd, stdir, qdir, 1024 * 1024, 3600}; + check(UploadStore::validate(p).empty(), "store policy valid"); + UploadStore::Policy bad = p; + bad.store_dir = tmpd; + check(!UploadStore::validate(bad).empty(), "store dirs distinct"); + auto clean = std::make_shared(); + UploadStore store(p, clean, sink); + StoredUpload out; + check(store.store(mk("a.png", png_bytes()), "png", 1000, out, req), "store clean"); + check(out.full_path.compare(0, std::string(stdir).size(), stdir) == 0 && + out.storage_name != "a.png", + "store generated name outside webroot"); + struct stat st{}; + check(::stat(out.full_path.c_str(), &st) == 0 && (st.st_mode & 0777) == 0600, + "store 0600 perms"); + UploadFile big = mk("b.png", std::vector(2 * 1024 * 1024, 0)); + check(!store.store(big, "png", 1000, out, req), "store quota"); + auto evil = std::make_shared(); + UploadStore store2(p, evil, sink); + check(!store2.store(mk("c.png", png_bytes()), "png", 1000, out, req), + "store infected quarantined"); + } + // Temp sweep removes stale files only. + { + UploadStore::Policy p{tmpd, stdir, qdir, 1024 * 1024, 10}; + UploadStore store(p, std::make_shared(), sink); + FILE* fp = ::fopen("/tmp/molo_up_tmp/AAAA-ttl.tmp", "w"); + if (fp) { + ::fputs("x", fp); + ::fclose(fp); + } + // Far-future clock makes the just-written file stale by TTL. + check(store.sweep_temp(4000000000ULL) >= 1, "sweep stale temp"); + } + + std::printf(failures == 0 ? "ALL UPLOAD TESTS PASSED\n" : "FAILURES: %d\n", failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/src/supervisor/test_bus.cpp b/tests/src/supervisor/test_bus.cpp new file mode 100644 index 0000000..0bca6dc --- /dev/null +++ b/tests/src/supervisor/test_bus.cpp @@ -0,0 +1,205 @@ +// V7-5: durable streams (persist, reconnect, redeliver) and ephemeral topics. +#include +#include +#include + +#include "supervisor/bus/stream_bus.hpp" +#include "supervisor/bus/topic_bus.hpp" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int failures = 0; + +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ + ++failures; \ + } \ + } while (0) + +const std::filesystem::path kTmp = "build/supervisor/tmp/test-bus"; + +void test_durable_stream_reconnect_no_loss() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::FrozenClock clock(1760000000000LL, 1000); + auto obs = molo::common::null_observer(); + + { + molo::bus::StreamBus bus(kTmp / "streams", clock, obs); + bus.create_stream("events"); + bus.create_consumer("events", "worker", 60000); + for (int i = 1; i <= 10000; ++i) { + bus.publish("events", "msg-" + std::to_string(i)); + } + // Consume half, then "disconnect" by destroying the bus mid-stream. + int acked = 0; + while (acked < 5000) { + auto batch = bus.poll("events", "worker", 500); + CHECK(!batch.empty(), "poll returns messages"); + for (const auto& msg : batch) { + CHECK(msg.data == "msg-" + std::to_string(msg.seq), "payload matches seq"); + CHECK(bus.ack("events", "worker", msg.seq), "ack works"); + ++acked; + } + } + } + // Reconnect: a fresh bus over the same directory replays the log and + // the persisted cursor resumes exactly where the acked prefix ended. + { + molo::bus::StreamBus bus(kTmp / "streams", clock, obs); + bus.create_stream("events"); + bus.create_consumer("events", "worker", 60000); + std::set seen; + std::uint64_t expect = 5001; + while (seen.size() < 5000) { + auto batch = bus.poll("events", "worker", 500); + CHECK(!batch.empty(), "reconnected poll returns messages"); + for (const auto& msg : batch) { + CHECK(msg.seq == expect, "no gaps after reconnect"); + ++expect; + seen.insert(msg.seq); + CHECK(bus.ack("events", "worker", msg.seq), "reconnected ack works"); + } + } + CHECK(seen.size() == 5000 && *seen.begin() == 5001 && *seen.rbegin() == 10000, + "zero loss, zero gaps across reconnect"); + CHECK(bus.lag("events", "worker") == 0, "caught up, lag zero"); + } + std::cout << "PASS test_durable_stream_reconnect_no_loss" << std::endl; +} + +void test_redelivery_until_acked() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::FrozenClock clock(1760000000000LL, 1000); + auto obs = std::make_shared(); + molo::bus::StreamBus bus(kTmp / "streams", clock, obs); + bus.create_stream("jobs"); + bus.create_consumer("jobs", "w", 1000); // 1s ack deadline + bus.publish("jobs", "a"); + bus.publish("jobs", "b"); + + auto first = bus.poll("jobs", "w", 10); + CHECK(first.size() == 2, "two messages delivered"); + CHECK(first[0].redeliveries == 0, "first delivery is fresh"); + // No acks: advance past the deadline, the same messages redeliver. + clock.advance(1500); + auto second = bus.poll("jobs", "w", 10); + CHECK(second.size() == 2, "both redeliver after deadline"); + CHECK(second[0].seq == first[0].seq && second[0].redeliveries == 1, "redelivery counted"); + CHECK(obs->counter_value("molo_bus_stream_redelivered_total") == 2, "redelivery metric"); + CHECK(bus.ack("jobs", "w", first[0].seq), "ack first"); + CHECK(!bus.ack("jobs", "w", first[0].seq), "double ack fails"); + clock.advance(1500); + auto third = bus.poll("jobs", "w", 10); + CHECK(third.size() == 1 && third[0].seq == first[1].seq, "only unacked redelivers"); + CHECK(bus.renew("jobs", "w", first[1].seq), "renew extends the deadline"); + clock.advance(500); + CHECK(bus.poll("jobs", "w", 10).empty(), "renewed message waits"); + clock.advance(600); + CHECK(bus.poll("jobs", "w", 10).size() == 1, "redelivers after renewed deadline"); + std::cout << "PASS test_redelivery_until_acked" << std::endl; +} + +void test_retention_and_caps() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::FrozenClock clock(1760000000000LL, 1000); + auto obs = molo::common::null_observer(); + molo::bus::StreamBus bus(kTmp / "streams", clock, obs); + + molo::bus::StreamSpec tiny; + tiny.retention_bytes = 1024; + tiny.retention_hours = 0; // size-only retention + bus.create_stream("tiny", tiny); + bus.create_consumer("tiny", "c"); + for (int i = 0; i < 100; ++i) { + bus.publish("tiny", std::string(100, 'x')); + } + CHECK(bus.last_seq("tiny") == 100, "seqs assigned across retention"); + auto retained = bus.poll("tiny", "c", 200); + CHECK(retained.size() < 100 && !retained.empty(), "retention evicted the oldest"); + if (!retained.empty()) { + CHECK(retained.front().seq > 1, "oldest seqs gone"); + CHECK(retained.back().seq == 100, "tail intact"); + } + + molo::bus::StreamSpec timed; + timed.retention_hours = 1; + timed.retention_bytes = 1024 * 1024 * 1024; + bus.create_stream("timed", timed); + bus.publish("timed", "old"); + clock.advance(2 * 3600 * 1000); + CHECK(bus.enforce_retention("timed") == 1, "time retention evicts one"); + bus.publish("timed", "new"); + + bool threw = false; + try { + bus.publish("tiny", std::string(1048576 + 1, 'y')); + } catch (const molo::bus::BusError&) { + threw = true; + } + CHECK(threw, "oversize stream payload rejected"); + threw = false; + try { + bus.create_stream("../escape"); + } catch (const molo::bus::BusError&) { + threw = true; + } + CHECK(threw, "path escape in stream name rejected"); + std::cout << "PASS test_retention_and_caps" << std::endl; +} + +void test_ephemeral_topics() { + molo::common::SystemClock clock; + molo::bus::TopicBus bus(clock); + + CHECK(bus.publish("a:b", "lost") == 0, "no-subscriber publish is a counted no-op"); + const auto sub1 = bus.subscribe("a:b"); + const auto sub2 = bus.subscribe("a:*"); + const auto late = bus.subscribe("a:b"); + CHECK(bus.publish("a:b", "hi") == 3, "exact plus wildcard plus late deliver"); + auto got = bus.poll(sub1, 10); + CHECK(got.size() == 1 && got[0].payload == "hi", "exact subscriber receives"); + CHECK(bus.poll(sub2, 10).size() == 1, "wildcard subscriber receives"); + CHECK(bus.poll(late, 10).size() == 1, "subscriber present at publish receives"); + + // Offline means offline: subscribe after the publish, get nothing. + CHECK(bus.publish("a:b", "before") == 3, "three live subscribers"); + const auto offline = bus.subscribe("a:b"); + bus.poll(sub1, 10); + bus.poll(sub2, 10); + bus.poll(late, 10); + CHECK(bus.poll(offline, 10).empty(), "offline subscriber gets nothing past"); + CHECK(bus.publish("a:b:c", "deep") == 0, "wildcard matches one segment only"); + + // Backpressure: oldest-drop with a counter, publisher never blocks. + for (int i = 0; i < 1034; ++i) { + bus.publish("a:b", "flood-" + std::to_string(i)); + } + CHECK(bus.dropped_total() > 0, "overflow counted"); + CHECK(bus.pending(sub1) == 1024, "subscriber queue capped at 1024"); + auto tail = bus.poll(sub1, 2000); + CHECK(tail.front().payload == "flood-10", "oldest dropped first"); + CHECK(bus.unsubscribe(sub1) && !bus.unsubscribe(sub1), "unsubscribe once"); + std::cout << "PASS test_ephemeral_topics" << std::endl; +} + +} // namespace + +int main() { + test_durable_stream_reconnect_no_loss(); + test_redelivery_until_acked(); + test_retention_and_caps(); + test_ephemeral_topics(); + if (failures == 0) { + std::cout << "ALL BUS TESTS PASSED" << std::endl; + return 0; + } + std::cout << failures << " FAILURES" << std::endl; + return 1; +} diff --git a/tests/src/supervisor/test_durability.cpp b/tests/src/supervisor/test_durability.cpp new file mode 100644 index 0000000..0f35e54 --- /dev/null +++ b/tests/src/supervisor/test_durability.cpp @@ -0,0 +1,253 @@ +// V7-6: durable queue survives a hard kill; unacked work replays. +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/kv_store.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/tasks/scheduler.hpp" +#include "supervisor/tasks/task_queue.hpp" +#include "supervisor/tasks/task_registry.hpp" + +namespace { + +int failures = 0; + +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ + ++failures; \ + } \ + } while (0) + +const std::filesystem::path kTmp = "build/supervisor/tmp/test-durability"; + +struct IdempotentCounter : molo::tasks::TaskHandler { + std::mutex* mutex; + std::set* completed; + std::set* seen_twice; + molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override { + std::lock_guard lock(*mutex); + if (!completed->insert(ctx.envelope.id).second) { + seen_twice->insert(ctx.envelope.id); // redelivery tolerated + } + return molo::tasks::TaskOutcome::Success; + } +}; + +void test_hard_kill_replays_unacked() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + + std::set ids; + { + molo::tasks::QueueOptions options; + options.durable_dir = (kTmp / "q").string(); + options.lease_ms = 60000; + molo::tasks::TaskQueue queue(options, clock, obs); + for (int i = 0; i < 50; ++i) { + molo::tasks::TaskEnvelope env; + env.name = "work"; + ids.insert(queue.enqueue(std::move(env))); + } + CHECK(queue.stats().queued == 50, "50 queued before the kill"); + // Lease 10 without ack: in-flight at crash time. + for (int i = 0; i < 10; ++i) { + CHECK(queue.lease(0).has_value(), "leases before the kill"); + } + // Hard kill: the queue object just dies here. No shutdown, no acks. + } + { + molo::tasks::QueueOptions options; + options.durable_dir = (kTmp / "q").string(); + molo::tasks::TaskQueue queue(options, clock, obs); + CHECK(queue.stats().queued == 50, "all 50 replay as queued after reboot"); + // Every replayed id matches a pre-crash id. + auto live = queue.list_queued(100); + CHECK(live.size() == 50, "50 live envelopes"); + for (const auto& env : live) { + CHECK(ids.count(env.id) == 1, "replayed id is pre-crash"); + } + molo::tasks::TaskRegistry registry; + std::mutex mutex; + std::set completed, seen_twice; + registry.register_handler("work", [&] { + auto h = std::make_unique(); + h->mutex = &mutex; + h->completed = &completed; + h->seen_twice = &seen_twice; + return h; + }); + int ran = 0; + while (queue.run_one(registry, 0)) { + ++ran; + } + CHECK(ran == 50 && completed.size() == 50, "all replayed work completes"); + CHECK(completed == ids, "completed set equals the pre-crash set"); + CHECK(queue.stats().queued == 0, "queue drains fully"); + } + std::cout << "PASS test_hard_kill_replays_unacked" << std::endl; +} + +void test_retry_state_survives_reboot() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + std::string id; + { + molo::tasks::QueueOptions options; + options.durable_dir = (kTmp / "q").string(); + molo::tasks::TaskQueue queue(options, clock, obs); + molo::tasks::TaskEnvelope env; + env.name = "flaky"; + id = queue.enqueue(std::move(env)); + auto leased = queue.lease(0); + CHECK(leased.has_value(), "leases the task"); + CHECK(queue.fail(leased->lease_id, true, "boom"), "fails retryable"); + auto stored = queue.get(id); + CHECK(stored.has_value() && stored->attempts == 1, "attempts persisted"); + CHECK(stored.has_value() && stored->run_at_ms.has_value(), "backoff run_at persisted"); + } + { + molo::tasks::QueueOptions options; + options.durable_dir = (kTmp / "q").string(); + molo::tasks::TaskQueue queue(options, clock, obs); + auto stored = queue.get(id); + CHECK(stored.has_value(), "retried task survives reboot"); + if (stored.has_value()) { + CHECK(stored->attempts == 1, "attempt count survives"); + CHECK(stored->run_at_ms.has_value(), "backoff deadline survives"); + } + } + std::cout << "PASS test_retry_state_survives_reboot" << std::endl; +} + +void test_compact_and_torn_tail() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + { + molo::tasks::QueueOptions options; + options.durable_dir = (kTmp / "q").string(); + molo::tasks::TaskQueue queue(options, clock, obs); + molo::tasks::TaskRegistry registry; + registry.register_handler("w", [] { + struct H : molo::tasks::TaskHandler { + molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override { + return molo::tasks::TaskOutcome::Success; + } + }; + return std::make_unique(); + }); + for (int i = 0; i < 20; ++i) { + molo::tasks::TaskEnvelope env; + env.name = "w"; + queue.enqueue(std::move(env)); + } + while (queue.run_one(registry, 0)) { + } + const auto log = kTmp / "q" / "task_queue.log"; + const std::uint64_t before = fs::file_size(log); + CHECK(before > 0, "op log written"); + queue.compact(); + CHECK(fs::file_size(log) < before, "compact shrinks the log"); + } + { + // Torn tail: crash garbage at the end of the log is truncated on open. + molo::tasks::QueueOptions options; + options.durable_dir = (kTmp / "q").string(); + { + molo::tasks::TaskQueue queue(options, clock, obs); + for (int i = 0; i < 5; ++i) { + molo::tasks::TaskEnvelope env; + env.name = "w"; + queue.enqueue(std::move(env)); + } + } + const auto log = kTmp / "q" / "task_queue.log"; + { + std::ofstream out(log, std::ios::app | std::ios::binary); + out << "junk"; + } + molo::tasks::TaskQueue queue(options, clock, obs); + CHECK(queue.stats().queued == 5, "torn tail truncated, 5 tasks replay"); + } + std::cout << "PASS test_compact_and_torn_tail" << std::endl; +} + +} // namespace + +void test_schedule_entries_survive_reboot() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::common::InMemoryKvStore kv(clock); + molo::tasks::QueueOptions options; + molo::tasks::TaskQueue queue(options, clock, obs); + { + molo::tasks::SchedulerOptions sched_options; + sched_options.instance_id = "sched-durable"; + sched_options.durable_dir = (kTmp / "sched").string(); + molo::tasks::Scheduler sched(queue, kv, clock, obs, sched_options); + molo::tasks::ScheduleEntry interval; + interval.key = "every-minute"; + interval.task_name = "work"; + interval.kind = molo::tasks::ScheduleKind::Interval; + interval.interval_secs = 60; + sched.put(interval); + molo::tasks::ScheduleEntry cron; + cron.key = "nightly"; + cron.task_name = "work"; + cron.kind = molo::tasks::ScheduleKind::Cron; + cron.cron = "0 2 * * *"; + sched.put(cron); + molo::tasks::ScheduleEntry temp; + temp.key = "temp"; + temp.task_name = "work"; + temp.kind = molo::tasks::ScheduleKind::Interval; + sched.put(temp); + CHECK(sched.remove("temp"), "temp entry removed before reboot"); + } + { + molo::tasks::SchedulerOptions sched_options; + sched_options.instance_id = "sched-durable"; + sched_options.durable_dir = (kTmp / "sched").string(); + molo::tasks::Scheduler sched(queue, kv, clock, obs, sched_options); + auto entries = sched.entries(); + CHECK(entries.size() == 2, "two entries replay after reboot"); + bool has_interval = false, has_cron = false; + for (const auto& entry : entries) { + if (entry.key == "every-minute" && entry.kind == molo::tasks::ScheduleKind::Interval) { + has_interval = true; + } + if (entry.key == "nightly" && entry.cron == "0 2 * * *") { + has_cron = true; + } + CHECK(entry.key != "temp", "removed entry stays removed"); + } + CHECK(has_interval && has_cron, "entry kinds and crons survive"); + } + std::cout << "PASS test_schedule_entries_survive_reboot" << std::endl; +} + +int main() { + test_hard_kill_replays_unacked(); + test_retry_state_survives_reboot(); + test_compact_and_torn_tail(); + test_schedule_entries_survive_reboot(); + if (failures == 0) { + std::cout << "ALL DURABILITY TESTS PASSED" << std::endl; + return 0; + } + std::cout << failures << " FAILURES" << std::endl; + return 1; +} diff --git a/tests/src/supervisor/test_multi_instance.cpp b/tests/src/supervisor/test_multi_instance.cpp new file mode 100644 index 0000000..07c70e0 --- /dev/null +++ b/tests/src/supervisor/test_multi_instance.cpp @@ -0,0 +1,111 @@ +// V7-8: two nodes behind one shared bus; broadcast plus catch-up. +#include +#include + +#include "supervisor/bus/stream_bus.hpp" +#include "supervisor/bus/topic_bus.hpp" +#include "supervisor/common/clock.hpp" +#include "supervisor/common/kv_store.hpp" +#include "supervisor/common/observer.hpp" + +namespace { + +int failures = 0; + +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ + ++failures; \ + } \ + } while (0) + +const std::filesystem::path kTmp = "build/supervisor/tmp/test-multi"; + +// One stateless app node (S7-9): local sockets hang off the ephemeral bus, +// catch-up replays off the durable bus, sessions live in shared KV. +struct Node { + std::string id; + molo::bus::TopicBus::SubscriberId room_sub = 0; + int received_live = 0; + std::vector caught_up; +}; + +void test_broadcast_reaches_both_nodes() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + + // Shared backends both nodes attach to. + molo::bus::TopicBus live(clock); + molo::bus::StreamBus bus(kTmp / "bus", clock, obs); + bus.create_stream("roomlog"); + molo::common::InMemoryKvStore kv(clock); + + Node a; + a.id = "node-a"; + Node b; + b.id = "node-b"; + a.room_sub = live.subscribe("room:lobby"); + b.room_sub = live.subscribe("room:lobby"); + bus.create_consumer("roomlog", "node-a"); + bus.create_consumer("roomlog", "node-b"); + + // Broadcast emit: publish once to the bus; every node re-emits to its + // local sockets (here: each node polls its own subscription). + auto broadcast = [&](const std::string& payload) { + const std::size_t delivered = live.publish("room:lobby", payload); + bus.publish("roomlog", payload); // durable copy for reconnect catch-up + return delivered; + }; + CHECK(broadcast("hello") == 2, "one publish, two nodes"); + + for (Node* node : {&a, &b}) { + auto got = live.poll(node->room_sub, 10); + CHECK(got.size() == 1 && got[0].payload == "hello", "live delivery on " + node->id); + node->received_live += static_cast(got.size()); + // Nodes also drain the durable log to keep their cursors current. + for (const auto& msg : bus.poll("roomlog", node->id, 10)) { + bus.ack("roomlog", node->id, msg.seq); + } + } + + // Shared session store: written via node A, visible via node B. + kv.set("session:u1", "{\"user\":\"u1\"}", 3600); + std::string session; + CHECK(kv.get("session:u1", session) && session == "{\"user\":\"u1\"}", "sessions shared"); + + // Kill node B: its live subscription dies with it. + CHECK(live.unsubscribe(b.room_sub), "node B dies"); + CHECK(broadcast("msg-2") == 1, "dead node gets nothing live"); + auto got_a = live.poll(a.room_sub, 10); + CHECK(got_a.size() == 1 && got_a[0].payload == "msg-2", "node A receives msg-2 live"); + for (const auto& msg : bus.poll("roomlog", "node-a", 10)) { + bus.ack("roomlog", "node-a", msg.seq); + } + + // Node B restarts: live messages sent while dead are gone (ephemeral + // tier), but the durable log replays them in order. + b.room_sub = live.subscribe("room:lobby"); + CHECK(live.poll(b.room_sub, 10).empty(), "no ephemeral replay while dead"); + for (const auto& msg : bus.poll("roomlog", "node-b", 10)) { + b.caught_up.push_back(msg.data); + bus.ack("roomlog", "node-b", msg.seq); + } + CHECK(b.caught_up.size() == 1 && b.caught_up[0] == "msg-2", "durable catch-up replays msg-2"); + CHECK(bus.lag("roomlog", "node-b") == 0, "node B caught up"); + std::cout << "PASS test_broadcast_reaches_both_nodes" << std::endl; +} + +} // namespace + +int main() { + test_broadcast_reaches_both_nodes(); + if (failures == 0) { + std::cout << "ALL MULTI-INSTANCE TESTS PASSED" << std::endl; + return 0; + } + std::cout << failures << " FAILURES" << std::endl; + return 1; +} diff --git a/tests/src/supervisor/test_replication.cpp b/tests/src/supervisor/test_replication.cpp new file mode 100644 index 0000000..547eabb --- /dev/null +++ b/tests/src/supervisor/test_replication.cpp @@ -0,0 +1,304 @@ +// V7-7: two-instance replication, optimistic plus LWW plus hook conflicts. +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/replication/conflict_resolver.hpp" +#include "supervisor/replication/hybrid_clock.hpp" +#include "supervisor/replication/replicator.hpp" +#include "supervisor/replication/sync_session.hpp" +#include "supervisor/replication/version_vector.hpp" + +namespace { + +int failures = 0; + +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ + ++failures; \ + } \ + } while (0) + +const std::filesystem::path kTmp = "build/supervisor/tmp/test-replication"; + +// In-process SyncPeer over another Replicator (production binds this to +// the bus transport; the sync logic is identical). +class ReplicatorSyncPeer : public molo::replication::SyncPeer { + public: + explicit ReplicatorSyncPeer(molo::replication::Replicator& remote) : remote_(remote) {} + + std::vector fetch_since( + const molo::replication::VersionVector& have, std::size_t max_entries, + std::size_t max_bytes) override { + return remote_.entries_since(have.get(remote_.instance()) + 1, max_entries, max_bytes); + } + + void submit(const std::vector& entries) override { + for (const auto& entry : entries) { + try { + remote_.apply_remote(entry); + } catch (const molo::replication::ConflictError&) { + // Per-entry independence: one loser's ConflictError never blocks + // the rest of the batch. The resolver already logged and counted it. + } + } + } + + void peer_versions(molo::replication::VersionVector& out) const override { + remote_.versions(out); + } + + std::uint64_t last_seq() const override { return remote_.last_seq(); } + std::string instance() const override { return remote_.instance(); } + + private: + molo::replication::Replicator& remote_; +}; + +void sync_both(molo::replication::Replicator& a, molo::replication::Replicator& b, + std::shared_ptr obs) { + ReplicatorSyncPeer peer_b(b); + molo::replication::SyncSession(a, peer_b, obs).round_trip(); + ReplicatorSyncPeer peer_a(a); + molo::replication::SyncSession(b, peer_a, obs).round_trip(); +} + +void test_optimistic_conflict_and_recovery() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::FrozenClock clock(1760000000000LL, 1000); + auto obs = std::make_shared(); + molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs); + molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs); + a.declare_model("orders", molo::replication::ConflictPolicy::Optimistic); + b.declare_model("orders", molo::replication::ConflictPolicy::Optimistic); + + a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 1, "{\"v\":1}"); + sync_both(a, b, obs); + CHECK(b.current("orders", "k1").has_value(), "base write replicates"); + CHECK(obs->counter_value("molo_replication_conflicts_total") == 0, "no conflict on fast-forward"); + + // Concurrent write-write at the same version with different data. + clock.advance(10); + a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"a\"}"); + b.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"b\"}"); + + // Direct resolver check: exactly one winner plus ConflictError semantics. + auto la = a.current("orders", "k1"); + auto lb = b.current("orders", "k1"); + auto outcome = a.resolver().resolve(*la, *lb); + CHECK(outcome.conflict && outcome.conflict_error, "write-write conflict flagged"); + CHECK(outcome.winner_version == 2, "winner version carried"); + + bool threw = false; + try { + a.apply_remote(*lb); + } catch (const molo::replication::ConflictError& e) { + threw = true; + CHECK(e.winner_version() == 2, "ConflictError carries winner version"); + CHECK(e.model() == "orders" && e.key() == "k1", "ConflictError names model and key"); + } + CHECK(threw, "loser gets ConflictError"); + + // Sync keeps each incumbent; the conflict is counted, nothing is lost. + sync_both(a, b, obs); + CHECK(obs->counter_value("molo_replication_conflicts_total") > 0, "sync counts conflicts"); + CHECK(a.resolver().conflict_log_size() > 0, "conflict logged"); + + // Documented recovery: the loser re-reads and retries with a higher version. + a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 3, "{\"by\":\"a3\"}"); + sync_both(a, b, obs); + auto ca = a.current("orders", "k1"); + auto cb = b.current("orders", "k1"); + CHECK(ca.has_value() && cb.has_value(), "both converge"); + if (ca.has_value() && cb.has_value()) { + CHECK(ca->version == 3 && cb->version == 3, "higher version wins everywhere"); + CHECK(ca->data == cb->data, "converged data agrees"); + } + std::cout << "PASS test_optimistic_conflict_and_recovery" << std::endl; +} + +void test_lww_cache_conflict() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::FrozenClock clock(1760000000000LL, 1000); + auto obs = molo::common::null_observer(); + molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs); + molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs); + a.declare_model("cache", molo::replication::ConflictPolicy::LastWriterWins); + b.declare_model("cache", molo::replication::ConflictPolicy::LastWriterWins); + + // Same frozen millisecond: hybrid counters tie at first write, so the + // instance id tie-break decides deterministically (node-b wins). + a.local_write("cache", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}"); + b.local_write("cache", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"b\"}"); + sync_both(a, b, obs); + auto ca = a.current("cache", "k"); + auto cb = b.current("cache", "k"); + CHECK(ca.has_value() && cb.has_value(), "both resolve LWW"); + if (ca.has_value() && cb.has_value()) { + CHECK(ca->data == "{\"by\":\"b\"}" && cb->data == "{\"by\":\"b\"}", + "higher hybrid timestamp wins on both"); + CHECK(ca->instance == "node-b", "winner instance recorded"); + } + CHECK(a.resolver().conflict_log_size() == 1, "loser kept in conflict log"); + auto log = a.resolver().conflict_log(10); + CHECK(!log.empty() && log[0].loser.data == "{\"by\":\"a\"}", "log holds the loser"); + + // A genuinely later write wins regardless of instance. + clock.advance(5); + a.local_write("cache", "k", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"a2\"}"); + sync_both(a, b, obs); + CHECK(b.current("cache", "k")->data == "{\"by\":\"a2\"}", "later wall clock wins"); + std::cout << "PASS test_lww_cache_conflict" << std::endl; +} + +void test_hook_policy_and_fallback() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::FrozenClock clock(1760000000000LL, 1000); + auto obs = molo::common::null_observer(); + molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs); + + // Hook that always prefers the remote side. + a.declare_model("special", molo::replication::ConflictPolicy::Hook, + [](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) { + return 1; + }); + a.local_write("special", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}"); + molo::replication::OplogEntry remote; + remote.instance = "node-b"; + remote.ts = molo::replication::HybridTs{1760000000000LL, 0, "node-b"}; + remote.model = "special"; + remote.key = "k"; + remote.op = molo::replication::OplogOp::Upsert; + remote.version = 1; + remote.data = "{\"by\":\"b\"}"; + auto outcome = a.apply_remote(remote); + CHECK(outcome.remote_won && outcome.winner.data == "{\"by\":\"b\"}", "hook decision honored"); + CHECK(a.resolver().conflict_log(10)[0].policy == "hook", "hook conflict logged"); + + // Throwing hook falls back to LWW immediately. + a.declare_model("broken", molo::replication::ConflictPolicy::Hook, + [](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) { + throw std::runtime_error("hook bug"); + return 0; + }); + a.local_write("broken", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}"); + molo::replication::OplogEntry remote2 = remote; + remote2.model = "broken"; + remote2.ts = molo::replication::HybridTs{1760000000000LL, 99, "node-b"}; + auto outcome2 = a.apply_remote(remote2); + CHECK(outcome2.detail.find("hook fallback") != std::string::npos, "throw falls back to LWW"); + CHECK(outcome2.remote_won, "LWW fallback picks the higher ts"); + + // Slow hook past the 5s budget falls back to LWW (S7-8). + a.declare_model("slow", molo::replication::ConflictPolicy::Hook, + [](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) { + std::this_thread::sleep_for(std::chrono::milliseconds(5200)); + return 0; + }); + a.local_write("slow", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}"); + molo::replication::OplogEntry remote3 = remote; + remote3.model = "slow"; + auto outcome3 = a.apply_remote(remote3); + CHECK(outcome3.detail.find("hook fallback") != std::string::npos, "5s timeout falls back"); + std::cout << "PASS test_hook_policy_and_fallback" << std::endl; +} + +void test_sync_batches_and_lag() { + namespace fs = std::filesystem; + fs::remove_all(kTmp); + molo::common::FrozenClock clock(1760000000000LL, 1000); + auto obs = std::make_shared(); + molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs); + molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs); + for (int i = 0; i < 600; ++i) { + a.local_write("bulk", "k" + std::to_string(i), molo::replication::OplogOp::Upsert, 1, "{}"); + } + ReplicatorSyncPeer peer_a(a); + molo::replication::SyncSession session(b, peer_a, obs); + auto s1 = session.round_trip(); + CHECK(s1.pulled == 500, "first round honors the 500-entry batch cap"); + CHECK(s1.lag == 100, "lag counts the remaining 100"); + auto s2 = session.round_trip(); + CHECK(s2.pulled == 100, "second round completes the catch-up"); + CHECK(s2.lag == 0, "lag zero when caught up"); + CHECK(b.current("bulk", "k599").has_value(), "tail entry arrived"); + CHECK(obs->gauges().count("molo_replication_sync_lag") == 1, "lag gauge exported"); + + // Deletes replicate as tombstones. + a.local_write("bulk", "k0", molo::replication::OplogOp::Delete, 2, "{}"); + session.round_trip(); + auto tomb = b.current("bulk", "k0"); + CHECK(tomb.has_value() && tomb->op == molo::replication::OplogOp::Delete, "delete replicates"); + std::cout << "PASS test_sync_batches_and_lag" << std::endl; +} + +void test_vectors_clocks_and_codecs() { + molo::replication::VersionVector v1, v2; + v1.update("a", 5); + v1.update("b", 3); + v2.update("a", 5); + v2.update("b", 4); + CHECK(v1.compare(v2) == molo::replication::VectorOrder::Before, "vector before"); + CHECK(v2.compare(v1) == molo::replication::VectorOrder::After, "vector after"); + v1.update("c", 1); + CHECK(v1.compare(v2) == molo::replication::VectorOrder::Concurrent, "vector concurrent"); + v1.merge(v2); + CHECK(v1.get("b") == 4 && v1.get("c") == 1, "merge takes maxima"); + molo::replication::VersionVector back; + CHECK(molo::replication::VersionVector::decode(v1.encode(), back), "vector codec roundtrip"); + CHECK(back.compare(v1) == molo::replication::VectorOrder::Equal, "vector codec preserves"); + + molo::common::FrozenClock clock(1000, 1000); + molo::replication::HybridClock hybrid("n1", clock); + auto t1 = hybrid.now(); + auto t2 = hybrid.now(); + CHECK(t2.compare(t1) > 0, "same-ms counter orders"); + clock.advance(10); + auto t3 = hybrid.now(); + CHECK(t3.wall_ms == 1010 && t3.counter == 0, "wall advance resets the counter"); + molo::replication::HybridTs parsed; + CHECK(molo::replication::HybridTs::decode(t3.encode(), parsed) && parsed == t3, "ts codec"); + + molo::replication::OplogEntry entry; + entry.seq = 7; + entry.instance = "n1"; + entry.ts = t3; + entry.model = "m"; + entry.key = "k"; + entry.op = molo::replication::OplogOp::Upsert; + entry.version = 2; + entry.data = "{\"x\":1}"; + molo::replication::OplogEntry entry_back; + CHECK(molo::replication::OplogEntry::decode(entry.encode(), entry_back), "oplog codec"); + CHECK(entry_back.seq == 7 && entry_back.version == 2 && entry_back.data == "{\"x\":1}", + "oplog fields survive"); + CHECK(!molo::replication::OplogEntry::decode("nope", entry_back), "oplog rejects garbage"); + + molo::replication::ConflictResolver resolver(clock, molo::common::null_observer()); + CHECK(resolver.policy_for("undeclared") == molo::replication::ConflictPolicy::Optimistic, + "default policy is optimistic"); + std::cout << "PASS test_vectors_clocks_and_codecs" << std::endl; +} + +} // namespace + +int main() { + test_optimistic_conflict_and_recovery(); + test_lww_cache_conflict(); + test_hook_policy_and_fallback(); + test_sync_batches_and_lag(); + test_vectors_clocks_and_codecs(); + if (failures == 0) { + std::cout << "ALL REPLICATION TESTS PASSED" << std::endl; + return 0; + } + std::cout << failures << " FAILURES" << std::endl; + return 1; +} diff --git a/tests/src/supervisor/test_scheduler.cpp b/tests/src/supervisor/test_scheduler.cpp new file mode 100644 index 0000000..a02125f --- /dev/null +++ b/tests/src/supervisor/test_scheduler.cpp @@ -0,0 +1,258 @@ +// V7-4: scheduler intervals, cron, leader failover without duplicates. +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/kv_store.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/tasks/cron.hpp" +#include "supervisor/tasks/scheduler.hpp" +#include "supervisor/tasks/task_queue.hpp" +#include "supervisor/tasks/task_registry.hpp" + +namespace { + +int failures = 0; + +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ + ++failures; \ + } \ + } while (0) + +constexpr std::int64_t kT0 = 1760000000000LL; // fixed frozen origin + +struct Harness { + molo::common::FrozenClock clock{kT0, 1000}; + molo::common::InMemoryKvStore kv{clock}; + molo::tasks::TaskQueue queue{molo::tasks::QueueOptions{}, clock, + molo::common::null_observer()}; + molo::tasks::TaskRegistry registry; + std::atomic runs{0}; + + Harness() { + registry.register_handler("tick", [&] { + struct H : molo::tasks::TaskHandler { + std::atomic* runs; + molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override { + ++(*runs); + return molo::tasks::TaskOutcome::Success; + } + }; + auto h = std::make_unique(); + h->runs = &runs; + return h; + }); + } + + int drain() { + int n = 0; + while (queue.run_one(registry, 0)) { + ++n; + } + return n; + } + + // Advances the frozen clock one second per scheduler tick, like the + // production 1s tick loop. Time jumps longer than the leader TTL would + // (correctly) drop leadership, so tests step through time instead. + void step(molo::tasks::Scheduler& sched, int seconds) { + for (int i = 0; i < seconds; ++i) { + clock.advance(1000); + sched.tick_once(); + } + } +}; + +void test_interval_fires_per_tick() { + Harness h; + molo::tasks::SchedulerOptions options; + options.instance_id = "sched-a"; + molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options); + + molo::tasks::ScheduleEntry entry; + entry.key = "every-minute"; + entry.task_name = "tick"; + entry.kind = molo::tasks::ScheduleKind::Interval; + entry.interval_secs = 60; + sched.put(entry); + + sched.tick_once(); + CHECK(sched.is_leader(), "single scheduler wins the lease"); + CHECK(h.queue.stats().queued == 0, "no fire before the interval"); + for (int i = 0; i < 5; ++i) { + h.step(sched, 60); + CHECK(h.drain() == 1, "one run per 60s tick"); + } + CHECK(h.runs.load() == 5, "frozen clock advances 5 ticks, 5 runs"); + sched.tick_once(); + CHECK(h.drain() == 0, "no fire without time passing"); + std::cout << "PASS test_interval_fires_per_tick" << std::endl; +} + +void test_leader_failover_no_duplicates() { + Harness h; + molo::tasks::SchedulerOptions a_opts; + a_opts.instance_id = "sched-a"; + a_opts.ttl_secs = 2; + a_opts.renew_secs = 1; + molo::tasks::Scheduler a(h.queue, h.kv, h.clock, molo::common::null_observer(), a_opts); + molo::tasks::SchedulerOptions b_opts; + b_opts.instance_id = "sched-b"; + b_opts.ttl_secs = 2; + b_opts.renew_secs = 1; + molo::tasks::Scheduler b(h.queue, h.kv, h.clock, molo::common::null_observer(), b_opts); + + molo::tasks::ScheduleEntry entry; + entry.key = "job"; + entry.task_name = "tick"; + entry.kind = molo::tasks::ScheduleKind::Interval; + entry.interval_secs = 60; + a.put(entry); + b.put(entry); + + a.tick_once(); + b.tick_once(); + CHECK(a.is_leader() && !b.is_leader(), "first scheduler wins the lease"); + + for (int i = 0; i < 60; ++i) { + h.clock.advance(1000); + a.tick_once(); // leader fires at the window edge + b.tick_once(); // follower stays passive + } + CHECK(h.drain() == 1, "exactly one fire with two schedulers"); + + // Kill the leader: it stops ticking, its lease expires, the follower + // takes over within the TTL (V7-4 bound is 15s; TTL here is 2s). + for (int i = 0; i < 3; ++i) { + h.clock.advance(1000); + b.tick_once(); + } + CHECK(b.is_leader(), "follower takes over after lease expiry"); + const long runs_before = h.runs.load(); + b.tick_once(); // takeover tick must not refire the shared last-fire window + CHECK(h.drain() == 0, "no duplicate run after failover"); + CHECK(h.runs.load() == runs_before, "run count unchanged by failover"); + + h.step(b, 60); + CHECK(h.drain() == 1, "new leader fires the next window"); + std::cout << "PASS test_leader_failover_no_duplicates" << std::endl; +} + +void test_cron_entries() { + // Unit checks on the expression engine first. + molo::tasks::CronExpr expr; + std::string error; + CHECK(molo::tasks::CronExpr::parse("* * * * *", expr, error), "every-minute parses"); + CHECK(molo::tasks::CronExpr::parse("*/15 9-17 * * 1-5", expr, error), "business hours parse"); + CHECK(!molo::tasks::CronExpr::parse("* * * *", expr, error), "4 fields rejected"); + CHECK(!molo::tasks::CronExpr::parse("61 * * * *", expr, error), "minute 61 rejected"); + CHECK(molo::tasks::CronExpr::parse("0 0 * * 7", expr, error), "dow 7 means sunday"); + + // 2026-10-23 is a Friday. 12:00 UTC that day: + const std::int64_t friday_noon = 1792756800000LL; // 2026-10-23 12:00 UTC, a Friday + molo::tasks::CronExpr noon; + CHECK(molo::tasks::CronExpr::parse("0 12 * * *", noon, error), "noon parses"); + CHECK(noon.matches(friday_noon + 30000), "noon matches inside the minute"); + CHECK(!noon.matches(friday_noon + 60000), "noon misses the next minute"); + CHECK(noon.next_after(friday_noon) == friday_noon + 86400000LL, "next noon is tomorrow"); + + // End to end through the scheduler with a frozen clock. + Harness h; + h.clock.set(friday_noon - 60000, 1000); // 11:59 + molo::tasks::SchedulerOptions options; + options.instance_id = "sched-cron"; + molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options); + molo::tasks::ScheduleEntry entry; + entry.key = "noon-job"; + entry.task_name = "tick"; + entry.kind = molo::tasks::ScheduleKind::Cron; + entry.cron = "0 12 * * *"; + sched.put(entry); + sched.tick_once(); + CHECK(h.drain() == 0, "no fire at 11:59"); + h.step(sched, 60); // step into 12:00 + CHECK(h.drain() == 1, "fires at 12:00"); + sched.tick_once(); + CHECK(h.drain() == 0, "fires once per minute"); + std::cout << "PASS test_cron_entries" << std::endl; +} + +void test_once_and_missed_policy() { + Harness h; + auto obs = std::make_shared(); + molo::tasks::SchedulerOptions options; + options.instance_id = "sched-once"; + molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, obs, options); + + molo::tasks::ScheduleEntry skip; + skip.key = "once-skip"; + skip.task_name = "tick"; + skip.kind = molo::tasks::ScheduleKind::Once; + skip.at_ms = kT0; + skip.missed = molo::tasks::MissedPolicy::Skip; + molo::tasks::ScheduleEntry run; + run.key = "once-run"; + run.task_name = "tick"; + run.kind = molo::tasks::ScheduleKind::Once; + run.at_ms = kT0; + run.missed = molo::tasks::MissedPolicy::RunOnce; + sched.put(skip); + sched.put(run); + + h.clock.advance(2 * 3600 * 1000); // miss both by 2h + sched.tick_once(); + CHECK(h.drain() == 1, "run-once fires, skip drops"); + CHECK(obs->event_count("tasks.schedule.missed") == 1, "miss recorded"); + sched.tick_once(); + CHECK(h.drain() == 0, "once entries never refire"); + std::cout << "PASS test_once_and_missed_policy" << std::endl; +} + +void test_interval_floor_and_bad_cron() { + Harness h; + molo::tasks::SchedulerOptions options; + options.instance_id = "sched-floor"; + molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options); + molo::tasks::ScheduleEntry fast; + fast.key = "fast"; + fast.task_name = "tick"; + fast.kind = molo::tasks::ScheduleKind::Interval; + fast.interval_secs = 5; + sched.put(fast); + CHECK(sched.entries()[0].interval_secs == 60, "interval floored at 60s"); + + molo::tasks::ScheduleEntry bad; + bad.key = "bad"; + bad.task_name = "tick"; + bad.kind = molo::tasks::ScheduleKind::Cron; + bad.cron = "not a cron"; + bool threw = false; + try { + sched.put(bad); + } catch (const molo::tasks::TaskError&) { + threw = true; + } + CHECK(threw, "bad cron rejected at put"); + CHECK(sched.remove("fast"), "remove works"); + CHECK(!sched.remove("fast"), "double remove fails"); + std::cout << "PASS test_interval_floor_and_bad_cron" << std::endl; +} + +} // namespace + +int main() { + test_interval_fires_per_tick(); + test_leader_failover_no_duplicates(); + test_cron_entries(); + test_once_and_missed_policy(); + test_interval_floor_and_bad_cron(); + if (failures == 0) { + std::cout << "ALL SCHEDULER TESTS PASSED" << std::endl; + return 0; + } + std::cout << failures << " FAILURES" << std::endl; + return 1; +} diff --git a/tests/src/supervisor/test_supervisor.cpp b/tests/src/supervisor/test_supervisor.cpp new file mode 100644 index 0000000..d375e31 --- /dev/null +++ b/tests/src/supervisor/test_supervisor.cpp @@ -0,0 +1,262 @@ +// V7-1: supervisor restarts, intensity caps, strategies, health, status. +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/supervisor/supervisor.hpp" +#include "supervisor/supervisor/supervisor_error.hpp" +#include "supervisor/supervisor/worker_group.hpp" + +namespace { + +int failures = 0; + +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ + ++failures; \ + } \ + } while (0) + +bool wait_for(const std::function& pred, long timeout_ms) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + while (std::chrono::steady_clock::now() < deadline) { + if (pred()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return pred(); +} + +int run_until_stopped(const std::atomic& stop) { + while (!stop.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return 0; +} + +void test_three_workers_restart_and_status() { + molo::common::SystemClock clock; + auto obs = std::make_shared(); + molo::supervisor::Supervisor sup(clock, obs); + + molo::supervisor::GroupSpec spec; + spec.name = "web"; + spec.size = 3; + spec.monitor_ms = 10; + spec.backoff_min_ms = 10; + spec.backoff_max_ms = 50; + const std::size_t idx = sup.add_group(spec, run_until_stopped); + CHECK(idx == 0, "first group index is 0"); + sup.start_all(); + + CHECK(wait_for([&] { + const std::string status = sup.status_json(); + return status.find("\"state\":\"running\"") != std::string::npos; + }, 2000), "workers reach running"); + + const std::string status = sup.status_json(); + CHECK(status.find("\"name\":\"web\"") != std::string::npos, "status names the group"); + CHECK(status.find("\"phase\":\"running\"") != std::string::npos, "status shows running phase"); + CHECK(status.size() <= 1048576, "status under the 1 MiB cap"); + CHECK(obs->counter_value("molo_supervisor_worker_starts_total") >= 3, "3 starts counted"); + + sup.shutdown(2000); + CHECK(sup.phase() == molo::supervisor::ShutdownPhase::Stopped, "shutdown reaches stopped"); + std::cout << "PASS test_three_workers_restart_and_status" << std::endl; +} + +void test_crash_restart_then_fatal() { + molo::common::SystemClock clock; + auto obs = std::make_shared(); + molo::supervisor::Supervisor sup(clock, obs); + + std::atomic runs{0}; + auto crasher = [&](const std::atomic&) { + ++runs; + return 1; // always crashes + }; + molo::supervisor::GroupSpec spec; + spec.name = "crasher"; + spec.size = 1; + spec.monitor_ms = 5; + spec.backoff_min_ms = 10; + spec.backoff_max_ms = 20; + spec.restart.policy = molo::supervisor::RestartPolicy::OnFailure; + spec.restart.max_restarts = 2; + spec.restart.window_secs = 60; + sup.add_group(spec, crasher); + + const auto t0 = std::chrono::steady_clock::now(); + sup.start_all(); + // Restart latency: the first restart must happen within 2s (V7-1). + CHECK(wait_for([&] { return runs.load() >= 2; }, 2000), "crasher restarted under 2s"); + const auto restarted_ms = + std::chrono::duration_cast(std::chrono::steady_clock::now() - t0) + .count(); + CHECK(restarted_ms < 2000, "restart latency under 2s"); + // Intensity cap: initial + 2 restarts, then fatal. + CHECK(wait_for([&] { return sup.status_json().find("\"state\":\"fatal\"") != std::string::npos; }, + 5000), + "crasher goes fatal after the cap"); + CHECK(runs.load() == 3, "exactly initial plus max_restarts runs"); + CHECK(obs->counter_value("molo_supervisor_worker_fatal_total") == 1, "fatal counted"); + sup.shutdown(1000); + std::cout << "PASS test_crash_restart_then_fatal" << std::endl; +} + +void test_explicit_restart_clears_fatal() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::supervisor::WorkerGroup group( + [] { + molo::supervisor::GroupSpec spec; + spec.name = "flaky"; + spec.monitor_ms = 5; + spec.backoff_min_ms = 5; + spec.backoff_max_ms = 10; + spec.restart.max_restarts = 1; + return spec; + }(), + [](const std::atomic&) { return 1; }, clock, obs); + group.start(); + CHECK(wait_for([&] { return group.fatal(); }, 5000), "group goes fatal"); + group.restart_child(0); // explicit restart clears Fatal + CHECK(wait_for([&] { return group.fatal(); }, 5000), "fatal again after crash loop"); + group.stop(); + std::cout << "PASS test_explicit_restart_clears_fatal" << std::endl; +} + +void test_strategies_recover() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + const molo::supervisor::GroupStrategy strategies[] = { + molo::supervisor::GroupStrategy::OneForOne, + molo::supervisor::GroupStrategy::OneForAll, + molo::supervisor::GroupStrategy::RestForOne, + }; + for (auto strategy : strategies) { + std::atomic failures_left{1}; + auto sometimes_fails = [&](const std::atomic& stop) { + if (failures_left.load() > 0 && failures_left.fetch_sub(1) == 1) { + return 1; // exactly one crash across the whole group + } + return run_until_stopped(stop); + }; + molo::supervisor::GroupSpec spec; + spec.name = "strat"; + spec.strategy = strategy; + spec.size = 2; + spec.monitor_ms = 5; + spec.backoff_min_ms = 5; + spec.backoff_max_ms = 10; + molo::supervisor::WorkerGroup group(spec, sometimes_fails, clock, obs); + group.start(); + CHECK(wait_for([&] { + auto states = group.states(); + return states.size() == 2 && states[0] == molo::supervisor::WorkerState::Running && + states[1] == molo::supervisor::WorkerState::Running; + }, 5000), "strategy recovers to all running: " + molo::supervisor::to_string(strategy)); + CHECK(!group.fatal(), "no fatal after single failure"); + group.stop(); + } + std::cout << "PASS test_strategies_recover" << std::endl; +} + +void test_health_probe_triggers_restart() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::supervisor::GroupSpec spec; + spec.name = "sick"; + spec.monitor_ms = 5; + spec.backoff_min_ms = 5; + spec.backoff_max_ms = 10; + spec.restart.max_restarts = 1; + spec.health.interval_ms = 20; + spec.health.max_failures = 2; + spec.probe = [] { return false; }; // always sick + std::atomic runs{0}; + auto worker = [&](const std::atomic& stop) { + ++runs; + return run_until_stopped(stop); + }; + molo::supervisor::WorkerGroup group(spec, worker, clock, obs); + group.start(); + CHECK(wait_for([&] { return group.fatal(); }, 8000), "sick worker restarts then fatal"); + CHECK(runs.load() >= 2, "unhealthy worker was restarted"); + group.stop(); + + // Healthy probe: worker stays up. + molo::supervisor::GroupSpec good = spec; + good.name = "healthy"; + good.probe = [] { return true; }; + molo::supervisor::WorkerGroup group2(good, run_until_stopped, clock, obs); + group2.start(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + auto states = group2.states(); + CHECK(states[0] == molo::supervisor::WorkerState::Running, "healthy worker stays running"); + CHECK(!group2.fatal(), "healthy group not fatal"); + group2.stop(); + std::cout << "PASS test_health_probe_triggers_restart" << std::endl; +} + +void test_scale_and_restart_group() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::supervisor::Supervisor sup(clock, obs); + molo::supervisor::GroupSpec spec; + spec.name = "elastic"; + spec.monitor_ms = 5; + spec.backoff_min_ms = 5; + const std::size_t idx = sup.add_group(spec, run_until_stopped); + sup.start_all(); + sup.scale_group(idx, 4); + CHECK(wait_for([&] { + const std::string status = sup.status_json(); + std::size_t count = 0, pos = 0; + while ((pos = status.find("\"state\":\"running\"", pos)) != std::string::npos) { + ++count; + ++pos; + } + return count == 4; + }, 3000), "scales to 4 running"); + sup.scale_group(idx, 1); + CHECK(wait_for([&] { return sup.status_json().find("elastic#1") == std::string::npos; }, 3000), + "scales down to 1"); + bool threw = false; + try { + sup.scale_group(idx, 65); + } catch (const molo::supervisor::SupervisorError&) { + threw = true; + } + CHECK(threw, "scale past 64 throws"); + sup.restart_group(idx); + CHECK(wait_for([&] { + return sup.status_json().find("\"state\":\"running\"") != std::string::npos; + }, 3000), "group restarts"); + sup.shutdown(2000); + std::cout << "PASS test_scale_and_restart_group" << std::endl; +} + +} // namespace + +int main() { + test_three_workers_restart_and_status(); + test_crash_restart_then_fatal(); + test_explicit_restart_clears_fatal(); + test_strategies_recover(); + test_health_probe_triggers_restart(); + test_scale_and_restart_group(); + if (failures == 0) { + std::cout << "ALL SUPERVISOR TESTS PASSED" << std::endl; + return 0; + } + std::cout << failures << " FAILURES" << std::endl; + return 1; +} diff --git a/tests/src/supervisor/test_tasks.cpp b/tests/src/supervisor/test_tasks.cpp new file mode 100644 index 0000000..8919c41 --- /dev/null +++ b/tests/src/supervisor/test_tasks.cpp @@ -0,0 +1,348 @@ +// V7-2, V7-3: task throughput, retries, backoff, dead letters, timeouts. +#include +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/tasks/retry_policy.hpp" +#include "supervisor/tasks/task_envelope.hpp" +#include "supervisor/tasks/task_queue.hpp" +#include "supervisor/tasks/task_registry.hpp" + +namespace { + +int failures = 0; + +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ + ++failures; \ + } \ + } while (0) + +class CountingHandler : public molo::tasks::TaskHandler { + public: + explicit CountingHandler(std::atomic& count) : count_(count) {} + molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override { + ++count_; + return molo::tasks::TaskOutcome::Success; + } + + private: + std::atomic& count_; +}; + +void test_throughput_off_loop() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::tasks::QueueOptions options; + molo::tasks::TaskQueue queue(options, clock, obs); + molo::tasks::TaskRegistry registry; + std::atomic done{0}; + registry.register_handler("count", [&] { return std::make_unique(done); }); + + // Baseline scheduling latency of this thread. + auto probe = [] { + const auto t0 = std::chrono::steady_clock::now(); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + return std::chrono::duration_cast(std::chrono::steady_clock::now() - + t0) + .count(); + }; + long baseline = 0; + for (int i = 0; i < 10; ++i) { + baseline += probe(); + } + baseline /= 10; + + std::atomic stop{false}; + std::vector workers; + for (int i = 0; i < 4; ++i) { + workers.emplace_back([&] { queue.run_worker([&] { return stop.load(); }, registry); }); + } + const int total = 500; + const auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < total; ++i) { + molo::tasks::TaskEnvelope env; + env.name = "count"; + queue.enqueue(std::move(env)); + } + const auto enqueue_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0) + .count(); + long loaded = 0; + for (int i = 0; i < 10; ++i) { + loaded += probe(); + } + loaded /= 10; + while (done.load() < total) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + stop = true; + for (auto& w : workers) { + w.join(); + } + CHECK(done.load() == total, "all 500 tasks succeed"); + CHECK(queue.stats().succeeded_total == total, "stats count all successes"); + CHECK(enqueue_ms < 2000, "enqueue never blocks on execution"); + CHECK(loaded < std::max(50000, baseline * 10), "producer thread stays responsive"); + std::cout << "PASS test_throughput_off_loop" << std::endl; +} + +void test_retry_backoff_schedule() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::tasks::QueueOptions options; + options.lease_ms = 60000; + options.retry = molo::tasks::RetrySpec{100, 1000, 0, 5}; // base 100ms, no jitter + molo::tasks::TaskQueue queue(options, clock, obs); + molo::tasks::TaskRegistry registry; + std::vector runs; + std::mutex runs_mutex; + registry.register_handler("flaky", [&] { + struct H : molo::tasks::TaskHandler { + std::vector* runs; + std::mutex* mutex; + molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override { + std::lock_guard lock(*mutex); + runs->push_back(ctx.clock.steady_ms()); + return runs->size() < 3 ? molo::tasks::TaskOutcome::RetryableFailure + : molo::tasks::TaskOutcome::Success; + } + }; + auto h = std::make_unique(); + h->runs = &runs; + h->mutex = &runs_mutex; + return h; + }); + + molo::tasks::TaskEnvelope env; + env.name = "flaky"; + env.max_retries = 5; + const std::string id = queue.enqueue(std::move(env)); + for (int i = 0; i < 3; ++i) { + CHECK(queue.run_one(registry, 5000), "run_one executes the retry"); + } + CHECK(runs.size() == 3, "handler ran 3 times"); + if (runs.size() == 3) { + const long d1 = static_cast(runs[1] - runs[0]); + const long d2 = static_cast(runs[2] - runs[1]); + // delay(0)=100ms, delay(1)=200ms, within 20 percent plus 40ms slack. + CHECK(d1 >= 80 && d1 <= 100 * 1.2 + 40, "first backoff near 100ms"); + CHECK(d2 >= 160 && d2 <= 200 * 1.2 + 40, "second backoff near 200ms"); + } + CHECK(queue.stats().succeeded_total == 1, "eventual success counted"); + CHECK(queue.stats().retried_total == 2, "two retries counted"); + CHECK(!queue.get(id).has_value(), "acked task leaves the queue"); + std::cout << "PASS test_retry_backoff_schedule" << std::endl; +} + +void test_permanent_failure_dead_letters() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs); + molo::tasks::TaskRegistry registry; + registry.register_handler("doomed", [] { + struct H : molo::tasks::TaskHandler { + molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override { + return molo::tasks::TaskOutcome::PermanentFailure; + } + }; + return std::make_unique(); + }); + molo::tasks::TaskEnvelope env; + env.name = "doomed"; + const std::string id = queue.enqueue(std::move(env)); + CHECK(queue.run_one(registry, 1000), "doomed task executes"); + CHECK(queue.dead_letters().size() == 1, "permanent failure dead-letters"); + auto entries = queue.dead_letters().list(10); + CHECK(entries[0].envelope.id == id, "dead entry keeps the task id"); + CHECK(entries[0].reason == "handler-permanent", "dead entry keeps the reason"); + + // Unknown handler names dead-letter too, never crash the worker. + molo::tasks::TaskEnvelope ghost; + ghost.name = "no.such.handler"; + queue.enqueue(std::move(ghost)); + CHECK(queue.run_one(registry, 1000), "ghost task executes"); + CHECK(queue.dead_letters().size() == 2, "unknown handler dead-letters"); + std::cout << "PASS test_permanent_failure_dead_letters" << std::endl; +} + +void test_idempotency_keys() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs); + molo::tasks::TaskRegistry registry; + std::atomic done{0}; + registry.register_handler("once", [&] { return std::make_unique(done); }); + + molo::tasks::TaskEnvelope a; + a.name = "once"; + a.idempotency_key = "order-123"; + const std::string first = queue.enqueue(a); + molo::tasks::TaskEnvelope b; + b.name = "once"; + b.idempotency_key = "order-123"; + const std::string second = queue.enqueue(std::move(b)); + CHECK(first == second, "duplicate key returns the existing id"); + CHECK(queue.stats().queued == 1, "duplicate does not re-enqueue"); + CHECK(queue.run_one(registry, 1000), "task runs"); + molo::tasks::TaskEnvelope c; + c.name = "once"; + c.idempotency_key = "order-123"; + CHECK(queue.enqueue(std::move(c)) == first, "key reserved after completion"); + CHECK(done.load() == 1, "handler ran exactly once"); + std::cout << "PASS test_idempotency_keys" << std::endl; +} + +void test_caps_and_lease_expiry() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::tasks::QueueOptions options; + options.max_depth = 2; + options.lease_ms = 60; + molo::tasks::TaskQueue queue(options, clock, obs); + + molo::tasks::TaskEnvelope big; + big.name = "big"; + big.payload = std::string(1048576 + 1, 'x'); + bool threw = false; + try { + queue.enqueue(std::move(big)); + } catch (const molo::tasks::TaskError&) { + threw = true; + } + CHECK(threw, "oversize payload rejected"); + + molo::tasks::TaskEnvelope t1; + t1.name = "t"; + queue.enqueue(std::move(t1)); + molo::tasks::TaskEnvelope t2; + t2.name = "t"; + queue.enqueue(std::move(t2)); + threw = false; + try { + molo::tasks::TaskEnvelope t3; + t3.name = "t"; + queue.enqueue(std::move(t3)); + } catch (const molo::tasks::TaskError&) { + threw = true; + } + CHECK(threw, "depth cap enforced"); + CHECK(!queue.heartbeat("nope"), "unknown heartbeat fails"); + + // Lease expiry on a fresh queue with a single task: the expired lease + // redelivers the same envelope with attempts incremented. + molo::tasks::QueueOptions exp_options; + exp_options.lease_ms = 60; + molo::tasks::TaskQueue exp_queue(exp_options, clock, obs); + molo::tasks::TaskEnvelope solo; + solo.name = "solo"; + exp_queue.enqueue(std::move(solo)); + auto leased = exp_queue.lease(0); + CHECK(leased.has_value(), "leases one task"); + std::this_thread::sleep_for(std::chrono::milliseconds(120)); + auto redelivered = exp_queue.lease(0); + CHECK(redelivered.has_value(), "expired lease redelivers"); + if (redelivered.has_value()) { + CHECK(redelivered->envelope.id == leased->envelope.id, "same task redelivered"); + CHECK(redelivered->envelope.attempts == 1, "expiry increments attempts"); + CHECK(!exp_queue.ack(leased->lease_id), "stale lease cannot ack"); + CHECK(exp_queue.ack(redelivered->lease_id), "fresh lease acks"); + } + std::cout << "PASS test_caps_and_lease_expiry" << std::endl; +} + +void test_hard_timeout_recycles() { + molo::common::SystemClock clock; + auto obs = molo::common::null_observer(); + molo::tasks::QueueOptions options; + options.orphan_grace_ms = 2000; + molo::tasks::TaskQueue queue(options, clock, obs); + molo::tasks::TaskRegistry registry; + std::atomic saw_cancel{false}; + registry.register_handler("slow", [&] { + struct H : molo::tasks::TaskHandler { + std::atomic* saw_cancel; + molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override { + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + if (ctx.cancel_requested.load()) { + saw_cancel->store(true); + } + return molo::tasks::TaskOutcome::Success; + } + }; + auto h = std::make_unique(); + h->saw_cancel = &saw_cancel; + return h; + }); + molo::tasks::TaskEnvelope env; + env.name = "slow"; + env.timeout_soft_ms = 50; + env.timeout_hard_ms = 100; + env.max_retries = 0; // one strike, then dead + queue.enqueue(std::move(env)); + const auto t0 = std::chrono::steady_clock::now(); + CHECK(queue.run_one(registry, 1000), "slow task executes"); + const auto took = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0) + .count(); + CHECK(took < 2000, "hard timeout bounds the run"); + CHECK(queue.recycle_requested(), "worker recycle requested"); + CHECK(saw_cancel.load(), "soft timeout sets the cancel flag"); + CHECK(queue.dead_letters().size() == 1, "exhausted task dead-letters"); + auto entries = queue.dead_letters().list(10); + CHECK(entries[0].reason == "hard-timeout", "dead reason names the hard timeout"); + std::cout << "PASS test_hard_timeout_recycles" << std::endl; +} + +void test_envelope_json_roundtrip() { + molo::tasks::TaskEnvelope env; + env.id = "abc"; + env.name = "billing.charge"; + env.payload = "{\"amount\":42}"; + env.attempts = 2; + env.max_retries = 5; + env.run_at_ms = 1700000000000LL; + env.timeout_soft_ms = 30000; + env.timeout_hard_ms = 60000; + env.idempotency_key = "k"; + env.enqueued_at_ms = 1700000001000LL; + const std::string json = env.encode(); + molo::tasks::TaskEnvelope back; + CHECK(molo::tasks::TaskEnvelope::decode(json, back), "envelope decodes"); + CHECK(back.id == "abc" && back.name == "billing.charge", "identity survives"); + CHECK(back.payload == "{\"amount\":42}", "payload survives"); + CHECK(back.attempts == 2 && back.max_retries == 5, "counters survive"); + CHECK(back.run_at_ms.has_value() && *back.run_at_ms == 1700000000000LL, "run_at survives"); + CHECK(back.idempotency_key == "k", "key survives"); + CHECK(!molo::tasks::TaskEnvelope::decode("{oops", back), "garbage rejected"); + CHECK(!molo::tasks::TaskEnvelope::decode("{\"id\":1}", back), "typeless rejected"); + CHECK(!molo::tasks::TaskEnvelope::generate_id().empty(), "ids generated"); + CHECK(molo::tasks::TaskEnvelope::generate_id() != molo::tasks::TaskEnvelope::generate_id(), + "ids unique"); + std::cout << "PASS test_envelope_json_roundtrip" << std::endl; +} + +} // namespace + +int main() { + test_throughput_off_loop(); + test_retry_backoff_schedule(); + test_permanent_failure_dead_letters(); + test_idempotency_keys(); + test_caps_and_lease_expiry(); + test_hard_timeout_recycles(); + test_envelope_json_roundtrip(); + if (failures == 0) { + std::cout << "ALL TASK TESTS PASSED" << std::endl; + return 0; + } + std::cout << failures << " FAILURES" << std::endl; + return 1; +} diff --git a/tests/src/supplychain/golden/advisories.json b/tests/src/supplychain/golden/advisories.json new file mode 100644 index 0000000..3ea783c --- /dev/null +++ b/tests/src/supplychain/golden/advisories.json @@ -0,0 +1,18 @@ +{ + "advisories": [ + { + "cvss": 7.5, + "fixed": "5.7.1", + "id": "GHSA-test-0001", + "package": "wolfssl", + "summary": "test advisory with fix below critical" + }, + { + "cvss": 4.2, + "fixed": "", + "id": "GHSA-test-0002", + "package": "moloapp", + "summary": "low test advisory without fix" + } + ] +} diff --git a/tests/src/supplychain/golden/provenance.json b/tests/src/supplychain/golden/provenance.json new file mode 100644 index 0000000..f0aa290 --- /dev/null +++ b/tests/src/supplychain/golden/provenance.json @@ -0,0 +1,50 @@ +{ + "_type": "https://in-toto.io/Statement/v1", + "predicate": { + "buildDefinition": { + "buildType": "https://moloapp.dev/build/v1", + "externalParameters": { + "commit": "9f3c1a2b4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8", + "invocation": "molo release --test", + "materials": [ + "gcc@15.2.0", + "cmake@3.30.0" + ] + }, + "resolvedDependencies": [ + { + "digest": { + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "uri": "sbom.cdx.json" + }, + { + "digest": { + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "uri": "sbom.spdx.json" + } + ] + }, + "runDetails": { + "builder": { + "id": "https://moloapp.dev/builder/test" + } + } + }, + "predicateType": "https://slsa.dev/provenance/v1", + "subject": [ + { + "digest": { + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "name": "sbom.cdx.json" + }, + { + "digest": { + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "name": "sbom.spdx.json" + } + ] +} diff --git a/tests/src/supplychain/golden/sbom.cdx.json b/tests/src/supplychain/golden/sbom.cdx.json new file mode 100644 index 0000000..2be9371 --- /dev/null +++ b/tests/src/supplychain/golden/sbom.cdx.json @@ -0,0 +1,95 @@ +{ + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "moloapp@1.2.3-test", + "hashes": [ + { + "alg": "SHA-256", + "content": "9892f106e6f5ca6500d39edb73913cd9cb0d9ea6b25686bb10e270dec3c9532e" + } + ], + "licenses": [ + { + "license": { + "name": "proprietary" + } + } + ], + "name": "moloapp", + "supplier": { + "name": "moloapp" + }, + "type": "library", + "version": "1.2.3-test" + }, + { + "bom-ref": "wolfssl@5.7.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + ], + "licenses": [ + { + "license": { + "name": "GPL-2.0" + } + } + ], + "name": "wolfssl", + "supplier": { + "name": "wolfSSL Inc." + }, + "type": "library", + "version": "5.7.0" + }, + { + "bom-ref": "zlib@1.3.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + ], + "licenses": [ + { + "license": { + "name": "Zlib" + } + } + ], + "name": "zlib", + "supplier": { + "name": "https://zlib.net" + }, + "type": "library", + "version": "1.3.1" + } + ], + "metadata": { + "component": { + "bom-ref": "moloapp@1.2.3-test", + "name": "moloapp", + "type": "application", + "version": "1.2.3-test" + }, + "timestamp": "2026-01-01T00:00:00Z", + "tools": { + "components": [ + { + "name": "gcc", + "version": "15.2.0" + }, + { + "name": "cmake", + "version": "3.30.0" + } + ] + } + }, + "serialNumber": "urn:uuid:00000000-0000-4000-8000-8682df12a0d3", + "specVersion": "1.6", + "version": 1 +} diff --git a/tests/src/supplychain/golden/sbom.spdx.json b/tests/src/supplychain/golden/sbom.spdx.json new file mode 100644 index 0000000..8a2383b --- /dev/null +++ b/tests/src/supplychain/golden/sbom.spdx.json @@ -0,0 +1,98 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "creationInfo": { + "created": "2026-01-01T00:00:00Z", + "creators": [ + "Tool: moloapp-1.2.3-test" + ] + }, + "dataLicense": "CC0-1.0", + "documentDescribes": [ + "SPDXRef-DOCUMENT" + ], + "documentNamespace": "https://moloapp.dev/sbom/9f3c1a2b4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8", + "name": "moloapp-sbom", + "packages": [ + { + "SPDXID": "SPDXRef-Package-moloapp", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "9892f106e6f5ca6500d39edb73913cd9cb0d9ea6b25686bb10e270dec3c9532e" + } + ], + "downloadLocation": "https://moloapp.dev", + "externalRefs": [ + { + "category": "PACKAGE-MANAGER", + "locator": "https://moloapp.dev", + "referenceType": "purl" + } + ], + "licenseDeclared": "proprietary", + "name": "moloapp", + "supplier": "Organization: moloapp", + "versionInfo": "1.2.3-test" + }, + { + "SPDXID": "SPDXRef-Package-wolfssl", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + ], + "downloadLocation": "https://www.wolfssl.com/download", + "externalRefs": [ + { + "category": "PACKAGE-MANAGER", + "locator": "https://www.wolfssl.com/download", + "referenceType": "purl" + } + ], + "licenseDeclared": "GPL-2.0", + "name": "wolfssl", + "supplier": "Organization: wolfSSL Inc.", + "versionInfo": "5.7.0" + }, + { + "SPDXID": "SPDXRef-Package-zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + ], + "downloadLocation": "https://zlib.net", + "externalRefs": [ + { + "category": "PACKAGE-MANAGER", + "locator": "https://zlib.net", + "referenceType": "purl" + } + ], + "licenseDeclared": "Zlib", + "name": "zlib", + "supplier": "Organization: https://zlib.net", + "versionInfo": "1.3.1" + } + ], + "relationships": [ + { + "relatedSpdxElement": "SPDXRef-Package-moloapp", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DOCUMENT" + }, + { + "relatedSpdxElement": "SPDXRef-Package-wolfssl", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DOCUMENT" + }, + { + "relatedSpdxElement": "SPDXRef-Package-zlib", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DOCUMENT" + } + ], + "spdxVersion": "SPDX-2.3" +} diff --git a/tests/src/supplychain/test_cdx_equiv.cpp b/tests/src/supplychain/test_cdx_equiv.cpp new file mode 100644 index 0000000..cb3f8a1 --- /dev/null +++ b/tests/src/supplychain/test_cdx_equiv.cpp @@ -0,0 +1,145 @@ +// test_cdx_equiv: CycloneDX emission carries the same triples as SPDX. +#include +#include +#include +#include +#include +#include +#include + +#include "src/supplychain/common/Json.hpp" +#include "src/supplychain/policy/DependencyPolicy.hpp" +#include "src/supplychain/policy/WolfPin.hpp" +#include "src/supplychain/sbom/CycloneDxEmitter.hpp" +#include "src/supplychain/sbom/SbomGenerator.hpp" +#include "src/supplychain/sbom/SpdxEmitter.hpp" + +namespace { + +int g_fail = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL line " << __LINE__ << ": " #cond "\n"; \ + ++g_fail; \ + } \ + } while (0) + +const char* kPinSha = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const char* kCommit = "9f3c1a2b4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8"; + +bool read_all(const std::string& path, std::string& text) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) return false; + char chunk[4096]; + std::size_t n = 0; + while ((n = std::fread(chunk, 1, sizeof(chunk), f)) > 0) text.append(chunk, n); + std::fclose(f); + return true; +} + +using Triple = std::tuple; + +std::set spdx_triples(const molo::supply::common::Json::Value& root) { + using molo::supply::common::Json; + std::set out; + const Json::Value* pkgs = Json::find(root, "packages"); + if (pkgs == nullptr || pkgs->type != Json::Value::Type::Array) return out; + for (const auto& p : pkgs->items) { + const Json::Value* name = Json::find(p, "name"); + const Json::Value* ver = Json::find(p, "versionInfo"); + const Json::Value* sums = Json::find(p, "checksums"); + std::string sha; + if (sums != nullptr && sums->type == Json::Value::Type::Array && !sums->items.empty()) { + const Json::Value* v = Json::find(sums->items[0], "checksumValue"); + if (v != nullptr) sha = Json::string_or(*v, ""); + } + out.emplace(Json::string_or(*name, ""), Json::string_or(*ver, ""), sha); + } + return out; +} + +std::set cdx_triples(const molo::supply::common::Json::Value& root) { + using molo::supply::common::Json; + std::set out; + const Json::Value* comps = Json::find(root, "components"); + if (comps == nullptr || comps->type != Json::Value::Type::Array) return out; + for (const auto& p : comps->items) { + const Json::Value* name = Json::find(p, "name"); + const Json::Value* ver = Json::find(p, "version"); + const Json::Value* hashes = Json::find(p, "hashes"); + std::string sha; + if (hashes != nullptr && hashes->type == Json::Value::Type::Array && !hashes->items.empty()) { + const Json::Value* v = Json::find(hashes->items[0], "content"); + if (v != nullptr) sha = Json::string_or(*v, ""); + } + out.emplace(Json::string_or(*name, ""), Json::string_or(*ver, ""), sha); + } + return out; +} + +} // namespace + +int main(int argc, char** argv) { + using namespace molo::supply; + const std::string golden = + (argc > 1 ? std::string(argv[1]) : std::string("tests/src/supplychain/golden")) + + "/sbom.cdx.json"; + const bool regen = std::getenv("MOLO_REGEN_GOLDEN") != nullptr; + + std::string error; + policy::WolfPin::Pin pin{"5.7.0", kPinSha, "https://www.wolfssl.com/download", "GPL-2.0", + "vendor/wolfssl-5.7.0.receipt"}; + policy::DependencyPolicy manifest; + std::string allow = std::string("{\"dependencies\": [{\"name\": \"zlib\", \"version\": " + "\"1.3.1\", \"sha256\": \"") + + kPinSha + "\", \"source\": \"https://zlib.net\", \"license\": \"Zlib\"}]}"; + CHECK(manifest.parse(allow, error)); + std::vector toolchain = {{"gcc", "15.2.0"}, {"cmake", "3.30.0"}}; + sbom::SbomDocument doc; + CHECK(sbom::SbomGenerator::generate(kCommit, toolchain, pin, manifest, "1.2.3-test", + "2026-01-01T00:00:00Z", doc, error)); + + std::string cdx; + CHECK(sbom::CycloneDxEmitter::emit_json(doc, cdx, error)); + CHECK(!cdx.empty() && cdx.back() == '\n'); + CHECK(cdx.find('\r') == std::string::npos); + CHECK(cdx.find("\"bomFormat\": \"CycloneDX\"") != std::string::npos); + CHECK(cdx.find("\"specVersion\": \"1.6\"") != std::string::npos); + CHECK(cdx.find(" \"bomFormat\"") != std::string::npos); // 2-space indent + common::Json::Value cdx_root; + CHECK(common::Json::parse(cdx, cdx_root, error)); + + std::string spdx; + CHECK(sbom::SpdxEmitter::emit_json(doc, spdx, error)); + common::Json::Value spdx_root; + CHECK(common::Json::parse(spdx, spdx_root, error)); + + std::set a = spdx_triples(spdx_root); + std::set b = cdx_triples(cdx_root); + CHECK(a.size() == 3 && b.size() == 3); + CHECK(a == b); // identical (name, version, sha256) triples + + if (regen) { + std::FILE* f = std::fopen(golden.c_str(), "wb"); + CHECK(f != nullptr); + if (f != nullptr) { + CHECK(std::fwrite(cdx.data(), 1, cdx.size(), f) == cdx.size()); + std::fclose(f); + std::cout << "test_cdx_equiv: regenerated " << golden << "\n"; + } + } else { + std::string want; + CHECK(read_all(golden, want)); + if (want != cdx) { + std::size_t at = 0; + while (at < want.size() && at < cdx.size() && want[at] == cdx[at]) ++at; + std::cerr << "FAIL golden mismatch at byte " << at << " want=" << want.size() + << " got=" << cdx.size() << "\n"; + ++g_fail; + } + } + + if (g_fail == 0) std::cout << "test_cdx_equiv: all checks passed\n"; + return g_fail == 0 ? 0 : 1; +} diff --git a/tests/src/supplychain/test_pin.cpp b/tests/src/supplychain/test_pin.cpp new file mode 100644 index 0000000..fd73968 --- /dev/null +++ b/tests/src/supplychain/test_pin.cpp @@ -0,0 +1,105 @@ +// test_pin: wolfSSL pin plus allow-list parsing and checks. +#include +#include +#include + +#include "src/supplychain/policy/DependencyPolicy.hpp" +#include "src/supplychain/policy/WolfPin.hpp" + +namespace { + +int g_fail = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL line " << __LINE__ << ": " #cond "\n"; \ + ++g_fail; \ + } \ + } while (0) + +const char* kPinSha = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +std::string valid_allow() { + return std::string( + "{\"wolfssl\": {\"version\": \"5.7.0\", \"sha256\": \"") + + kPinSha + + "\", \"source\": \"https://www.wolfssl.com/download\", \"license\": \"GPL-2.0\", " + "\"receipt\": \"vendor/wolfssl-5.7.0.receipt\"}, \"dependencies\": [{\"name\": " + "\"zlib\", \"version\": \"1.3.1\", \"sha256\": \"" + + kPinSha + + "\", \"source\": \"https://zlib.net\", \"license\": \"Zlib\"}]}"; +} + +} // namespace + +int main() { + using molo::supply::policy::DependencyPolicy; + using molo::supply::policy::WolfPin; + + std::string error; + WolfPin::Pin pin; + CHECK(WolfPin::parse_pin(valid_allow(), pin, error)); + CHECK(error.empty()); + CHECK(pin.version == "5.7.0"); + CHECK(pin.sha256 == kPinSha); + CHECK(pin.source == "https://www.wolfssl.com/download"); + CHECK(pin.license == "GPL-2.0"); + + CHECK(WolfPin::check(pin, "5.7.0", kPinSha) == WolfPin::Check::Ok); + CHECK(WolfPin::check(pin, "5.6.0", kPinSha) == WolfPin::Check::VersionMismatch); + CHECK(WolfPin::check(pin, "5.7.0", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") == + WolfPin::Check::HashMismatch); + CHECK(std::string(WolfPin::to_string(WolfPin::Check::Ok)) == "ok"); + + DependencyPolicy manifest; + CHECK(manifest.parse(valid_allow(), error)); + CHECK(manifest.size() == 1); + const DependencyPolicy::Entry* zlib = manifest.find("zlib"); + CHECK(zlib != nullptr && zlib->version == "1.3.1"); + CHECK(manifest.find("nope") == nullptr); + + // wolfssl-only file has zero generic entries but still parses. + DependencyPolicy wolf_only; + CHECK(wolf_only.parse("{\"wolfssl\": {\"version\": \"5.7.0\"}}", error)); + CHECK(wolf_only.size() == 0); + + // Rejections. + WolfPin::Pin bad; + CHECK(!WolfPin::parse_pin("{\"wolfssl\": {\"version\": \"5.7.0\"}}", bad, error)); + CHECK(!WolfPin::parse_pin("{\"wolfssl\": {\"version\": \"5.7.0\", \"sha256\": \"xyz\", " + "\"source\": \"s\", \"license\": \"l\", \"receipt\": \"r\"}}", + bad, error)); + CHECK(!WolfPin::parse_pin("{\"dependencies\": []}", bad, error)); + CHECK(!WolfPin::parse_pin("{oops", bad, error)); + std::string huge(70000, 'x'); + CHECK(!WolfPin::parse_pin(huge, bad, error)); + + DependencyPolicy over; + std::string many = "{\"dependencies\": ["; + for (int i = 0; i < 65; ++i) { + if (i > 0) many += ","; + many += "{\"name\": \"d" + std::to_string(i) + "\", \"version\": \"1\", \"sha256\": \"" + + std::string(kPinSha) + "\", \"source\": \"s\", \"license\": \"l\"}"; + } + many += "]}"; + CHECK(!over.parse(many, error)); + + // load_file roundtrip through a temp file. + const std::string tmp = "/tmp/supply_test_pin_allow.json"; + { + std::FILE* f = std::fopen(tmp.c_str(), "wb"); + CHECK(f != nullptr); + std::string body = valid_allow(); + CHECK(std::fwrite(body.data(), 1, body.size(), f) == body.size()); + std::fclose(f); + } + WolfPin::Pin from_file; + CHECK(WolfPin::load_file(tmp, from_file, error)); + CHECK(from_file.version == "5.7.0"); + CHECK(!WolfPin::load_file("/tmp/supply_test_pin_missing.json", from_file, error)); + std::remove(tmp.c_str()); + + if (g_fail == 0) std::cout << "test_pin: all checks passed\n"; + return g_fail == 0 ? 0 : 1; +} diff --git a/tests/src/supplychain/test_provenance.cpp b/tests/src/supplychain/test_provenance.cpp new file mode 100644 index 0000000..267318c --- /dev/null +++ b/tests/src/supplychain/test_provenance.cpp @@ -0,0 +1,112 @@ +// test_provenance: SLSA predicate plus in-toto statement wrapping. +#include +#include +#include +#include +#include + +#include "src/supplychain/attest/Attestor.hpp" +#include "src/supplychain/attest/ProvenanceBuilder.hpp" +#include "src/supplychain/caps.hpp" +#include "src/supplychain/common/Json.hpp" + +namespace { + +int g_fail = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL line " << __LINE__ << ": " #cond "\n"; \ + ++g_fail; \ + } \ + } while (0) + +const char* kShaA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const char* kShaB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + +bool read_all(const std::string& path, std::string& text) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) return false; + char chunk[4096]; + std::size_t n = 0; + while ((n = std::fread(chunk, 1, sizeof(chunk), f)) > 0) text.append(chunk, n); + std::fclose(f); + return true; +} + +} // namespace + +int main(int argc, char** argv) { + using molo::supply::attest::Attestor; + using molo::supply::attest::ProvenanceBuilder; + using molo::supply::caps::kMoloBuildType; + using molo::supply::caps::kSlsaPredicateType; + using molo::supply::common::Json; + + const std::string golden = + (argc > 1 ? std::string(argv[1]) : std::string("tests/src/supplychain/golden")) + + "/provenance.json"; + const bool regen = std::getenv("MOLO_REGEN_GOLDEN") != nullptr; + + // Subjects intentionally unsorted: output must sort by path. + std::vector subjects = {{"sbom.spdx.json", kShaB}, + {"sbom.cdx.json", kShaA}}; + std::vector toolchain = {"gcc@15.2.0", "cmake@3.30.0"}; + std::string predicate = + ProvenanceBuilder::build("https://moloapp.dev/builder/test", + "9f3c1a2b4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8", + toolchain, subjects, "molo release --test"); + CHECK(predicate.find(kMoloBuildType) != std::string::npos); + Json::Value pred_root; + std::string error; + CHECK(Json::parse(predicate, pred_root, error)); + + std::string statement; + CHECK(Attestor::statement(subjects, kSlsaPredicateType, predicate, statement, error)); + CHECK(!statement.empty() && statement.back() == '\n'); + CHECK(statement.find(kSlsaPredicateType) != std::string::npos); + CHECK(statement.find("https://in-toto.io/Statement/v1") != std::string::npos); + std::size_t cdx = statement.find("sbom.cdx.json"); + std::size_t spdx = statement.find("sbom.spdx.json"); + CHECK(cdx != std::string::npos && spdx != std::string::npos && cdx < spdx); // sorted + Json::Value stmt_root; + CHECK(Json::parse(statement, stmt_root, error)); + + // 33rd subject rejected. + std::vector many; + for (int i = 0; i < 33; ++i) many.push_back({"f" + std::to_string(i), kShaA}); + std::string rejected; + CHECK(!Attestor::statement(many, kSlsaPredicateType, predicate, rejected, error)); + // Exactly 32 accepted. + many.pop_back(); + CHECK(Attestor::statement(many, kSlsaPredicateType, predicate, rejected, error)); + + // Bad inputs rejected. + CHECK(!Attestor::statement(subjects, "", predicate, rejected, error)); + CHECK(!Attestor::statement(subjects, kSlsaPredicateType, "{oops", rejected, error)); + std::vector bad_sub = {{"x", "short"}}; + CHECK(!Attestor::statement(bad_sub, kSlsaPredicateType, predicate, rejected, error)); + + if (regen) { + std::FILE* f = std::fopen(golden.c_str(), "wb"); + CHECK(f != nullptr); + if (f != nullptr) { + CHECK(std::fwrite(statement.data(), 1, statement.size(), f) == statement.size()); + std::fclose(f); + std::cout << "test_provenance: regenerated " << golden << "\n"; + } + } else { + std::string want; + CHECK(read_all(golden, want)); + if (want != statement) { + std::size_t at = 0; + while (at < want.size() && at < statement.size() && want[at] == statement[at]) ++at; + std::cerr << "FAIL golden mismatch at byte " << at << " want=" << want.size() + << " got=" << statement.size() << "\n"; + ++g_fail; + } + } + + if (g_fail == 0) std::cout << "test_provenance: all checks passed\n"; + return g_fail == 0 ? 0 : 1; +} diff --git a/tests/src/supplychain/test_scan.cpp b/tests/src/supplychain/test_scan.cpp new file mode 100644 index 0000000..2a20554 --- /dev/null +++ b/tests/src/supplychain/test_scan.cpp @@ -0,0 +1,156 @@ +// test_scan: OSV client (fake transport) plus vulnerability gate verdicts. +#include +#include +#include +#include + +#include "src/supplychain/scan/OsvClient.hpp" +#include "src/supplychain/scan/VulnHook.hpp" + +namespace { + +int g_fail = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL line " << __LINE__ << ": " #cond "\n"; \ + ++g_fail; \ + } \ + } while (0) + +bool read_all(const std::string& path, std::string& text) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) return false; + char chunk[4096]; + std::size_t n = 0; + while ((n = std::fread(chunk, 1, sizeof(chunk), f)) > 0) text.append(chunk, n); + std::fclose(f); + return true; +} + +const char* kOsvBody = + "{\"vulns\": [" + "{\"id\": \"GHSA-crit-1\", \"summary\": \"critical\", \"severity\": [{\"type\": " + "\"CVSS_V3\", \"score\": \"9.8\"}], \"affected\": [{\"ranges\": [{\"events\": [{" + "\"fixed\": \"5.7.1\"}]}]}]}," + "{\"id\": \"GHSA-low-2\", \"summary\": \"low\", \"severity\": [{\"type\": " + "\"CVSS_V3\", \"score\": \"4.1\"}], \"affected\": []}," + "{\"id\": \"GHSA-nosev-3\", \"details\": \"no score\"}]}"; + +} // namespace + +int main(int argc, char** argv) { + using molo::supply::scan::OsvClient; + using molo::supply::scan::VulnHook; + + std::string error; + std::vector findings; + + // Fake transport: first call fails, retry succeeds (1 retry cap). + int calls = 0; + OsvClient::Transport flaky = [&](const std::string& url, const std::string& body, + std::string& resp, std::string& terr) { + (void)url; + (void)body; + ++calls; + if (calls == 1) { + terr = "boom"; + return false; + } + resp = kOsvBody; + return true; + }; + { + OsvClient client(flaky); + CHECK(client.query("wolfssl", "5.7.0", findings, error)); + CHECK(findings.size() == 3); + CHECK(findings[0].id == "GHSA-crit-1"); + CHECK(findings[0].cvss > 9.7 && findings[0].cvss < 9.9); + CHECK(findings[0].fixed_version == "5.7.1"); + CHECK(findings[1].cvss > 4.0 && findings[1].cvss < 4.2); + CHECK(findings[2].cvss == 0.0); + CHECK(client.queries_made() == 2); // initial plus one retry + } + + // Offline default transport fails; clean empty response parses. + { + OsvClient offline; + CHECK(!offline.query("wolfssl", "5.7.0", findings, error)); + std::vector none; + CHECK(OsvClient::parse_response("wolfssl", "{}", none, error)); + CHECK(none.empty()); + CHECK(!OsvClient::parse_response("wolfssl", "{bad", none, error)); + CHECK(!offline.query("", "1", findings, error)); + } + + // Request body shape. + CHECK(OsvClient::request_body("wolfssl", "5.7.0").find("\"version\": \"5.7.0\"") != + std::string::npos); + + // Query cap: 64 allowed, 65th rejected. + { + int n = 0; + OsvClient::Transport ok = [&](const std::string&, const std::string&, std::string& resp, + std::string&) { + ++n; + resp = "{}"; + return true; + }; + OsvClient client(ok); + for (int i = 0; i < 64; ++i) CHECK(client.query("p", "v", findings, error)); + CHECK(!client.query("p", "v", findings, error)); + CHECK(n == 64); + } + + // Gate verdicts with fixed clock. + const long long now = 1780000000LL; + std::string reason; + std::vector empty; + std::vector no_exc; + CHECK(VulnHook::evaluate(empty, no_exc, now, false, reason) == VulnHook::Verdict::Clean); + CHECK(VulnHook::evaluate(empty, no_exc, now, true, reason) == VulnHook::Verdict::Blocked); + + std::vector crit = {{"GHSA-crit-1", "c", "wolfssl", "5.7.1", 9.8}}; + CHECK(VulnHook::is_blocking(crit[0])); + CHECK(VulnHook::evaluate(crit, no_exc, now, false, reason) == VulnHook::Verdict::Blocked); + + std::vector wolf_fix = {{"GHSA-w-1", "w", "wolfssl", "5.7.1", 7.0}}; + CHECK(VulnHook::is_blocking(wolf_fix[0])); // wolfssl fix always blocks + CHECK(VulnHook::evaluate(wolf_fix, no_exc, now, false, reason) == VulnHook::Verdict::Blocked); + + std::vector low = {{"GHSA-low-2", "l", "zlib", "", 4.1}}; + CHECK(!VulnHook::is_blocking(low[0])); + CHECK(VulnHook::evaluate(low, no_exc, now, false, reason) == VulnHook::Verdict::Disclosed); + + std::vector valid_exc = {{"GHSA-crit-1", now + 3600, "accepted risk"}}; + CHECK(VulnHook::evaluate(crit, valid_exc, now, false, reason) == VulnHook::Verdict::Disclosed); + std::vector stale_exc = {{"GHSA-crit-1", now - 1, "too old"}}; + CHECK(VulnHook::evaluate(crit, stale_exc, now, false, reason) == VulnHook::Verdict::Blocked); + CHECK(reason.find("exception-expired") != std::string::npos); + CHECK(std::string(VulnHook::to_string(VulnHook::Verdict::Clean)) == "clean"); + + // Golden advisories file drives the same gate end to end. + const std::string golden = + (argc > 1 ? std::string(argv[1]) : std::string("tests/src/supplychain/golden")) + + "/advisories.json"; + std::string adv_text; + CHECK(read_all(golden, adv_text)); + std::vector from_file; + CHECK(VulnHook::parse_advisories(adv_text, from_file, error)); + CHECK(from_file.size() == 2); + CHECK(VulnHook::evaluate(from_file, no_exc, now, false, reason) == VulnHook::Verdict::Blocked); + std::vector cover = + {{"GHSA-test-0001", now + 604800, "patch scheduled"}}; + CHECK(VulnHook::evaluate(from_file, cover, now, false, reason) == + VulnHook::Verdict::Disclosed); + + std::vector exc; + CHECK(VulnHook::parse_exceptions("{\"exceptions\": [{\"id\": \"a\", \"expires\": 9, " + "\"reason\": \"r\"}]}", + exc, error)); + CHECK(exc.size() == 1 && exc[0].expires_unix == 9); + CHECK(!VulnHook::parse_exceptions("{\"exceptions\": [{\"id\": \"a\"}]}", exc, error)); + + if (g_fail == 0) std::cout << "test_scan: all checks passed\n"; + return g_fail == 0 ? 0 : 1; +} diff --git a/tests/src/supplychain/test_spdx.cpp b/tests/src/supplychain/test_spdx.cpp new file mode 100644 index 0000000..8168167 --- /dev/null +++ b/tests/src/supplychain/test_spdx.cpp @@ -0,0 +1,139 @@ +// test_spdx: SBOM generation plus SPDX 2.3 canonical emission. +#include +#include +#include +#include +#include + +#include "src/supplychain/common/Json.hpp" +#include "src/supplychain/policy/DependencyPolicy.hpp" +#include "src/supplychain/policy/WolfPin.hpp" +#include "src/supplychain/sbom/SbomCheck.hpp" +#include "src/supplychain/sbom/SbomGenerator.hpp" +#include "src/supplychain/sbom/SpdxEmitter.hpp" + +namespace { + +int g_fail = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL line " << __LINE__ << ": " #cond "\n"; \ + ++g_fail; \ + } \ + } while (0) + +const char* kPinSha = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const char* kCommit = "9f3c1a2b4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8"; + +bool read_all(const std::string& path, std::string& text) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) return false; + char chunk[4096]; + std::size_t n = 0; + while ((n = std::fread(chunk, 1, sizeof(chunk), f)) > 0) text.append(chunk, n); + std::fclose(f); + return true; +} + +bool make_doc(molo::supply::sbom::SbomDocument& doc, std::string& error) { + using namespace molo::supply; + policy::WolfPin::Pin pin{"5.7.0", kPinSha, "https://www.wolfssl.com/download", "GPL-2.0", + "vendor/wolfssl-5.7.0.receipt"}; + policy::DependencyPolicy manifest; + std::string allow = std::string("{\"dependencies\": [{\"name\": \"zlib\", \"version\": " + "\"1.3.1\", \"sha256\": \"") + + kPinSha + "\", \"source\": \"https://zlib.net\", \"license\": \"Zlib\"}]}"; + if (!manifest.parse(allow, error)) return false; + std::vector toolchain = {{"gcc", "15.2.0"}, {"cmake", "3.30.0"}}; + return sbom::SbomGenerator::generate(kCommit, toolchain, pin, manifest, "1.2.3-test", + "2026-01-01T00:00:00Z", doc, error); +} + +} // namespace + +int main(int argc, char** argv) { + using namespace molo::supply; + const std::string golden = + (argc > 1 ? std::string(argv[1]) : std::string("tests/src/supplychain/golden")) + + "/sbom.spdx.json"; + const bool regen = std::getenv("MOLO_REGEN_GOLDEN") != nullptr; + + std::string error; + sbom::SbomDocument doc; + CHECK(make_doc(doc, error)); + CHECK(doc.components.size() == 3); + CHECK(doc.components[0].name == "moloapp"); + CHECK(doc.components[1].name == "wolfssl"); + CHECK(doc.components[2].name == "zlib"); + + std::vector missing; + CHECK(sbom::SbomCheck::check_ntia(doc, missing, error)); + CHECK(missing.empty()); + + // Negative NTIA case. + sbom::SbomDocument bad_doc = doc; + bad_doc.components[2].sha256 = "nope"; + CHECK(!sbom::SbomCheck::check_ntia(bad_doc, missing, error)); + CHECK(missing.size() == 1); + + // Generator rejections. + sbom::SbomDocument out; + policy::WolfPin::Pin pin{"5.7.0", kPinSha, "s", "l", "r"}; + policy::DependencyPolicy manifest; + std::vector toolchain = {{"gcc", "15.2.0"}}; + CHECK(!sbom::SbomGenerator::generate("", toolchain, pin, manifest, "1", "t", out, error)); + std::vector too_many(101, {"t", "v"}); + CHECK(!sbom::SbomGenerator::generate(kCommit, too_many, pin, manifest, "1", "t", out, error)); + + // Canonical emission checks. + std::string spdx; + CHECK(sbom::SpdxEmitter::emit_json(doc, spdx, error)); + CHECK(!spdx.empty() && spdx.back() == '\n'); + CHECK(spdx.find('\r') == std::string::npos); + CHECK(spdx.find(" \"SPDXID\": \"SPDXRef-DOCUMENT\"") != std::string::npos); // 2-space indent + std::size_t a = spdx.find("\"SPDXID\""); + std::size_t b = spdx.find("\"creationInfo\""); + std::size_t c = spdx.find("\"dataLicense\""); + std::size_t d = spdx.find("\"spdxVersion\""); + CHECK(a < b && b < c && c < d); // sorted keys + CHECK(spdx.find("\"spdxVersion\": \"SPDX-2.3\"") != std::string::npos); + // Parses back as an object. + common::Json::Value back; + CHECK(common::Json::parse(spdx, back, error)); + + // 1 MiB cap: 64 components with huge sources must be rejected, not cut. + sbom::SbomDocument fat = doc; + for (int i = 0; i < 61; ++i) { + sbom::SbomComponent comp{"dep" + std::to_string(i), "1", + std::string(kPinSha), std::string(70000, 's'), "MIT", "acme"}; + fat.components.push_back(comp); + } + CHECK(fat.components.size() == 64); + std::string fat_out; + CHECK(!sbom::SpdxEmitter::emit_json(fat, fat_out, error)); + + // Golden comparison. + if (regen) { + std::FILE* f = std::fopen(golden.c_str(), "wb"); + CHECK(f != nullptr); + if (f != nullptr) { + CHECK(std::fwrite(spdx.data(), 1, spdx.size(), f) == spdx.size()); + std::fclose(f); + std::cout << "test_spdx: regenerated " << golden << "\n"; + } + } else { + std::string want; + CHECK(read_all(golden, want)); + if (want != spdx) { + std::size_t at = 0; + while (at < want.size() && at < spdx.size() && want[at] == spdx[at]) ++at; + std::cerr << "FAIL golden mismatch at byte " << at << " want=" << want.size() + << " got=" << spdx.size() << "\n"; + ++g_fail; + } + } + + if (g_fail == 0) std::cout << "test_spdx: all checks passed\n"; + return g_fail == 0 ? 0 : 1; +} diff --git a/tests/src/supplychain/test_verify.cpp b/tests/src/supplychain/test_verify.cpp new file mode 100644 index 0000000..4ff6b5c --- /dev/null +++ b/tests/src/supplychain/test_verify.cpp @@ -0,0 +1,243 @@ +// test_verify: hashing, offline verification, fake signing, CLI handlers. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if __has_include("cli/embed/sha256.hpp") +#include "cli/embed/sha256.hpp" +#elif __has_include("embed/sha256.hpp") +#include "embed/sha256.hpp" +#else +#include "src/supplychain/adapters/cli/embed/sha256.hpp" // ADAPTER fallback for isolated builds +#endif +#include "src/supplychain/attest/Signer.hpp" +#include "src/supplychain/attest/Verify.hpp" +#include "src/supplychain/cli/SupplyCli.hpp" +#include "src/supplychain/policy/WolfPin.hpp" + +namespace { + +int g_fail = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::cerr << "FAIL line " << __LINE__ << ": " #cond "\n"; \ + ++g_fail; \ + } \ + } while (0) + +const char* kPinSha = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +std::string g_root; + +bool write_all(const std::string& path, const std::string& text) { + std::FILE* f = std::fopen(path.c_str(), "wb"); + if (f == nullptr) return false; + bool ok = std::fwrite(text.data(), 1, text.size(), f) == text.size(); + ok = ok && std::fclose(f) == 0; + return ok; +} + +bool read_all(const std::string& path, std::string& text) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) return false; + char chunk[4096]; + std::size_t n = 0; + while ((n = std::fread(chunk, 1, sizeof(chunk), f)) > 0) text.append(chunk, n); + std::fclose(f); + return true; +} + +molo::cli::ParsedArgs make_args(std::vector positionals, + std::map flags = {}) { + molo::cli::ParsedArgs args; + args.positionals = std::move(positionals); + args.flags = std::move(flags); + return args; +} + +} // namespace + +int main() { + using molo::cli::ExitCode; + using molo::cli::Output; + using molo::supply::attest::Signer; + using molo::supply::attest::Verify; + using molo::supply::cli::SupplyCli; + using molo::supply::policy::WolfPin; + + g_root = "/tmp/supply_test_verify_" + std::to_string(::getpid()); + std::filesystem::remove_all(g_root); + CHECK(std::filesystem::create_directories(g_root)); + + // Independent SHA-256 check vector ("abc"). + CHECK(molo::embed::Sha256::hex_of_string("abc") == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + CHECK(molo::embed::Sha256::hex_of_string("").size() == 64); + std::string missing_hex; + CHECK(!molo::embed::Sha256::hex_of_file(g_root + "/nope", missing_hex)); + + // Fixture files. + const std::string rel = g_root + "/release.bin"; + const std::string wolf = g_root + "/wolfssl-5.7.0.tar.gz"; + CHECK(write_all(rel, "release-bytes-v1")); + CHECK(write_all(wolf, "wolfssl-bytes-v1")); + std::string dig_rel, dig_wolf; + CHECK(molo::embed::Sha256::hex_of_file(rel, dig_rel)); + CHECK(molo::embed::Sha256::hex_of_file(wolf, dig_wolf)); + CHECK(dig_rel != dig_wolf); + + const std::string subjects_path = g_root + "/subjects.txt"; + CHECK(write_all(subjects_path, dig_rel + " release.bin\n" + dig_wolf + " wolfssl-5.7.0.tar.gz\n")); + const std::string bundle_path = g_root + "/bundle.json"; + CHECK(write_all(bundle_path, "{\"subjects\": [\"" + dig_rel + "\", \"" + dig_wolf + + "\"], \"signature\": \"FAKE\"}\n")); + + std::vector> expected = {{rel, dig_rel}, {wolf, dig_wolf}}; + WolfPin::Pin pin{"5.7.0", dig_wolf, "https://www.wolfssl.com/download", "GPL-2.0", "receipt"}; + Verify::Result r = Verify::verify_files(expected, bundle_path, &pin); + CHECK(r.ok); + + // Tamper with one byte: recompute must fail. + CHECK(write_all(rel, "release-bytes-v2")); + r = Verify::verify_files(expected, bundle_path, &pin); + CHECK(!r.ok); + CHECK(write_all(rel, "release-bytes-v1")); + CHECK(Verify::verify_files(expected, bundle_path, &pin).ok); + + // Bundle that drops a digest fails the chain check. + const std::string thin_bundle = g_root + "/thin.json"; + CHECK(write_all(thin_bundle, "{\"subjects\": [\"" + dig_rel + "\"]}\n")); + r = Verify::verify_files(expected, thin_bundle, &pin); + CHECK(!r.ok); + + // Pin mismatch on the wolfssl subject fails. + WolfPin::Pin wrong_pin{"5.7.0", std::string(kPinSha), "s", "l", "r"}; + r = Verify::verify_files(expected, bundle_path, &wrong_pin); + CHECK(!r.ok); + + // Empty subjects and missing bundle fail loudly. + std::vector> none; + CHECK(!Verify::verify_files(none, bundle_path, nullptr).ok); + CHECK(!Verify::verify_files(expected, g_root + "/absent.json", nullptr).ok); + std::vector> short_hex = {{rel, "abc"}}; + CHECK(!Verify::verify_files(short_hex, bundle_path, nullptr).ok); + + // Fake cosign path: deterministic offline bundle. + ::setenv("MOLO_FAKE_COSIGN", "1", 1); + const std::string fake_bundle = g_root + "/fake.bundle.json"; + std::string sign_error; + CHECK(Signer::sign(rel, fake_bundle, sign_error)); + std::string fake_text; + CHECK(read_all(fake_bundle, fake_text)); + CHECK(fake_text.find(dig_rel) != std::string::npos); + std::vector> single = {{rel, dig_rel}}; + CHECK(Verify::verify_files(single, fake_bundle, nullptr).ok); + CHECK(Signer::quote_path("a'b") == "'a'\\''b'"); + CHECK(!Signer::sign("", fake_bundle, sign_error)); + + // CLI: run_verify end to end through the adapter Output. + { + Output out(false, 0, true, false); + CHECK(SupplyCli::run_verify(make_args({g_root}), out) == ExitCode::Ok); + } + { + Output out(false, 0, true, false); + CHECK(SupplyCli::run_verify(make_args({g_root}, {{"manifest", "subjects.txt"}}), out) == + ExitCode::Ok); + } + { + Output out(false, 0, true, false); + CHECK(SupplyCli::run_verify(make_args({g_root}, {{"bogus", "1"}}), out) == ExitCode::Usage); + } + { + Output out(false, 0, true, false); + molo::cli::ParsedArgs args = make_args({g_root + "/absent-dir"}); + CHECK(SupplyCli::run_verify(args, out) == ExitCode::Runtime); + } + { + // JSON mode prints a status object. + Output out(true, 0, true, false); + molo::cli::ParsedArgs args = make_args({g_root}); + args.global.format = "json"; + CHECK(SupplyCli::run_verify(args, out) == ExitCode::Ok); + } + + // CLI: run_check with clean advisories passes, blocking ones fail. + const std::string allow_path = g_root + "/supply.allow.json"; + CHECK(write_all(allow_path, "{\"wolfssl\": {\"version\": \"5.7.0\", \"sha256\": \"" + + std::string(kPinSha) + + "\", \"source\": \"https://www.wolfssl.com/download\", " + "\"license\": \"GPL-2.0\", \"receipt\": \"r\"}}")); + CHECK(write_all(g_root + "/advisories.json", "{\"advisories\": []}")); + { + Output out(false, 0, true, false); + CHECK(SupplyCli::run_check(make_args({g_root}), out) == ExitCode::Ok); + } + CHECK(write_all(g_root + "/advisories.json", + "{\"advisories\": [{\"id\": \"GHSA-x\", \"package\": \"wolfssl\", \"cvss\": " + "9.9, \"fixed\": \"5.7.1\", \"summary\": \"s\"}]}")); + { + Output out(false, 0, true, false); + CHECK(SupplyCli::run_check(make_args({g_root}), out) == ExitCode::Runtime); + } + // Unexpired exception downgrades to disclosed (still exit 0). + CHECK(write_all(g_root + "/exceptions.json", + "{\"exceptions\": [{\"id\": \"GHSA-x\", \"expires\": 4100000000, \"reason\": " + "\"scheduled\"}]}")); + { + Output out(false, 0, true, false); + CHECK(SupplyCli::run_check(make_args({g_root}, {{"exceptions", "exceptions.json"}}), out) == + ExitCode::Ok); + } + + // CLI: run_release full six steps with fake cosign. + CHECK(write_all(g_root + "/advisories.json", "{\"advisories\": []}")); + CHECK(write_all(g_root + "/suite.stamp", "PASS\n")); + CHECK(write_all(g_root + "/toolchain.txt", "gcc@15.2.0\ncmake@3.30.0\n")); + { + Output out(false, 0, true, false); + molo::cli::ParsedArgs args = + make_args({}, {{"dir", g_root}, + {"commit", "9f3c1a2b4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8"}, + {"suite-stamp", "suite.stamp"}}); + CHECK(SupplyCli::run_release(args, out) == ExitCode::Ok); + std::string tag; + CHECK(read_all(g_root + "/TAG", tag)); + CHECK(tag.find("9f3c1a2b") != std::string::npos); + std::string prov; + CHECK(read_all(g_root + "/provenance.json", prov)); + CHECK(prov.find("https://slsa.dev/provenance/v1") != std::string::npos); + std::string bundle; + CHECK(read_all(g_root + "/bundle.json", bundle)); + CHECK(!bundle.empty()); + std::string spdx_text; + CHECK(read_all(g_root + "/sbom.spdx.json", spdx_text)); + } + { + // Failing stamp stops at step 1. + CHECK(write_all(g_root + "/suite.stamp", "FAIL\n")); + Output out(false, 0, true, false); + molo::cli::ParsedArgs args = + make_args({}, {{"dir", g_root}, {"commit", "abc"}, {"suite-stamp", "suite.stamp"}}); + CHECK(SupplyCli::run_release(args, out) == ExitCode::Runtime); + CHECK(write_all(g_root + "/suite.stamp", "PASS\n")); + } + { + // Missing required flags is usage, not runtime. + Output out(false, 0, true, false); + CHECK(SupplyCli::run_release(make_args({}), out) == ExitCode::Usage); + } + ::unsetenv("MOLO_FAKE_COSIGN"); + + std::filesystem::remove_all(g_root); + + if (g_fail == 0) std::cout << "test_verify: all checks passed\n"; + return g_fail == 0 ? 0 : 1; +} diff --git a/tests/src/tenancy/mini.hpp b/tests/src/tenancy/mini.hpp new file mode 100644 index 0000000..03b3f44 --- /dev/null +++ b/tests/src/tenancy/mini.hpp @@ -0,0 +1,63 @@ +#pragma once +// Minimal CHECK harness. Each test file registers TEST cases and ends with +// TEST_MAIN(). Prints one PASS/FAIL line per case, exit 0 when all pass. + +#include +#include +#include +#include +#include + +namespace mini { + +struct Case { + std::string name; + std::function fn; +}; + +inline std::vector& Registry() { + static std::vector r; + return r; +} + +inline int& Failures() { + static int f = 0; + return f; +} + +inline void Register(std::string name, std::function fn) { + Registry().push_back(Case{std::move(name), std::move(fn)}); +} + +inline void Check(bool cond, const char* expr, const char* file, int line) { + if (!cond) { + ++Failures(); + std::cerr << "FAIL " << file << ":" << line << ": " << expr << "\n"; + } +} + +} // namespace mini + +#define CHECK(cond) ::mini::Check(static_cast(cond), #cond, __FILE__, __LINE__) +#define CHECK_EQ(a, b) ::mini::Check(((a) == (b)), #a " == " #b, __FILE__, __LINE__) +#define CHECK_NE(a, b) ::mini::Check(((a) != (b)), #a " != " #b, __FILE__, __LINE__) + +#define TEST(name) \ + static void name(); \ + static bool _reg_##name = ([]() { \ + ::mini::Register(#name, &name); \ + return true; \ + })(); \ + static void name() + +#define TEST_MAIN() \ + int main() { \ + for (auto& c : ::mini::Registry()) { \ + int before = ::mini::Failures(); \ + c.fn(); \ + std::cout << (before == ::mini::Failures() ? "PASS " \ + : "FAIL ") \ + << c.name << "\n"; \ + } \ + return ::mini::Failures() == 0 ? 0 : 1; \ + } diff --git a/tests/src/tenancy/test_bridge.cpp b/tests/src/tenancy/test_bridge.cpp new file mode 100644 index 0000000..bab3dd1 --- /dev/null +++ b/tests/src/tenancy/test_bridge.cpp @@ -0,0 +1,75 @@ +// Bridge: per-database isolation, cross-file transaction refusal, txn +// commit/rollback staging, and open caps. +#include "mini.hpp" + +#include + +#include "tenancy/bridge.h" + +namespace { + +using molo::tenancy::DatabaseBridge; + +} // namespace + +TEST(per_db_isolation) { + DatabaseBridge b("/var/molo/tenants"); + CHECK(b.open("acme")); + CHECK(b.open("globex")); + CHECK(b.file_for("acme") == "/var/molo/tenants/acme.db"); + CHECK(b.put("acme", "theme", "dark")); + CHECK(b.put("globex", "theme", "light")); + CHECK(b.get("acme", "theme") == "dark"); + CHECK(b.get("globex", "theme") == "light"); + CHECK(!b.get("acme", "missing").has_value()); + CHECK(!b.put("unknown", "k", "v")); + CHECK(!b.get("unknown", "k").has_value()); + CHECK(!b.open("Bad Slug!!")); +} + +TEST(txn_commit_rollback) { + DatabaseBridge b("/var/molo/tenants"); + CHECK(b.open("acme")); + auto t = b.begin("acme"); + CHECK(t.active()); + CHECK(t.put("k", "staged")); + CHECK(t.get("k") == "staged"); + CHECK(!b.get("acme", "k").has_value()); + CHECK(t.commit()); + CHECK(!t.active()); + CHECK(b.get("acme", "k") == "staged"); + auto t2 = b.begin("acme"); + CHECK(t2.put("k2", "nope")); + t2.rollback(); + CHECK(!b.get("acme", "k2").has_value()); + auto missing = b.begin("ghost"); + CHECK(!missing.active()); +} + +TEST(cross_file_txn_refused) { + DatabaseBridge b("/var/molo/tenants"); + CHECK(b.open("acme")); + CHECK(b.open("globex")); + auto same = b.begin2("acme", "acme"); + CHECK(same.active()); + bool threw = false; + try { + auto cross = b.begin2("acme", "globex"); + } catch (const DatabaseBridge::CrossFileError&) { + threw = true; + } + CHECK(threw); +} + +TEST(open_caps) { + DatabaseBridge b("/var/molo/tenants"); + for (int i = 0; i < 1024; ++i) { + CHECK(b.open("d" + std::to_string(i))); + } + CHECK_EQ(b.db_count(), 1024u); + CHECK_EQ(b.file_count(), 2048u); + CHECK(!b.open("one-too-many")); + CHECK_EQ(b.db_count(), 1024u); +} + +TEST_MAIN() diff --git a/tests/src/tenancy/test_flags_config.cpp b/tests/src/tenancy/test_flags_config.cpp new file mode 100644 index 0000000..f43d14d --- /dev/null +++ b/tests/src/tenancy/test_flags_config.cpp @@ -0,0 +1,76 @@ +// Feature flags: defaults, tenant overrides, unknown reads false, 256 cap. +// Config overlay: global plus tenant precedence, typed getters, key cap. +#include "mini.hpp" + +#include + +#include "tenancy/config_overlay.h" +#include "tenancy/feature_flags.h" + +namespace { + +using molo::tenancy::ConfigOverlay; +using molo::tenancy::FeatureFlags; + +} // namespace + +TEST(flags_defaults_and_overrides) { + FeatureFlags f; + CHECK(f.define("new_ui", false)); + CHECK(f.define("fast_path", true)); + CHECK(!f.enabled("acme", "new_ui")); + CHECK(f.enabled("acme", "fast_path")); + CHECK(f.set_tenant("acme", "new_ui", true)); + CHECK(f.enabled("acme", "new_ui")); + CHECK(!f.enabled("globex", "new_ui")); + CHECK(f.set_tenant("acme", "fast_path", false)); + CHECK(!f.enabled("acme", "fast_path")); + CHECK(!f.enabled("acme", "undefined_flag")); + CHECK(!f.set_tenant("acme", "undefined_flag", true)); +} + +TEST(flags_cap) { + FeatureFlags f; + for (int i = 0; i < 256; ++i) { + CHECK(f.define("flag" + std::to_string(i), false)); + } + CHECK_EQ(f.count(), 256u); + CHECK(!f.define("flag256", false)); + CHECK(f.define("flag0", true)); + CHECK(f.enabled("acme", "flag0")); +} + +TEST(config_precedence_and_typed) { + ConfigOverlay c; + c.set_global("timeout", "30"); + c.set_global("debug", "false"); + CHECK(c.get("acme", "timeout", "?") == "30"); + CHECK(c.get("acme", "missing", "fb") == "fb"); + CHECK(c.set_tenant("acme", "timeout", "5")); + CHECK(c.get("acme", "timeout", "?") == "5"); + CHECK(c.get("globex", "timeout", "?") == "30"); + CHECK_EQ(c.get_int("acme", "timeout", 99), 5); + CHECK_EQ(c.get_int("globex", "timeout", 99), 30); + CHECK_EQ(c.get_int("acme", "missing", 99), 99); + CHECK(c.set_tenant("acme", "bad_int", "12x")); + CHECK_EQ(c.get_int("acme", "bad_int", 99), 99); + CHECK(!c.get_bool("acme", "debug", true)); + CHECK(c.set_tenant("acme", "debug", "yes")); + CHECK(c.get_bool("acme", "debug", false)); + CHECK(c.get_bool("acme", "missing", true)); + CHECK(c.clear_tenant("acme")); + CHECK(c.get("acme", "timeout", "?") == "30"); + CHECK(!c.clear_tenant("acme")); +} + +TEST(config_key_cap) { + ConfigOverlay c; + for (int i = 0; i < 1024; ++i) { + CHECK(c.set_tenant("acme", "k" + std::to_string(i), "v")); + } + CHECK(!c.set_tenant("acme", "k1024", "v")); + CHECK(c.set_tenant("acme", "k0", "v2")); + CHECK(c.get("acme", "k0", "") == "v2"); +} + +TEST_MAIN() diff --git a/tests/src/tenancy/test_metrics.cpp b/tests/src/tenancy/test_metrics.cpp new file mode 100644 index 0000000..3fa6cd4 --- /dev/null +++ b/tests/src/tenancy/test_metrics.cpp @@ -0,0 +1,47 @@ +// Tenant metrics: 512 tenants keep labels, the 513th folds into "other", +// counts accumulate per label. +#include "mini.hpp" + +#include +#include + +#include "tenancy/metrics.h" + +namespace { + +using molo::tenancy::TenantMetrics; + +} // namespace + +TEST(label_allowlist) { + TenantMetrics m; + for (int i = 0; i < 512; ++i) { + std::string slug = "t" + std::to_string(i); + CHECK(m.label_for(slug) == slug); + } + CHECK(m.label_for("t512") == "other"); + CHECK(m.label_for("t999") == "other"); + CHECK(m.label_for("t0") == "t0"); + CHECK(m.label_for("other") == "other"); +} + +TEST(counts_fold_into_other) { + TenantMetrics m; + for (int i = 0; i < 512; ++i) m.label_for("t" + std::to_string(i)); + m.record_request("t0"); + m.record_request("t0"); + m.record_error("t0"); + m.record_request("late-a"); + m.record_request("late-b"); + m.record_error("late-c"); + auto snap = m.snapshot(); + CHECK_EQ(snap["t0"].requests, 2u); + CHECK_EQ(snap["t0"].errors, 1u); + CHECK_EQ(snap["other"].requests, 2u); + CHECK_EQ(snap["other"].errors, 1u); + std::uint64_t total = 0; + for (const auto& [label, c] : snap) total += c.requests + c.errors; + CHECK_EQ(total, 6u); +} + +TEST_MAIN() diff --git a/tests/src/tenancy/test_pool_scope.cpp b/tests/src/tenancy/test_pool_scope.cpp new file mode 100644 index 0000000..9d52cd8 --- /dev/null +++ b/tests/src/tenancy/test_pool_scope.cpp @@ -0,0 +1,151 @@ +// Pool scope: two tenants across pg and sqlite checkouts, GUC set/reset, +// dirty slot refusal, registry cap, 100k checkout leak check, ORM scoping +// and RLS proof strings. +#include "mini.hpp" + +#include + +#include "tenancy/adapters/db.h" +#include "tenancy/orm_scope.h" +#include "tenancy/pool_factory.h" +#include "tenancy/rls_policy.h" + +namespace { + +using molo::tenancy::adapters::db::Backend; +using molo::tenancy::adapters::db::DbError; +using molo::tenancy::OrmScope; +using molo::tenancy::PoolFactory; +using molo::tenancy::RlsPolicy; + +} // namespace + +TEST(two_tenant_checkout_both_backends) { + PoolFactory f; + { + auto a = f.acquire("acme", Backend::Postgres); + CHECK(a.active()); + CHECK(a.slug() == "acme"); + CHECK(a.guc() == "acme"); + CHECK_EQ(f.active_count(), 1u); + auto b = f.acquire("globex", Backend::Sqlite); + CHECK(b.guc() == "globex"); + CHECK(b.backend() == Backend::Sqlite); + CHECK_EQ(f.active_count(), 2u); + } + CHECK_EQ(f.active_count(), 0u); + CHECK_EQ(f.tenant_count(), 2u); + auto c = f.acquire("globex", Backend::Postgres); + CHECK(c.guc() == "globex"); + c.release(); + CHECK(!c.active()); + CHECK_EQ(f.active_count(), 0u); +} + +TEST(dirty_slot_refused) { + PoolFactory f; + { + auto a = f.acquire("acme", Backend::Postgres); + } + CHECK_EQ(f.slot_count(), 1u); + f.DebugDirtySlotForTest(0, "stale-tenant"); + bool threw = false; + try { + auto b = f.acquire("globex", Backend::Postgres); + } catch (const DbError& e) { + threw = true; + CHECK(e.code() == "scope_dirty"); + } + CHECK(threw); + f.DebugDirtySlotForTest(0, ""); + auto c = f.acquire("globex", Backend::Postgres); + CHECK(c.guc() == "globex"); +} + +TEST(invalid_slug_rejected) { + PoolFactory f; + bool threw = false; + try { + auto a = f.acquire("Bad Slug!!", Backend::Postgres); + } catch (const DbError& e) { + threw = true; + CHECK(e.code() == "invalid_tenant"); + } + CHECK(threw); + CHECK_EQ(f.tenant_count(), 0u); +} + +TEST(registry_cap) { + PoolFactory f; + for (int i = 0; i < 100000; ++i) { + auto l = f.acquire("t" + std::to_string(i), Backend::Postgres); + } + CHECK_EQ(f.tenant_count(), 100000u); + CHECK_EQ(f.active_count(), 0u); + bool threw = false; + try { + auto l = f.acquire("one-too-many", Backend::Postgres); + } catch (const DbError& e) { + threw = true; + CHECK(e.code() == "too_many_tenants"); + } + CHECK(threw); +} + +TEST(checkout_leak_loop) { + PoolFactory f; + for (int i = 0; i < 100000; ++i) { + auto a = f.acquire(i % 2 == 0 ? "acme" : "globex", + i % 3 == 0 ? Backend::Sqlite : Backend::Postgres); + CHECK(a.guc() == (i % 2 == 0 ? "acme" : "globex")); + } + CHECK_EQ(f.active_count(), 0u); + CHECK_EQ(f.tenant_count(), 2u); + CHECK(f.slot_count() <= 4u); +} + +TEST(orm_scope_injection) { + std::string scoped = OrmScope::ApplySelect("SELECT * FROM orders", "acme", false); + CHECK(scoped.find("tenant_id = 'acme'") != std::string::npos); + std::string with_where = + OrmScope::ApplySelect("SELECT * FROM orders WHERE total > 10", "acme", false); + CHECK(with_where.find("WHERE tenant_id = 'acme' AND") != std::string::npos); + std::string with_order = + OrmScope::ApplySelect("SELECT * FROM orders ORDER BY id", "acme", false); + CHECK(with_order.find("WHERE tenant_id = 'acme'") != std::string::npos); + CHECK(with_order.find("ORDER BY") != std::string::npos); + std::string upd = + OrmScope::ApplyUpdate("UPDATE orders SET total = 1", "acme", false); + CHECK(upd.find("tenant_id = 'acme'") != std::string::npos); + std::string priv = OrmScope::ApplySelect("SELECT * FROM orders", "acme", true); + CHECK(priv == "SELECT * FROM orders"); + std::string already = + OrmScope::ApplySelect("SELECT * FROM orders WHERE tenant_id = 'x'", "acme", false); + CHECK(already.find("AND") == std::string::npos); + std::string ins = OrmScope::InsertWithTenant("orders", "total", "?", "acme"); + CHECK(ins.find("tenant_id") != std::string::npos); + CHECK(ins.find("'acme'") != std::string::npos); + CHECK(OrmScope::KeyPrefix("acme") == "tenant:acme:"); + CHECK(OrmScope::ScopedChannel("acme", "events") == "tenant:acme:events"); + std::string quote = OrmScope::FilterClause("o'brien"); + CHECK(quote == "tenant_id = 'o''brien'"); +} + +TEST(rls_proof) { + PoolFactory f; + auto lease = f.acquire("acme", Backend::Postgres); + std::string set_local = RlsPolicy::SetLocal(lease.guc()); + CHECK(set_local == "SET LOCAL app.current_tenant = 'acme'"); + std::string scoped = OrmScope::ApplySelect("SELECT * FROM orders", lease.guc(), false); + CHECK(scoped.find("tenant_id = 'acme'") != std::string::npos); + CHECK(RlsPolicy::ResetLocal() == "RESET app.current_tenant"); + std::string policy = RlsPolicy::PolicyDdl("orders"); + CHECK(policy.find("current_setting('app.current_tenant', true)") != std::string::npos); + CHECK(policy.find("tenant_id") != std::string::npos); + CHECK(RlsPolicy::EnableRls("orders").find("ENABLE ROW LEVEL SECURITY") != + std::string::npos); + lease.release(); + CHECK(lease.guc() == ""); +} + +TEST_MAIN() diff --git a/tests/src/tenancy/test_provision.cpp b/tests/src/tenancy/test_provision.cpp new file mode 100644 index 0000000..694cd84 --- /dev/null +++ b/tests/src/tenancy/test_provision.cpp @@ -0,0 +1,194 @@ +// Provisioner: happy path, duplicate/invalid rejection, hook rollback, +// tombstone and purge gates. DbCatalog round trip over a test-local fake +// Connection emulating the tenants table. +#include "mini.hpp" + +#include +#include +#include +#include +#include + +#include "tenancy/adapters/db.h" +#include "tenancy/catalog.h" +#include "tenancy/provisioner.h" +#include "tenancy/store/db_catalog.h" +#include "tenancy/store/memory_catalog.h" +#include "tenancy/strategy.h" +#include "tenancy/tenant.h" + +namespace { + +using molo::tenancy::adapters::db::Backend; +using molo::tenancy::adapters::db::BoundStatement; +using molo::tenancy::adapters::db::Connection; +using molo::tenancy::adapters::db::Context; +using molo::tenancy::adapters::db::Result; +using molo::tenancy::adapters::db::Row; +using molo::tenancy::adapters::db::Value; +using molo::tenancy::Catalog; +using molo::tenancy::Provisioner; +using molo::tenancy::PurgeResult; +using molo::tenancy::Strategy; +using molo::tenancy::Tenant; + +class FakeConnection : public Connection { + public: + Result exec(const Context&, const BoundStatement& stmt) override { + const std::string& sql = stmt.sql(); + if (sql.rfind("CREATE TABLE", 0) == 0) return Result{}; + if (sql.rfind("UPDATE tenants", 0) == 0) { + std::string id = molo::tenancy::adapters::db::AsString(stmt.params()[6]); + auto it = rows_.find(id); + if (it == rows_.end()) return Result{}; + it->second["slug"] = stmt.params()[0]; + it->second["strategy"] = stmt.params()[1]; + it->second["status"] = stmt.params()[2]; + it->second["storage_bytes"] = stmt.params()[3]; + it->second["created_unix"] = stmt.params()[4]; + it->second["tombstoned_unix"] = stmt.params()[5]; + Result r; + r.affected = 1; + return r; + } + if (sql.rfind("INSERT INTO tenants", 0) == 0) { + Row row; + row["id"] = stmt.params()[0]; + row["slug"] = stmt.params()[1]; + row["strategy"] = stmt.params()[2]; + row["status"] = stmt.params()[3]; + row["storage_bytes"] = stmt.params()[4]; + row["created_unix"] = stmt.params()[5]; + row["tombstoned_unix"] = stmt.params()[6]; + rows_[molo::tenancy::adapters::db::AsString(stmt.params()[0])] = row; + Result r; + r.affected = 1; + return r; + } + if (sql.rfind("SELECT ", 0) == 0) { + Result r; + if (sql.find("WHERE id=?") != std::string::npos) { + auto it = rows_.find(molo::tenancy::adapters::db::AsString(stmt.params()[0])); + if (it != rows_.end()) r.rows.push_back(it->second); + return r; + } + if (sql.find("WHERE slug=?") != std::string::npos) { + std::string slug = molo::tenancy::adapters::db::AsString(stmt.params()[0]); + for (const auto& [id, row] : rows_) { + if (molo::tenancy::adapters::db::AsString(row.at("slug")) == slug) { + r.rows.push_back(row); + } + } + return r; + } + std::vector ids; + for (const auto& [id, row] : rows_) ids.push_back(id); + std::sort(ids.begin(), ids.end()); + long limit = molo::tenancy::adapters::db::AsInt(stmt.params()[0]); + for (const auto& id : ids) { + if (static_cast(r.rows.size()) >= limit) break; + r.rows.push_back(rows_.at(id)); + } + return r; + } + if (sql.rfind("DELETE FROM tenants", 0) == 0) { + Result r; + r.affected = rows_.erase(molo::tenancy::adapters::db::AsString(stmt.params()[0])) > 0 ? 1 : 0; + return r; + } + return Result{}; + } + void prepare(const std::string&, const std::string&) override {} + Result exec_prepared(const Context&, const std::string&, const std::vector&) override { + return Result{}; + } + bool ping() override { return true; } + void close() override {} + Backend backend() const override { return Backend::Postgres; } + + private: + std::map rows_; +}; + +} // namespace + +TEST(provision_happy_path) { + Catalog cat(std::make_unique()); + int created = 0; + Provisioner::Hooks hooks; + hooks.on_create = [&created](const Tenant&) { ++created; return true; }; + Provisioner p(cat, hooks); + auto t = p.provision("acme", Strategy::SharedSchema, 1000); + CHECK(t.has_value()); + CHECK_EQ(created, 1); + CHECK(t->id() == "ten-acme"); + CHECK(cat.find_by_slug("acme").has_value()); + CHECK(!p.provision("acme", Strategy::SharedSchema, 1000).has_value()); + CHECK(!p.provision("Bad Slug!!", Strategy::SharedSchema, 1000).has_value()); +} + +TEST(provision_hook_failure_rolls_back) { + Catalog cat(std::make_unique()); + int dropped = 0; + Provisioner::Hooks hooks; + hooks.on_create = [](const Tenant&) { return false; }; + hooks.on_drop = [&dropped](const Tenant&) { ++dropped; return true; }; + Provisioner p(cat, hooks); + CHECK(!p.provision("acme", Strategy::SharedSchema, 1000).has_value()); + CHECK_EQ(dropped, 1); + CHECK(!cat.find_by_slug("acme").has_value()); + CHECK(cat.list(10).empty()); +} + +TEST(rollback_removes_tenant) { + Catalog cat(std::make_unique()); + Provisioner p(cat); + CHECK(p.provision("acme", Strategy::SharedSchema, 1000).has_value()); + CHECK(p.rollback("acme", 1001)); + CHECK(!cat.find_by_slug("acme").has_value()); + CHECK(!p.rollback("ghost", 1001)); +} + +TEST(purge_gates) { + Catalog cat(std::make_unique()); + int dropped = 0; + Provisioner::Hooks hooks; + hooks.on_drop = [&dropped](const Tenant&) { ++dropped; return true; }; + Provisioner p(cat, hooks); + CHECK(p.provision("acme", Strategy::SharedSchema, 1000).has_value()); + CHECK(p.purge("acme", "PURGE:ten-acme", 2000, 100) == PurgeResult::NotTombstone); + CHECK_EQ(dropped, 0); + CHECK(p.tombstone("acme", 1500)); + CHECK(p.purge("acme", "PURGE:ten-acme", 1550, 100) == PurgeResult::RetentionHold); + CHECK(p.purge("acme", "wrong", 1700, 100) == PurgeResult::BadConfirm); + CHECK(p.purge("acme", "PURGE:ten-acme", 1700, 100) == PurgeResult::Ok); + CHECK_EQ(dropped, 1); + CHECK(!cat.find_by_slug("acme").has_value()); + CHECK(p.purge("acme", "PURGE:ten-acme", 9999, 100) == PurgeResult::NotFound); +} + +TEST(db_catalog_round_trip) { + FakeConnection conn; + molo::tenancy::store::DbCatalog db(conn); + db.ensure_schema(); + Tenant t("ten-acme", "acme", Strategy::SchemaPerTenant, + molo::tenancy::TenantStatus::Active, 10, 1000, 0); + CHECK(db.upsert(t)); + auto by_id = db.get_by_id("ten-acme"); + CHECK(by_id.has_value()); + CHECK(by_id->slug() == "acme"); + CHECK(by_id->strategy() == Strategy::SchemaPerTenant); + CHECK(by_id->storage_bytes() == 10); + auto by_slug = db.get_by_slug("acme"); + CHECK(by_slug.has_value()); + CHECK(by_slug->id() == "ten-acme"); + t.suspend(); + CHECK(db.upsert(t)); + CHECK(db.get_by_id("ten-acme")->status() == molo::tenancy::TenantStatus::Suspended); + CHECK_EQ(db.list(10).size(), 1u); + CHECK(db.remove("ten-acme")); + CHECK(!db.get_by_id("ten-acme").has_value()); + CHECK(!db.remove("ten-acme")); +} + +TEST_MAIN() diff --git a/tests/src/tenancy/test_quota.cpp b/tests/src/tenancy/test_quota.cpp new file mode 100644 index 0000000..ec30a4a --- /dev/null +++ b/tests/src/tenancy/test_quota.cpp @@ -0,0 +1,79 @@ +// Quota: burst then 429 with retry_after, refill over time, storage 507, +// tenant independence and reset. +#include "mini.hpp" + +#include +#include + +#include "tenancy/quota.h" +#include "tenancy/tenant_buckets.h" + +namespace { + +using molo::tenancy::Quota; +using molo::tenancy::TenantBuckets; + +} // namespace + +TEST(burst_then_429) { + std::int64_t now = 1000000; + Quota::Limits limits; + limits.requests_per_minute = 60.0; + limits.burst = 3.0; + Quota q(limits, [&now]() { return now; }); + CHECK(q.check_request("acme").allowed); + CHECK(q.check_request("acme").allowed); + CHECK(q.check_request("acme").allowed); + Quota::Verdict v = q.check_request("acme"); + CHECK(!v.allowed); + CHECK_EQ(v.status, 429); + CHECK(v.retry_after_sec > 0); + now += 61000; + CHECK(q.check_request("acme").allowed); +} + +TEST(tenants_independent_and_reset) { + std::int64_t now = 1000000; + Quota::Limits limits; + limits.requests_per_minute = 60.0; + limits.burst = 1.0; + Quota q(limits, [&now]() { return now; }); + CHECK(q.check_request("acme").allowed); + CHECK(!q.check_request("acme").allowed); + CHECK(q.check_request("globex").allowed); + q.reset("acme"); + CHECK(q.check_request("acme").allowed); +} + +TEST(storage_507) { + Quota::Limits limits; + limits.storage_bytes = 100; + Quota q(limits); + CHECK(q.add_storage("acme", 60).allowed); + CHECK(q.add_storage("acme", 40).allowed); + CHECK_EQ(q.storage_used("acme"), 100); + Quota::Verdict v = q.add_storage("acme", 1); + CHECK(!v.allowed); + CHECK_EQ(v.status, 507); + CHECK_EQ(q.storage_used("acme"), 100); + CHECK(q.add_storage("globex", 100).allowed); +} + +TEST(tenant_buckets) { + TenantBuckets b; + TenantBuckets::Spec spec; + spec.capacity = 2.0; + spec.refill_per_sec = 1.0; + CHECK(b.configure("api", spec)); + CHECK(b.take("api", "acme", 1000)); + CHECK(b.take("api", "acme", 1000)); + CHECK(!b.take("api", "acme", 1000)); + CHECK(b.take("api", "globex", 1000)); + CHECK(!b.take("unknown-bucket", "acme", 1000)); + CHECK(b.take("api", "acme", 3000)); + b.reset("api", "acme"); + CHECK_EQ(b.entries(), 1u); + CHECK(b.take("api", "acme", 3000)); +} + +TEST_MAIN() diff --git a/tests/src/tenancy/test_tenant_id_resolve.cpp b/tests/src/tenancy/test_tenant_id_resolve.cpp new file mode 100644 index 0000000..d09be54 --- /dev/null +++ b/tests/src/tenancy/test_tenant_id_resolve.cpp @@ -0,0 +1,185 @@ +// TenantId validation matrix, strategy round trip, resolver source matrix, +// conflict rule, dev default, and middleware attach/strip behavior. +#include "mini.hpp" + +#include +#include + +#include "tenancy/adapters/http.h" +#include "tenancy/catalog.h" +#include "tenancy/context.h" +#include "tenancy/middleware.h" +#include "tenancy/resolver.h" +#include "tenancy/store/memory_catalog.h" +#include "tenancy/strategy.h" +#include "tenancy/tenant.h" +#include "tenancy/tenant_id.h" + +namespace { + +using molo::tenancy::Catalog; +using molo::tenancy::ResolveOutcome; +using molo::tenancy::ResolveRequest; +using molo::tenancy::ResolveStatus; +using molo::tenancy::Resolver; +using molo::tenancy::Strategy; +using molo::tenancy::TenantId; +using molo::tenancy::TenantMiddleware; + +std::unique_ptr SeedCatalog() { + auto cat = std::make_unique(std::make_unique()); + CHECK(cat->create("acme", Strategy::SharedSchema, 1000).has_value()); + CHECK(cat->create("globex", Strategy::SchemaPerTenant, 1000).has_value()); + CHECK(cat->create("dev", Strategy::SharedSchema, 1000).has_value()); + return cat; +} + +} // namespace + +TEST(tenant_id_matrix) { + CHECK(TenantId::TryParse("a").has_value()); + CHECK(TenantId::TryParse("acme").has_value()); + CHECK(TenantId::TryParse("acme-2").has_value()); + CHECK(TenantId::TryParse("0").has_value()); + CHECK(TenantId::TryParse(std::string(63, 'x')).has_value()); + CHECK(!TenantId::TryParse("").has_value()); + CHECK(!TenantId::TryParse(std::string(64, 'x')).has_value()); + CHECK(!TenantId::TryParse("Acme").has_value()); + CHECK(!TenantId::TryParse("acme inc").has_value()); + CHECK(!TenantId::TryParse("-acme").has_value()); + CHECK(!TenantId::TryParse("acme-").has_value()); + CHECK(!TenantId::TryParse("acme_inc").has_value()); + CHECK(!TenantId::TryParse("acme.inc").has_value()); + CHECK(TenantId::TryParse("acme")->slug() == "acme"); + CHECK(TenantId::TryParse("acme") == TenantId::TryParse("acme")); + CHECK(TenantId::TryParse("acme") != TenantId::TryParse("globex")); +} + +TEST(strategy_round_trip) { + CHECK(molo::tenancy::ToString(Strategy::SharedSchema) == "shared_schema"); + CHECK(molo::tenancy::ToString(Strategy::SchemaPerTenant) == "schema_per_tenant"); + CHECK(molo::tenancy::ToString(Strategy::DatabasePerTenant) == "database_per_tenant"); + CHECK(molo::tenancy::ToString(Strategy::FilePerTenant) == "file_per_tenant"); + CHECK(molo::tenancy::TryParseStrategy("shared_schema") == Strategy::SharedSchema); + CHECK(molo::tenancy::TryParseStrategy("schema_per_tenant") == Strategy::SchemaPerTenant); + CHECK(molo::tenancy::TryParseStrategy("database_per_tenant") == Strategy::DatabasePerTenant); + CHECK(molo::tenancy::TryParseStrategy("file_per_tenant") == Strategy::FilePerTenant); + CHECK(!molo::tenancy::TryParseStrategy("nope").has_value()); + CHECK(molo::tenancy::ParseStrategy("nope") == Strategy::SharedSchema); +} + +TEST(strip_prefix) { + CHECK(Resolver::StripTenantPrefix("/t/acme/dash").first == "/dash"); + CHECK(Resolver::StripTenantPrefix("/t/acme/dash").second == "acme"); + CHECK(Resolver::StripTenantPrefix("/t/acme").first == "/"); + CHECK(Resolver::StripTenantPrefix("/t/acme").second == "acme"); + CHECK(Resolver::StripTenantPrefix("/dash").first == "/dash"); + CHECK(Resolver::StripTenantPrefix("/dash").second == ""); + CHECK(Resolver::StripTenantPrefix("/t/Acme/x").second == ""); + CHECK(Resolver::StripTenantPrefix("/t//x").second == ""); + CHECK(Resolver::SlugFromHost("acme.example.com:8080", {"example.com"}) == "acme"); + CHECK(Resolver::SlugFromHost("example.com", {"example.com"}) == ""); + CHECK(Resolver::SlugFromHost("globex.localhost", {"localhost"}) == "globex"); + CHECK(Resolver::SlugFromHost("localhost", {"localhost"}) == ""); + CHECK(Resolver::SlugFromHost("ACME.Example.COM", {"example.com"}) == "acme"); + CHECK(Resolver::SlugFromHost("a.b.example.com", {"example.com"}) == ""); + CHECK(Resolver::SlugFromHost("acme.other.com", {"example.com"}) == ""); +} + +TEST(resolve_matrix) { + auto cat = SeedCatalog(); + Resolver::Options ropts; + ropts.base_domains = {"example.com"}; + Resolver r(ropts); + ResolveOutcome o = r.Resolve(ResolveRequest{"example.com", "/x", "acme"}, *cat); + CHECK(o.status == ResolveStatus::Resolved); + CHECK(o.tenant->slug() == "acme"); + o = r.Resolve(ResolveRequest{"example.com", "/t/globex/dash", ""}, *cat); + CHECK(o.status == ResolveStatus::Resolved); + CHECK(o.tenant->slug() == "globex"); + o = r.Resolve(ResolveRequest{"acme.example.com", "/x", ""}, *cat); + CHECK(o.status == ResolveStatus::Resolved); + CHECK(o.tenant->slug() == "acme"); + o = r.Resolve(ResolveRequest{"acme.example.com", "/t/acme/x", "acme"}, *cat); + CHECK(o.status == ResolveStatus::Resolved); + o = r.Resolve(ResolveRequest{"example.com", "/x", "unknown-tenant"}, *cat); + CHECK(o.status == ResolveStatus::Unknown); + o = r.Resolve(ResolveRequest{"example.com", "/x", "Bad Header!!"}, *cat); + CHECK(o.status == ResolveStatus::Unknown); +} + +TEST(resolve_conflict) { + auto cat = SeedCatalog(); + Resolver::Options ropts; + ropts.base_domains = {"example.com"}; + Resolver r(ropts); + ResolveOutcome o = r.Resolve(ResolveRequest{"example.com", "/t/globex/x", "acme"}, *cat); + CHECK(o.status == ResolveStatus::Conflict); + o = r.Resolve(ResolveRequest{"globex.example.com", "/x", "acme"}, *cat); + CHECK(o.status == ResolveStatus::Conflict); + o = r.Resolve(ResolveRequest{"globex.example.com", "/t/acme", ""}, *cat); + CHECK(o.status == ResolveStatus::Conflict); +} + +TEST(resolve_dev_default) { + auto cat = SeedCatalog(); + Resolver::Options opts; + opts.dev_mode = true; + Resolver dev(opts); + ResolveOutcome o = dev.Resolve(ResolveRequest{"localhost:8080", "/x", ""}, *cat); + CHECK(o.status == ResolveStatus::Resolved); + CHECK(o.dev_default); + CHECK(o.tenant->slug() == "dev"); + Resolver prod; + o = prod.Resolve(ResolveRequest{"localhost:8080", "/x", ""}, *cat); + CHECK(o.status == ResolveStatus::Unknown); + CHECK(!o.dev_default); + auto empty = std::make_unique(std::make_unique()); + o = dev.Resolve(ResolveRequest{"localhost", "/x", ""}, *empty); + CHECK(o.status == ResolveStatus::Unknown); +} + +TEST(middleware_attach_and_errors) { + auto cat = SeedCatalog(); + cat->suspend("globex"); + Resolver::Options opts; + opts.base_domains = {"example.com"}; + TenantMiddleware mw(*cat, Resolver(opts)); + CHECK(mw.name() == "tenant"); + CHECK(mw.category() == molo::tenancy::adapters::Category::Security); + + auto next = [](molo::tenancy::adapters::HttpRequest& req) + -> molo::tenancy::adapters::Task { + auto ext = req.get_ext("tenancy.context"); + molo::tenancy::adapters::HttpResponse r; + r.status = 200; + r.body = (ext.has_value() ? ext->slug() : "missing") + "|" + req.path(); + co_return r; + }; + + molo::tenancy::adapters::HttpRequest ok; + ok.set_path("/t/acme/dash"); + ok.set_header("host", "example.com"); + molo::tenancy::adapters::HttpResponse r = mw.handle(ok, next).Get(); + CHECK_EQ(r.status, 200); + CHECK(r.body == "acme|/dash"); + + molo::tenancy::adapters::HttpRequest suspended; + suspended.set_path("/x"); + suspended.set_header("x-tenant", "globex"); + r = mw.handle(suspended, next).Get(); + CHECK_EQ(r.status, 403); + + molo::tenancy::adapters::HttpRequest unknown; + unknown.set_path("/x"); + r = mw.handle(unknown, next).Get(); + CHECK_EQ(r.status, 404); + + molo::tenancy::adapters::HttpRequest conflict; + conflict.set_path("/t/acme/x"); + conflict.set_header("x-tenant", "globex"); + r = mw.handle(conflict, next).Get(); + CHECK_EQ(r.status, 400); +} + +TEST_MAIN()