From c53e2a331961d818e657b4400bbc9c5c83d7afb0 Mon Sep 17 00:00:00 2001 From: retoor Date: Sun, 19 Jul 2026 18:57:43 +0200 Subject: [PATCH] Update --- .gitignore | 7 + CLAUDE.md | 4 +- Dockerfile | 1 + README.md | 2 + devplacepy/attachments.py | 16 +- devplacepy/cli/devii.py | 80 ++- devplacepy/content.py | 14 +- devplacepy/customization.py | 12 + devplacepy/database/CLAUDE.md | 2 +- devplacepy/database/__init__.py | 6 +- devplacepy/database/awards.py | 15 +- devplacepy/database/comments.py | 2 +- devplacepy/database/content.py | 37 ++ devplacepy/database/core.py | 14 +- devplacepy/database/pagination.py | 15 +- devplacepy/database/ranking.py | 20 +- devplacepy/database/remote.py | 348 ++++++++++++ devplacepy/database/schema.py | 133 +++-- devplacepy/database/soft_delete.py | 29 +- devplacepy/db_client.py | 29 + devplacepy/docs_api/groups/gateway.py | 44 +- devplacepy/docs_api/groups/profiles.py | 51 +- devplacepy/docs_api/render.py | 1 + devplacepy/docs_devrant.py | 2 +- devplacepy/main.py | 23 +- devplacepy/models.py | 13 +- devplacepy/rendering.py | 53 +- devplacepy/routers/CLAUDE.md | 2 +- devplacepy/routers/avatar.py | 7 +- devplacepy/routers/devii.py | 12 +- devplacepy/routers/gists.py | 1 + devplacepy/routers/messages.py | 2 +- devplacepy/routers/profile/__init__.py | 2 + devplacepy/routers/profile/index.py | 21 +- devplacepy/routers/profile/interactions.py | 61 +++ devplacepy/routers/xmlrpc.py | 7 +- devplacepy/schemas/auth.py | 11 + devplacepy/schemas/profile.py | 4 + devplacepy/services/CLAUDE.md | 10 +- devplacepy/services/access_tokens.py | 7 + devplacepy/services/ai_modifier.py | 9 +- devplacepy/services/audit/service.py | 1 + devplacepy/services/audit/store.py | 12 +- devplacepy/services/base.py | 26 +- devplacepy/services/bot/engage.py | 2 +- devplacepy/services/bot/social.py | 2 +- devplacepy/services/containers/files/bot.py | 2 +- .../services/containers/files/bot.py.bak | 2 +- devplacepy/services/correction.py | 50 +- devplacepy/services/dbapi/nl2sql.py | 6 +- devplacepy/services/deepsearch/chat.py | 2 +- devplacepy/services/deepsearch/embeddings.py | 8 +- devplacepy/services/deepsearch/llm.py | 17 +- devplacepy/services/deepsearch/store.py | 58 +- devplacepy/services/devii/CLAUDE.md | 56 +- .../devii/actions/ai_correction_actions.py | 2 +- .../devii/actions/ai_modifier_actions.py | 2 +- .../devii/actions/catalog/comments.py | 2 +- .../services/devii/actions/dispatcher.py | 12 + devplacepy/services/devii/actions/spec.py | 27 +- devplacepy/services/devii/agentic/actions.py | 28 + .../services/devii/agentic/compaction.py | 49 +- .../services/devii/agentic/controller.py | 27 + devplacepy/services/devii/agentic/lessons.py | 237 +++++++- .../devii/ai_correction/controller.py | 2 +- .../services/devii/ai_modifier/controller.py | 2 +- devplacepy/services/devii/config.py | 1 + .../services/devii/interaction/__init__.py | 37 ++ .../services/devii/interaction/actions.py | 130 +++++ .../services/devii/interaction/broker.py | 513 ++++++++++++++++++ .../devii/interaction/capabilities.py | 217 ++++++++ .../services/devii/interaction/controller.py | 131 +++++ .../services/devii/interaction/markdown.py | 106 ++++ .../services/devii/interaction/parse.py | 187 +++++++ .../services/devii/interaction/prefs.py | 119 ++++ .../services/devii/interaction/schema.py | 260 +++++++++ devplacepy/services/devii/registry.py | 2 + devplacepy/services/devii/service.py | 48 +- devplacepy/services/devii/session/core.py | 207 ++++++- devplacepy/services/devii/text.py | 19 + devplacepy/services/devrant/tokens.py | 3 + devplacepy/services/gitea/enhance.py | 5 +- devplacepy/services/gitea/planning.py | 5 +- devplacepy/services/jobs/award_service.py | 5 +- devplacepy/services/jobs/base.py | 31 +- devplacepy/services/jobs/deepsearch/crawl.py | 209 +++---- .../services/jobs/deepsearch/enhance.py | 1 + .../services/jobs/deepsearch/orchestrate.py | 15 +- devplacepy/services/jobs/deepsearch/worker.py | 2 +- devplacepy/services/jobs/isslop/agent/llm.py | 6 +- devplacepy/services/jobs/queue.py | 43 ++ devplacepy/services/messaging/persist.py | 4 +- devplacepy/services/messaging/relay.py | 2 +- devplacepy/services/news/service.py | 3 +- devplacepy/services/openai_gateway/CLAUDE.md | 15 +- devplacepy/services/openai_gateway/gateway.py | 54 +- devplacepy/services/openai_gateway/routing.py | 16 + devplacepy/services/openai_gateway/service.py | 49 +- devplacepy/services/openai_gateway/usage.py | 5 + devplacepy/services/openai_gateway/vision.py | 3 + devplacepy/services/pubsub/hub.py | 32 +- devplacepy/services/registry.py | 54 ++ devplacepy/services/telegram/backend.py | 87 ++- devplacepy/services/telegram/bridge.py | 259 ++++++++- devplacepy/services/telegram/service.py | 71 ++- devplacepy/services/telegram/worker.py | 53 +- devplacepy/services/weblock.py | 28 + .../static/css/components/ai-choice.css | 60 ++ .../static/css/components/ai-confirm.css | 27 + devplacepy/static/css/components/ai-field.css | 28 + .../static/css/components/ai-interaction.css | 148 +++++ devplacepy/static/css/devii.css | 15 + devplacepy/static/css/gists.css | 4 + devplacepy/static/css/landing.css | 497 ++++++++++++++++- devplacepy/static/css/messages.css | 26 +- devplacepy/static/js/Application.js | 17 + devplacepy/static/js/ContentRenderer.js | 67 ++- devplacepy/static/js/InteractionsPref.js | 67 +++ devplacepy/static/js/MessagesLayout.js | 144 ++++- devplacepy/static/js/autoload/AiAutoload.js | 114 ++++ devplacepy/static/js/components/AiActions.js | 52 ++ devplacepy/static/js/components/AiChoice.js | 137 +++++ .../static/js/components/AiChoiceMulti.js | 111 ++++ devplacepy/static/js/components/AiConfirm.js | 103 ++++ devplacepy/static/js/components/AiField.js | 207 +++++++ devplacepy/static/js/components/AiGroup.js | 38 ++ devplacepy/static/js/components/AiHelp.js | 22 + .../static/js/components/AiInteraction.js | 217 ++++++++ .../js/components/AiMarkdownFallback.js | 23 + devplacepy/static/js/components/AiOption.js | 15 + devplacepy/static/js/components/AiSelect.js | 16 + devplacepy/static/js/components/AiStatus.js | 32 ++ devplacepy/static/js/components/AppTitle.js | 3 +- devplacepy/static/js/devii/devii-terminal.js | 307 ++++++++++- devplacepy/templates/_comment.html | 2 +- devplacepy/templates/_comment_form.html | 2 +- devplacepy/templates/_post_card.html | 4 +- devplacepy/templates/docs/ai-correction.html | 2 +- devplacepy/templates/docs/ai-modifier.html | 2 +- devplacepy/templates/docs/devii-tools.html | 1 + devplacepy/templates/gist_detail.html | 8 +- devplacepy/templates/gists.html | 4 +- devplacepy/templates/landing.html | 140 +++-- devplacepy/templates/messages.html | 4 +- devplacepy/templates/post.html | 6 +- devplacepy/templates/profile.html | 38 +- devplacepy/templates/project_detail.html | 4 +- devplacepy/templating.py | 19 +- devplacepy/utils/accounts.py | 1 + devplacepy/utils/auth.py | 20 +- devplacepy/utils/authcache.py | 5 +- devplacepy/utils/rewards.py | 2 +- tests/api/admin/awards.py | 110 ++++ tests/api/awards/index.py | 116 ++++ tests/api/profile/award.py | 190 +++++++ tests/api/profile/awards_tab.py | 93 ++++ tests/api/profile/interactions.py | 109 ++++ tests/unit/awards/images.py | 40 ++ tests/unit/database/awards.py | 198 +++++++ tests/unit/services/containers.py.bak | 334 ------------ tests/unit/services/devii/actions/catalog.py | 14 + .../services/devii/interaction/__init__.py | 1 + .../services/devii/interaction/answer_text.py | 48 ++ .../unit/services/devii/interaction/broker.py | 93 ++++ .../devii/interaction/capabilities.py | 61 +++ .../services/devii/interaction/controller.py | 55 ++ .../services/devii/interaction/markdown.py | 57 ++ .../unit/services/devii/interaction/parse.py | 117 ++++ .../unit/services/devii/interaction/prefs.py | 61 +++ .../unit/services/devii/interaction/schema.py | 96 ++++ tests/unit/services/devii/session.py | 28 +- tests/unit/services/devii/text_normalize.py | 23 + tests/unit/services/jobs/award_service.py | 174 ++++++ tests/unit/services/openai_gateway/gateway.py | 282 +++++++++- tests/unit/services/openai_gateway/routing.py | 16 + tests/unit/services/openai_gateway/usage.py | 29 + tests/unit/services/telegram/worker.py | 30 +- tests/unit/services/test_compounds.py | 212 ++++++++ tests/unit/services/test_conformance.py | 56 ++ 179 files changed, 9307 insertions(+), 897 deletions(-) create mode 100644 devplacepy/database/remote.py create mode 100644 devplacepy/db_client.py create mode 100644 devplacepy/routers/profile/interactions.py create mode 100644 devplacepy/services/devii/interaction/__init__.py create mode 100644 devplacepy/services/devii/interaction/actions.py create mode 100644 devplacepy/services/devii/interaction/broker.py create mode 100644 devplacepy/services/devii/interaction/capabilities.py create mode 100644 devplacepy/services/devii/interaction/controller.py create mode 100644 devplacepy/services/devii/interaction/markdown.py create mode 100644 devplacepy/services/devii/interaction/parse.py create mode 100644 devplacepy/services/devii/interaction/prefs.py create mode 100644 devplacepy/services/devii/interaction/schema.py create mode 100644 devplacepy/services/registry.py create mode 100644 devplacepy/services/weblock.py create mode 100644 devplacepy/static/css/components/ai-choice.css create mode 100644 devplacepy/static/css/components/ai-confirm.css create mode 100644 devplacepy/static/css/components/ai-field.css create mode 100644 devplacepy/static/css/components/ai-interaction.css create mode 100644 devplacepy/static/js/InteractionsPref.js create mode 100644 devplacepy/static/js/autoload/AiAutoload.js create mode 100644 devplacepy/static/js/components/AiActions.js create mode 100644 devplacepy/static/js/components/AiChoice.js create mode 100644 devplacepy/static/js/components/AiChoiceMulti.js create mode 100644 devplacepy/static/js/components/AiConfirm.js create mode 100644 devplacepy/static/js/components/AiField.js create mode 100644 devplacepy/static/js/components/AiGroup.js create mode 100644 devplacepy/static/js/components/AiHelp.js create mode 100644 devplacepy/static/js/components/AiInteraction.js create mode 100644 devplacepy/static/js/components/AiMarkdownFallback.js create mode 100644 devplacepy/static/js/components/AiOption.js create mode 100644 devplacepy/static/js/components/AiSelect.js create mode 100644 devplacepy/static/js/components/AiStatus.js create mode 100644 tests/api/admin/awards.py create mode 100644 tests/api/awards/index.py create mode 100644 tests/api/profile/award.py create mode 100644 tests/api/profile/awards_tab.py create mode 100644 tests/api/profile/interactions.py create mode 100644 tests/unit/awards/images.py create mode 100644 tests/unit/database/awards.py delete mode 100644 tests/unit/services/containers.py.bak create mode 100644 tests/unit/services/devii/interaction/__init__.py create mode 100644 tests/unit/services/devii/interaction/answer_text.py create mode 100644 tests/unit/services/devii/interaction/broker.py create mode 100644 tests/unit/services/devii/interaction/capabilities.py create mode 100644 tests/unit/services/devii/interaction/controller.py create mode 100644 tests/unit/services/devii/interaction/markdown.py create mode 100644 tests/unit/services/devii/interaction/parse.py create mode 100644 tests/unit/services/devii/interaction/prefs.py create mode 100644 tests/unit/services/devii/interaction/schema.py create mode 100644 tests/unit/services/devii/text_normalize.py create mode 100644 tests/unit/services/jobs/award_service.py create mode 100644 tests/unit/services/test_compounds.py create mode 100644 tests/unit/services/test_conformance.py diff --git a/.gitignore b/.gitignore index de5fa194..876dabca 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,10 @@ var/ .coverage .coverage.* htmlcov/ + +# local environments and scratch +.venv/ +tmp/ +*.log +*.bak +test.db diff --git a/CLAUDE.md b/CLAUDE.md index 904773af..328958fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -155,7 +155,7 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror | `/auth` | auth/ package | | `/feed`, `/posts`, `/comments` | flat files | | `/projects`, `/projects/{slug}/files`, `/projects/{slug}/containers` | projects/ package - see `routers/projects/CLAUDE.md` | -| `/profile` | profile/ package (customization, notifications, ai-correction, ai-modifier, telegram, usage) | +| `/profile` | profile/ package (customization, notifications, ai-correction, ai-modifier, interactions, telegram, usage) | | `/messages` | messages.py - see `services/messaging/CLAUDE.md` | | `/notifications`, `/votes`, `/reactions`, `/bookmarks`, `/polls`, `/avatar`, `/follow`, `/leaderboard` | flat files | | (none) | relations.py - block/mute | @@ -193,7 +193,7 @@ The CLIENT pipeline (`ContentRenderer.js`, `dp-content`/`dp-title`) is retained **Emoji shortcodes are the full GitHub/Discord `:name:` set** (~4869 names), generated once from the `emoji` library by `rendering.py` `build_emoji_shortcodes()`; the frontend gets the identical map via the generated `static/js/emoji-shortcodes.js` (regenerate with `devplace emoji-sync` after bumping the dependency, never hand-edit). -**Template em-dash normalization:** the shared `templates.env.template_class` runs every rendered template's final HTML through `normalize_em_dash` - the em-dash character and its HTML entity forms all become a plain hyphen, application-wide. One hook; never re-strip em-dash per template. +**Em-dash normalization:** `_normalize_dashes` in `rendering.py` replaces all forms (em dash `\u2014`, en dash `\u2013`, and their HTML entities) with a hyphen BEFORE mistune processes the text. Runs inside `_render_content`/`_render_title` which are `@lru_cache`d, so each unique text is normalized once. No per-template overhead. ### Auth diff --git a/Dockerfile b/Dockerfile index 9d61bd53..add00885 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ RUN pip install --no-cache-dir ".[bots]" \ EXPOSE 10500 ENV DEVPLACE_WEB_WORKERS=2 +ENV DEVPLACE_TEMPLATE_AUTO_RELOAD=0 HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \ CMD curl -f http://localhost:10500/ || exit 1 diff --git a/README.md b/README.md index dd3e7ca8..6513aff4 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,8 @@ Member progression is driven by activity and peer recognition. - **AI modifier.** Enabled by default and applied synchronously by default. It works like AI content correction, except it runs **only** where the prose you author contains an inline `@ai ` directive: the configured prompt tells the model to execute that instruction and replace the marked part, removing the `@ai` marker. Text with no `@ai ...` directive is left exactly as written. It is **context-aware**: the model is given a grounding summary of who is asking (your username, role, level, stars, post count, rank, followers, and bio), the current date, and where the directive sits - the post a comment replies to, the conversation a direct message belongs to, the gist's language and code, and so on - so directives like `@ai answer the question above`, `@ai write my bio from my stats`, or `@ai reply to this` work. It uses your own API key for per-user attribution, is fail-soft (the original is kept on any error), and applies across the web UI, the REST and devRant APIs, and Devii, on the same prose fields as correction (posts, projects, gists, comments, direct messages, and your bio). Code and source files are never touched. In direct messages it runs live: typing `@ai ` in a message executes it and the resolved result appears in the chat for both participants without a reload. You can switch the apply mode to background or disable it on your profile or via the Devii `ai_modifier_set` tool; the settings are saved at `POST /profile/{username}/ai-modifier`. The default instruction is "Execute what is behind `@ai` (the prompt) and replace that part including `@ai`". Successful modifications accumulate per-user running totals - modifications, token counts, cost, and timing/performance (average latency, average speed in tokens per second, and total processing time) - shown on the profile page; token, call, and performance figures are visible to the member, while the dollar figures (total and average cost) are shown to administrators only. +- **Devii interactive widgets.** Administrators set the site default on the Devii service (`devii_interactions_default`, default on). Guests always use that default. Signed-in members inherit it until they override it on their profile or via the Devii `interactions_set` tool (`POST /profile/{username}/interactions`; owner or admin). When enabled, Devii may present decisions with channel-aware controls (`ui_prompt`); when disabled, it falls back to plain numbered menus. + Every AI gateway response (`/openai/v1/*`) also returns per-call `X-Gateway-*` headers with the full token breakdown and the dollar cost of that call, so any client can read its own usage. ## Code Farm diff --git a/devplacepy/attachments.py b/devplacepy/attachments.py index 07f56954..63977db2 100644 --- a/devplacepy/attachments.py +++ b/devplacepy/attachments.py @@ -444,12 +444,13 @@ def link_attachments(uids, target_type, target_uid): return placeholders = ",".join(f":p{i}" for i in range(len(flat))) params = {f"p{i}": uid for i, uid in enumerate(flat)} - db.query( - f"UPDATE attachments SET target_type=:tt, target_uid=:tu WHERE uid IN ({placeholders})", - tt=target_type, - tu=target_uid, - **params, - ) + with db: + db.query( + f"UPDATE attachments SET target_type=:tt, target_uid=:tu WHERE uid IN ({placeholders})", + tt=target_type, + tu=target_uid, + **params, + ) def set_gitea_asset_id(uid, asset_id): @@ -617,7 +618,8 @@ def delete_attachments_for(target_type, target_uids): for row in rows: _unlink_attachment_files(row) ids = ",".join(str(row["id"]) for row in rows) - db.query(f"DELETE FROM attachments WHERE id IN ({ids})") + with db: + db.query(f"DELETE FROM attachments WHERE id IN ({ids})") def get_attachments(target_type, target_uid): diff --git a/devplacepy/cli/devii.py b/devplacepy/cli/devii.py index 3c30122a..b2881944 100644 --- a/devplacepy/cli/devii.py +++ b/devplacepy/cli/devii.py @@ -1,13 +1,11 @@ # retoor import sys -from devplacepy.database import get_table +from devplacepy.database import db, get_table from devplacepy.cli._shared import _audit_cli def cmd_devii_reset_quota(args): - from devplacepy.database import db - table_name = "devii_usage_ledger" if table_name not in db.tables: print(f"Table '{table_name}' does not exist, nothing to reset") @@ -48,6 +46,67 @@ def cmd_devii_reset_quota(args): print(f"Reset AI quota for '{args.username}' ({count} ledger rows deleted)") +def _active_count() -> int: + if "devii_lessons" not in db.tables: + return 0 + return db["devii_lessons"].count(deleted_at=None) + + +def _soft_deleted_count() -> int: + if "devii_lessons" not in db.tables: + return 0 + return db["devii_lessons"].count(deleted_at={"!=": None}) + + +def cmd_devii_lessons_count(args): + active = _active_count() + deleted = _soft_deleted_count() + print(f"Lessons: {active} active, {deleted} soft-deleted ({active + deleted} total)") + + +def cmd_devii_lessons_clear(args): + from devplacepy.services.devii.agentic.lessons import TABLE + + if TABLE not in db.tables: + print("No devii_lessons table exists") + return + active = _active_count() + deleted = _soft_deleted_count() + total = active + deleted + if not args.force: + print(f"Will delete {total} lesson(s) ({active} active, {deleted} soft-deleted). Pass --force to confirm.") + return + db[TABLE].delete() + _audit_cli("cli.devii.lessons.clear", "CLI cleared all devii_lessons", metadata={"active": active, "soft_deleted": deleted}) + print(f"Deleted {total} lesson(s)") + + +def cmd_devii_lessons_prune(args): + from devplacepy.services.devii.agentic.lessons import LessonStore, _read_retention_settings + + if "devii_lessons" not in db.tables: + print("No devii_lessons table exists") + return + active_before = _active_count() + if args.all_owners: + _, max_age = _read_retention_settings(db) + store = LessonStore(db, "_global", "_global") + pruned = store.prune_all_owners(max_age) + elif args.username: + user = get_table("users").find_one(username=args.username) + if not user: + print(f"User '{args.username}' not found") + sys.exit(1) + _, max_age = _read_retention_settings(db) + store = LessonStore(db, "user", user["uid"]) + pruned = store.prune(max_age) + else: + print("Provide --all-owners, or --username USER") + sys.exit(1) + _audit_cli("cli.devii.lessons.prune", "CLI pruned devii_lessons", metadata={"pruned": pruned, "active_before": active_before}) + print(f"Pruned {pruned} lesson(s) (active before: {active_before}, now: {_active_count()})") + + def register_devii(subparsers): devii = subparsers.add_parser("devii", help="Devii assistant management") devii_sub = devii.add_subparsers(title="action", dest="action") @@ -64,3 +123,18 @@ def register_devii(subparsers): "--all", action="store_true", help="Reset every quota (users and guests)" ) devii_reset.set_defaults(func=cmd_devii_reset_quota) + + devii_lessons = devii_sub.add_parser("lessons", help="Manage persisted Devii lesson data") + lessons_sub = devii_lessons.add_subparsers(title="sub-action", dest="sub_action") + + lessons_count = lessons_sub.add_parser("count", help="Count active and soft-deleted lessons") + lessons_count.set_defaults(func=cmd_devii_lessons_count) + + lessons_prune = lessons_sub.add_parser("prune", help="Soft-delete lessons older than the configured max age") + lessons_prune.add_argument("--all-owners", action="store_true", help="Prune across every owner") + lessons_prune.add_argument("--username", help="Prune for a specific user") + lessons_prune.set_defaults(func=cmd_devii_lessons_prune) + + lessons_clear = lessons_sub.add_parser("clear", help="Hard-delete every devii_lessons row") + lessons_clear.add_argument("--force", action="store_true", help="Required to confirm hard deletion") + lessons_clear.set_defaults(func=cmd_devii_lessons_clear) diff --git a/devplacepy/content.py b/devplacepy/content.py index 3984f90f..e6062687 100644 --- a/devplacepy/content.py +++ b/devplacepy/content.py @@ -13,6 +13,7 @@ from devplacepy.database import ( resolve_by_slug, get_users_by_uids, get_vote_counts, + STAR_TARGETS, get_user_votes, get_reactions_by_targets, get_user_bookmarks, @@ -20,6 +21,7 @@ from devplacepy.database import ( get_poll_for_post, update_target_stars, clear_user_stars, + clear_user_post_count, get_target_owner_uid, resolve_object_url, soft_delete, @@ -162,6 +164,8 @@ def create_content_item( **fields, } ) + if table_name == "posts": + clear_user_post_count(user["uid"]) if table_name == "projects": from devplacepy.templating import clear_user_projects_cache @@ -615,6 +619,8 @@ def delete_content_item( soft_delete_engagement(target_type, [item["uid"]], actor) if comment_uids: soft_delete_engagement("comment", comment_uids, actor) + if target_type == "post": + clear_user_post_count(item["user_uid"]) if target_type == "project": from devplacepy.project_files import soft_delete_all_project_files from devplacepy.templating import clear_user_projects_cache @@ -647,7 +653,11 @@ def load_detail( if user and item["user_uid"] in get_blocked_uids(user["uid"]): return None author = get_users_by_uids([item["user_uid"]]).get(item["user_uid"]) - ups, downs = get_vote_counts([item["uid"]]) + if target_type in STAR_TARGETS: + star_count = item.get("stars") or 0 + else: + ups, downs = get_vote_counts([item["uid"]]) + star_count = ups.get(item["uid"], 0) - downs.get(item["uid"], 0) reactions = ( get_reactions_by_targets(target_type, [item["uid"]], user).get( item["uid"], {"counts": {}, "mine": []} @@ -664,7 +674,7 @@ def load_detail( "item": item, "author": author, "is_owner": bool(user and user["uid"] == item["user_uid"]), - "star_count": ups.get(item["uid"], 0) - downs.get(item["uid"], 0), + "star_count": star_count, "my_vote": get_user_votes(user["uid"], [item["uid"]]).get(item["uid"], 0) if user else 0, diff --git a/devplacepy/customization.py b/devplacepy/customization.py index 85e8c72d..a4437f7c 100644 --- a/devplacepy/customization.py +++ b/devplacepy/customization.py @@ -36,6 +36,18 @@ def owner_for(request: Request) -> tuple[str, str] | None: def _overrides_for(request: Request) -> dict: + cached = getattr(request.state, "_custom_overrides", None) + if cached is not None: + return cached + overrides = _resolve_overrides(request) + try: + request.state._custom_overrides = overrides + except Exception: + pass + return overrides + + +def _resolve_overrides(request: Request) -> dict: if get_setting("customization_enabled", "1") != "1": return {"css": "", "js": ""} owner = owner_for(request) diff --git a/devplacepy/database/CLAUDE.md b/devplacepy/database/CLAUDE.md index c85eba1b..03de8142 100644 --- a/devplacepy/database/CLAUDE.md +++ b/devplacepy/database/CLAUDE.md @@ -99,7 +99,7 @@ if "comments" not in db.tables: - **NEVER index the bare `deleted_at` column - use a PARTIAL trash index `WHERE deleted_at IS NOT NULL`.** `ensure_soft_delete_columns` creates `idx__trash ON (deleted_at) WHERE deleted_at IS NOT NULL` (and drops any legacy full `idx_
_deleted`). A full `deleted_at` index is a planner hazard: the column is one giant `NULL` bucket plus many unique delete-timestamps, so `sqlite_stat1` mis-estimates `deleted_at IS NULL` as returning ~2 rows and the planner picks that index for live reads, then `USE TEMP B-TREE FOR ORDER BY` to sort the whole live set (the global feed was doing exactly this, with 82% of posts soft-deleted). The partial index serves the admin Trash view (`deleted_at IS NOT NULL`) cheaply and stops poisoning live `IS NULL` queries. -- **For "live, newest-first" listings add a composite or live-partial index that includes the sort column.** A `WHERE deleted_at IS NULL ORDER BY created_at` query needs the ordering in the index or it filesorts. Posts use a partial `idx_posts_live_created ON (created_at) WHERE deleted_at IS NULL` (feed) plus `idx_posts_user_created (user_uid, created_at)` (profile); comments use `idx_comments_target_created (target_type, target_uid, created_at)`; votes use `idx_votes_user_target (user_uid, target_uid)` (the per-user "my_vote" check on every card); notifications/gists/projects use `(user_uid, created_at)`. All were verified to drop the `USE TEMP B-TREE FOR ORDER BY` step. +- **For "live, newest-first" listings add a composite or live-partial index that includes the sort column.** A `WHERE deleted_at IS NULL ORDER BY created_at` query needs the ordering in the index or it filesorts. Posts use a partial `idx_posts_live_created ON (created_at) WHERE deleted_at IS NULL` (feed) plus `idx_posts_user_created (user_uid, created_at)` (profile); comments use `idx_comments_target_created (target_type, target_uid, created_at)`; votes use `idx_votes_user_target (user_uid, target_uid)` (the per-user "my_vote" check on every card); notifications/gists/projects use `(user_uid, created_at)`; follows use `idx_follows_follower_created (follower_uid, created_at)` + `idx_follows_following_created (following_uid, created_at)` (the followers/following tabs sort newest-first; the legacy single-column follower/following indexes were dropped as redundant prefixes). All were verified to drop the `USE TEMP B-TREE FOR ORDER BY` step. - **Index the non-`uid` lookup keys too, not just the sort/owner columns.** A demand-vs-supply audit added the last missing single-key lookups: the `resolve_by_slug` hot path filters `slug` on content detail pages, so posts/gists/news/projects each get `idx_
_slug (slug)`; `get_setting`/`set_setting` filter `key`, so `idx_site_settings_key (key)`; the container store's `find_one(slug=)`/`find_one(name=)` fallbacks get `idx_instances_slug`/`idx_instances_name`. The DM thread load `find(sender_uid=, receiver_uid=)` gets the covering composites `idx_messages_conversation (sender_uid, receiver_uid)` + `idx_messages_conversation_rev (receiver_uid, sender_uid)` (the read-flag `UPDATE` uses the reverse); the badge-has check gets `idx_badges_user_name (user_uid, badge_name)`; the admin user list `ORDER BY -created_at` gets `idx_users_created_at (created_at)` (the existing `(role, created_at)` cannot serve a full-table created_at sort). All are non-unique so `_index` always creates them even if legacy duplicate data exists. Column sets already resolved to ~1 row by an existing prefix index (votes `+target_type`, game_quests `+kind`, poll_options `position`) are intentionally left uncovered - a trailing column there only adds write cost. diff --git a/devplacepy/database/__init__.py b/devplacepy/database/__init__.py index 6b0aaefc..5764e0ba 100644 --- a/devplacepy/database/__init__.py +++ b/devplacepy/database/__init__.py @@ -5,7 +5,7 @@ from .core import refresh_snapshot, _local_cache_versions, _cache_version_cache, from .settings import _settings_cache, get_setting, get_int_setting, set_setting, clear_settings_cache, internal_gateway_key from .users import get_users_by_uids, _admins_cache, invalidate_admins_cache, get_admin_uids, set_user_timezone, set_last_seen, get_online_users, get_primary_admin_uid, search_users_by_username from .relations import _relations_cache, get_user_relations, get_blocked_uids, get_muted_uids, get_silenced_uids, invalidate_user_relations -from .pagination import PAGE_SIZE, paginate, interleave_by_author, paginate_diverse, get_user_post_count, build_pagination +from .pagination import PAGE_SIZE, paginate, interleave_by_author, paginate_diverse, get_user_post_count, clear_user_post_count, build_pagination from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns, soft_delete, soft_delete_in, restore, purge, list_deleted, count_deleted, restore_event, purge_event from .engagement import _comment_count_cache, get_comment_counts_by_post_uids, get_post_counts_by_user_uids, get_vote_counts, get_user_votes, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_poll_for_post from .usage import _add_usage, _get_usage, add_correction_usage, get_correction_usage, add_modifier_usage, get_modifier_usage, NEWS_USAGE_KEY, add_news_usage, get_news_usage, ISSUE_USAGE_KEY, add_issue_usage, get_issue_usage, SEO_USAGE_KEY, add_seo_usage, get_seo_usage, AWARD_USAGE_KEY, add_award_usage, get_award_usage @@ -36,7 +36,7 @@ from .follows import get_follow_counts, get_follow_list, get_following_among from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache from .ranking import VOTABLE_TARGETS, STAR_TARGETS, _authors_cache, _ranked_authors, _rank_map, get_top_authors, get_leaderboard, get_user_rank, get_user_stars, clear_user_stars, update_target_stars, soft_delete_engagement, delete_engagement, get_target_owner_uid from .comments import _drop_blocked, _build_comment_items, load_comments, get_recent_comments_by_target_uids, get_recent_comments_by_post_uids, load_comments_by_target_uids -from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news +from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news, get_trending_topics from .attachments_data import get_attachments, get_attachments_by_type, get_news_images_by_uids, delete_attachment_record, delete_attachments, _delete_attachment_file, get_user_media, get_deleted_media from .stats import _stats_cache, get_site_stats, _analytics_cache, get_platform_analytics, _gist_languages_cache, get_gist_languages from .schema import BUG_TABLE_RENAMES, migrate_bug_tables_to_issue_tables, init_db, _refresh_query_planner_stats, OLD_GATEWAY_URL, migrate_ai_gateway_settings, backfill_api_keys, _backfill_gamification @@ -98,6 +98,7 @@ __all__ = [ "interleave_by_author", "paginate_diverse", "get_user_post_count", + "clear_user_post_count", "build_pagination", "SOFT_DELETE_TABLES", "ensure_soft_delete_columns", @@ -225,6 +226,7 @@ __all__ = [ "text_search_clause", "get_daily_topic", "get_featured_news", + "get_trending_topics", "get_attachments", "get_attachments_by_type", "get_news_images_by_uids", diff --git a/devplacepy/database/awards.py b/devplacepy/database/awards.py index c0936fc9..51c35b32 100644 --- a/devplacepy/database/awards.py +++ b/devplacepy/database/awards.py @@ -2,6 +2,7 @@ from datetime import datetime, timedelta, timezone +from devplacepy.cache import TTLCache from devplacepy.config import ( AWARD_DISPLAY_HOURS_DEFAULT, AWARD_GIVE_COOLDOWN_HOURS_DEFAULT, @@ -111,10 +112,22 @@ def recompute_user_award_stats(receiver_uid: str) -> None: users.update(payload, ["uid"]) +_prominence_cache = TTLCache(ttl=15, max_size=500) + + def award_is_prominent(user: dict | None) -> bool: if not user or not user.get("last_award_at") or not user.get("last_award_uid"): return False - award = resolve_by_slug(_awards_table(), user["last_award_uid"]) + cached = _prominence_cache.get(user["last_award_uid"]) + if cached is not None: + return cached + prominent = _compute_prominence(user["last_award_uid"]) + _prominence_cache.set(user["last_award_uid"], prominent) + return prominent + + +def _compute_prominence(award_uid: str) -> bool: + award = resolve_by_slug(_awards_table(), award_uid) if not award or not award.get("generated_at"): return False try: diff --git a/devplacepy/database/comments.py b/devplacepy/database/comments.py index 31180a11..b2e7c7da 100644 --- a/devplacepy/database/comments.py +++ b/devplacepy/database/comments.py @@ -126,7 +126,7 @@ def load_comments_by_target_uids(target_type, target_uids, user=None): params["tt"] = target_type raw = list( db.query( - f"SELECT * FROM comments WHERE target_type=:tt AND target_uid IN ({placeholders}) ORDER BY created_at", + f"SELECT * FROM comments WHERE target_type=:tt AND target_uid IN ({placeholders}) AND deleted_at IS NULL ORDER BY created_at", **params, ) ) diff --git a/devplacepy/database/content.py b/devplacepy/database/content.py index 8428e321..c84bb6f2 100644 --- a/devplacepy/database/content.py +++ b/devplacepy/database/content.py @@ -1,7 +1,14 @@ # retoor +from collections import Counter + +from devplacepy.cache import TTLCache + from .core import db, get_table, or_ +_daily_topic_cache = TTLCache(ttl=60, max_size=1) +_trending_cache = TTLCache(ttl=15, max_size=1) + def resolve_by_slug(table, slug, include_deleted=False): has_soft_delete = table.has_column("deleted_at") @@ -77,6 +84,15 @@ def text_search_clause( def get_daily_topic(): + cached = _daily_topic_cache.get("topic") + if cached is not None: + return cached + topic = _load_daily_topic() + _daily_topic_cache.set("topic", topic) + return topic + + +def _load_daily_topic(): if "news" in db.tables: article = db["news"].find_one( status="published", deleted_at=None, order_by=["-synced_at"] @@ -128,3 +144,24 @@ def get_featured_news(limit=5): } ) return articles + + +def get_trending_topics(limit: int = 6) -> list[dict]: + cached = _trending_cache.get("topics") + if cached is not None: + return cached[:limit] + if "posts" not in db.tables or "topic" not in db["posts"].columns: + return [] + rows = db.query( + "SELECT topic FROM posts WHERE deleted_at IS NULL " + "AND topic IS NOT NULL AND topic != '' " + "ORDER BY created_at DESC LIMIT 200" + ) + counter: Counter[str] = Counter() + for row in rows: + topic = (row["topic"] or "").strip() + if topic: + counter[topic] += 1 + topics = [{"topic": t, "count": c} for t, c in counter.most_common(limit)] + _trending_cache.set("topics", topics) + return topics diff --git a/devplacepy/database/core.py b/devplacepy/database/core.py index 47029ed9..45b99bc8 100644 --- a/devplacepy/database/core.py +++ b/devplacepy/database/core.py @@ -76,18 +76,14 @@ def get_cache_version(name: str) -> int: try: _ensure_cache_state() with db: - row = next( - iter( - db.query( - "SELECT version FROM cache_state WHERE name = :name", name=name - ) - ), - None, - ) - version = int(row["version"]) if row else 0 + rows = list(db.query("SELECT name, version FROM cache_state")) + versions = {row["name"]: int(row["version"]) for row in rows} except Exception as e: logger.warning(f"Could not read cache version {name}: {e}") return 0 + for key, version in versions.items(): + _cache_version_cache.set(key, version) + version = versions.get(name, 0) _cache_version_cache.set(name, version) return version diff --git a/devplacepy/database/pagination.py b/devplacepy/database/pagination.py index 54aa2d4d..cdd45fb6 100644 --- a/devplacepy/database/pagination.py +++ b/devplacepy/database/pagination.py @@ -1,5 +1,6 @@ # retoor +from devplacepy.cache import TTLCache from .core import db, get_table from .relations import get_blocked_uids @@ -7,6 +8,9 @@ from .relations import get_blocked_uids PAGE_SIZE = 25 +_user_post_count_cache = TTLCache(ttl=15, max_size=2000) + + def paginate( table, *clauses, @@ -74,10 +78,19 @@ def paginate_diverse( return interleave_by_author(rows, uid_key=uid_key), next_cursor +def clear_user_post_count(user_uid: str) -> None: + _user_post_count_cache.pop(user_uid) + + def get_user_post_count(user_uid: str) -> int: + cached = _user_post_count_cache.get(user_uid) + if cached is not None: + return cached if "posts" not in db.tables: return 0 - return get_table("posts").count(user_uid=user_uid, deleted_at=None) + count = get_table("posts").count(user_uid=user_uid, deleted_at=None) + _user_post_count_cache.set(user_uid, count) + return count def build_pagination(page, total, per_page=25): diff --git a/devplacepy/database/ranking.py b/devplacepy/database/ranking.py index b9f89aa1..3b4e86e6 100644 --- a/devplacepy/database/ranking.py +++ b/devplacepy/database/ranking.py @@ -16,7 +16,7 @@ VOTABLE_TARGETS: dict[str, str] = { STAR_TARGETS: set[str] = {"post", "project", "gist"} -_authors_cache = TTLCache(ttl=15, max_size=200) +_authors_cache = TTLCache(ttl=60, max_size=200) _stars_cache = TTLCache(ttl=15, max_size=2000) @@ -151,17 +151,19 @@ def delete_engagement(target_type: str, target_uids: list) -> None: if "reactions" in db.tables: placeholders, params = _in_clause(uids) params["tt"] = target_type - db.query( - f"DELETE FROM reactions WHERE target_type=:tt AND target_uid IN ({placeholders})", - **params, - ) + with db: + db.query( + f"DELETE FROM reactions WHERE target_type=:tt AND target_uid IN ({placeholders})", + **params, + ) if "bookmarks" in db.tables: placeholders, params = _in_clause(uids) params["tt"] = target_type - db.query( - f"DELETE FROM bookmarks WHERE target_type=:tt AND target_uid IN ({placeholders})", - **params, - ) + with db: + db.query( + f"DELETE FROM bookmarks WHERE target_type=:tt AND target_uid IN ({placeholders})", + **params, + ) if target_type == "post" and "polls" in db.tables: for uid in uids: for poll in db["polls"].find(post_uid=uid): diff --git a/devplacepy/database/remote.py b/devplacepy/database/remote.py new file mode 100644 index 00000000..29a8e609 --- /dev/null +++ b/devplacepy/database/remote.py @@ -0,0 +1,348 @@ +# retoor + +import inspect +import os + +import httpx + +from devplacepy.cache import TTLCache +from devplacepy_services.base.db_codec import ( + decode_value, + encode_args, + is_write, + is_write_sql, +) + +_SERVICE_URL = os.environ.get("DEVPLACE_DB_SERVICE_URL", "http://127.0.0.1:10601").rstrip("/") +_INTERNAL_KEY = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", "").strip() +_CLIENT: httpx.Client | None = None + +# Section 7.3: settings reads tolerate up to 5s staleness. patch_module() +# generically RPCs every devplacepy.database call, bypassing the local +# TTL cache get_setting/get_int_setting had in-process - without this, +# every settings read (rate limiting, maintenance mode, admin dashboards) +# pays a full HTTP round trip to the database broker. +_SETTINGS_CACHE_TTL_SECONDS = 5 +_SETTINGS_CACHE = TTLCache(ttl=_SETTINGS_CACHE_TTL_SECONDS, max_size=512) +_CACHED_SETTINGS_FNS = frozenset({"get_setting", "get_int_setting"}) + + +def _service_url() -> str: + return os.environ.get("DEVPLACE_DB_SERVICE_URL", _SERVICE_URL).rstrip("/") + + +def _headers() -> dict[str, str]: + headers: dict[str, str] = {} + key = os.environ.get("DEVPLACE_GATEWAY_INTERNAL_KEY", _INTERNAL_KEY).strip() + if key: + headers["X-Internal-Key"] = key + return headers + + +def _client() -> httpx.Client: + global _CLIENT + if _CLIENT is None: + _CLIENT = httpx.Client(timeout=30.0) + return _CLIENT + + +def _post(path: str, body: dict) -> object: + response = _client().post( + f"{_service_url()}/{path.lstrip('/')}", + json=body, + headers=_headers(), + ) + if response.status_code >= 400: + payload = response.json() if response.content else {} + message = payload.get("error", "Database service request failed") + raise RuntimeError(message) + if not response.content: + return None + return decode_value(response.json()) + + +def _invoke_cached(fn_name: str, args, kwargs): + cache_key = f"{fn_name}:{args!r}:{sorted(kwargs.items())!r}" + cached = _SETTINGS_CACHE.get(cache_key) + if cached is not None: + return cached + value = _invoke(fn_name, args, kwargs, write=False) + _SETTINGS_CACHE.set(cache_key, value) + return value + + +def _invoke(fn_name: str, args, kwargs, *, write: bool = False): + encoded_args, encoded_kwargs = encode_args(args, kwargs) + payload = { + "fn": fn_name, + "args": encoded_args, + "kwargs": encoded_kwargs, + "write": write, + } + result = _post("internal/invoke", payload) + if isinstance(result, dict) and "result" in result: + return result["result"] + return result + + +class RemoteSearchClause: + def __init__(self, term, fields, author_field=None): + self.term = term.strip() + self.fields = tuple(fields) + self.author_field = author_field + + +class RemoteUidInClause: + def __init__(self, field, uids): + self.field = field + self.uids = frozenset(uids) + + +class RemoteTable: + def __init__(self, db: "RemoteDb", name: str) -> None: + self._db = db + self._name = name + self._column_cache = None + + def __getattr__(self, name: str): + def caller(*args, **kwargs): + return self._db._table_op(self._name, name, args, kwargs) + + return caller + + def has_column(self, name: str) -> bool: + cache = self._column_cache + if cache is None: + sample = self.find(_limit=1) + row = next(iter(sample), None) + cache = set(row.keys()) if row else set() + self._column_cache = cache + return name in cache + + def count(self, **kwargs): + return self._db._table_op(self._name, "count", [], kwargs) + + @property + def table(self): + return self + + @property + def exists(self) -> bool: + return self._name in self._db.tables + +class RemoteDb: + def __init__(self) -> None: + self._tables_cache: list[str] | None = None + + @property + def tables(self) -> list[str]: + if self._tables_cache is None: + result = _post("internal/db-op", {"op": "tables"}) + self._tables_cache = list(result or []) + return self._tables_cache + + def __getitem__(self, name: str) -> RemoteTable: + return RemoteTable(self, name) + + def query(self, sql: str, **params): + encoded_args, encoded_kwargs = encode_args((sql,), params) + result = _post( + "internal/db-op", + { + "op": "query", + "args": encoded_args, + "kwargs": encoded_kwargs, + "write": is_write_sql(sql), + }, + ) + return result or [] + + def _table_op(self, table: str, method: str, args, kwargs, *, write: bool = False): + encoded_args, encoded_kwargs = encode_args(args, kwargs) + result = _post( + "internal/db-op", + { + "op": "table_op", + "table": table, + "method": method, + "args": encoded_args, + "kwargs": encoded_kwargs, + "write": write, + }, + ) + if method in {"insert", "update", "delete"}: + self._tables_cache = None + return result + + @property + def executable(self): + return self + + @property + def in_transaction(self) -> bool: + return False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +_LOCAL_REMOTE = frozenset( + { + "get_table", + "refresh_snapshot", + "_in_clause", + "_now_iso", + "text_search_clause", + } +) + + +def _remote_text_search_clause( + table, search, fields=("title", "description"), author_field=None +): + term = (search or "").strip() + if not term: + return None + if type(table).__name__ == "RemoteTable": + return RemoteSearchClause(term, fields, author_field) + from devplacepy.database.content import text_search_clause as local_clause + + return local_clause(table, search, fields, author_field=author_field) + + +def _remote_get_table(name: str): + import devplacepy.database.core as core + + return core.db[name] + + +def _remote_refresh_snapshot() -> None: + return None + + +def patch_module(module) -> None: + import devplacepy.database as db_module + + for name in db_module.__all__: + if name in _LOCAL_REMOTE: + continue + target = getattr(module, name, None) + if target is None or not callable(target): + continue + if inspect.isclass(target): + continue + + def make_wrapper(fn_name: str, fn_write: bool): + if fn_name in _CACHED_SETTINGS_FNS: + + def wrapper(*args, **kwargs): + return _invoke_cached(fn_name, args, kwargs) + + wrapper.__name__ = fn_name + return wrapper + + def wrapper(*args, **kwargs): + return _invoke(fn_name, args, kwargs, write=fn_write) + + wrapper.__name__ = fn_name + return wrapper + + setattr(module, name, make_wrapper(name, is_write(name))) + + +def activate() -> None: + import devplacepy.database.core as core + + core.db = RemoteDb() + import devplacepy.database as db_module + + patch_module(db_module) + for submodule_name in ( + "settings", + "users", + "relations", + "pagination", + "soft_delete", + "engagement", + "usage", + "awards", + "seo_meta", + "activity", + "customization", + "email", + "notifications", + "forks", + "follows", + "deepsearch", + "ranking", + "comments", + "content", + "attachments_data", + "stats", + "schema", + ): + try: + submodule = __import__( + f"devplacepy.database.{submodule_name}", + fromlist=[submodule_name], + ) + except ImportError: + continue + patch_module(submodule) + for external_name in ( + "devplacepy.services.statistics.tracking", + "devplacepy.services.base", + "devplacepy.attachments", + "devplacepy.project_files", + ): + try: + external = __import__(external_name, fromlist=[external_name.split(".")[-1]]) + except ImportError: + continue + if hasattr(external, "db"): + external.db = RemoteDb() + db_module.db = core.db + db_module.get_table = _remote_get_table + core.get_table = _remote_get_table + db_module.refresh_snapshot = _remote_refresh_snapshot + core.refresh_snapshot = _remote_refresh_snapshot + db_module.text_search_clause = _remote_text_search_clause + import devplacepy.database.content as content_module + + content_module.text_search_clause = _remote_text_search_clause + for submodule_name in ( + "settings", + "users", + "relations", + "pagination", + "soft_delete", + "engagement", + "usage", + "awards", + "seo_meta", + "activity", + "customization", + "email", + "notifications", + "forks", + "follows", + "deepsearch", + "ranking", + "comments", + "content", + "attachments_data", + "stats", + "schema", + ): + try: + submodule = __import__( + f"devplacepy.database.{submodule_name}", + fromlist=[submodule_name], + ) + except ImportError: + continue + if hasattr(submodule, "db"): + submodule.db = core.db \ No newline at end of file diff --git a/devplacepy/database/schema.py b/devplacepy/database/schema.py index e6208010..c5845de8 100644 --- a/devplacepy/database/schema.py +++ b/devplacepy/database/schema.py @@ -1,6 +1,6 @@ # retoor -from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _index, _uid_index, db, defaultdict, get_table, logger +from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger from .settings import get_setting, set_setting from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns from .ranking import _authors_cache @@ -183,8 +183,10 @@ def init_db(): ) _index(db, "badges", "idx_badges_user", ["user_uid"]) _index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"]) - _index(db, "follows", "idx_follows_follower", ["follower_uid"]) - _index(db, "follows", "idx_follows_following", ["following_uid"]) + _drop_index(db, "idx_follows_follower") + _drop_index(db, "idx_follows_following") + _index(db, "follows", "idx_follows_follower_created", ["follower_uid", "created_at"]) + _index(db, "follows", "idx_follows_following_created", ["following_uid", "created_at"]) user_relations = get_table("user_relations") for column, example in ( ("uid", ""), @@ -363,9 +365,10 @@ def init_db(): if not conversations.has_column("channel"): conversations.create_column_by_example("channel", "main") try: - db.query( - "UPDATE devii_conversations SET channel='main' WHERE channel IS NULL" - ) + with db: + db.query( + "UPDATE devii_conversations SET channel='main' WHERE channel IS NULL" + ) except Exception as e: # noqa: BLE001 logger.warning(f"Could not backfill devii_conversations.channel: {e}") _index( @@ -391,6 +394,7 @@ def init_db(): db, "devii_tasks", "idx_devii_tasks_due", ["enabled", "status", "next_run_at"] ) _index(db, "devii_lessons", "idx_devii_lessons_owner", ["owner_kind", "owner_id"]) + _index(db, "devii_lessons", "idx_devii_lessons_owner_created", ["owner_kind", "owner_id", "created_at"]) _index( db, "devii_virtual_tools", "idx_devii_vtools_owner", ["owner_kind", "owner_id"] ) @@ -430,6 +434,12 @@ def init_db(): "idx_gw_usage_endpoint_time", ["endpoint", "created_at"], ) + _index( + db, + "gateway_usage_ledger", + "idx_gw_usage_appref_time", + ["app_reference", "created_at"], + ) _index(db, "gateway_concurrency_samples", "idx_gw_conc_time", ["created_at"]) jobs_table = get_table("jobs") for column, example in ( @@ -559,10 +569,11 @@ def init_db(): correction_usage.create_column_by_example(column, example) try: if "correction_usage" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user " - "ON correction_usage (user_uid)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user " + "ON correction_usage (user_uid)" + ) except Exception as e: logger.warning(f"Could not create unique index on correction_usage: {e}") @@ -582,10 +593,11 @@ def init_db(): modifier_usage.create_column_by_example(column, example) try: if "modifier_usage" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user " - "ON modifier_usage (user_uid)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user " + "ON modifier_usage (user_uid)" + ) except Exception as e: logger.warning(f"Could not create unique index on modifier_usage: {e}") @@ -605,10 +617,11 @@ def init_db(): news_usage.create_column_by_example(column, example) try: if "news_usage" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user " - "ON news_usage (user_uid)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user " + "ON news_usage (user_uid)" + ) except Exception as e: logger.warning(f"Could not create unique index on news_usage: {e}") @@ -628,10 +641,11 @@ def init_db(): issue_usage.create_column_by_example(column, example) try: if "issue_usage" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user " - "ON issue_usage (user_uid)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user " + "ON issue_usage (user_uid)" + ) except Exception as e: logger.warning(f"Could not create unique index on issue_usage: {e}") @@ -651,10 +665,11 @@ def init_db(): seo_usage.create_column_by_example(column, example) try: if "seo_usage" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user " - "ON seo_usage (user_uid)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user " + "ON seo_usage (user_uid)" + ) except Exception as e: logger.warning(f"Could not create unique index on seo_usage: {e}") @@ -674,10 +689,11 @@ def init_db(): award_usage.create_column_by_example(column, example) try: if "award_usage" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_award_usage_user " - "ON award_usage (user_uid)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_award_usage_user " + "ON award_usage (user_uid)" + ) except Exception as e: logger.warning(f"Could not create unique index on award_usage: {e}") @@ -701,21 +717,22 @@ def init_db(): awards.create_column_by_example(column, example) try: if "awards" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_awards_slug ON awards (slug)" - ) - db.query( - "CREATE INDEX IF NOT EXISTS idx_awards_receiver_created " - "ON awards (receiver_uid, created_at)" - ) - db.query( - "CREATE INDEX IF NOT EXISTS idx_awards_giver_created " - "ON awards (giver_uid, created_at)" - ) - db.query( - "CREATE INDEX IF NOT EXISTS idx_awards_generated " - "ON awards (receiver_uid, generated_at)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_awards_slug ON awards (slug)" + ) + db.query( + "CREATE INDEX IF NOT EXISTS idx_awards_receiver_created " + "ON awards (receiver_uid, created_at)" + ) + db.query( + "CREATE INDEX IF NOT EXISTS idx_awards_giver_created " + "ON awards (giver_uid, created_at)" + ) + db.query( + "CREATE INDEX IF NOT EXISTS idx_awards_generated " + "ON awards (receiver_uid, generated_at)" + ) except Exception as e: logger.warning(f"Could not create awards indexes: {e}") @@ -788,10 +805,11 @@ def init_db(): user_activity.create_column_by_example(column, example) try: if "user_activity" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action " - "ON user_activity (user_uid, action)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action " + "ON user_activity (user_uid, action)" + ) except Exception as e: logger.warning(f"Could not create unique index on user_activity: {e}") @@ -806,10 +824,11 @@ def init_db(): user_activity_seen.create_column_by_example(column, example) try: if "user_activity_seen" in db.tables: - db.query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique " - "ON user_activity_seen (user_uid, action, target)" - ) + with db: + db.query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique " + "ON user_activity_seen (user_uid, action, target)" + ) except Exception as e: logger.warning(f"Could not create unique index on user_activity_seen: {e}") @@ -1219,6 +1238,8 @@ def init_db(): "statistics_tracking_enabled": "1", "docs_search_mode": "agent", "outbound_proxy_url": "", + "devii_lessons_max_per_owner": "500", + "devii_lessons_max_age_days": "90", } for key, value in operational_defaults.items(): existing = db["site_settings"].find_one(key=key) @@ -1372,6 +1393,8 @@ def backfill_api_keys() -> int: users.create_column_by_example("ai_modifier_sync", 1) if not users.has_column("ai_modifier_prompt"): users.create_column_by_example("ai_modifier_prompt", DEFAULT_MODIFIER_PROMPT) + if not users.has_column("interactions_enabled"): + users.create_column_by_example("interactions_enabled", -1) if not users.has_column("timezone"): users.create_column_by_example("timezone", "") if not users.has_column("avatar_seed"): @@ -1396,6 +1419,10 @@ def backfill_api_keys() -> int: "WHERE ai_modifier_prompt IS NULL OR ai_modifier_prompt = ''", prompt=DEFAULT_MODIFIER_PROMPT, ) + db.query( + "UPDATE users SET interactions_enabled = -1 " + "WHERE interactions_enabled IS NULL" + ) import uuid_utils updated = 0 diff --git a/devplacepy/database/soft_delete.py b/devplacepy/database/soft_delete.py index b74923bb..187f2afe 100644 --- a/devplacepy/database/soft_delete.py +++ b/devplacepy/database/soft_delete.py @@ -94,11 +94,12 @@ def soft_delete_in(table_name, column, uids, deleted_by, *, stamp=None, **extra) for index, (key, value) in enumerate(extra.items()): params[f"x{index}"] = value extra_sql += f" AND {key} = :x{index}" - db.query( - f"UPDATE {table_name} SET deleted_at = :dat, deleted_by = :dby " - f"WHERE {column} IN ({placeholders}) AND deleted_at IS NULL{extra_sql}", - **params, - ) + with db: + db.query( + f"UPDATE {table_name} SET deleted_at = :dat, deleted_by = :dby " + f"WHERE {column} IN ({placeholders}) AND deleted_at IS NULL{extra_sql}", + **params, + ) return len(uids) @@ -161,11 +162,12 @@ def restore_event(stamp): s=stamp, ).__next__()["n"] ) - db.query( - f"UPDATE {table_name} SET deleted_at = NULL, deleted_by = NULL " - f"WHERE deleted_at = :s", - s=stamp, - ) + with db: + db.query( + f"UPDATE {table_name} SET deleted_at = NULL, deleted_by = NULL " + f"WHERE deleted_at = :s", + s=stamp, + ) return restored @@ -182,7 +184,8 @@ def purge_event(stamp): ) if rows: purged.append((table_name, rows)) - db.query( - f"DELETE FROM {table_name} WHERE deleted_at = :s", s=stamp - ) + with db: + db.query( + f"DELETE FROM {table_name} WHERE deleted_at = :s", s=stamp + ) return purged diff --git a/devplacepy/db_client.py b/devplacepy/db_client.py new file mode 100644 index 00000000..d8b3f997 --- /dev/null +++ b/devplacepy/db_client.py @@ -0,0 +1,29 @@ +# retoor + +import os + + +def _activate() -> None: + if os.environ.get("DEVPLACE_DB_SERVICE") == "1": + return + if os.environ.get("DEVPLACE_REMOTE_DB") == "1": + from devplacepy.database.remote import activate + + activate() + + +_activate() + +import devplacepy.database as _database + + +def _remote_table(table) -> bool: + return type(table).__name__ == "RemoteTable" + + +def __getattr__(name: str): + return getattr(_database, name) + + +def __dir__(): + return sorted(name for name in dir(_database) if not name.startswith("_")) \ No newline at end of file diff --git a/devplacepy/docs_api/groups/gateway.py b/devplacepy/docs_api/groups/gateway.py index 6446d1d0..683f6d68 100644 --- a/devplacepy/docs_api/groups/gateway.py +++ b/devplacepy/docs_api/groups/gateway.py @@ -5,7 +5,6 @@ from .._shared import endpoint, field GROUP = { "slug": "gateway", "title": "OpenAI Gateway", - "admin": True, "intro": """ # OpenAI Gateway @@ -37,6 +36,26 @@ generic model `molodetz-img-small`, which the gateway maps to the configured ima Flux 1.1 Pro by default). Cost is tracked per call with a flat per-image price when the upstream returns no native cost. +## Quick start + +Copy the command below and paste it into a terminal. If you are signed in the `{{ api_key }}` +and `{{ app_reference }}` placeholders are already filled in with your own values; otherwise +replace them with the API key from your [profile](/profile) page and any application identifier. + +```bash +curl -X POST "{{ base }}/openai/v1/chat/completions" \ + -H "Authorization: Bearer {{ api_key }}" \ + -H "X-App-Reference: {{ app_reference }}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "molodetz", + "messages": [{"role": "user", "content": "Hello, how are you?"}] + }' +``` + +The response carries `X-Gateway-*` headers with token counts and dollar cost for the call. +For streaming, add `"stream": true` to the JSON body. + ## Model routing and providers On top of the single default upstream above, an administrator can register additional named @@ -90,6 +109,17 @@ Dollar costs use the upstream's native `cost` field when it returns one model route, falling back to the prices configured on the `openai` service when no route matches. The denied paths that make no upstream call (embeddings or image generation disabled) return no usage headers. +## Request header `X-App-Reference` + +Clients **SHOULD** send an `X-App-Reference` header to identify themselves for cost attribution. +The value is a free-form slug (max 30 characters, letters, digits, `_`, `.`, `-`). When missing or +invalid, the gateway defaults to `default`. The value is recorded in every usage ledger row and can +be queried alongside owner-kind and owner-id to attribute spending per application. + +``` +X-App-Reference: devplace-devii-v-1-0-0 +``` + Administrators enable and configure this gateway under [Background Services](/docs/services.html) (the `openai` service). @@ -104,7 +134,7 @@ for signing DevPlace's own requests. path="/openai/v1/chat/completions", title="Chat completions", summary="OpenAI-compatible chat completion. Supports streaming.", - auth="user", + auth="public", encoding="json", params=[ field( @@ -113,7 +143,7 @@ for signing DevPlace's own requests. "string", False, "gpt-4o-mini", - "Model id. When it matches a configured model route the gateway forwards to that route's provider and upstream model; otherwise it uses the default upstream model.", + "Model id. When it matches a configured model route the gateway forwards to that route's provider and upstream model; otherwise it falls back to the configured default upstream model.", ), field( "messages", @@ -144,7 +174,7 @@ for signing DevPlace's own requests. path="/openai/v1/embeddings", title="Embeddings", summary="OpenAI-compatible text embeddings. Request model molodetz~embed.", - auth="user", + auth="public", encoding="json", params=[ field( @@ -175,6 +205,7 @@ for signing DevPlace's own requests. notes=[ "Returns `503` when the gateway service is not running or embeddings are disabled.", "Every response carries the `X-Gateway-*` token and dollar-cost headers (see Per-call cost and usage headers above).", + "If `model` matches a configured embed model route it is forwarded to that route's provider and upstream model; otherwise it falls back to the configured default embedding model.", ], ), endpoint( @@ -183,7 +214,7 @@ for signing DevPlace's own requests. path="/openai/v1/images/generations", title="Image generation", summary="OpenAI-compatible image generation. Request model molodetz-img-small.", - auth="user", + auth="public", encoding="json", params=[ field( @@ -222,6 +253,7 @@ for signing DevPlace's own requests. notes=[ "Returns `503` when the gateway service is not running or image generation is disabled.", "Every response carries the `X-Gateway-*` token and dollar-cost headers (see Per-call cost and usage headers above).", + "If `model` matches a configured image model route it is forwarded to that route's provider and upstream model; otherwise it falls back to the configured default image model.", ], ), endpoint( @@ -230,7 +262,7 @@ for signing DevPlace's own requests. path="/openai/v1/{path}", title="Passthrough", summary="Any other /v1 path is forwarded to the upstream as-is.", - auth="user", + auth="public", interactive=False, params=[ field( diff --git a/devplacepy/docs_api/groups/profiles.py b/devplacepy/docs_api/groups/profiles.py index 545c83e0..b7dddd24 100644 --- a/devplacepy/docs_api/groups/profiles.py +++ b/devplacepy/docs_api/groups/profiles.py @@ -136,7 +136,7 @@ four ways to sign requests. "textarea", False, "Leave literary as is, only do punctuation and casing", - "Correction instruction, up to 2000 characters.", + "Correction instruction, up to 20000 characters.", ), ], sample_response={ @@ -150,6 +150,53 @@ four ways to sign requests. }, }, ), + endpoint( + id="profile-interactions", + method="POST", + path="/profile/{username}/interactions", + title="Configure Devii interactive widgets", + summary="Enable or disable CA-IWP interactive prompts (ui_prompt) for this account, or reset to the administrator default. Guests always use the site default. Admins may target any user.", + auth="user", + encoding="form", + destructive=True, + params=[ + field( + "username", + "path", + "string", + True, + "bob_test", + "Profile owner. Must be yourself unless you are an admin.", + ), + field( + "enabled", + "form", + "boolean", + False, + "true", + "true to enable interactive widgets, false to disable. Ignored when reset is true.", + ), + field( + "reset", + "form", + "boolean", + False, + "false", + "true to clear the user override and inherit the administrator default.", + ), + ], + sample_response={ + "ok": True, + "redirect": "/profile/bob_test", + "data": { + "url": "/profile/bob_test", + "enabled": True, + "source": "user", + "default": True, + "override": True, + }, + }, + ), endpoint( id="profile-ai-modifier", method="POST", @@ -190,7 +237,7 @@ four ways to sign requests. "textarea", False, "Execute what is behind `@ai` (the prompt) and replace that part including `@ai`", - "Modifier instruction, up to 2000 characters.", + "Modifier instruction, up to 20000 characters.", ), ], sample_response={ diff --git a/devplacepy/docs_api/render.py b/devplacepy/docs_api/render.py index 0db74e03..f07174ff 100644 --- a/devplacepy/docs_api/render.py +++ b/devplacepy/docs_api/render.py @@ -43,5 +43,6 @@ def render_group(slug, base, username, api_key): "{{ base }}": base, "{{ username }}": username or "YOUR_USERNAME", "{{ api_key }}": api_key or "YOUR_API_KEY", + "{{ app_reference }}": f"user-{username}-app-v-1-0-0" if username else "user-app-v-13.37.0", } return _substitute(group, replacements) diff --git a/devplacepy/docs_devrant.py b/devplacepy/docs_devrant.py index 99b2e251..12aa8f9f 100644 --- a/devplacepy/docs_devrant.py +++ b/devplacepy/docs_devrant.py @@ -237,7 +237,7 @@ DEVRANT_GROUPS = { encoding="form", params=[ field("rant_id", "path", type="int", required=True, example="1", description="Rant id."), - field("comment", "body", type="textarea", required=True, example="Great rant!", description="Comment text, 1-1000 chars."), + field("comment", "body", type="textarea", required=True, example="Great rant!", description="Comment text, 1-125000 chars."), ], sample_response={"success": True}, ), diff --git a/devplacepy/main.py b/devplacepy/main.py index 63a817d0..dcaa7787 100644 --- a/devplacepy/main.py +++ b/devplacepy/main.py @@ -37,8 +37,10 @@ from devplacepy.database import ( get_user_post_count, get_user_stars, get_blocked_uids, + get_top_authors, + get_trending_topics, ) -from devplacepy.templating import templates +from devplacepy.templating import templates, jinja_unread_count from devplacepy.cache import TTLCache from devplacepy.responses import respond, wants_json, json_error from devplacepy.schemas import LandingOut, ValidationErrorOut @@ -465,7 +467,8 @@ app.include_router(game.router, prefix="/game") @app.middleware("http") async def refresh_db_snapshot(request: Request, call_next): - refresh_snapshot() + if not request.url.path.startswith(("/static", "/avatar")): + refresh_snapshot() return await call_next(request) @@ -608,7 +611,7 @@ async def response_timing(request: Request, call_next): return response -app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=6) +app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5) _home_cache = TTLCache(ttl=int(os.environ.get("DEVPLACE_HOME_CACHE_TTL", "60")), max_size=4) @@ -700,6 +703,14 @@ async def landing(request: Request): breadcrumbs=[], schemas=[website_schema(base)], ) + user_xp = user.get("xp", 0) or 0 if user else 0 + user_level = user.get("level", 1) or 1 if user else 1 + xp_progress_pct = (user_xp % 100) if user_xp else 0 + unread_count = jinja_unread_count(user["uid"]) if user else 0 + + top_contributors = get_top_authors(5) if not blocked else [] + trending_topics = get_trending_topics(6) if not blocked else [] + return respond( request, "landing.html", @@ -710,8 +721,14 @@ async def landing(request: Request): "is_authenticated": bool(user), "user_post_count": get_user_post_count(user["uid"]) if user else 0, "user_stars": get_user_stars(user["uid"]) if user else 0, + "user_xp": user_xp, + "user_level": user_level, + "xp_progress_pct": xp_progress_pct, + "unread_count": unread_count, "landing_articles": landing_articles, "landing_posts": landing_posts, + "top_contributors": top_contributors, + "trending_topics": trending_topics, }, model=LandingOut, ) diff --git a/devplacepy/models.py b/devplacepy/models.py index 68749e1d..5f00fd92 100644 --- a/devplacepy/models.py +++ b/devplacepy/models.py @@ -158,7 +158,7 @@ class PostEditForm(BaseModel): class CommentForm(BaseModel): - content: str = Field(min_length=3, max_length=1000) + content: str = Field(min_length=3, max_length=125000) target_uid: str = Field(default="", max_length=36) post_uid: str = Field(default="", max_length=36) target_type: Literal["post", "project", "news", "issue", "gist"] = "post" @@ -173,7 +173,7 @@ class CommentForm(BaseModel): class CommentEditForm(BaseModel): - content: str = Field(min_length=3, max_length=1000) + content: str = Field(min_length=3, max_length=125000) class ProjectForm(BaseModel): @@ -258,13 +258,18 @@ class NotificationDefaultForm(BaseModel): class AiCorrectionForm(BaseModel): enabled: bool = False sync: bool = False - prompt: str = Field(default=DEFAULT_CORRECTION_PROMPT, max_length=2000) + prompt: str = Field(default=DEFAULT_CORRECTION_PROMPT, max_length=20000) class AiModifierForm(BaseModel): enabled: bool = False sync: bool = False - prompt: str = Field(default=DEFAULT_MODIFIER_PROMPT, max_length=2000) + prompt: str = Field(default=DEFAULT_MODIFIER_PROMPT, max_length=20000) + + +class InteractionsForm(BaseModel): + enabled: bool = True + reset: bool = False class TelegramPairForm(BaseModel): diff --git a/devplacepy/rendering.py b/devplacepy/rendering.py index e1bc14c6..86e05418 100644 --- a/devplacepy/rendering.py +++ b/devplacepy/rendering.py @@ -45,6 +45,9 @@ def write_emoji_module(path: Path = EMOJI_JS_PATH) -> int: EMOJI_MAP = build_emoji_shortcodes() +_WIDGET_RE = re.compile(r"(.*?)", re.DOTALL | re.IGNORECASE) +_WIDGET_PH = "\x00WIDGET_{}\x00" + _SHORTCODE_RE = re.compile(r":([A-Za-z0-9_+\-]+):") _YOUTUBE_RE = re.compile( r"(?:https?://)?(?:www\.)?" @@ -102,7 +105,17 @@ _content_markdown = mistune.create_markdown( def _normalize_dashes(text: str) -> str: - return text.replace("\u2014", "-") + text = text.replace("\u2014", "-") + text = text.replace("\u2013", "-") + text = text.replace("—", "-") + text = text.replace("–", "-") + text = text.replace("—", "-") + text = text.replace("–", "-") + text = text.replace("—", "-") + text = text.replace("—", "-") + text = text.replace("–", "-") + text = text.replace("–", "-") + return text def _replace_shortcodes(text: str) -> str: @@ -252,16 +265,42 @@ def _render_title(text: str) -> str: return _keep_inline(_content_markdown(text)) -def render_content(text) -> Markup: - if not text: - return Markup("") - return Markup(_render_content(str(text))) +def _extract_widgets(text: str) -> tuple[str, list[str]]: + widgets: list[str] = [] + def _replacer(m: re.Match) -> str: + widgets.append(m.group(1)) + return _WIDGET_PH.format(len(widgets) - 1) + return _WIDGET_RE.sub(_replacer, text), widgets -def render_title(text) -> Markup: +def _reinsert_widgets(text: str, widgets: list[str]) -> str: + for i, widget in enumerate(widgets): + text = text.replace(_WIDGET_PH.format(i), widget) + return text + + +def render_content(text, author_is_admin: bool = False) -> Markup: if not text: return Markup("") - return Markup(_render_title(str(text))) + text_str = str(text) + if author_is_admin and _WIDGET_RE.search(text_str): + modified, widgets = _extract_widgets(text_str) + rendered = _render_content(modified) + result = _reinsert_widgets(rendered, widgets) + return Markup(result) + return Markup(_render_content(text_str)) + + +def render_title(text, author_is_admin: bool = False) -> Markup: + if not text: + return Markup("") + text_str = str(text) + if author_is_admin and _WIDGET_RE.search(text_str): + modified, widgets = _extract_widgets(text_str) + rendered = _render_title(modified) + result = _reinsert_widgets(rendered, widgets) + return Markup(result) + return Markup(_render_title(text_str)) def content_preview(text, length: int = 60) -> str: diff --git a/devplacepy/routers/CLAUDE.md b/devplacepy/routers/CLAUDE.md index c90a1f9d..c3bfbecf 100644 --- a/devplacepy/routers/CLAUDE.md +++ b/devplacepy/routers/CLAUDE.md @@ -14,7 +14,7 @@ Prefixes are wired in `main.py`: | `/comments` | comments.py | | `/projects` | projects/ package - `index.py` (listing/detail/create/delete plus owner-only visibility toggles `POST /{slug}/private` and `POST /{slug}/readonly`, and async `POST /{slug}/fork`), `files.py`, and the `containers/` subpackage (below). `main.py` mounts the whole `/projects` tree from this one package. See `routers/projects/CLAUDE.md` for the deep detail on this tree | | `/projects/{slug}/files` | projects/files.py - per-project virtual filesystem (CRUD dirs/files, upload, inline edit, plus line-range ops: `lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append` for large text files) | -| `/profile` | profile/ package - `index.py` (page/search/followers/update/api-key; the **Posts** tab shows only the 10 most recent posts, while the sidebar `posts_count` stat always uses the real `count()` so it is never capped to the rendered 10), `customization.py`, `notifications.py`, `ai_correction.py` (`POST /{username}/ai-correction`, owner-or-admin AI content correction toggle + prompt), `ai_modifier.py` (`POST /{username}/ai-modifier`, owner-or-admin AI modifier toggle + prompt), `telegram.py` (`POST /{username}/telegram`, owner-or-admin Telegram pairing-code request/unpair), `usage.py` (`_ai_quota`, re-exported from the package) | +| `/profile` | profile/ package - `index.py` (page/search/followers/update/api-key; the **Posts** tab shows only the 10 most recent posts, while the sidebar `posts_count` stat always uses the real `count()` so it is never capped to the rendered 10), `customization.py`, `notifications.py`, `ai_correction.py` (`POST /{username}/ai-correction`, owner-or-admin AI content correction toggle + prompt), `ai_modifier.py` (`POST /{username}/ai-modifier`, owner-or-admin AI modifier toggle + prompt), `interactions.py` (`POST /{username}/interactions`, owner-or-admin Devii interactive widgets preference; admin default via `devii_interactions_default`), `telegram.py` (`POST /{username}/telegram`, owner-or-admin Telegram pairing-code request/unpair), `usage.py` (`_ai_quota`, re-exported from the package) | | `/messages` | messages.py - real-time DM chat. `GET ""` (page; the conversation sidebar is one windowed SQL aggregate and an opened thread loads only the newest `CONVERSATION_MESSAGE_LIMIT` = 500 messages - keep these queries bounded), `GET /search`, `POST /send` (no-JS fallback), and `WS /messages/ws` (NOT lock-owner gated - accepts on every worker; closes `1008` for guests). The WS adds live bidirectional delivery, typing, read receipts, and in-process presence on top of the existing `messages` table; both the WS `send` and HTTP `POST /send` route through the shared `services/messaging/persist.py` `persist_message` choke point (identical audit/notification/mention) and then through `_finalize_and_broadcast`, which awaits any pending SYNC AI correction/modifier futures (DM content runs through both, so typing `@ai ` executes live) and broadcasts the FINAL corrected/modified content (the WS path passes `request=websocket` and awaits directly since HTTP middleware does not run for websockets; the frontend swaps the sender's optimistic bubble to the authoritative content). Same-worker delivery is instant via the per-worker `message_hub`; cross-worker delivery is filled by `services/messaging/relay.py` `message_relay` (per-worker ~1s DB poll on the `messages.id` watermark, deduped against the hub's delivered set). See `devplacepy/services/messaging/CLAUDE.md` | | `/notifications` | notifications.py | | `/votes` | votes.py | diff --git a/devplacepy/routers/avatar.py b/devplacepy/routers/avatar.py index da865b19..f32f8022 100644 --- a/devplacepy/routers/avatar.py +++ b/devplacepy/routers/avatar.py @@ -17,15 +17,14 @@ _CACHE_CONTROL = f"public, max-age={SECONDS_PER_DAY}, immutable" @router.get("/{style}/{seed}") async def avatar_proxy(request: Request, style: str, seed: str, size: int = 128): - cache_key = f"{seed}:{size}" - etag = '"' + hashlib.md5(cache_key.encode("utf-8")).hexdigest() + '"' + etag = '"' + hashlib.md5(f"{seed}:{size}".encode("utf-8")).hexdigest() + '"' headers = {"ETag": etag, "Cache-Control": _CACHE_CONTROL} if request.headers.get("if-none-match") == etag: return Response(status_code=304, headers=headers) - svg = _cache.get(cache_key) + svg = _cache.get(seed) if svg is None: svg = generate_avatar_svg(seed) - _cache.set(cache_key, svg) + _cache.set(seed, svg) return Response(content=svg, media_type="image/svg+xml", headers=headers) diff --git a/devplacepy/routers/devii.py b/devplacepy/routers/devii.py index 6ab7b939..1ede21df 100644 --- a/devplacepy/routers/devii.py +++ b/devplacepy/routers/devii.py @@ -185,7 +185,10 @@ async def clippy_proxy(request: Request): return JSONResponse({"error": "Devii is unavailable"}, status_code=503) cfg = svc.effective_config() body = await request.body() - headers = {"Content-Type": "application/json"} + headers = { + "Content-Type": "application/json", + "X-App-Reference": "devplace-devii-v-1-0-0", + } if cfg.get("devii_ai_key"): headers["Authorization"] = f"Bearer {cfg['devii_ai_key']}" async with stealth.stealth_async_client(timeout=45.0) as client: @@ -260,6 +263,8 @@ async def devii_ws(websocket: WebSocket): if command == "reset": await session.reset() continue + if await session.try_answer_interaction(text): + continue if svc.quota_exceeded(owner_kind, owner_id, owner_is_admin): limit = svc.daily_limit_for(owner_kind, owner_is_admin) audit.record_system( @@ -302,6 +307,11 @@ async def devii_ws(websocket: WebSocket): ) elif kind in ("avatar_result", "client_result"): session.resolve_query(str(data.get("id", "")), data.get("result")) + elif kind == "interaction_result": + session.resolve_interaction( + str(data.get("id", data.get("interaction_id", ""))), + data.get("result") or data, + ) except WebSocketDisconnect: pass except Exception: # noqa: BLE001 - never let the socket loop crash the worker diff --git a/devplacepy/routers/gists.py b/devplacepy/routers/gists.py index aeab6b5c..b7124b57 100644 --- a/devplacepy/routers/gists.py +++ b/devplacepy/routers/gists.py @@ -57,6 +57,7 @@ LANGUAGES = [ ("yaml", "YAML"), ("json", "JSON"), ("markdown", "Markdown"), + ("markdown_rendered", "Markdown Rendered"), ("swift", "Swift"), ("php", "PHP"), ("ruby", "Ruby"), diff --git a/devplacepy/routers/messages.py b/devplacepy/routers/messages.py index cf09ec0d..754ce4fa 100644 --- a/devplacepy/routers/messages.py +++ b/devplacepy/routers/messages.py @@ -231,7 +231,7 @@ async def send_message(request: Request, data: Annotated[MessageForm, Depends(js async def broadcast_message( sender: dict, message: dict, client_id: Optional[str] = None ) -> None: - frame = message_frame(message, sender.get("username", ""), client_id) + frame = message_frame(message, sender.get("username", ""), client_id, sender_role=sender.get("role")) message_hub.mark_delivered(message["uid"]) targets = [message["sender_uid"], message["receiver_uid"]] await message_hub.send_to_users(targets, frame) diff --git a/devplacepy/routers/profile/__init__.py b/devplacepy/routers/profile/__init__.py index a05a4ca7..5d95f05d 100644 --- a/devplacepy/routers/profile/__init__.py +++ b/devplacepy/routers/profile/__init__.py @@ -6,6 +6,7 @@ from devplacepy.routers.profile import ( avatar, award, customization, + interactions, notifications, telegram, ) @@ -17,6 +18,7 @@ router.include_router(customization.router) router.include_router(notifications.router) router.include_router(ai_correction.router) router.include_router(ai_modifier.router) +router.include_router(interactions.router) router.include_router(avatar.router) router.include_router(telegram.router) diff --git a/devplacepy/routers/profile/index.py b/devplacepy/routers/profile/index.py index f5146744..d4e8fc4c 100644 --- a/devplacepy/routers/profile/index.py +++ b/devplacepy/routers/profile/index.py @@ -12,6 +12,7 @@ from devplacepy.database import ( get_notification_prefs, get_user_stars, get_user_rank, + get_user_post_count, get_comment_counts_by_post_uids, get_reactions_by_targets, get_user_bookmarks, @@ -211,9 +212,7 @@ async def profile_page( ) for g in gists_raw: gists.append({"gist": g, "time_ago": time_ago(g["created_at"])}) - posts_count = get_table("posts").count( - user_uid=profile_user["uid"], deleted_at=None - ) + posts_count = get_user_post_count(profile_user["uid"]) activities = [] if tab == "activity": @@ -302,6 +301,18 @@ async def profile_page( if is_owner else None ) + from devplacepy.services.devii.interaction import prefs as interaction_prefs + + interactions_snap = ( + interaction_prefs.snapshot("user", profile_user["uid"], profile_user) + if is_owner + else { + "enabled": True, + "source": None, + "default": True, + "override": None, + } + ) from devplacepy.services.telegram import store as telegram_store telegram_paired = ( @@ -402,6 +413,10 @@ async def profile_page( "ai_modifier_enabled": ai_modifier_enabled, "ai_modifier_sync": ai_modifier_sync, "ai_modifier_prompt": ai_modifier_prompt, + "interactions_enabled": interactions_snap["enabled"], + "interactions_source": interactions_snap["source"], + "interactions_default": interactions_snap["default"], + "interactions_override": interactions_snap["override"], "telegram_paired": telegram_paired, "notif_telegram_paired": notif_telegram_paired, "can_manage_customization": can_manage_customization, diff --git a/devplacepy/routers/profile/interactions.py b/devplacepy/routers/profile/interactions.py new file mode 100644 index 00000000..092f74c2 --- /dev/null +++ b/devplacepy/routers/profile/interactions.py @@ -0,0 +1,61 @@ +# retoor + +import logging +from typing import Annotated + +from fastapi import Depends, APIRouter, Request + +from devplacepy.models import InteractionsForm +from devplacepy.responses import action_result +from devplacepy.services.audit import record as audit +from devplacepy.services.devii.interaction import prefs +from devplacepy.routers.profile._shared import resolve_customization_target +from devplacepy.dependencies import json_or_form + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.post("/{username}/interactions") +async def set_interactions( + request: Request, + username: str, + data: Annotated[InteractionsForm, Depends(json_or_form(InteractionsForm))], +): + target, denied = resolve_customization_target(request, username) + if denied is not None: + return denied + if data.reset: + snap = prefs.set_user_pref(target["uid"], None) + summary = f"reset interactive widgets to admin default for {target['username']}" + new_value = -1 + else: + snap = prefs.set_user_pref(target["uid"], bool(data.enabled)) + summary = ( + f"{'enabled' if data.enabled else 'disabled'} interactive widgets " + f"for {target['username']}" + ) + new_value = 1 if data.enabled else 0 + logger.info(summary) + audit.record( + request, + "profile.interactions", + target_type="user", + target_uid=target["uid"], + target_label=target["username"], + new_value=new_value, + summary=summary, + links=[audit.target("user", target["uid"], target["username"])], + ) + url = f"/profile/{target['username']}" + return action_result( + request, + url, + data={ + "url": url, + "enabled": snap["enabled"], + "source": snap["source"], + "default": snap["default"], + "override": snap["override"], + }, + ) diff --git a/devplacepy/routers/xmlrpc.py b/devplacepy/routers/xmlrpc.py index 4ab8f443..b38a3ab9 100644 --- a/devplacepy/routers/xmlrpc.py +++ b/devplacepy/routers/xmlrpc.py @@ -4,6 +4,7 @@ import logging import httpx from fastapi import APIRouter, Request +from starlette.requests import ClientDisconnect from starlette.responses import PlainTextResponse, Response from devplacepy.config import XMLRPC_BIND, XMLRPC_PORT @@ -49,7 +50,11 @@ async def info(request: Request, path: str = "") -> PlainTextResponse: @router.post("/") @router.post("/{path:path}") async def proxy(request: Request, path: str = "") -> Response: - body = await request.body() + try: + body = await request.body() + except ClientDisconnect: + logger.warning("XML-RPC client disconnected before request body was read") + return PlainTextResponse("Client disconnected", status_code=400) headers = { key: value for key, value in request.headers.items() diff --git a/devplacepy/schemas/auth.py b/devplacepy/schemas/auth.py index 44e15430..52ab7bff 100644 --- a/devplacepy/schemas/auth.py +++ b/devplacepy/schemas/auth.py @@ -40,12 +40,23 @@ class LandingPostOut(_Out): slug: str = "" +class TrendingTopicOut(_Out): + topic: str = "" + count: int = 0 + + class LandingOut(_Out): is_authenticated: bool = False user_post_count: int = 0 user_stars: int = 0 + user_xp: int = 0 + user_level: int = 1 + xp_progress_pct: int = 0 + unread_count: int = 0 landing_articles: list[LandingArticleOut] = [] landing_posts: list[LandingPostOut] = [] + top_contributors: list = [] + trending_topics: list[TrendingTopicOut] = [] class DeviiPageOut(_Out): diff --git a/devplacepy/schemas/profile.py b/devplacepy/schemas/profile.py index 1c5b7415..d19d4465 100644 --- a/devplacepy/schemas/profile.py +++ b/devplacepy/schemas/profile.py @@ -50,6 +50,10 @@ class ProfileOut(_Out): ai_modifier_enabled: bool = False ai_modifier_sync: bool = False ai_modifier_prompt: Optional[str] = None + interactions_enabled: bool = True + interactions_source: Optional[str] = None + interactions_default: bool = True + interactions_override: Optional[bool] = None telegram_paired: bool = False notif_telegram_paired: bool = False can_manage_customization: bool = False diff --git a/devplacepy/services/CLAUDE.md b/devplacepy/services/CLAUDE.md index 18544225..a6b78d0d 100644 --- a/devplacepy/services/CLAUDE.md +++ b/devplacepy/services/CLAUDE.md @@ -22,9 +22,9 @@ Generic, lightweight, **fire-and-forget** offload for non-critical side-effects **Opt-in, default off, server-side.** When a user turns it on (profile **AI content correction** block, owner-only, saved at `POST /profile/{username}/ai-correction`), the prose they author is rewritten by the AI gateway. The per-user `ai_correction_sync` flag (0 = background default, 1 = sync) picks the **apply mode**: background rewrites the stored fields a moment after the write (never slows the request); sync makes the HTTP response wait until the correction is applied. -- **Sync must never block the event loop (the self-deadlock trap).** The correction POSTs to the in-process gateway (`INTERNAL_GATEWAY_URL` = `localhost:{PORT}`), and the hooked content helpers are synchronous on the loop thread, so a blocking inline call self-deadlocks the (single) worker against its own gateway request - on a single worker that loop is the only thing that can serve the gateway request it is waiting on, so a blocking inline call deadlocks the worker until the httpx timeout, then fail-softs (server hangs, no correction). The fix: sync runs `_run_correction` via `loop.run_in_executor` (loop stays free), stashes the future on `request.scope[PENDING_SCOPE_KEY]`, and the `await_pending_corrections` middleware in `main.py` awaits it after the handler. Never reintroduce a blocking inline correction on the loop thread. +- **Sync must never block the event loop (the self-deadlock trap).** The correction POSTs to the in-process gateway (`INTERNAL_GATEWAY_URL` = `localhost:{PORT}`), and the hooked content helpers are synchronous on the loop thread, so a blocking inline call self-deadlocks the (single) worker against its own gateway request - on a single worker that loop is the only thing that can serve the gateway request it is waiting on, so a blocking inline call deadlocks the worker until the httpx timeout, then fail-softs (server hangs, no correction). The fix: sync runs `_run_correction` on the dedicated `AI_APPLY_EXECUTOR` thread pool (`correction.py`, 4 workers) via `loop.run_in_executor` (loop stays free, and the default executor - shared with `asyncio.to_thread` password hashing at login/signup - is never occupied by blocking AI HTTP calls), stashes the future on `request.scope[PENDING_SCOPE_KEY]`, and the `await_pending_corrections` middleware in `main.py` awaits it after the handler. Never reintroduce a blocking inline correction on the loop thread. - **Field registry is the single source of truth.** `CORRECTABLE_FIELDS: dict[str, tuple[str, ...]]` maps each correctable table to its prose columns: `posts` -> `(title, content)`, `projects`/`gists` -> `(title, description)`, `comments`/`messages` -> `(content,)`, `users` -> `(bio,)`. `gists.source_code`, project files, and Gitea issues are intentionally excluded - code and external systems are never corrected. -- **Choke entrypoint.** `schedule_correction(user, table, uid, request=None)` is the only thing handlers call (every hook passes `request`). It is a no-op unless: a user dict is present, `table` is in the registry, `user["ai_correction_enabled"]` is truthy, and the user has a non-empty `api_key`. In **sync** mode (`user["ai_correction_sync"]`) it calls `_run_inline_awaited`, which - only when a `request.scope` and a running event loop exist - submits `_run_correction` to `loop.run_in_executor` (off the loop thread) and stashes the future on `request.scope[PENDING_SCOPE_KEY]` for the middleware to await; if there is no request/loop it returns False and falls back to background. In **background** mode (default) it `background.submit`s `_run_correction` (per-worker queue; also runs inline under `DEVPLACE_DISABLE_SERVICES=1`). The same `_run_correction` worker function runs in all cases. +- **Choke entrypoint.** `schedule_correction(user, table, uid, request=None)` is the only thing handlers call (every hook passes `request`). It is a no-op unless: a user dict is present, `table` is in the registry, `user["ai_correction_enabled"]` is truthy, and the user has a non-empty `api_key`. In **sync** mode (`user["ai_correction_sync"]`) it calls `_run_inline_awaited`, which - only when a `request.scope` and a running event loop exist - submits `_run_correction` to `loop.run_in_executor(AI_APPLY_EXECUTOR, ...)` (off the loop thread, on the dedicated AI pool) and stashes the future on `request.scope[PENDING_SCOPE_KEY]` for the middleware to await; if there is no request/loop it returns False and falls back to background. In **background** mode (default) it `background.submit`s `_run_correction` (per-worker queue; also runs inline under `DEVPLACE_DISABLE_SERVICES=1`). The same `_run_correction` worker function runs in all cases. - **The hooks (covers UI + REST + Devii + devRant in one place):** `content.create_content_item` (posts/projects/gists), `content.edit_content_item`, `content.create_comment_record`, `content.edit_comment_record`, `services/messaging/persist.persist_message` (DMs - sender is the user), `routers/profile/index.update_profile` (bio), and the two devRant direct-update edit paths (`routers/devrant/rants.edit_rant`, `routers/devrant/comments.edit_comment`). devRant create paths route through the shared content cores, so they are already hooked. Code-only and external paths (project files, Gitea) are deliberately not hooked. - **The gateway call is fail-soft.** `correct_text(api_key, prompt, text)` is synchronous (runs on the background worker thread), POSTs to `INTERNAL_GATEWAY_URL` with `model=INTERNAL_MODEL` via `stealth.stealth_sync_client`, authenticated with the user's own `Bearer` api_key (per-user attribution). It returns the ORIGINAL text on any error, empty output, or suspiciously large output (`len > len(text) * MAX_GROWTH_FACTOR + 200`, rejecting hallucinated expansion). `_run_correction` reads the row back, corrects each registry field that has non-blank text, writes only changed fields via `table.update(updates, ["uid"])` without touching `updated_at` (auto-correction is not a user edit) or the slug (slugs are permanent), and `clear_user_cache(user_uid)` when `table == "users"` so the corrected bio re-caches. - **Per-user usage aggregation (only on success).** `correct_text` returns `(text, usage)`; `usage` is parsed from the gateway's `X-Gateway-*` response headers (`_usage_from_headers`) whenever the upstream call returned 200 (cost was incurred, even if the corrected output was rejected), else `None` on any failure. `_usage_from_headers` captures the token and cost headers PLUS the timing headers `X-Gateway-Upstream-Latency-Ms` and `X-Gateway-Total-Latency-Ms` (as `upstream_latency_ms`/`total_latency_ms`), so each call's timing is metered. `_run_correction` accumulates the per-field `usage` into one `totals` dict and, when `totals["calls"] > 0`, makes ONE call to `database.add_correction_usage(user_uid, totals)` (a single `totals` dict, not positional args) - so a 2-field content item is a single aggregated write, and a failed/empty correction records nothing. `add_correction_usage` (and `add_modifier_usage`) delegate to the shared `database._add_usage(usage_table, user_uid, totals)`: a single atomic `INSERT ... ON CONFLICT(user_uid) DO UPDATE SET col = col + excluded.col` upsert against the `correction_usage` table (per-user running SUMS: `calls`/`prompt_tokens`/`completion_tokens`/`total_tokens`/`cost_usd`/`upstream_latency_ms`/`total_latency_ms`/`updated_at`, unique index `idx_correction_usage_user`, the two latency columns REAL default 0.0, all ensured in `init_db`). It is a derived counter table (NOT in `SOFT_DELETE_TABLES`, like `gateway_usage_ledger`) and is deliberately separate from `users` so accumulating never invalidates the auth/user cache. `database.get_correction_usage(user_uid)` (via `_get_usage`) returns the stored sums PLUS computed averages: `avg_tokens` (total_tokens/calls), `avg_upstream_latency_ms`, `avg_total_latency_ms`, `avg_tokens_per_second` (completion_tokens over total upstream seconds), and `avg_cost_usd`. @@ -57,7 +57,7 @@ This section covers only the shared machinery. The individual services built on In production (`make prod`, 2 workers) only the lock-holding worker runs services, but an admin request can hit either worker. State therefore lives in the DB, not process memory: - **Desired/config state** in `site_settings` (via `get_setting`/`set_setting`): `service__enabled` (`"0"`/`"1"` - also the boot flag), `service__command` (`":"`, verbs `run`/`clear`), `service__log_size`, plus each declared config field's own key. -- **Observed state** in the `service_state` table (one row per service): `status`, `last_run`, `next_run`, `started_at`, `heartbeat`, `logs` (JSON), `updated_at`. Written by the supervising worker, read by any worker. +- **Observed state** in the `service_state` table (one row per service): `status`, `last_run`, `next_run`, `started_at`, `heartbeat`, `logs` (JSON), `updated_at`. Written by the supervising worker, read by any worker. The heartbeat persists every `PERSIST_SECONDS` (8s, under the 15s `STALE_SECONDS` liveness window) and caches the row id so a persist is one UPDATE, not a `find_one` + UPDATE. `collect_metrics()` runs on its own slower `METRICS_SECONDS` cadence (15s default, per-class overridable - `AuditService` uses 300s because its metric is a `COUNT(*)` over the ever-growing audit table); between refreshes the last snapshot is re-persisted, and a `force` persist (transitions, run-now) always recomputes. Keep expensive aggregates out of the 1s tick: put them in `collect_metrics` and, if still heavy, raise the subclass `METRICS_SECONDS`. `main.py` registers every service in **all** workers (so `describe_all()` works anywhere) but only the lock worker calls `service_manager.supervise()`. @@ -118,7 +118,7 @@ devplace devii reset-quota --all # Reset every quota (users and guests) `uvicorn --workers N` = N independent processes sharing only the filesystem and SQLite DB. Module-global caches/counters are per-process, so a local `clear()` is invisible to siblings. Full reference: admin docs `Production -> Multi-worker and concurrency` (`templates/docs/production-concurrency.html`). Enforce these: -- **In-process caches are never authoritative across workers.** Coordinate invalidation through the `cache_state` version table in `database.py`: call `bump_cache_version(name)` at every write path and `sync_local_cache(name, cache)` before every read path. Existing names: `auth` (guards `_user_cache`; bumped by `clear_user_cache`/`clear_session_cache` on logout, ban, role/password change), `settings` (guards `_settings_cache`; bumped by `set_setting`/`clear_settings_cache`), `relations` (`_relations_cache`), `customizations` (`_customizations_cache`), `notif_prefs` (`_notification_prefs_cache`), `gateway_routing` (`_ROUTING_CACHE`), and **`admins`** (guards `_admins_cache`, which memoizes `get_admin_uids()` / `get_primary_admin_uid()`). The `_cache_version_cache` (`ttl=1`) caches the version reads themselves, so the bump is seen by other workers within ~1s on their next read (the originating worker is immediate); the version read/bump fail open (logged, never raise). When you add a per-process cache whose staleness matters, give it a name and wire both calls - do not invent a second invalidation mechanism. +- **In-process caches are never authoritative across workers.** Coordinate invalidation through the `cache_state` version table in `database.py`: call `bump_cache_version(name)` at every write path and `sync_local_cache(name, cache)` before every read path. Existing names: `auth` (guards `_user_cache`; bumped by `clear_user_cache`/`clear_session_cache` on logout, ban, role/password change - a version bump makes EVERY worker clear its WHOLE `_user_cache`, so display-only refreshes (XP/level in `award_xp`, AI-corrected bio) call `clear_user_cache(uid, propagate=False)` for a local pop without the global bump; identity/authz changes (logout, ban, role, password, api-key/token revoke) MUST keep the default propagate=True), `settings` (guards `_settings_cache`; bumped by `set_setting`/`clear_settings_cache`), `relations` (`_relations_cache`), `customizations` (`_customizations_cache`), `notif_prefs` (`_notification_prefs_cache`), `gateway_routing` (`_ROUTING_CACHE`), and **`admins`** (guards `_admins_cache`, which memoizes `get_admin_uids()` / `get_primary_admin_uid()`). The `_cache_version_cache` (`ttl=1`) caches the version reads themselves, so the bump is seen by other workers within ~1s on their next read (the originating worker is immediate); the version read/bump fail open (logged, never raise). When you add a per-process cache whose staleness matters, give it a name and wire both calls - do not invent a second invalidation mechanism. - **Admin-set cache invariant (load-bearing).** `_admins_cache` makes `get_admin_uids()` / `get_primary_admin_uid()` (hit on the projects-listing visibility filter, `_owner_is_admin`, and every primary-admin gate: `/dbapi`, backup-archive download) a dict lookup instead of a per-call `SELECT`. It is NOT the authorization gate - `is_admin(user)`/`require_admin` read the role off the user object (independently invalidated via `clear_user_cache`), so a stale admin set cannot grant access. But **any code that writes `users.role` MUST call `database.invalidate_admins_cache()`** (clears local + bumps the `admins` version), exactly like the soft-delete and `with db:` rules. Current role-write sites all do: `routers/admin/users.py` (role change), `cli.py` (`role set`), and `utils._create_account` (first user -> Admin). Omitting the call leaves the admin set stale for up to the 300s TTL. - **Deterministic / eventual caches skip version-sync on purpose.** Two caches need no `cache_state` name because correctness does not depend on cross-worker freshness: (1) `docs_prose._render_markdown` is an `@lru_cache` on the **static** prose source string (pure function of template content, so identical on every worker; changes only on deploy), and (2) `main._home_cache` (`TTLCache ttl=60`) memoizes the guest `/` blocks - the featured-news block (identical for everyone) and the latest-posts block **only for the no-block case**; a viewer with a non-empty block set bypasses the cache and is computed fresh, so the original block-filter semantics are preserved byte-for-byte and there is zero cross-user leakage. Worst case is a soft-deleted/edited public post lingering on `/` for <=60s. Use a plain `TTLCache`/`lru_cache` (no version name) ONLY when the value is deterministic or its staleness is purely cosmetic; anything whose staleness affects correctness or permissions MUST use the version-sync pattern above. - **Authoritative state lives in the DB**, memory is only a short-TTL accelerator (sessions, the Devii 24h cap via `devii_usage_ledger`, cost counters). @@ -132,7 +132,7 @@ Read-mostly aggregates that were recomputed per request or per vote sit behind s | Cache | Where | Key | TTL | Invalidation | |---|---|---|---|---| -| `_authors_cache` | `database/ranking.py` | `ranked`/`rank_map` | 15s | TTL only - `update_target_stars` deliberately does NOT clear it per vote anymore (the old per-vote `clear()` forced a full UNION-JOIN ranking recompute on the next feed/leaderboard/landing request and never propagated cross-worker anyway) | +| `_authors_cache` | `database/ranking.py` | `ranked`/`rank_map` | 60s | TTL only - `update_target_stars` deliberately does NOT clear it per vote anymore (the old per-vote `clear()` forced a full UNION-JOIN ranking recompute on the next feed/leaderboard/landing request and never propagated cross-worker anyway) | | `_stars_cache` | `database/ranking.py` | user uid | 15s | TTL + `clear_user_stars(owner_uid)` from `content.apply_vote`, so a user's own total updates immediately on the voting worker | | `_leaderboard_cache` | `services/game/store/farm.py` | `top:{limit}` | 15s | TTL only (the farm scan + Python `farm_score` sort runs at most once per 15s per worker) | | `_projects_cache` | `templating.py` | user uid | 10s | TTL + `clear_user_projects_cache(uid)` from the `content.py` project create/delete choke points (in-function import - `templating` imports `content` at module level). `jinja_user_projects` also filters `deleted_at=None` now (the composer dropdown previously listed soft-deleted projects) | diff --git a/devplacepy/services/access_tokens.py b/devplacepy/services/access_tokens.py index fad10f5c..55dafa3e 100644 --- a/devplacepy/services/access_tokens.py +++ b/devplacepy/services/access_tokens.py @@ -90,6 +90,9 @@ def revoke_token(uid: str) -> bool: return False stamp = datetime.now(timezone.utc).isoformat() tokens.update({"id": row["id"], "deleted_at": stamp, "deleted_by": "manual"}, ["id"]) + from devplacepy.utils.authcache import clear_user_cache + + clear_user_cache(row["user_uid"]) return True @@ -101,6 +104,10 @@ def revoke_all(user_uid: str) -> int: for row in list(tokens.find(user_uid=user_uid, deleted_at=None)): tokens.update({"id": row["id"], "deleted_at": stamp, "deleted_by": "manual"}, ["id"]) count += 1 + if count: + from devplacepy.utils.authcache import clear_user_cache + + clear_user_cache(user_uid) return count diff --git a/devplacepy/services/ai_modifier.py b/devplacepy/services/ai_modifier.py index 666d6e77..1edf639d 100644 --- a/devplacepy/services/ai_modifier.py +++ b/devplacepy/services/ai_modifier.py @@ -61,19 +61,20 @@ def schedule_modification( prompt = (user.get("ai_modifier_prompt") or DEFAULT_MODIFIER_PROMPT).strip() user_uid = user.get("uid") or "" if user.get("ai_modifier_sync") and schedule_pending( - _run_modification, request, api_key, prompt, table, uid, user_uid + _run_modification, request, api_key, prompt, table, uid, user_uid, row ): return background.submit(_run_modification, api_key, prompt, table, uid, user_uid) def _run_modification( - api_key: str, prompt: str, table: str, uid: str, user_uid: str + api_key: str, prompt: str, table: str, uid: str, user_uid: str, row: dict | None = None ) -> None: fields = CORRECTABLE_FIELDS.get(table) if not fields: return - row = get_table(table).find_one(uid=uid) + if row is None: + row = get_table(table).find_one(uid=uid) if not row: return updates: dict = {} @@ -97,6 +98,6 @@ def _run_modification( if table == "users" and user_uid: from devplacepy.utils import clear_user_cache - clear_user_cache(user_uid) + clear_user_cache(user_uid, propagate=False) if totals["calls"] and user_uid: add_modifier_usage(user_uid, totals) diff --git a/devplacepy/services/audit/service.py b/devplacepy/services/audit/service.py index d7f2346f..e601650a 100644 --- a/devplacepy/services/audit/service.py +++ b/devplacepy/services/audit/service.py @@ -18,6 +18,7 @@ class AuditService(BaseService): description = "Prunes audit_log rows and their links older than the retention window." default_enabled = True min_interval = 3600 + METRICS_SECONDS = 300 config_fields = [ ConfigField( RETENTION_KEY, diff --git a/devplacepy/services/audit/store.py b/devplacepy/services/audit/store.py index 3f7f3c35..b16432c6 100644 --- a/devplacepy/services/audit/store.py +++ b/devplacepy/services/audit/store.py @@ -101,16 +101,15 @@ def insert_event(row: dict) -> str: record["created_at"] = now() if record.get("via_agent") is None: record["via_agent"] = 0 - get_table(AUDIT_TABLE).insert(record) + get_table(AUDIT_TABLE).insert(record, ensure=False) return record["uid"] def insert_links(audit_uid: str, links: list[dict]) -> int: if not links: return 0 - table = get_table(LINKS_TABLE) stamp = now() - count = 0 + records = [] for link in links: record = _empty_link() record.update( @@ -121,9 +120,10 @@ def insert_links(audit_uid: str, links: list[dict]) -> int: record["created_at"] = stamp if not record.get("object_uid"): continue - table.insert(record) - count += 1 - return count + records.append(record) + if records: + get_table(LINKS_TABLE).insert_many(records, ensure=False) + return len(records) def get_event(uid: str) -> Optional[dict]: diff --git a/devplacepy/services/base.py b/devplacepy/services/base.py index 288c4145..852e900e 100644 --- a/devplacepy/services/base.py +++ b/devplacepy/services/base.py @@ -123,7 +123,8 @@ class BaseService(ABC): details = "" DEFAULT_LOG_SIZE = 20 TICK_SECONDS = 1 - PERSIST_SECONDS = 3 + PERSIST_SECONDS = 8 + METRICS_SECONDS = 15 STALE_SECONDS = 15 def __init__(self, name: str, interval_seconds: int = 3600): @@ -144,6 +145,9 @@ class BaseService(ABC): self._run_pending = False self._last_command = None self._last_persist = None + self._last_metrics = None + self._metrics_snapshot = {} + self._state_row_id = None self.enabled_field = ConfigField( self.enabled_key, "Enabled", @@ -234,6 +238,14 @@ class BaseService(ABC): logger.warning(f"Could not collect metrics for {self.name}: {e}") return {} + def _current_metrics(self, now, force: bool = False) -> dict: + if not force and self._last_metrics is not None: + if (now - self._last_metrics).total_seconds() < self.METRICS_SECONDS: + return self._metrics_snapshot + self._last_metrics = now + self._metrics_snapshot = self._safe_metrics() + return self._metrics_snapshot + def start_supervisor(self) -> None: if self._task is not None: return @@ -344,16 +356,18 @@ class BaseService(ABC): "started_at": self._started_at.isoformat() if self._started_at else "", "heartbeat": now.isoformat(), "logs": json.dumps(list(self.log_buffer)), - "metrics": json.dumps(self._safe_metrics()), + "metrics": json.dumps(self._current_metrics(now, force=force)), "updated_at": now.isoformat(), } try: table = get_table("service_state") - existing = table.find_one(name=self.name) - if existing: - table.update({**record, "id": existing["id"]}, ["id"]) + if self._state_row_id is None: + existing = table.find_one(name=self.name) + self._state_row_id = existing["id"] if existing else None + if self._state_row_id is not None: + table.update({**record, "id": self._state_row_id}, ["id"]) else: - table.insert(record) + self._state_row_id = table.insert(record) except Exception as e: logger.warning(f"Could not persist state for {self.name}: {e}") diff --git a/devplacepy/services/bot/engage.py b/devplacepy/services/bot/engage.py index 3b81d687..051a3e28 100644 --- a/devplacepy/services/bot/engage.py +++ b/devplacepy/services/bot/engage.py @@ -387,7 +387,7 @@ class BotEngageMixin: idx = random.randrange(limit) target = comments.nth(idx) try: - text = (await target.inner_text() or "")[:1000] + text = (await target.inner_text() or "")[:125000] except Exception: continue if not text: diff --git a/devplacepy/services/bot/social.py b/devplacepy/services/bot/social.py index 2adf3e91..090a097c 100644 --- a/devplacepy/services/bot/social.py +++ b/devplacepy/services/bot/social.py @@ -433,7 +433,7 @@ class BotSocialMixin: mentioner = "" if comment_loc is not None: try: - parent_text = (await comment_loc.inner_text() or "")[:1000] + parent_text = (await comment_loc.inner_text() or "")[:125000] except Exception: parent_text = "" try: diff --git a/devplacepy/services/containers/files/bot.py b/devplacepy/services/containers/files/bot.py index e888f1db..db247e80 100644 --- a/devplacepy/services/containers/files/bot.py +++ b/devplacepy/services/containers/files/bot.py @@ -72,7 +72,7 @@ MENTION_POLL_SECONDS = int(os.environ.get("MENTION_POLL_SECONDS", "30")) DM_POLL_SECONDS = int(os.environ.get("DM_POLL_SECONDS", "10")) BOT_USERNAME = os.environ.get("BOT_USERNAME", "") -COMMENT_CHAR_LIMIT = 1000 +COMMENT_CHAR_LIMIT = 125000 MESSAGE_CHAR_LIMIT = 2000 PART_SUFFIX_RESERVE = 12 PART_DELIVERY_DELAY = 0.5 diff --git a/devplacepy/services/containers/files/bot.py.bak b/devplacepy/services/containers/files/bot.py.bak index c7445559..6fe2f154 100644 --- a/devplacepy/services/containers/files/bot.py.bak +++ b/devplacepy/services/containers/files/bot.py.bak @@ -72,7 +72,7 @@ MENTION_POLL_SECONDS = int(os.environ.get("MENTION_POLL_SECONDS", "30")) DM_POLL_SECONDS = int(os.environ.get("DM_POLL_SECONDS", "10")) BOT_USERNAME = os.environ.get("BOT_USERNAME", "") -COMMENT_CHAR_LIMIT = 1000 +COMMENT_CHAR_LIMIT = 125000 MESSAGE_CHAR_LIMIT = 2000 PART_SUFFIX_RESERVE = 12 PART_DELIVERY_DELAY = 0.5 diff --git a/devplacepy/services/correction.py b/devplacepy/services/correction.py index 9a2b35e9..5c9b7358 100644 --- a/devplacepy/services/correction.py +++ b/devplacepy/services/correction.py @@ -2,6 +2,8 @@ import asyncio import logging +import threading +from concurrent.futures import ThreadPoolExecutor import httpx @@ -30,6 +32,22 @@ CORRECTION_TIMEOUT_SECONDS = 20.0 MAX_GROWTH_FACTOR = 3 PENDING_SCOPE_KEY = "devplace_pending_corrections" +AI_APPLY_EXECUTOR = ThreadPoolExecutor(max_workers=4, thread_name_prefix="ai-apply") + +_gateway_client: httpx.Client | None = None +_gateway_client_lock = threading.Lock() + + +def _client() -> httpx.Client: + global _gateway_client + if _gateway_client is None: + with _gateway_client_lock: + if _gateway_client is None: + _gateway_client = stealth.stealth_sync_client( + timeout=CORRECTION_TIMEOUT_SECONDS + ) + return _gateway_client + def _usage_from_headers(response_headers) -> dict: parsed = parse_usage_headers(response_headers) @@ -72,21 +90,25 @@ def gateway_complete( ], "temperature": 0.1, } - headers = {"Content-Type": "application/json"} + headers = { + "Content-Type": "application/json", + "X-App-Reference": "devplace-correction-v-1-0-0", + } if api_key: headers["Authorization"] = f"Bearer {api_key}" try: - with stealth.stealth_sync_client(timeout=timeout) as client: - response = client.post(INTERNAL_GATEWAY_URL, json=payload, headers=headers) - response.raise_for_status() - usage = _usage_from_headers(response.headers) - content = ( - response.json() - .get("choices", [{}])[0] - .get("message", {}) - .get("content", "") - .strip() - ) + response = _client().post( + INTERNAL_GATEWAY_URL, json=payload, headers=headers, timeout=timeout + ) + response.raise_for_status() + usage = _usage_from_headers(response.headers) + content = ( + response.json() + .get("choices", [{}])[0] + .get("message", {}) + .get("content", "") + .strip() + ) except (httpx.HTTPError, ValueError, KeyError, IndexError) as exc: logger.warning("AI gateway completion failed, keeping original: %s", exc) return text, None @@ -138,7 +160,7 @@ def schedule_pending(fn, request: object, *args) -> bool: loop = asyncio.get_running_loop() except RuntimeError: return False - future = loop.run_in_executor(None, fn, *args) + future = loop.run_in_executor(AI_APPLY_EXECUTOR, fn, *args) scope.setdefault(PENDING_SCOPE_KEY, []).append(future) return True @@ -182,6 +204,6 @@ def _run_correction( if table == "users" and user_uid: from devplacepy.utils import clear_user_cache - clear_user_cache(user_uid) + clear_user_cache(user_uid, propagate=False) if totals["calls"] and user_uid: add_correction_usage(user_uid, totals) diff --git a/devplacepy/services/dbapi/nl2sql.py b/devplacepy/services/dbapi/nl2sql.py index 3f07fead..b60b36b3 100644 --- a/devplacepy/services/dbapi/nl2sql.py +++ b/devplacepy/services/dbapi/nl2sql.py @@ -92,7 +92,11 @@ async def _complete(messages: list[dict], api_key: str, model: str) -> str: "max_tokens": MAX_TOKENS, "temperature": 0.0, } - headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "X-App-Reference": "devplace-dbapi-v-1-0-0", + } async with stealth.stealth_async_client(timeout=GATEWAY_TIMEOUT_SECONDS) as client: response = await client.post(INTERNAL_GATEWAY_URL, json=payload, headers=headers) if response.status_code >= 400: diff --git a/devplacepy/services/deepsearch/chat.py b/devplacepy/services/deepsearch/chat.py index 44978945..73968ed0 100644 --- a/devplacepy/services/deepsearch/chat.py +++ b/devplacepy/services/deepsearch/chat.py @@ -82,7 +82,7 @@ class DeepsearchChat: stored_dim, ) return [] - return self.store.hybrid_search(question, query_vector, top_k=CHAT_TOP_K) + return await self.store.hybrid_search(question, query_vector, top_k=CHAT_TOP_K) async def answer(self, question: str, history: list[dict] | None = None) -> ChatAnswer: chunks = await self.retrieve(question) diff --git a/devplacepy/services/deepsearch/embeddings.py b/devplacepy/services/deepsearch/embeddings.py index fbdfc229..8c6f87b5 100644 --- a/devplacepy/services/deepsearch/embeddings.py +++ b/devplacepy/services/deepsearch/embeddings.py @@ -9,8 +9,8 @@ import re import time from dataclasses import dataclass, field -from devplacepy import stealth from devplacepy.config import INTERNAL_EMBED_MODEL, INTERNAL_EMBED_URL +from devplacepy.services.deepsearch.llm import gateway_client logger = logging.getLogger(__name__) @@ -112,13 +112,15 @@ async def embed_texts( headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "X-App-Reference": "devplace-deepsearch-v-1-0-0", } payload = {"model": INTERNAL_EMBED_MODEL, "input": pending_text} start = time.monotonic() backend = "gateway" try: - async with stealth.stealth_async_client(timeout=EMBED_TIMEOUT_SECONDS) as client: - response = await client.post(gateway_url, json=payload, headers=headers) + response = await gateway_client().post( + gateway_url, json=payload, headers=headers, timeout=EMBED_TIMEOUT_SECONDS + ) if response.status_code >= 400: raise RuntimeError(f"embed gateway returned {response.status_code}") data = response.json() diff --git a/devplacepy/services/deepsearch/llm.py b/devplacepy/services/deepsearch/llm.py index 1697ae3f..17e39739 100644 --- a/devplacepy/services/deepsearch/llm.py +++ b/devplacepy/services/deepsearch/llm.py @@ -5,6 +5,8 @@ from __future__ import annotations import logging import time +import httpx + from devplacepy import stealth from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL @@ -13,6 +15,15 @@ logger = logging.getLogger(__name__) CHAT_TIMEOUT_SECONDS = 120.0 DEFAULT_MAX_TOKENS = 1200 +_gateway_client: httpx.AsyncClient | None = None + + +def gateway_client() -> httpx.AsyncClient: + global _gateway_client + if _gateway_client is None or _gateway_client.is_closed: + _gateway_client = stealth.stealth_async_client(timeout=CHAT_TIMEOUT_SECONDS) + return _gateway_client + async def request_completion( messages: list[dict], @@ -33,10 +44,12 @@ async def request_completion( headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "X-App-Reference": "devplace-deepsearch-v-1-0-0", } start: float = time.monotonic() - async with stealth.stealth_async_client(timeout=timeout) as client: - response = await client.post(gateway_url, json=payload, headers=headers) + response = await gateway_client().post( + gateway_url, json=payload, headers=headers, timeout=timeout + ) elapsed_ms: int = int((time.monotonic() - start) * 1000) if response.status_code >= 400: raise RuntimeError(f"chat gateway returned {response.status_code}") diff --git a/devplacepy/services/deepsearch/store.py b/devplacepy/services/deepsearch/store.py index 5efa767d..129a253b 100644 --- a/devplacepy/services/deepsearch/store.py +++ b/devplacepy/services/deepsearch/store.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import logging import math import re @@ -19,6 +20,7 @@ BM25_B = 0.75 RRF_K = 60.0 DEFAULT_TOP_K = 8 CANDIDATE_MULTIPLIER = 4 +CHROMADB_TIMEOUT = 30.0 @dataclass @@ -45,6 +47,20 @@ class VectorStore: self._collection = None self._dims: int | None = None + async def _run_sync(self, func, *args, timeout: float = CHROMADB_TIMEOUT): + """Run a synchronous ChromaDB call in a thread executor with a timeout.""" + try: + return await asyncio.wait_for( + asyncio.to_thread(func, *args), timeout=timeout + ) + except asyncio.TimeoutError: + logger.error( + "deepsearch ChromaDB operation timed out after %.1fs on collection %s", + timeout, + self.collection_name, + ) + raise + def _ensure(self): if self._collection is not None: return self._collection @@ -71,7 +87,7 @@ class VectorStore: return self._dims return self._dims - def add(self, chunks: list[Chunk], vectors: list[list[float]]) -> None: + def _add_sync(self, chunks: list[Chunk], vectors: list[list[float]]) -> None: if not chunks: return keep_chunks: list[Chunk] = [] @@ -111,9 +127,17 @@ class VectorStore: ], ) - def all_chunks(self) -> list[Chunk]: + async def add(self, chunks: list[Chunk], vectors: list[list[float]]) -> None: + if not chunks: + return + await self._run_sync(self._add_sync, chunks, vectors) + + def _all_chunks_sync(self, limit: int = 0) -> list[Chunk]: collection = self._ensure() - data = collection.get(include=["documents", "metadatas"]) + kwargs: dict = {"include": ["documents", "metadatas"]} + if limit > 0: + kwargs["limit"] = limit + data = collection.get(**kwargs) chunks: list[Chunk] = [] ids = data.get("ids") or [] documents = data.get("documents") or [] @@ -134,13 +158,17 @@ class VectorStore: ) return chunks - def count(self) -> int: + async def all_chunks(self, limit: int = 1000) -> list[Chunk]: + return await self._run_sync(self._all_chunks_sync, limit) + + async def count(self) -> int: try: - return self._ensure().count() + collection = self._ensure() + return await self._run_sync(collection.count) except Exception: return 0 - def vector_search( + def _vector_search_sync( self, query_vector: list[float], top_k: int, where: dict | None = None ) -> list[Chunk]: collection = self._ensure() @@ -173,6 +201,13 @@ class VectorStore: ) return chunks + async def vector_search( + self, query_vector: list[float], top_k: int, where: dict | None = None + ) -> list[Chunk]: + return await self._run_sync( + self._vector_search_sync, query_vector, top_k, where + ) + def keyword_scores(self, query: str, chunks: list[Chunk]) -> dict[str, float]: terms = _tokenize(query) if not terms or not chunks: @@ -205,14 +240,14 @@ class VectorStore: scores[chunk.uid] = score return scores - def hybrid_search( + async def hybrid_search( self, query: str, query_vector: list[float], top_k: int = DEFAULT_TOP_K, where: dict | None = None, ) -> list[Chunk]: - candidates = self.vector_search( + candidates = await self.vector_search( query_vector, top_k * CANDIDATE_MULTIPLIER, where ) if not candidates: @@ -234,10 +269,11 @@ class VectorStore: candidates.sort(key=lambda chunk: chunk.score, reverse=True) return candidates[:top_k] - def coverage_analytics(self) -> dict: - chunks = self.all_chunks() + async def coverage_analytics(self, sample_limit: int = 5000) -> dict: + chunks = await self.all_chunks(limit=sample_limit) if not chunks: - return {"chunks": 0, "domains": 0, "sources": 0, "avg_chunk_chars": 0} + total = await self.count() + return {"chunks": total, "domains": 0, "sources": 0, "avg_chunk_chars": 0} domains = { urlparse(chunk.metadata.get("url", "")).netloc for chunk in chunks } diff --git a/devplacepy/services/devii/CLAUDE.md b/devplacepy/services/devii/CLAUDE.md index 557ec734..6709d8e5 100644 --- a/devplacepy/services/devii/CLAUDE.md +++ b/devplacepy/services/devii/CLAUDE.md @@ -35,6 +35,12 @@ Conversation history persists to `devii_conversations` (rehydrated on reconnect, **Per-owner self-learning memory is privacy-critical.** The `LessonStore` (reflect/recall) is **owner-scoped, never shared**: `LessonStore(db, owner_kind, owner_id)` filters every read/write by owner. The hub builds one per session over the same `owned_db` as the task store - the main `db` (table `devii_lessons`) for signed-in users (persistent, isolated, survives restarts) and a fresh `memory_db()` for guests (ephemeral, scoped to that web session). `forget_lessons` (agentic tool -> `LessonStore.clear()`/`delete()`) lets the user purge them; the system prompt forbids storing credentials/secrets in lessons. **Regression to avoid: do NOT share one `LessonStore` across sessions** - that leaked one user's reflected lessons (including credentials) into every other user's recall. +**Lesson retention and deduplication.** Every `add()` deduplicates against existing active lessons via Jaccard similarity (threshold 0.70): a near-duplicate bumps the original's `hits` counter and refreshes `created_at` instead of inserting a new row. A per-owner cap (`devii_lessons_max_per_owner`, default 500, configurable on `/admin/services` and `site_settings`) is enforced on every insert: when exceeded, the oldest lessons are soft-deleted (`deleted_by="retention"`). Age-based pruning runs on `DeviiService.run_once()` (every 60s on the lock-owner worker): lessons older than `devii_lessons_max_age_days` (default 90, configurable) are soft-deleted across all owners. Both settings persist in `site_settings` and are seeded in `init_db`. The `devii_lessons` table is in `SOFT_DELETE_TABLES` for admin Trash restore/purge. + +**Quality signals.** Each lesson has a `rating` column (integer, default 0). The `lesson_rate(uid, value)` agentic tool (value 1 for useful, -1 for unhelpful) lets the agent self-rate lessons. In `_rebuild()`, lessons with `rating <= -3` are excluded from the BM25 index and therefore never returned by `recall()`. The `lesson_count()` tool reports active lesson count. + +**CLI:** `devplace devii lessons count` reports active/soft-deleted totals; `devplace devii lessons prune --all-owners|--username USER` soft-deletes old lessons; `devplace devii lessons clear --force` hard-deletes all rows. Rate-limiting guard: `run_once` swallows all exceptions so a bad schema never stops housekeeping. + ## Reminders and scheduled tasks (persistent across reboot) `create_task` queues a self-contained prompt that a fresh agent runs later (`services/devii/tasks/`): `kind=once` (`delay_seconds` for relative, `run_at` UTC for absolute), `interval` (`every_seconds`), or `cron`. The per-session `Scheduler` ticks every 1s and executes due rows through the session's executor, so the result is broadcast to any connected tab and buffered (`type:"task"` frame) when the terminal is closed. @@ -142,9 +148,57 @@ Devii can partially configure its OWN system message: every system prompt ends w - **Store** (`behavior/store.py`). `BehaviorStore(db, owner_kind, owner_id)` over `devii_behavior` (one upserted row per owner keyed on `owner_kind`/`owner_id`; `text()` reads, `set()` upserts; index `idx_devii_behavior_owner`). Persistent for users, `memory_db()` for guests (built in `hub.get_or_create` from the shared `owned_db`, like the other owner stores). - **Controller** (`behavior/controller.py`). `BehaviorController(store)`, `dispatch("update_behavior", args)` -> `store.set(behavior)`. Built in `DeviiSession` and passed to `Dispatcher(behavior=...)`; the dispatcher routes `handler="behavior"` to it and degrades gracefully ("not available in this context") when unwired (e.g. the standalone CLI, same as `virtual_tools`/`avatar`). -- **Injection and refresh** (`session.py`). `_compose_system_prompt()` = base prompt (`_system_prompt_for(is_admin)`) + `\n\n` + `BEHAVIOR_HEADER` (+ `\n` + body when non-empty; just the header when empty). Used to seed the `Agent` and the scheduler executor worker, and `_refresh_system_prompt()` rewrites `agent._messages[0]["content"]` **at the top of every `_run_turn`** (next to `_refresh_tools`), so a mid-conversation `update_behavior` takes effect on the following turn and the live system message is never overridden by the stale base. +- **Injection and refresh** (`session.py`). `_compose_system_prompt()` = base prompt (`_system_prompt_for(is_admin)`) + CA-IWP fragment + live `CHANNEL` block + `\n\n` + `BEHAVIOR_HEADER` (+ `\n` + body when non-empty; just the header when empty). Used to seed the `Agent` and the scheduler executor worker, and `_refresh_system_prompt()` rewrites `agent._messages[0]["content"]` **at the top of every `_run_turn`** (next to `_refresh_tools`), so a mid-conversation `update_behavior` takes effect on the following turn and the live system message is never overridden by the stale base. - Registered via `BEHAVIOR_ACTIONS` (`registry.py`); the system-prompt **SELF-CONFIGURED BEHAVIOR (TRUTH RULES)** section in `agent.py` steers when/how to call it. +## Channel-Aware Interactive Widget Protocol (CA-IWP / Speak With Buttons) + +Devii can ask the user for decisions through a gated tool surface instead of inventing HTML or channel-specific prose. Spec lives as the CA-IWP document; implementation is Devii-only under `services/devii/interaction/`. + +### Layers + +| Piece | Role | +|-------|------| +| `interaction/capabilities.py` | Maps session channel (`main`/`docs` -> `site-chat`, `telegram`, `cli`) to capability flags + limits; builds the compact `CHANNEL` / `CAPS` / `LIMITS` / `TOOLS` / `INTERACTION` fragment injected every turn. | +| `interaction/schema.py` | Pydantic validation for `ui_prompt` args (widget catalog: confirm, choice, choice_multi, text, number, date, select, `//`, group). Untrusted model input is sanitized and capped. | +| `interaction/broker.py` | Validates, assigns `interaction_id`, single-flight open interactions, routes to site / telegram / plain adapters, returns structured `{status, values, meta}`. | +| `interaction/controller.py` | Dispatches `ui_prompt` / `ui_cancel` / `ui_notify` (`handler="interaction"`). | +| `interaction/actions.py` | Catalog entries registered in `registry.py` as `INTERACTION_ACTIONS`. | +| `interaction/markdown.py` + `parse.py` | Dual-surface markdown projection and plain/CLI reply parsers. | + +### Tool gating and preferences (admin default + user override) + +`DeviiSession._builtin_tools()` filters UI tools through `channel_context().tools`. Docs channel stays search-only (no UI tools). Open interaction (single-flight) exposes only `ui_cancel` plus preference tools. + +**Admin default:** Devii service ConfigField `devii_interactions_default` (bool, default on) on `/admin/services`. Guests always use this default. + +**User override:** column `users.interactions_enabled` (`-1` = inherit default, `0` = off, `1` = on). New signups insert `-1`. Resolution: `interaction/prefs.py` `effective_for(owner_kind, owner_id)`. + +**Surfaces (same fan-out as AI correction):** +- Devii tools `interactions_get` / `interactions_set` (`requires_auth=True`; set accepts `enabled` or `reset=true` to inherit again). Always offered to signed-in users even when widgets are off, so they can re-enable. +- HTTP `POST /profile/{username}/interactions` (owner-or-admin, form `InteractionsForm`), profile card + `InteractionsPref.js`, `docs_api` endpoint, audit `profile.interactions`, profile JSON keys on `ProfileOut`. +- When effective is off, `ui_prompt` / `ui_notify` are absent from the tool list and the controller refuses them; the model must use degraded markdown menus. + +### Site-chat path + +`ui_prompt` blocks like client tools: session `_interaction_wait` emits `{type:"interaction", id, args}` on the WebSocket; `devii-terminal.js` mounts an `` tree via `createElement` (never model HTML), lazy-loads widget modules through `AiAutoload` (`static/js/autoload/AiAutoload.js`, allowlisted tag -> module map only), and replies with `{type:"interaction_result"}`. Router resolves via `session.resolve_interaction`. Custom elements live under `static/js/components/Ai*.js` with per-component CSS under `static/css/components/ai-*.css`. Light DOM only; `Application` boots the shell (`AiInteraction`, `AiStatus`, `AiActions`, `AiHelp`, `AiOption`) and starts the autoloader. + +### Telegram path + +`TelegramConnection` handles `type:"interaction"`: sends dual-surface plain text plus inline keyboard for confirm / single choice (`callback_data` short tokens). `TelegramBridge` registers a pending future **before** the chat lock so the next message or `callback_query` can resolve mid-turn. Worker `allowed_updates` includes `callback_query`; service routes `type:"callback"` to the bridge. + +### Plain / CLI path + +Broker uses `present_plain` or emits a plain menu and waits; `parse_plain_reply` accepts y/n, indices, value tokens, cancel. + +### System prompt + +Every non-docs turn appends `CA_IWP_SYSTEM_FRAGMENT` plus the live channel fragment from `_compose_system_prompt()`. Model rules: prefer `ui_prompt` when gated on; never invent HTML/CE tags; branch on `status` (submitted / cancelled / timeout / superseded / error). + +### Tests + +Unit coverage under `tests/unit/services/devii/interaction/` (capabilities, schema, markdown, parse, broker, controller) plus session tool-gating assertions and telegram callback emission. Mirror this layout for any extension. + ## Related Devii tool wrappers (customization, container) Two more Devii-side controllers live under `services/devii/` but their full mechanism is documented in their owning subsystem's file, not here: diff --git a/devplacepy/services/devii/actions/ai_correction_actions.py b/devplacepy/services/devii/actions/ai_correction_actions.py index e2d82c3a..0faafdb4 100644 --- a/devplacepy/services/devii/actions/ai_correction_actions.py +++ b/devplacepy/services/devii/actions/ai_correction_actions.py @@ -61,7 +61,7 @@ AI_CORRECTION_ACTIONS: tuple[Action, ...] = ( ), arg( "prompt", - "The correction instruction (max 2000 chars). Omit to keep the current one.", + "The correction instruction (max 20000 chars). Omit to keep the current one.", ), ), ), diff --git a/devplacepy/services/devii/actions/ai_modifier_actions.py b/devplacepy/services/devii/actions/ai_modifier_actions.py index a219a39b..907be0d6 100644 --- a/devplacepy/services/devii/actions/ai_modifier_actions.py +++ b/devplacepy/services/devii/actions/ai_modifier_actions.py @@ -64,7 +64,7 @@ AI_MODIFIER_ACTIONS: tuple[Action, ...] = ( ), arg( "prompt", - "The modifier instruction (max 2000 chars). Omit to keep the current one.", + "The modifier instruction (max 20000 chars). Omit to keep the current one.", ), ), ), diff --git a/devplacepy/services/devii/actions/catalog/comments.py b/devplacepy/services/devii/actions/catalog/comments.py index 3eac99ff..323562cf 100644 --- a/devplacepy/services/devii/actions/catalog/comments.py +++ b/devplacepy/services/devii/actions/catalog/comments.py @@ -28,7 +28,7 @@ COMMENTS_ACTIONS: tuple[Action, ...] = ( summary="Edit the body of one of your own comments", params=( path("comment_uid", "Uid of the comment."), - body("content", "New comment body, 3-1000 characters.", required=True), + body("content", "New comment body, 3-125000 characters.", required=True), ), ), Action( diff --git a/devplacepy/services/devii/actions/dispatcher.py b/devplacepy/services/devii/actions/dispatcher.py index 35665e19..bf00e782 100644 --- a/devplacepy/services/devii/actions/dispatcher.py +++ b/devplacepy/services/devii/actions/dispatcher.py @@ -116,6 +116,7 @@ _DEVII_MECHANIC_EVENTS = { "email_mark": "email.message.flag", "email_set_flags": "email.message.flag", "telegram_send": "telegram.send", + "interactions_set": "profile.interactions", } _DEVII_CONTAINER_EVENTS = { @@ -268,6 +269,7 @@ class Dispatcher: owner_id: str = "", virtual_tools: Any = None, behavior: Any = None, + interaction: Any = None, ) -> None: self._actions = catalog.by_name() self._client = client @@ -308,6 +310,7 @@ class Dispatcher: self._telegram = TelegramSendController(owner_kind, owner_id) self._virtual_tools = virtual_tools self._behavior = behavior + self._interaction = interaction self._read_files: set[tuple[str, str]] = set() @staticmethod @@ -500,6 +503,15 @@ class Dispatcher: ) return await self._behavior.dispatch(action.name, arguments) + if action.handler == "interaction": + if self._interaction is None: + return error_result( + ToolInputError( + "Interactive prompts are not available in this context." + ) + ) + return await self._interaction.dispatch(action.name, arguments) + if action.handler == "virtual_tool": if self._virtual_tools is None: return error_result( diff --git a/devplacepy/services/devii/actions/spec.py b/devplacepy/services/devii/actions/spec.py index fc0f96ea..ec903d8a 100644 --- a/devplacepy/services/devii/actions/spec.py +++ b/devplacepy/services/devii/actions/spec.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from functools import lru_cache from typing import Any, Literal ParamLocation = Literal["path", "query", "body", "file"] @@ -49,6 +50,7 @@ class Action: "virtual_tool", "email", "telegram", + "interaction", ] = "http" freeform_body: bool = False ajax: bool = False @@ -117,16 +119,27 @@ class Catalog: def tool_schemas(self) -> list[dict[str, Any]]: return [action.tool_schema() for action in self.actions] + @lru_cache(maxsize=16) + def _schemas_for( + self, + authenticated: bool, + is_admin: bool, + is_primary_admin: bool, + ) -> tuple[dict[str, Any], ...]: + return tuple( + action.tool_schema() + for action in self.actions + if (authenticated or not action.requires_auth) + and (is_admin or not action.requires_admin) + and (is_primary_admin or not action.requires_primary_admin) + ) + def tool_schemas_for( self, authenticated: bool, is_admin: bool = False, is_primary_admin: bool = False, ) -> list[dict[str, Any]]: - return [ - action.tool_schema() - for action in self.actions - if (authenticated or not action.requires_auth) - and (is_admin or not action.requires_admin) - and (is_primary_admin or not action.requires_primary_admin) - ] + return list( + self._schemas_for(bool(authenticated), bool(is_admin), bool(is_primary_admin)) + ) diff --git a/devplacepy/services/devii/agentic/actions.py b/devplacepy/services/devii/agentic/actions.py index 80723d1e..39a4a375 100644 --- a/devplacepy/services/devii/agentic/actions.py +++ b/devplacepy/services/devii/agentic/actions.py @@ -150,6 +150,34 @@ AGENTIC_ACTIONS: tuple[Action, ...] = ( ), ), ), + Action( + name="lesson_rate", + method="LOCAL", + path="", + summary="Vote on a lesson's quality to help the system keep the best ones and drop bad ones", + description=( + "Rate a lesson recorded by reflect(). A positive vote (1) marks it as useful; " + "a negative vote (-1) marks it as unhelpful. Lessons with a cumulative rating of " + "-3 or lower are excluded from recall() results. Use this when you notice a lesson " + "was particularly helpful or misleading." + ), + handler="agentic", + requires_auth=False, + params=( + arg("uid", "The uid of the lesson to rate, from a recall() result.", required=True), + arg("value", "1 for useful, -1 for unhelpful.", required=True, kind="integer"), + ), + ), + Action( + name="lesson_count", + method="LOCAL", + path="", + summary="Report how many learned lessons are stored for this session or account", + description="Returns the number of active lessons with an optional per-tag breakdown.", + handler="agentic", + requires_auth=False, + params=(), + ), Action( name="eval", method="LOCAL", diff --git a/devplacepy/services/devii/agentic/compaction.py b/devplacepy/services/devii/agentic/compaction.py index f2660593..8d4218ef 100644 --- a/devplacepy/services/devii/agentic/compaction.py +++ b/devplacepy/services/devii/agentic/compaction.py @@ -6,13 +6,17 @@ import json import logging from typing import Any +from ..text import normalize_newlines + logger = logging.getLogger("devii.agentic.compaction") SUMMARY_INPUT_CAP = 600_000 SUMMARY_PROMPT = ( "Summarize the following assistant conversation segment as a concise factual log of " "actions taken, tools called, entities created or changed, conclusions reached, and " - "outstanding work. Keep identifiers, slugs, uids, and decisions verbatim. Maximum 800 words.\n\n" + "outstanding work. Keep identifiers, slugs, uids, and decisions verbatim. " + "Write plain markdown with real line breaks (never the two-character sequence \\n). " + "Use headings and bullet lists where helpful. Maximum 800 words.\n\n" "---\n\n" ) @@ -32,6 +36,37 @@ def find_compaction_split(messages: list[dict[str, Any]], keep_tail: int) -> int return 1 +def _segment_plain(messages: list[dict[str, Any]]) -> str: + parts: list[str] = [] + for message in messages: + role = str(message.get("role") or "unknown") + content = message.get("content") + if isinstance(content, str) and content.strip(): + parts.append(f"{role}:\n{normalize_newlines(content)}") + continue + if isinstance(content, list): + chunks: list[str] = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + chunks.append(str(part.get("text") or "")) + elif isinstance(part, str): + chunks.append(part) + text = normalize_newlines("\n".join(c for c in chunks if c)) + if text.strip(): + parts.append(f"{role}:\n{text}") + continue + tool_calls = message.get("tool_calls") + if tool_calls: + names = [] + for call in tool_calls: + fn = (call or {}).get("function") or {} + name = fn.get("name") or "tool" + names.append(str(name)) + if names: + parts.append(f"{role}: called {', '.join(names)}") + return "\n\n".join(parts) + + async def compact_messages( llm: Any, messages: list[dict[str, Any]], keep_tail: int ) -> list[dict[str, Any]]: @@ -46,16 +81,24 @@ async def compact_messages( if not middle: return messages - segment = json.dumps(middle, default=str)[:SUMMARY_INPUT_CAP] + segment = _segment_plain(middle)[:SUMMARY_INPUT_CAP] + if not segment.strip(): + segment = json.dumps(middle, default=str)[:SUMMARY_INPUT_CAP] try: summary = await llm.summarize(SUMMARY_PROMPT + segment) except Exception: # noqa: BLE001 - compaction must never break the loop logger.exception("Compaction summary failed; keeping full context") return messages + summary = normalize_newlines(summary or "").strip() + if not summary: + return messages logger.info("Compacted %d messages into a summary", len(middle)) return [ system_message, - {"role": "assistant", "content": f"[compacted earlier turns]\n\n{summary}"}, + { + "role": "assistant", + "content": f"[compacted earlier turns]\n\n{summary}", + }, *tail, ] diff --git a/devplacepy/services/devii/agentic/controller.py b/devplacepy/services/devii/agentic/controller.py index 2e038d1c..6621a49e 100644 --- a/devplacepy/services/devii/agentic/controller.py +++ b/devplacepy/services/devii/agentic/controller.py @@ -63,6 +63,8 @@ class AgenticController: "reflect": self._reflect, "recall": self._recall, "forget_lessons": self._forget, + "lesson_rate": self._lesson_rate, + "lesson_count": self._lesson_count, "verify": self._verify, "delegate": self._delegate, "eval": self._eval, @@ -162,6 +164,31 @@ class AgenticController: ensure_ascii=False, ) + async def _lesson_rate(self, arguments: dict[str, Any]) -> str: + uid = str(arguments.get("uid", "")).strip() + if not uid: + raise ToolInputError("lesson_rate requires a lesson uid.") + raw_value = arguments.get("value") + if raw_value is None: + raise ToolInputError("lesson_rate requires a value (1 or -1).") + try: + value = int(raw_value) + except (ValueError, TypeError): + raise ToolInputError("lesson_rate value must be an integer (1 or -1).") from None + if value not in (1, -1): + raise ToolInputError("lesson_rate value must be 1 (useful) or -1 (unhelpful).") + ok = self._lessons.rate(uid, value) + if not ok: + raise ToolInputError(f"No lesson found with uid '{uid}'.") + return json.dumps({"status": "success", "lesson_uid": uid, "rated": value}, ensure_ascii=False) + + async def _lesson_count(self, arguments: dict[str, Any]) -> str: + count = self._lessons.count() + return json.dumps( + {"status": "success", "active_lessons": count}, + ensure_ascii=False, + ) + async def _verify(self, arguments: dict[str, Any]) -> str: summary = str(arguments.get("summary", "")).strip() if not summary: diff --git a/devplacepy/services/devii/agentic/lessons.py b/devplacepy/services/devii/agentic/lessons.py index 91d94233..3df4af7a 100644 --- a/devplacepy/services/devii/agentic/lessons.py +++ b/devplacepy/services/devii/agentic/lessons.py @@ -7,6 +7,7 @@ import logging import math import re import uuid +from datetime import timedelta from typing import Any from ..tasks.schedule import now_utc, to_iso @@ -18,6 +19,11 @@ TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|\d+") BM25_K1 = 1.5 BM25_B = 0.75 +DEDUP_JACCARD_THRESHOLD = 0.70 +DEFAULT_MAX_PER_OWNER = 500 +DEFAULT_MAX_AGE_DAYS = 90 +LOW_QUALITY_THRESHOLD = -3 + def tokenize(text: str) -> list[str]: tokens = TOKEN_RE.findall((text or "").lower()) @@ -27,6 +33,33 @@ def tokenize(text: str) -> list[str]: return list(dict.fromkeys(tokens + extra)) +def _jaccard(tokens_a: set[str], tokens_b: set[str]) -> float: + if not tokens_a and not tokens_b: + return 0.0 + if not tokens_a or not tokens_b: + return 0.0 + return len(tokens_a & tokens_b) / len(tokens_a | tokens_b) + + +def _read_retention_settings(db: Any) -> tuple[int, int]: + max_per = DEFAULT_MAX_PER_OWNER + max_age = DEFAULT_MAX_AGE_DAYS + if "site_settings" not in db.tables: + return max_per, max_age + for row in db["site_settings"].find(key={"in": ["devii_lessons_max_per_owner", "devii_lessons_max_age_days"]}): + if row["key"] == "devii_lessons_max_per_owner": + try: + max_per = int(row["value"]) + except (ValueError, TypeError): + pass + elif row["key"] == "devii_lessons_max_age_days": + try: + max_age = int(row["value"]) + except (ValueError, TypeError): + pass + return max_per, max_age + + class LessonStore: def __init__(self, db: Any, owner_kind: str, owner_id: str) -> None: self._db = db @@ -39,9 +72,10 @@ class LessonStore: self._idf: dict[str, float] = {} self._avgdl = 0.0 self._n = 0 + self._ensure_columns() self._ensure_indexes() - def _ensure_indexes(self) -> None: + def _ensure_columns(self) -> None: if TABLE not in self._db.tables: return table = self._db[TABLE] @@ -49,7 +83,15 @@ class LessonStore: table.create_column_by_example("deleted_at", "") if not table.has_column("deleted_by"): table.create_column_by_example("deleted_by", "") + if not table.has_column("rating"): + table.create_column_by_example("rating", 0) + + def _ensure_indexes(self) -> None: + if TABLE not in self._db.tables: + return + table = self._db[TABLE] table.create_index(["owner_kind", "owner_id"]) + table.create_index(["owner_kind", "owner_id", "created_at"]) @property def _table(self) -> Any: @@ -64,17 +106,99 @@ class LessonStore: return 0 return self._table.count(deleted_at=None, **self._scope) + def _row_text(self, row: dict[str, Any]) -> str: + return " ".join( + str(row.get(field) or "") + for field in ("observation", "conclusion", "next_action", "tags") + ) + + def _find_similar(self, text: str, threshold: float = DEDUP_JACCARD_THRESHOLD) -> dict[str, Any] | None: + query_tokens = set(tokenize(text)) + if not query_tokens: + return None + all_rows = self.all() + best_row: dict[str, Any] | None = None + best_score = 0.0 + for row in all_rows: + row_tokens = set(tokenize(self._row_text(row))) + score = _jaccard(query_tokens, row_tokens) + if score > best_score and score >= threshold: + best_score = score + best_row = row + return best_row + + def _enforce_cap(self, max_per_owner: int) -> int: + soft_deleted = 0 + while True: + current = self.count() + if current <= max_per_owner: + break + excess = current - max_per_owner + rows = list( + self._table.find( + deleted_at=None, + order_by=["created_at"], + _limit=excess, + **self._scope, + ) + ) + if not rows: + break + now = to_iso(now_utc()) + for row in rows: + self._table.update( + { + "id": row["id"], + "deleted_at": now, + "deleted_by": "retention", + }, + ["id"], + ) + soft_deleted += 1 + self._dirty = True + if soft_deleted: + logger.info( + "Retention cap pruned %d lesson(s) for owner=%s/%s", + soft_deleted, + self._owner_kind, + self._owner_id, + ) + return soft_deleted + def add( self, observation: str, conclusion: str, next_action: str, tags: str = "" ) -> dict[str, Any]: + text = " ".join([observation, conclusion, next_action, tags]) + similar = self._find_similar(text) + if similar and similar.get("id") is not None: + hits = (similar.get("hits") or 0) + 1 + self._table.update( + { + "id": similar["id"], + "hits": hits, + "created_at": to_iso(now_utc()), + }, + ["id"], + ) + self._dirty = True + logger.info( + "Lesson deduplicated owner=%s/%s hits=%d", + self._owner_kind, + self._owner_id, + hits, + ) + return {**similar, "hits": hits, "deduplicated": True} + + uid = uuid.uuid4().hex record = { - "uid": uuid.uuid4().hex, + "uid": uid, "observation": observation, "conclusion": conclusion, "next_action": next_action, "tags": tags, "created_at": to_iso(now_utc()), "hits": 0, + "rating": 0, "deleted_at": None, "deleted_by": None, **self._scope, @@ -84,6 +208,8 @@ class LessonStore: logger.info( "Lesson stored owner=%s/%s tags=%s", self._owner_kind, self._owner_id, tags ) + max_per, _ = _read_retention_settings(self._db) + self._enforce_cap(max_per) return record def all(self) -> list[dict[str, Any]]: @@ -118,17 +244,108 @@ class LessonStore: ) return n + def rate(self, uid: str, value: int) -> bool: + if TABLE not in self._db.tables: + return False + row = self._table.find_one(uid=uid, deleted_at=None, **self._scope) + if not row: + return False + current = row.get("rating") or 0 + self._table.update( + {"id": row["id"], "rating": current + value}, + ["id"], + ) + self._dirty = True + logger.info( + "Lesson %s rated %+d (now %d) owner=%s/%s", + uid, + value, + current + value, + self._owner_kind, + self._owner_id, + ) + return True + + def prune(self, max_age_days: int | None = None) -> int: + if TABLE not in self._db.tables: + return 0 + if max_age_days is None: + _, max_age_days = _read_retention_settings(self._db) + cutoff = now_utc() - timedelta(days=max_age_days) + cutoff_iso = to_iso(cutoff) + rows = list( + self._table.find( + deleted_at=None, + created_at={"<": cutoff_iso}, + **self._scope, + ) + ) + if not rows: + return 0 + now = to_iso(now_utc()) + soft_deleted = 0 + for row in rows: + self._table.update( + { + "id": row["id"], + "deleted_at": now, + "deleted_by": "retention", + }, + ["id"], + ) + soft_deleted += 1 + if soft_deleted: + self._dirty = True + logger.info( + "Pruned %d old lesson(s) for owner=%s/%s", + soft_deleted, + self._owner_kind, + self._owner_id, + ) + return soft_deleted + + def prune_all_owners(self, max_age_days: int | None = None) -> int: + if TABLE not in self._db.tables: + return 0 + if max_age_days is None: + _, max_age_days = _read_retention_settings(self._db) + cutoff = now_utc() - timedelta(days=max_age_days) + cutoff_iso = to_iso(cutoff) + rows = list( + self._table.find( + deleted_at=None, + created_at={"<": cutoff_iso}, + ) + ) + if not rows: + return 0 + now = to_iso(now_utc()) + soft_deleted = 0 + for row in rows: + self._table.update( + { + "id": row["id"], + "deleted_at": now, + "deleted_by": "retention", + }, + ["id"], + ) + soft_deleted += 1 + if soft_deleted: + logger.info("Pruned %d old lesson(s) across all owners", soft_deleted) + return soft_deleted + def _rebuild(self) -> None: rows = self.all() df: collections.Counter = collections.Counter() docs: list[dict[str, Any]] = [] tf_list: list[collections.Counter] = [] - dl: list[int] = [] + dl_list: list[int] = [] for row in rows: - text = " ".join( - str(row.get(field) or "") - for field in ("observation", "conclusion", "next_action", "tags") - ) + rating = row.get("rating") or 0 + if rating <= LOW_QUALITY_THRESHOLD: + continue + text = self._row_text(row) tokens = tokenize(text) if not tokens: continue @@ -137,12 +354,12 @@ class LessonStore: df[term] += 1 docs.append(row) tf_list.append(tf) - dl.append(len(tokens)) + dl_list.append(len(tokens)) self._docs = docs self._tf = tf_list - self._dl = dl + self._dl = dl_list self._n = len(docs) - self._avgdl = sum(dl) / max(self._n, 1) + self._avgdl = sum(dl_list) / max(self._n, 1) self._idf = { term: math.log((self._n - freq + 0.5) / (freq + 0.5) + 1) for term, freq in df.items() diff --git a/devplacepy/services/devii/ai_correction/controller.py b/devplacepy/services/devii/ai_correction/controller.py index 89159748..99b930cf 100644 --- a/devplacepy/services/devii/ai_correction/controller.py +++ b/devplacepy/services/devii/ai_correction/controller.py @@ -70,7 +70,7 @@ class AiCorrectionController: if prompt_raw is None: prompt = user.get("ai_correction_prompt") or DEFAULT_CORRECTION_PROMPT else: - prompt = str(prompt_raw).strip()[:2000] or DEFAULT_CORRECTION_PROMPT + prompt = str(prompt_raw).strip()[:20000] or DEFAULT_CORRECTION_PROMPT get_table("users").update( { "uid": self._owner_id, diff --git a/devplacepy/services/devii/ai_modifier/controller.py b/devplacepy/services/devii/ai_modifier/controller.py index eb247b9b..f03799e9 100644 --- a/devplacepy/services/devii/ai_modifier/controller.py +++ b/devplacepy/services/devii/ai_modifier/controller.py @@ -70,7 +70,7 @@ class AiModifierController: if prompt_raw is None: prompt = user.get("ai_modifier_prompt") or DEFAULT_MODIFIER_PROMPT else: - prompt = str(prompt_raw).strip()[:2000] or DEFAULT_MODIFIER_PROMPT + prompt = str(prompt_raw).strip()[:20000] or DEFAULT_MODIFIER_PROMPT get_table("users").update( { "uid": self._owner_id, diff --git a/devplacepy/services/devii/config.py b/devplacepy/services/devii/config.py index 078c7969..676dea62 100644 --- a/devplacepy/services/devii/config.py +++ b/devplacepy/services/devii/config.py @@ -178,6 +178,7 @@ FIELD_PRICE_CACHE_HIT = "devii_price_cache_hit" FIELD_PRICE_CACHE_MISS = "devii_price_cache_miss" FIELD_PRICE_OUTPUT = "devii_price_output" FIELD_ALLOW_EVAL = "devii_allow_eval" +FIELD_INTERACTIONS_DEFAULT = "devii_interactions_default" FIELD_BROWSER_CONTROL = "devii_browser_control" FIELD_TIMEOUT = "devii_timeout" FIELD_FETCH_TIMEOUT = "devii_fetch_timeout" diff --git a/devplacepy/services/devii/interaction/__init__.py b/devplacepy/services/devii/interaction/__init__.py new file mode 100644 index 00000000..3a475f32 --- /dev/null +++ b/devplacepy/services/devii/interaction/__init__.py @@ -0,0 +1,37 @@ +# retoor + +from __future__ import annotations + +from .capabilities import ( + CHANNEL_SITE_CHAT, + CHANNEL_TELEGRAM, + CHANNEL_CLI, + CHANNEL_API, + CHANNEL_UNKNOWN, + ChannelContext, + channel_id_for_session, + context_for, + fragment_for, + interactions_enabled, +) +from .controller import InteractionController +from . import prefs +from .schema import InteractionRequest, InteractionResult, validate_prompt_args + +__all__ = [ + "CHANNEL_SITE_CHAT", + "CHANNEL_TELEGRAM", + "CHANNEL_CLI", + "CHANNEL_API", + "CHANNEL_UNKNOWN", + "ChannelContext", + "channel_id_for_session", + "context_for", + "fragment_for", + "interactions_enabled", + "InteractionController", + "InteractionRequest", + "InteractionResult", + "validate_prompt_args", + "prefs", +] diff --git a/devplacepy/services/devii/interaction/actions.py b/devplacepy/services/devii/interaction/actions.py new file mode 100644 index 00000000..8a2f6be5 --- /dev/null +++ b/devplacepy/services/devii/interaction/actions.py @@ -0,0 +1,130 @@ +# retoor + +from __future__ import annotations + +from ..actions.spec import Action, Param + + +def arg( + name: str, description: str, required: bool = False, kind: str = "string" +) -> Param: + return Param( + name=name, + location="body", + description=description, + required=required, + type=kind, + ) + + +UI = ( + "Presents a channel-aware interactive prompt (CA-IWP). The host renders the best " + "affordance for the current channel (site custom elements, Telegram inline keys, or " + "plain numbered menus). Blocks until the user submits, cancels, or the timeout fires. " + "Never invent HTML or custom-element tags in your prose - call this tool instead." +) + +INTERACTION_ACTIONS: tuple[Action, ...] = ( + Action( + name="ui_prompt", + method="LOCAL", + path="", + summary="Ask the user an interactive question with widgets (confirm, choice, text, form)", + description=( + UI + + " Prefer this for decisions and short forms. Returns structured " + "{interaction_id, status, values, meta}. Branch on status: submitted, cancelled, " + "timeout, superseded, or error. Keep labels short and value tokens stable." + ), + handler="interaction", + requires_auth=False, + params=( + arg("title", "Short question title (≤120 chars).", required=True), + arg("description", "Optional help prose (≤500 chars)."), + arg( + "widgets", + "Ordered list of widget objects. Types: confirm, choice, choice_multi, " + "text, number, date, select, // (help), group. Each input needs name+label; " + "choice types need options[{value,label}].", + required=True, + kind="array", + ), + arg("submit_label", "Primary action label (default Confirm)."), + arg("cancel_label", "Cancel action label (default Cancel)."), + arg("cancelable", "Whether the user may cancel (default true).", kind="boolean"), + arg( + "timeout_sec", + "Optional timeout in seconds (0 = none, max 86400).", + kind="integer", + ), + arg("id", "Optional stable interaction id (a-z, digits, hyphens)."), + ), + ), + Action( + name="ui_cancel", + method="LOCAL", + path="", + summary="Cancel the open interactive prompt", + description="Closes the current open interaction (or the one named by id) with status cancelled.", + handler="interaction", + requires_auth=False, + params=( + arg("id", "Optional interaction id to cancel; defaults to the open one."), + arg("reason", "Optional cancel reason."), + ), + ), + Action( + name="ui_notify", + method="LOCAL", + path="", + summary="Show a non-blocking ephemeral status on channels that support it", + description=( + "Sends a short status line to the user without waiting. Only effective when the " + "channel advertises ephemeral_status (site-chat). Elsewhere it is skipped." + ), + handler="interaction", + requires_auth=False, + params=(arg("text", "Status text to show (≤200 chars).", required=True),), + ), + Action( + name="interactions_get", + method="LOCAL", + path="", + summary="Show whether interactive widgets (ui_prompt) are enabled for the user", + description=( + "Returns the effective interactive-widgets preference: enabled (bool), source " + "('user' override or 'default'), the admin default, and the user's override if set. " + "The site default is set by administrators on the Devii service; users may override " + "it on their profile or with interactions_set." + ), + handler="interaction", + requires_auth=True, + read_only=True, + ), + Action( + name="interactions_set", + method="LOCAL", + path="", + summary="Enable, disable, or reset interactive widgets for the user", + description=( + "Sets whether this account accepts interactive ui_prompt widgets. Pass enabled=true " + "or false to override the admin default; pass reset=true (or omit enabled and set " + "reset) to clear the override and inherit the admin default again. Guests cannot " + "set a preference." + ), + handler="interaction", + requires_auth=True, + params=( + arg( + "enabled", + "true to enable interactive widgets, false to disable them. Omit when reset=true.", + kind="boolean", + ), + arg( + "reset", + "true to clear the user override and inherit the administrator default again.", + kind="boolean", + ), + ), + ), +) diff --git a/devplacepy/services/devii/interaction/broker.py b/devplacepy/services/devii/interaction/broker.py new file mode 100644 index 00000000..68dda94b --- /dev/null +++ b/devplacepy/services/devii/interaction/broker.py @@ -0,0 +1,513 @@ +# retoor + +from __future__ import annotations + +import asyncio +import logging +import time +from typing import Any, Awaitable, Callable, Optional + +import uuid_utils + +from .capabilities import ( + CHANNEL_CLI, + CHANNEL_SITE_CHAT, + CHANNEL_TELEGRAM, + ChannelContext, + channel_id_for_session, + context_for, +) +from .markdown import project_markdown, project_plain_menu +from .parse import parse_plain_reply +from .schema import ( + InteractionRequest, + InteractionResult, + count_input_fields, + validate_prompt_args, +) + +logger = logging.getLogger("devii.interaction") + +EmitFn = Callable[[dict[str, Any]], Awaitable[None]] +WaitFn = Callable[[str, dict[str, Any], float], Awaitable[Any]] +TelegramPresentFn = Callable[[dict[str, Any]], Awaitable[Any]] +PlainPresentFn = Callable[[str], Awaitable[str]] + + +class InteractionBroker: + def __init__( + self, + session_channel: str = "main", + *, + owner_kind: str = "guest", + owner_id: str = "", + emit: Optional[EmitFn] = None, + wait_site: Optional[WaitFn] = None, + present_telegram: Optional[TelegramPresentFn] = None, + present_plain: Optional[PlainPresentFn] = None, + ) -> None: + self._session_channel = session_channel + self._channel_id = channel_id_for_session(session_channel) + self._owner_kind = owner_kind + self._owner_id = owner_id + self._emit = emit + self._wait_site = wait_site + self._present_telegram = present_telegram + self._present_plain = present_plain + self._open: dict[str, dict[str, Any]] = {} + self._lock = asyncio.Lock() + self._single_flight = True + + @property + def channel_id(self) -> str: + return self._channel_id + + def set_channel_id(self, channel_id: str) -> None: + self._channel_id = channel_id + + def open_id(self) -> str | None: + if not self._open: + return None + return next(iter(self._open)) + + def open_request(self) -> InteractionRequest | None: + open_id = self.open_id() + if not open_id: + return None + payload = self._open.get(open_id) or {} + request = payload.get("request_model") + if isinstance(request, InteractionRequest): + return request + raw = payload.get("request") + if isinstance(raw, dict): + try: + return InteractionRequest.model_validate(raw) + except Exception: + return None + return None + + def answer_text(self, text: str) -> dict[str, Any] | None: + open_id = self.open_id() + if not open_id: + return None + request = self.open_request() + if request is None: + return { + "handled": True, + "status": "error", + "error": "Open interaction has no parseable schema.", + "interaction_id": open_id, + } + status, values, error = parse_plain_reply(request, text) + if status == "error": + return { + "handled": True, + "status": "error", + "error": error or "Could not parse reply.", + "interaction_id": open_id, + } + result = { + "status": status, + "interaction_id": open_id, + "values": values if status == "submitted" else {}, + "meta": { + "adapter": self._channel_id, + "degraded": True, + "via": "text", + }, + "error": error, + } + return { + "handled": True, + "status": status, + "interaction_id": open_id, + "result": result, + } + + def channel_context(self) -> ChannelContext: + owner_kind = getattr(self, "_owner_kind", "guest") + owner_id = getattr(self, "_owner_id", "") + return context_for( + self._channel_id, + open_interaction_id=self.open_id(), + owner_kind=owner_kind, + owner_id=owner_id, + ) + + def bind( + self, + *, + emit: Optional[EmitFn] = None, + wait_site: Optional[WaitFn] = None, + present_telegram: Optional[TelegramPresentFn] = None, + present_plain: Optional[PlainPresentFn] = None, + ) -> None: + if emit is not None: + self._emit = emit + if wait_site is not None: + self._wait_site = wait_site + if present_telegram is not None: + self._present_telegram = present_telegram + if present_plain is not None: + self._present_plain = present_plain + + async def prompt(self, arguments: dict[str, Any]) -> InteractionResult: + request = validate_prompt_args(arguments) + async with self._lock: + if self._single_flight and self._open: + for open_id in list(self._open): + await self._finish( + open_id, + status="superseded", + values={}, + error=None, + ) + interaction_id = request.id or f"ix-{uuid_utils.uuid7().hex[:8]}" + if not interaction_id.startswith("ix-") and not request.id: + interaction_id = f"ix-{interaction_id}" + started = time.monotonic() + payload = { + "interaction_id": interaction_id, + "request": request.model_dump(), + "request_model": request, + "markdown": project_markdown(request, interaction_id), + "plain": project_plain_menu(request), + "channel_id": self._channel_id, + "started": started, + "future": asyncio.get_event_loop().create_future(), + } + self._open[interaction_id] = payload + logger.info( + "interaction open id=%s channel=%s widgets=%d", + interaction_id, + self._channel_id, + count_input_fields(request.widgets), + ) + + try: + result = await self._present(interaction_id, request, payload) + except Exception as exc: + logger.exception("interaction failed id=%s", interaction_id) + result = InteractionResult( + interaction_id=interaction_id, + status="error", + channel_id=self._channel_id, + values={}, + meta={"adapter": self._channel_id, "degraded": True}, + error=str(exc), + ) + await self._resolve_local(interaction_id, result) + return result + + latency_ms = int((time.monotonic() - started) * 1000) + if "latency_ms" not in result.meta: + result.meta["latency_ms"] = latency_ms + if "adapter" not in result.meta: + result.meta["adapter"] = self._channel_id + self._open.pop(interaction_id, None) + logger.info( + "interaction closed id=%s status=%s latency_ms=%s", + interaction_id, + result.status, + result.meta.get("latency_ms"), + ) + return result + + async def cancel( + self, interaction_id: str = "", reason: str = "cancelled" + ) -> InteractionResult: + target = interaction_id or self.open_id() + if not target or target not in self._open: + return InteractionResult( + interaction_id=target or "", + status="error", + channel_id=self._channel_id, + error="No open interaction to cancel.", + ) + status = reason if reason in ( + "cancelled", + "timeout", + "superseded", + "error", + "channel_changed", + ) else "cancelled" + result = InteractionResult( + interaction_id=target, + status=status, # type: ignore[arg-type] + channel_id=self._channel_id, + values={}, + meta={"adapter": self._channel_id, "reason": reason}, + ) + await self._resolve_local(target, result) + return result + + def resolve( + self, + interaction_id: str, + *, + status: str, + values: dict[str, Any] | None = None, + error: str | None = None, + meta: dict[str, Any] | None = None, + ) -> bool: + payload = self._open.get(interaction_id) + if payload is None: + return False + result = InteractionResult( + interaction_id=interaction_id, + status=status, # type: ignore[arg-type] + channel_id=self._channel_id, + values=values or {}, + meta=meta or {"adapter": self._channel_id}, + error=error, + ) + future = payload.get("future") + if future is not None and not future.done(): + future.set_result(result) + return True + + async def _present( + self, + interaction_id: str, + request: InteractionRequest, + payload: dict[str, Any], + ) -> InteractionResult: + timeout = float(request.timeout_sec or 0) + if self._wait_site is not None and self._channel_id in ( + CHANNEL_SITE_CHAT, + CHANNEL_TELEGRAM, + ): + return await self._present_site(interaction_id, request, payload, timeout) + if self._channel_id == CHANNEL_TELEGRAM: + return await self._present_telegram_path( + interaction_id, request, payload, timeout + ) + return await self._present_plain_path( + interaction_id, request, payload, timeout + ) + + async def _present_site( + self, + interaction_id: str, + request: InteractionRequest, + payload: dict[str, Any], + timeout: float, + ) -> InteractionResult: + if self._wait_site is None: + return await self._present_plain_path( + interaction_id, request, payload, timeout, degraded=True + ) + frame = { + "interaction_id": interaction_id, + "channel": self._channel_id, + "title": request.title, + "description": request.description, + "widgets": [w.model_dump() for w in request.widgets], + "submit_label": request.submit_label, + "cancel_label": request.cancel_label, + "cancelable": request.cancelable, + "timeout_sec": request.timeout_sec, + "markdown": payload["markdown"], + "plain": payload["plain"], + "request": request.model_dump(), + } + try: + raw = await self._wait_site(interaction_id, frame, timeout) + except asyncio.TimeoutError: + return InteractionResult( + interaction_id=interaction_id, + status="timeout", + channel_id=self._channel_id, + meta={ + "adapter": self._channel_id, + "degraded": self._channel_id != CHANNEL_SITE_CHAT, + }, + ) + return self._normalize_site_result( + interaction_id, + raw, + adapter=self._channel_id, + degraded=self._channel_id != CHANNEL_SITE_CHAT, + ) + + async def _present_telegram_path( + self, + interaction_id: str, + request: InteractionRequest, + payload: dict[str, Any], + timeout: float, + ) -> InteractionResult: + if self._present_telegram is not None: + try: + raw = await self._present_telegram( + { + "interaction_id": interaction_id, + "request": request.model_dump(), + "markdown": payload["markdown"], + "plain": payload["plain"], + "timeout_sec": request.timeout_sec, + } + ) + return self._normalize_site_result( + interaction_id, raw, adapter=CHANNEL_TELEGRAM, degraded=True + ) + except asyncio.TimeoutError: + return InteractionResult( + interaction_id=interaction_id, + status="timeout", + channel_id=self._channel_id, + meta={"adapter": CHANNEL_TELEGRAM, "degraded": True}, + ) + except Exception as exc: + logger.exception("telegram adapter failed, falling back to plain") + payload["plain_error"] = str(exc) + return await self._present_plain_path( + interaction_id, request, payload, timeout, degraded=True + ) + + async def _present_plain_path( + self, + interaction_id: str, + request: InteractionRequest, + payload: dict[str, Any], + timeout: float, + *, + degraded: bool = False, + ) -> InteractionResult: + menu = payload["plain"] + if self._present_plain is not None: + try: + if timeout > 0: + reply = await asyncio.wait_for( + self._present_plain(menu), timeout=timeout + ) + else: + reply = await self._present_plain(menu) + except asyncio.TimeoutError: + return InteractionResult( + interaction_id=interaction_id, + status="timeout", + channel_id=self._channel_id, + meta={ + "adapter": self._channel_id or CHANNEL_CLI, + "degraded": True, + }, + ) + status, values, error = parse_plain_reply(request, reply) + return InteractionResult( + interaction_id=interaction_id, + status=status, # type: ignore[arg-type] + channel_id=self._channel_id, + values=values, + meta={ + "adapter": self._channel_id or CHANNEL_CLI, + "degraded": True, + }, + error=error, + ) + future: asyncio.Future = payload["future"] + if self._emit is not None: + await self._emit( + { + "type": "interaction", + "id": interaction_id, + "mode": "plain", + "text": menu, + "markdown": payload["markdown"], + } + ) + try: + if timeout > 0: + result = await asyncio.wait_for(future, timeout=timeout) + else: + result = await future + if isinstance(result, InteractionResult): + result.meta.setdefault("degraded", degraded or True) + return result + return self._normalize_site_result( + interaction_id, result, adapter=self._channel_id, degraded=True + ) + except asyncio.TimeoutError: + return InteractionResult( + interaction_id=interaction_id, + status="timeout", + channel_id=self._channel_id, + meta={"adapter": self._channel_id, "degraded": True}, + ) + + def _normalize_site_result( + self, + interaction_id: str, + raw: Any, + *, + adapter: str = CHANNEL_SITE_CHAT, + degraded: bool = False, + ) -> InteractionResult: + if isinstance(raw, InteractionResult): + return raw + if not isinstance(raw, dict): + return InteractionResult( + interaction_id=interaction_id, + status="error", + channel_id=self._channel_id, + error="Invalid interaction result payload.", + meta={"adapter": adapter, "degraded": degraded}, + ) + if raw.get("error") and not raw.get("status"): + return InteractionResult( + interaction_id=interaction_id, + status="error", + channel_id=self._channel_id, + error=str(raw.get("error")), + meta={"adapter": adapter, "degraded": degraded}, + ) + status = str(raw.get("status") or "submitted") + if status not in ( + "submitted", + "cancelled", + "timeout", + "superseded", + "error", + "channel_changed", + ): + status = "error" + values = raw.get("values") if isinstance(raw.get("values"), dict) else {} + meta = raw.get("meta") if isinstance(raw.get("meta"), dict) else {} + meta.setdefault("adapter", adapter) + meta.setdefault("degraded", degraded) + return InteractionResult( + interaction_id=str(raw.get("interaction_id") or interaction_id), + status=status, # type: ignore[arg-type] + channel_id=self._channel_id, + values=values, + meta=meta, + error=str(raw["error"]) if raw.get("error") else None, + ) + + async def _finish( + self, + interaction_id: str, + *, + status: str, + values: dict[str, Any], + error: str | None, + ) -> None: + result = InteractionResult( + interaction_id=interaction_id, + status=status, # type: ignore[arg-type] + channel_id=self._channel_id, + values=values, + meta={"adapter": self._channel_id}, + error=error, + ) + await self._resolve_local(interaction_id, result) + + async def _resolve_local( + self, interaction_id: str, result: InteractionResult + ) -> None: + payload = self._open.pop(interaction_id, None) + if payload is None: + return + future = payload.get("future") + if future is not None and not future.done(): + future.set_result(result) diff --git a/devplacepy/services/devii/interaction/capabilities.py b/devplacepy/services/devii/interaction/capabilities.py new file mode 100644 index 00000000..c0f97d71 --- /dev/null +++ b/devplacepy/services/devii/interaction/capabilities.py @@ -0,0 +1,217 @@ +# retoor + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +CHANNEL_SITE_CHAT = "site-chat" +CHANNEL_TELEGRAM = "telegram" +CHANNEL_CLI = "cli" +CHANNEL_API = "api" +CHANNEL_UNKNOWN = "unknown" + +UI_TOOL_NAMES = frozenset( + { + "ui_prompt", + "ui_cancel", + "ui_notify", + "interactions_get", + "interactions_set", + } +) + +DEFAULT_LIMITS = { + "max_options": 32, + "max_fields": 24, + "max_label_chars": 200, + "max_help_chars": 500, + "callback_data_bytes": 0, +} + +_SITE_CAPS = { + "rich_widgets": True, + "markdown": True, + "confirm": True, + "choice_single": True, + "choice_multi": True, + "text_input": True, + "number_input": True, + "date_input": True, + "file_input": False, + "inline_buttons": True, + "polls": False, + "web_app": False, + "streaming_partial": True, + "ephemeral_status": True, +} + +_TELEGRAM_CAPS = { + "rich_widgets": False, + "markdown": True, + "confirm": True, + "choice_single": True, + "choice_multi": True, + "text_input": True, + "number_input": True, + "date_input": True, + "file_input": False, + "inline_buttons": True, + "polls": True, + "web_app": False, + "streaming_partial": False, + "ephemeral_status": False, +} + +_PLAIN_CAPS = { + "rich_widgets": False, + "markdown": True, + "confirm": True, + "choice_single": True, + "choice_multi": True, + "text_input": True, + "number_input": True, + "date_input": True, + "file_input": False, + "inline_buttons": False, + "polls": False, + "web_app": False, + "streaming_partial": False, + "ephemeral_status": False, +} + +_EMPTY_CAPS = {key: False for key in _SITE_CAPS} + +_MATRICES: dict[str, dict[str, bool]] = { + CHANNEL_SITE_CHAT: _SITE_CAPS, + CHANNEL_TELEGRAM: _TELEGRAM_CAPS, + CHANNEL_CLI: _PLAIN_CAPS, + CHANNEL_API: _PLAIN_CAPS, + CHANNEL_UNKNOWN: _EMPTY_CAPS, +} + +_SESSION_CHANNEL_MAP = { + "main": CHANNEL_SITE_CHAT, + "docs": CHANNEL_SITE_CHAT, + "telegram": CHANNEL_TELEGRAM, + "cli": CHANNEL_CLI, + "api": CHANNEL_API, +} + + +@dataclass +class ChannelContext: + channel_id: str + capabilities: dict[str, bool] = field(default_factory=dict) + limits: dict[str, int] = field(default_factory=dict) + tools: list[str] = field(default_factory=list) + open_interaction_id: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "channel_id": self.channel_id, + "capabilities": dict(self.capabilities), + "limits": dict(self.limits), + "tools": list(self.tools), + "open_interaction_id": self.open_interaction_id, + } + + +def channel_id_for_session(session_channel: str) -> str: + return _SESSION_CHANNEL_MAP.get(str(session_channel or "").strip().lower(), CHANNEL_UNKNOWN) + + +def capabilities_for(channel_id: str) -> dict[str, bool]: + return dict(_MATRICES.get(channel_id, _EMPTY_CAPS)) + + +def limits_for(channel_id: str) -> dict[str, int]: + limits = dict(DEFAULT_LIMITS) + if channel_id == CHANNEL_TELEGRAM: + limits["callback_data_bytes"] = 64 + limits["max_options"] = 16 + return limits + + +def interactions_enabled( + owner_kind: str = "guest", owner_id: str = "" +) -> bool: + from .prefs import effective_for + + return effective_for(owner_kind, owner_id) + + +def context_for( + channel_id: str, + *, + open_interaction_id: str | None = None, + tools_allowed: bool = True, + owner_kind: str = "guest", + owner_id: str = "", +) -> ChannelContext: + caps = capabilities_for(channel_id) + limits = limits_for(channel_id) + tools: list[str] = [] + pref_tools = ["interactions_get", "interactions_set"] + if owner_kind == "user": + tools.extend(pref_tools) + if ( + tools_allowed + and interactions_enabled(owner_kind, owner_id) + and channel_id in (CHANNEL_SITE_CHAT, CHANNEL_TELEGRAM, CHANNEL_CLI, CHANNEL_API) + ): + if open_interaction_id: + tools = [t for t in tools if t in pref_tools] + ["ui_cancel"] + else: + tools = list(dict.fromkeys(tools + ["ui_prompt", "ui_cancel"])) + if caps.get("ephemeral_status"): + tools.append("ui_notify") + return ChannelContext( + channel_id=channel_id, + capabilities=caps, + limits=limits, + tools=tools, + open_interaction_id=open_interaction_id, + ) + + +def fragment_for(ctx: ChannelContext) -> str: + active_caps = [name for name, on in ctx.capabilities.items() if on] + caps_text = ",".join(active_caps) if active_caps else "none" + tools_text = ",".join(ctx.tools) if ctx.tools else "none" + interaction = ( + f"open:{ctx.open_interaction_id}" if ctx.open_interaction_id else "none" + ) + return ( + f"CHANNEL: {ctx.channel_id}\n" + f"CAPS: {caps_text}\n" + f"LIMITS: options≤{ctx.limits.get('max_options', 32)} " + f"fields≤{ctx.limits.get('max_fields', 24)}\n" + f"TOOLS: {tools_text}\n" + f"INTERACTION: {interaction}" + ) + + +CA_IWP_SYSTEM_FRAGMENT = ( + "# Interactive asks (CA-IWP)\n" + "You are channel-aware. Current channel and capabilities are injected each turn as " + "CHANNEL / CAPS / LIMITS / TOOLS / INTERACTION.\n" + "HARD RULES when TOOLS includes ui_prompt (this is the interactive mode):\n" + "1. You MUST call ui_prompt for every decision, confirm, choice, short form, yes/no, " + "pick-one, multi-select, text/number/date ask, or demo of interactive widgets. Do not " + "simulate buttons in markdown when ui_prompt is available.\n" + "2. Do NOT fall back to numbered lists, fake [Yes]/[No] prose, or 'I would show a widget' " + "while ui_prompt is in TOOLS. Only use degraded markdown menus when ui_prompt is absent " + "from TOOLS.\n" + "3. Never invent HTML, JavaScript, or custom-element tags in your visible reply. " + "The host renders UI from tool args.\n" + "4. Never claim buttons/widgets exist on a channel that lacks them; match wording to CHANNEL.\n" + "5. One open interaction at a time. Wait for the ui_prompt tool result before continuing.\n" + "6. The user may answer with the on-screen controls OR by typing a clear answer " + "(yes/no, option label/number, free text field value, DD/MM/YYYY, or cancel). " + "If they type a new request instead of an answer, treat the interaction as cancelled " + "and continue with their new request.\n" + "7. On status=cancelled or timeout, do not pretend the user agreed. Branch explicitly.\n" + "8. Keep tool arguments compact: short labels, stable value tokens, no essays inside options.\n" + "9. Do not use ui_prompt for long-form research answers or pure information delivery." +) diff --git a/devplacepy/services/devii/interaction/controller.py b/devplacepy/services/devii/interaction/controller.py new file mode 100644 index 00000000..02656af9 --- /dev/null +++ b/devplacepy/services/devii/interaction/controller.py @@ -0,0 +1,131 @@ +# retoor + +from __future__ import annotations + +import json +import logging +from typing import Any + +from devplacepy.services.devii.errors import ToolInputError + +from .broker import InteractionBroker +from .capabilities import interactions_enabled +from . import prefs +from .schema import sanitize_text + +logger = logging.getLogger("devii.interaction") + + +class InteractionController: + def __init__( + self, + broker: InteractionBroker, + owner_kind: str = "guest", + owner_id: str = "", + ) -> None: + self._broker = broker + self._owner_kind = owner_kind + self._owner_id = owner_id + + @property + def broker(self) -> InteractionBroker: + return self._broker + + async def dispatch(self, name: str, arguments: dict[str, Any]) -> str: + if name == "interactions_get": + return self._pref_get() + if name == "interactions_set": + return self._pref_set(arguments or {}) + if name in ("ui_prompt", "ui_notify") and not interactions_enabled( + self._owner_kind, self._owner_id + ): + raise ToolInputError( + "Interactive widgets are disabled for this account. " + "Enable them with interactions_set or on the profile, or use a short numbered menu." + ) + if name == "ui_prompt": + return await self._prompt(arguments) + if name == "ui_cancel": + return await self._cancel(arguments) + if name == "ui_notify": + return await self._notify(arguments) + raise ToolInputError(f"Unknown interaction tool: {name}") + + def _require_user(self) -> None: + if self._owner_kind != "user" or not self._owner_id: + raise ToolInputError( + "Interactive-widget preferences are only available for signed-in users." + ) + + def _pref_get(self) -> str: + self._require_user() + data = prefs.snapshot(self._owner_kind, self._owner_id) + return json.dumps({"status": "success", **data}, ensure_ascii=False) + + def _pref_set(self, arguments: dict[str, Any]) -> str: + self._require_user() + reset = arguments.get("reset") + if isinstance(reset, bool): + do_reset = reset + else: + do_reset = str(reset or "").strip().lower() in ("true", "1", "yes", "on") + if do_reset: + data = prefs.set_user_pref(self._owner_id, None) + return json.dumps({"status": "success", **data}, ensure_ascii=False) + if "enabled" not in arguments or arguments.get("enabled") is None: + raise ToolInputError( + "Pass enabled=true/false to override, or reset=true to inherit the admin default." + ) + raw = arguments.get("enabled") + if isinstance(raw, bool): + enabled = raw + else: + enabled = str(raw).strip().lower() in ("true", "1", "yes", "on") + data = prefs.set_user_pref(self._owner_id, enabled) + return json.dumps({"status": "success", **data}, ensure_ascii=False) + + async def _prompt(self, arguments: dict[str, Any]) -> str: + ctx = self._broker.channel_context() + if "ui_prompt" not in ctx.tools and self._broker.open_id(): + raise ToolInputError( + "An interaction is already open. Wait for it, or call ui_cancel first." + ) + if "ui_prompt" not in ctx.tools: + raise ToolInputError( + "ui_prompt is not available. Enable interactive widgets with interactions_set " + "or use a short numbered markdown menu." + ) + result = await self._broker.prompt(arguments or {}) + return json.dumps(result.to_dict(), ensure_ascii=False) + + async def _cancel(self, arguments: dict[str, Any]) -> str: + interaction_id = sanitize_text(arguments.get("id", ""), 64) + reason = sanitize_text(arguments.get("reason", "cancelled"), 40) or "cancelled" + result = await self._broker.cancel(interaction_id, reason=reason) + return json.dumps(result.to_dict(), ensure_ascii=False) + + async def _notify(self, arguments: dict[str, Any]) -> str: + text = sanitize_text(arguments.get("text", ""), 200) + if not text: + raise ToolInputError("'text' is required for ui_notify.") + ctx = self._broker.channel_context() + if not ctx.capabilities.get("ephemeral_status"): + return json.dumps( + { + "status": "skipped", + "channel_id": ctx.channel_id, + "message": "ephemeral status is not supported on this channel", + }, + ensure_ascii=False, + ) + emit = getattr(self._broker, "_emit", None) + if emit is not None: + await emit({"type": "status", "text": text}) + return json.dumps( + { + "status": "ok", + "channel_id": ctx.channel_id, + "text": text, + }, + ensure_ascii=False, + ) diff --git a/devplacepy/services/devii/interaction/markdown.py b/devplacepy/services/devii/interaction/markdown.py new file mode 100644 index 00000000..cfefa8ae --- /dev/null +++ b/devplacepy/services/devii/interaction/markdown.py @@ -0,0 +1,106 @@ +# retoor + +from __future__ import annotations + +from .schema import InteractionRequest, WidgetModel, flatten_widgets + + +def project_markdown(request: InteractionRequest, interaction_id: str) -> str: + lines: list[str] = [f"### {request.title}", ""] + if request.description: + lines.append(request.description) + lines.append("") + for widget in request.widgets: + lines.extend(_widget_lines(widget)) + actions = f"[{request.submit_label}]" + if request.cancelable: + actions += f" [{request.cancel_label}]" + lines.append("") + lines.append(actions) + lines.append(f"") + return "\n".join(lines).strip() + "\n" + + +def project_plain_menu(request: InteractionRequest) -> str: + lines: list[str] = [request.title] + if request.description: + lines.append(request.description) + lines.append("") + widgets = [ + w + for w in flatten_widgets(request.widgets) + if w.type != "//" + ] + if len(widgets) == 1 and widgets[0].type == "confirm": + lines.append(f"Reply: yes | no | cancel") + return "\n".join(lines).strip() + "\n" + if len(widgets) == 1 and widgets[0].type in ("choice", "select"): + options = widgets[0].options[:12] + lines.append(f"{widgets[0].label}") + for index, option in enumerate(options, start=1): + lines.append(f"{index}. {option.label} (`{option.value}`)") + lines.append("") + lines.append(f"Reply with a number (1-{len(options)}) or `cancel`.") + return "\n".join(lines).strip() + "\n" + for widget in widgets: + if widget.type in ("choice", "choice_multi", "select"): + lines.append(f"{widget.label}:") + for index, option in enumerate(widget.options[:12], start=1): + lines.append(f" {index}. {option.label} (`{option.value}`)") + elif widget.type == "confirm": + lines.append(f"{widget.label}: yes | no") + else: + lines.append(f"{widget.label}: (free text)") + lines.append("") + lines.append("Reply with values, or `cancel`.") + return "\n".join(lines).strip() + "\n" + + +def _widget_lines(widget: WidgetModel, indent: str = "") -> list[str]: + if widget.type == "//": + return [f"{indent}{widget.text or widget.label}", ""] + if widget.type == "group": + lines = [f"{indent}**{widget.label}**", ""] + for child in widget.widgets: + lines.extend(_widget_lines(child, indent + " ")) + return lines + req = "required" if widget.required else "optional" + lines = [f"{indent}- [ ] **{widget.name}** ({req}) — {widget.label}"] + if widget.help: + lines.append(f"{indent} _{widget.help}_") + if widget.type == "confirm": + default = widget.default + yes = "•" if default is True else " " + no = "•" if default is False else " " + lines.append(f"{indent} - ({yes}) `true` — Yes") + lines.append(f"{indent} - ({no}) `false` — No") + elif widget.type in ("choice", "select"): + default = str(widget.default) if widget.default is not None else "" + for option in widget.options: + mark = "•" if option.value == default else " " + desc = f" — {option.description}" if option.description else "" + lines.append( + f"{indent} - ({mark}) `{option.value}` — {option.label}{desc}" + ) + elif widget.type == "choice_multi": + defaults = set(widget.default or []) if isinstance(widget.default, list) else set() + for option in widget.options: + mark = "x" if option.value in defaults else " " + desc = f" — {option.description}" if option.description else "" + lines.append( + f"{indent} - [{mark}] `{option.value}` — {option.label}{desc}" + ) + elif widget.type == "text": + extra = f", max {widget.max_length}" if widget.max_length else "" + lines.append(f"{indent} - _free text{extra}_") + elif widget.type == "number": + parts = [] + if widget.min is not None: + parts.append(f"min {widget.min}") + if widget.max is not None: + parts.append(f"max {widget.max}") + hint = (", " + ", ".join(parts)) if parts else "" + lines.append(f"{indent} - _number{hint}_") + elif widget.type == "date": + lines.append(f"{indent} - _date DD/MM/YYYY_") + return lines diff --git a/devplacepy/services/devii/interaction/parse.py b/devplacepy/services/devii/interaction/parse.py new file mode 100644 index 00000000..cafbadda --- /dev/null +++ b/devplacepy/services/devii/interaction/parse.py @@ -0,0 +1,187 @@ +# retoor + +from __future__ import annotations + +import re +from datetime import datetime +from typing import Any + +from .schema import InteractionRequest, WidgetModel, flatten_widgets + +YES = frozenset({"y", "yes", "true", "1", "ok", "confirm", "ja", "oui", "j"}) +NO = frozenset({"n", "no", "false", "0", "nee", "non"}) +CANCEL = frozenset({"cancel", "c", "abort", "quit", "annuleren", "stop"}) +DATE_FORMATS = ( + "%d/%m/%Y", + "%d-%m-%Y", + "%d.%m.%Y", + "%d/%m/%y", + "%d-%m-%y", + "%Y-%m-%d", +) + + +def parse_plain_reply( + request: InteractionRequest, text: str +) -> tuple[str, dict[str, Any], str | None]: + raw = (text or "").strip() + if not raw: + return "error", {}, "Empty reply." + lower = raw.lower() + if lower in CANCEL: + return "cancelled", {}, None + + widgets = [w for w in flatten_widgets(request.widgets) if w.type != "//"] + if not widgets: + return "submitted", {}, None + + if len(widgets) == 1: + return _parse_single(widgets[0], raw, strict=True) + + if not _looks_like_multi_answer(raw, widgets): + return "error", {}, "Not a structured answer for the open form." + + values: dict[str, Any] = {} + chunks = [part.strip() for part in re.split(r"[;\n]+", raw) if part.strip()] + if len(chunks) == 1 and "," in chunks[0] and len(widgets) > 1: + chunks = [part.strip() for part in chunks[0].split(",") if part.strip()] + if len(chunks) != len(widgets): + return ( + "error", + {}, + f"Expected {len(widgets)} values (one per field), got {len(chunks)}.", + ) + for widget, chunk in zip(widgets, chunks): + status, partial, err = _parse_single(widget, chunk, strict=True) + if status != "submitted": + return status, {}, err + values.update(partial) + return "submitted", values, None + + +def _looks_like_multi_answer(raw: str, widgets: list[WidgetModel]) -> bool: + if ";" in raw or "\n" in raw: + return True + if "," in raw and len(widgets) > 1: + return True + return False + + +def _parse_single( + widget: WidgetModel, raw: str, *, strict: bool = False +) -> tuple[str, dict[str, Any], str | None]: + text = raw.strip() + lower = text.lower() + if lower in CANCEL: + return "cancelled", {}, None + name = widget.name + if widget.type == "confirm": + if lower in YES: + return "submitted", {name: True}, None + if lower in NO: + return "submitted", {name: False}, None + return "error", {}, "Reply yes/no (or ja/nee)." + if widget.type in ("choice", "select"): + value = _match_option(widget, text) + if value is None: + return "error", {}, f"Unknown option for {name}." + return "submitted", {name: value}, None + if widget.type == "choice_multi": + parts = [p.strip() for p in re.split(r"[,\s]+", text) if p.strip()] + values: list[str] = [] + for part in parts: + matched = _match_option(widget, part) + if matched is None: + return "error", {}, f"Unknown option {part!r} for {name}." + if matched not in values: + values.append(matched) + if widget.required and not values: + return "error", {}, f"{name} requires at least one option." + return "submitted", {name: values}, None + if widget.type == "number": + if strict and not re.fullmatch(r"[+-]?\d+(?:[.,]\d+)?", text): + return "error", {}, f"{name} must be a number." + normalized = text.replace(",", ".") + try: + number = float(normalized) if "." in normalized else int(normalized) + except ValueError: + return "error", {}, f"{name} must be a number." + if widget.min is not None and number < widget.min: + return "error", {}, f"{name} must be ≥ {widget.min}." + if widget.max is not None and number > widget.max: + return "error", {}, f"{name} must be ≤ {widget.max}." + return "submitted", {name: number}, None + if widget.type == "date": + if strict and not _looks_like_date(text): + return "error", {}, f"{name} must be a date as DD/MM/YYYY." + european = _parse_date_european(text) + if european is None: + return "error", {}, f"{name} must be a date as DD/MM/YYYY." + return "submitted", {name: european}, None + if widget.type == "text": + if strict and _looks_like_command(text): + return "error", {}, f"{name} does not look like a field answer." + if widget.max_length and len(text) > widget.max_length: + return "error", {}, f"{name} exceeds max length {widget.max_length}." + if widget.required and not text: + return "error", {}, f"{name} is required." + return "submitted", {name: text}, None + return "error", {}, f"Unsupported widget type {widget.type}." + + +def _looks_like_date(text: str) -> bool: + return bool( + re.fullmatch(r"\d{1,2}[/.-]\d{1,2}[/.-]\d{2,4}", text.strip()) + or re.fullmatch(r"\d{4}-\d{2}-\d{2}", text.strip()) + ) + + +def _looks_like_command(text: str) -> bool: + value = text.strip().lower() + if not value: + return False + if value.startswith("/"): + return True + starters = ( + "show me ", + "laat ", + "geef ", + "toon ", + "please ", + "can you ", + "could you ", + "i want ", + "ik wil ", + "volgende", + "next ", + ) + return any(value.startswith(prefix) for prefix in starters) + + +def _parse_date_european(text: str) -> str | None: + value = text.strip() + for fmt in DATE_FORMATS: + try: + return datetime.strptime(value, fmt).strftime("%d/%m/%Y") + except ValueError: + continue + return None + + +def _match_option(widget: WidgetModel, text: str) -> str | None: + raw = text.strip() + if raw.isdigit(): + index = int(raw) + if 1 <= index <= len(widget.options): + option = widget.options[index - 1] + if not option.disabled: + return option.value + lower = raw.lower() + for option in widget.options: + if option.disabled: + continue + if option.value == raw or option.value.lower() == lower: + return option.value + if option.label.lower() == lower: + return option.value + return None diff --git a/devplacepy/services/devii/interaction/prefs.py b/devplacepy/services/devii/interaction/prefs.py new file mode 100644 index 00000000..84fb9a0e --- /dev/null +++ b/devplacepy/services/devii/interaction/prefs.py @@ -0,0 +1,119 @@ +# retoor + +from __future__ import annotations + +from typing import Any + +INHERIT = -1 +FIELD_DEFAULT = "devii_interactions_default" +FIELD_LEGACY = "devii_interactions_enabled" + + +def _truthy(raw: Any, default: bool = True) -> bool: + if raw is None: + return default + if isinstance(raw, bool): + return raw + if isinstance(raw, (int, float)): + return raw != 0 + text = str(raw).strip().lower() + if text in ("",): + return default + return text not in ("0", "false", "off", "no") + + +def admin_default() -> bool: + try: + from devplacepy.database import get_setting + + raw = get_setting(FIELD_DEFAULT, "") + if raw == "" or raw is None: + raw = get_setting(FIELD_LEGACY, "1") + return _truthy(raw, True) + except Exception: + return True + + +def _coerce_user_pref(raw: Any) -> int | None: + if raw is None: + return None + try: + value = int(raw) + except (TypeError, ValueError): + return None + if value == INHERIT: + return None + if value in (0, 1): + return value + return None + + +def user_pref_raw(user: dict[str, Any] | None) -> int | None: + if not user: + return None + return _coerce_user_pref(user.get("interactions_enabled")) + + +def effective_for_user(user: dict[str, Any] | None) -> bool: + pref = user_pref_raw(user) + if pref is None: + return admin_default() + return bool(pref) + + +def effective_for(owner_kind: str, owner_id: str = "") -> bool: + if owner_kind != "user" or not owner_id: + return admin_default() + try: + from devplacepy.database import get_table + + user = get_table("users").find_one(uid=owner_id) + except Exception: + return admin_default() + return effective_for_user(user) + + +def snapshot(owner_kind: str, owner_id: str = "", user: dict[str, Any] | None = None) -> dict[str, Any]: + default = admin_default() + if owner_kind != "user": + return { + "enabled": default, + "source": "default", + "default": default, + "override": None, + } + if user is None and owner_id: + try: + from devplacepy.database import get_table + + user = get_table("users").find_one(uid=owner_id) + except Exception: + user = None + pref = user_pref_raw(user) + if pref is None: + return { + "enabled": default, + "source": "default", + "default": default, + "override": None, + } + return { + "enabled": bool(pref), + "source": "user", + "default": default, + "override": bool(pref), + } + + +def set_user_pref(user_uid: str, enabled: bool | None) -> dict[str, Any]: + from devplacepy.database import get_table + from devplacepy.utils import clear_user_cache + + value = INHERIT if enabled is None else (1 if enabled else 0) + get_table("users").update( + {"uid": user_uid, "interactions_enabled": value}, + ["uid"], + ) + clear_user_cache(user_uid) + user = get_table("users").find_one(uid=user_uid) or {"uid": user_uid} + return snapshot("user", user_uid, user) diff --git a/devplacepy/services/devii/interaction/schema.py b/devplacepy/services/devii/interaction/schema.py new file mode 100644 index 00000000..8bcc19be --- /dev/null +++ b/devplacepy/services/devii/interaction/schema.py @@ -0,0 +1,260 @@ +# retoor + +from __future__ import annotations + +import re +from typing import Any, Literal + +from pydantic import BaseModel, Field, field_validator, model_validator + +from .capabilities import DEFAULT_LIMITS + +WIDGET_TYPES = ( + "confirm", + "choice", + "choice_multi", + "text", + "number", + "date", + "select", + "//", + "group", +) + +VALUE_RE = re.compile(r"^[a-z0-9_][a-z0-9_-]{0,63}$") +NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$") +ID_RE = re.compile(r"^[a-z][a-z0-9-]{0,63}$") +CONTROL_CHARS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + + +def sanitize_text(value: Any, max_len: int) -> str: + text = CONTROL_CHARS.sub("", str(value if value is not None else "")) + text = " ".join(text.split()) + if len(text) > max_len: + text = text[:max_len] + return text + + +class OptionModel(BaseModel): + value: str + label: str + description: str = "" + disabled: bool = False + + @field_validator("value") + @classmethod + def _value(cls, v: str) -> str: + text = sanitize_text(v, 64).lower().replace(" ", "_") + if not VALUE_RE.match(text): + raise ValueError( + "option value must match ^[a-z0-9_][a-z0-9_-]{0,63}$" + ) + return text + + @field_validator("label") + @classmethod + def _label(cls, v: str) -> str: + text = sanitize_text(v, 80) + if not text: + raise ValueError("option label is required") + return text + + @field_validator("description") + @classmethod + def _description(cls, v: str) -> str: + return sanitize_text(v, 120) + + +class WidgetModel(BaseModel): + type: Literal[ + "confirm", + "choice", + "choice_multi", + "text", + "number", + "date", + "select", + "//", + "group", + ] + name: str = "" + label: str = "" + help: str = "" + text: str = "" + required: bool = True + default: Any = None + display: Literal["auto", "radio", "select", "buttons"] = "auto" + options: list[OptionModel] = Field(default_factory=list) + widgets: list[WidgetModel] = Field(default_factory=list) + max_length: int | None = None + placeholder: str = "" + pattern: str = "" + min: float | None = None + max: float | None = None + step: float | None = None + min_selected: int | None = None + max_selected: int | None = None + + @field_validator("name") + @classmethod + def _name(cls, v: str) -> str: + text = sanitize_text(v, 64) + return text + + @field_validator("label", "help", "text", "placeholder", "pattern") + @classmethod + def _strings(cls, v: str) -> str: + return sanitize_text(v, 500) + + @model_validator(mode="after") + def _shape(self) -> WidgetModel: + if self.type == "//": + if not self.text and not self.label: + raise ValueError("help widget (//) requires text") + if not self.text: + self.text = self.label + return self + if self.type == "group": + if not self.label: + raise ValueError("group requires label") + if not self.widgets: + raise ValueError("group requires nested widgets") + if len(self.widgets) > 24: + raise ValueError("group may contain at most 24 widgets") + return self + if not self.name or not NAME_RE.match(self.name): + raise ValueError( + f"widget name required and must match ^[a-zA-Z_][a-zA-Z0-9_]{{0,63}}$ (got {self.name!r})" + ) + if not self.label: + raise ValueError(f"widget '{self.name}' requires label") + self.label = sanitize_text(self.label, 200) + self.help = sanitize_text(self.help, 500) + if self.type in ("choice", "choice_multi", "select"): + if not self.options: + raise ValueError(f"widget '{self.name}' requires options") + if len(self.options) > DEFAULT_LIMITS["max_options"]: + raise ValueError( + f"widget '{self.name}' has too many options (max {DEFAULT_LIMITS['max_options']})" + ) + if self.type == "select": + self.display = "select" + return self + + +class InteractionRequest(BaseModel): + title: str + description: str = "" + widgets: list[WidgetModel] + submit_label: str = "Confirm" + cancel_label: str = "Cancel" + cancelable: bool = True + timeout_sec: int = 0 + id: str = "" + + @field_validator("title") + @classmethod + def _title(cls, v: str) -> str: + text = sanitize_text(v, 120) + if not text: + raise ValueError("title is required") + return text + + @field_validator("description") + @classmethod + def _description(cls, v: str) -> str: + return sanitize_text(v, 500) + + @field_validator("submit_label", "cancel_label") + @classmethod + def _labels(cls, v: str) -> str: + return sanitize_text(v, 40) or "Confirm" + + @field_validator("timeout_sec") + @classmethod + def _timeout(cls, v: int) -> int: + n = int(v or 0) + if n < 0 or n > 86400: + raise ValueError("timeout_sec must be between 0 and 86400") + return n + + @field_validator("id") + @classmethod + def _id(cls, v: str) -> str: + text = sanitize_text(v, 64).lower() + if text and not ID_RE.match(text): + raise ValueError("id must match ^[a-z][a-z0-9-]{0,63}$") + return text + + @model_validator(mode="after") + def _widgets(self) -> InteractionRequest: + if not self.widgets: + raise ValueError("widgets must contain at least one item") + if len(self.widgets) > DEFAULT_LIMITS["max_fields"]: + raise ValueError( + f"at most {DEFAULT_LIMITS['max_fields']} top-level widgets" + ) + depth = _max_depth(self.widgets) + if depth > 3: + raise ValueError("widget nesting depth must be ≤ 3") + return self + + +class InteractionResult(BaseModel): + interaction_id: str + status: Literal[ + "submitted", "cancelled", "timeout", "superseded", "error", "channel_changed" + ] + channel_id: str + values: dict[str, Any] = Field(default_factory=dict) + meta: dict[str, Any] = Field(default_factory=dict) + error: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "interaction_id": self.interaction_id, + "status": self.status, + "channel_id": self.channel_id, + "values": {} if self.status != "submitted" else dict(self.values), + "meta": dict(self.meta), + "error": self.error, + } + + +def _max_depth(widgets: list[WidgetModel], depth: int = 1) -> int: + deepest = depth + for widget in widgets: + if widget.type == "group" and widget.widgets: + deepest = max(deepest, _max_depth(widget.widgets, depth + 1)) + return deepest + + +def validate_prompt_args(arguments: dict[str, Any]) -> InteractionRequest: + try: + return InteractionRequest.model_validate(arguments or {}) + except Exception as exc: + from devplacepy.services.devii.errors import ToolInputError + + raise ToolInputError(f"Invalid ui_prompt arguments: {exc}") from exc + + +def count_input_fields(widgets: list[WidgetModel]) -> int: + total = 0 + for widget in widgets: + if widget.type == "//": + continue + if widget.type == "group": + total += count_input_fields(widget.widgets) + else: + total += 1 + return total + + +def flatten_widgets(widgets: list[WidgetModel]) -> list[WidgetModel]: + out: list[WidgetModel] = [] + for widget in widgets: + if widget.type == "group": + out.extend(flatten_widgets(widget.widgets)) + else: + out.append(widget) + return out diff --git a/devplacepy/services/devii/registry.py b/devplacepy/services/devii/registry.py index 4915e665..03f4be52 100644 --- a/devplacepy/services/devii/registry.py +++ b/devplacepy/services/devii/registry.py @@ -19,6 +19,7 @@ from .actions.notification_actions import NOTIFICATION_ACTIONS from .actions.rsearch_actions import RSEARCH_ACTIONS from .actions.spec import Catalog from .actions.telegram_actions import TELEGRAM_ACTIONS +from .interaction.actions import INTERACTION_ACTIONS from .virtual_tools.actions import VIRTUAL_TOOL_ACTIONS from .agentic.actions import AGENTIC_ACTIONS from .tasks.actions import TASK_ACTIONS @@ -42,6 +43,7 @@ CATALOG = Catalog( + AI_MODIFIER_ACTIONS + EMAIL_ACTIONS + TELEGRAM_ACTIONS + + INTERACTION_ACTIONS + VIRTUAL_TOOL_ACTIONS ) diff --git a/devplacepy/services/devii/service.py b/devplacepy/services/devii/service.py index 72dce5f9..02e5364c 100644 --- a/devplacepy/services/devii/service.py +++ b/devplacepy/services/devii/service.py @@ -126,6 +126,16 @@ class DeviiService(BaseService): "raw JavaScript execution; read-only tools (context, discover, read) are unaffected.", group="Agent", ), + ConfigField( + config.FIELD_INTERACTIONS_DEFAULT, + "Interactive widgets default", + type="bool", + default=True, + help="Site default for CA-IWP interactive prompts (ui_prompt). Guests always use " + "this default. Signed-in users inherit it until they override it on their profile " + "or with the interactions_set tool.", + group="Agent", + ), ConfigField( config.FIELD_USER_DAILY_USD, "Max USD per user / 24h", @@ -235,6 +245,24 @@ class DeviiService(BaseService): help="Connection/read timeout for IMAP and SMTP calls.", group="Email", ), + ConfigField( + "devii_lessons_max_per_owner", + "Max lessons per owner", + type="int", + default=500, + minimum=0, + help="Soft cap on active lessons per user/guest. When exceeded, the oldest lessons are pruned. 0 disables the cap.", + group="Memory", + ), + ConfigField( + "devii_lessons_max_age_days", + "Max lesson age (days)", + type="int", + default=90, + minimum=1, + help="Lessons older than this are soft-deleted on the periodic housekeeping pass.", + group="Memory", + ), ] def __init__(self): @@ -316,12 +344,28 @@ class DeviiService(BaseService): ensured = self._ensure_task_schedulers() pruned = hub.ledger.prune(48) removed = await hub.gc_idle() - if pruned or removed or ensured: + lessons_pruned = self._prune_old_lessons() + if pruned or removed or ensured or lessons_pruned: self.log( f"Housekeeping: ensured {ensured} task scheduler(s), pruned {pruned} " - f"ledger rows, closed {removed} idle sessions" + f"ledger rows, pruned {lessons_pruned} lessons, closed {removed} idle sessions" ) + def _prune_old_lessons(self) -> int: + try: + from devplacepy.database import db + from .agentic.lessons import LessonStore, _read_retention_settings + + _, max_age = _read_retention_settings(db) + store = LessonStore(db, "_global", "_global") + pruned = store.prune_all_owners(max_age) + if pruned: + logger.info("Devii lessons housekeeping: pruned %d across all owners", pruned) + return pruned + except Exception: + logger.exception("Devii lessons housekeeping failed") + return 0 + def _ensure_task_schedulers(self) -> int: from devplacepy.database import db from devplacepy.utils import is_admin, is_primary_admin diff --git a/devplacepy/services/devii/session/core.py b/devplacepy/services/devii/session/core.py index a56f3a7b..40dcb3f7 100644 --- a/devplacepy/services/devii/session/core.py +++ b/devplacepy/services/devii/session/core.py @@ -19,10 +19,18 @@ from ..config import Settings from ..cost import CostTracker from ..cost.tracker import Pricing from ..http_client import PlatformClient +from ..interaction import InteractionController +from ..interaction.broker import InteractionBroker +from ..interaction.capabilities import ( + CA_IWP_SYSTEM_FRAGMENT, + UI_TOOL_NAMES, + fragment_for, +) from ..llm import LLMClient from ..registry import CATALOG from ..tasks import Scheduler, TaskController, TaskStore from ..virtual_tools import VirtualToolController, VirtualToolStore +from ..text import normalize_newlines from ._helpers import _format_offset, _now_iso, _repair_history from .prompts import ( BEHAVIOR_HEADER, @@ -98,6 +106,14 @@ class DeviiSession: ) self._behavior_store = behavior_store self.behavior = BehaviorController(behavior_store) + self.interaction_broker = InteractionBroker( + session_channel=channel, + owner_kind=owner_kind, + owner_id=owner_id, + ) + self.interaction = InteractionController( + self.interaction_broker, owner_kind=owner_kind, owner_id=owner_id + ) self.dispatcher = Dispatcher( CATALOG, self.client, @@ -113,6 +129,7 @@ class DeviiSession: owner_id=owner_id, virtual_tools=self.virtual_tools, behavior=self.behavior, + interaction=self.interaction, ) self.tools = self._builtin_tools() self._system_prompt = ( @@ -164,8 +181,16 @@ class DeviiSession: self._turn_epoch = 0 def restore_history(self, messages: list[dict[str, Any]]) -> None: - if messages: - self.agent._messages = messages + if not messages: + return + repaired: list[dict[str, Any]] = [] + for message in messages: + item = dict(message) + content = item.get("content") + if isinstance(content, str): + item["content"] = normalize_newlines(content) + repaired.append(item) + self.agent._messages = repaired def history(self) -> list[dict[str, Any]]: visible: list[dict[str, Any]] = [] @@ -174,11 +199,12 @@ class DeviiSession: content = message.get("content") if role not in ("user", "assistant") or not content: continue - if ( - role == "user" - and isinstance(content, str) - and content.startswith(INTERNAL_PREFIXES) - ): + if not isinstance(content, str): + continue + content = normalize_newlines(content).strip() + if not content: + continue + if role == "user" and content.startswith(INTERNAL_PREFIXES): continue visible.append({"role": role, "content": content}) return visible @@ -228,6 +254,10 @@ class DeviiSession: self._disconnected.clear() self.avatar.bind(self._avatar_request) self.browser.bind(self._client_request) + self.interaction_broker.bind( + emit=self._emit_interaction, + wait_site=self._interaction_wait, + ) self.ensure_scheduler_started() if self._buffer: pending = self._buffer @@ -248,6 +278,14 @@ class DeviiSession: self._disconnected.set() self.avatar.unbind() self.browser.unbind() + open_id = self.interaction_broker.open_id() + if open_id: + self.interaction_broker.resolve( + open_id, + status="cancelled", + values={}, + error="Browser disconnected.", + ) logger.info( "Session %s/%s detached (%d conns)", self.owner_kind, @@ -457,7 +495,13 @@ class DeviiSession: for s in schemas if s.get("function", {}).get("name") in DOCS_TOOLS ] - return schemas + allowed = set(self.interaction_broker.channel_context().tools) + return [ + s + for s in schemas + if s.get("function", {}).get("name") not in UI_TOOL_NAMES + or s.get("function", {}).get("name") in allowed + ] def _refresh_tools(self) -> None: if self.channel == "docs": @@ -471,7 +515,14 @@ class DeviiSession: return self._system_prompt body = self._behavior_store.text().strip() section = BEHAVIOR_HEADER if not body else f"{BEHAVIOR_HEADER}\n{body}" - return f"{self._system_prompt}\n\n{self._clock_line()}\n\n{section}" + channel_block = fragment_for(self.interaction_broker.channel_context()) + return ( + f"{self._system_prompt}\n\n" + f"{CA_IWP_SYSTEM_FRAGMENT}\n\n" + f"{channel_block}\n\n" + f"{self._clock_line()}\n\n" + f"{section}" + ) def _clock_line(self) -> str: from datetime import datetime, timedelta, timezone @@ -745,6 +796,144 @@ class DeviiSession: if len(self._buffer) > 100: self._buffer = self._buffer[-100:] + async def _emit_interaction(self, payload: dict[str, Any]) -> None: + await self._emit(payload, buffer=False) + + async def _interaction_wait( + self, interaction_id: str, frame: dict[str, Any], timeout: float + ) -> Any: + loop = asyncio.get_event_loop() + deadline_seconds = timeout if timeout and timeout > 0 else BROWSER_REQUEST_DEADLINE_SECONDS + if timeout and timeout > 0: + deadline_seconds = max(timeout, 1.0) + else: + deadline_seconds = max(BROWSER_REQUEST_DEADLINE_SECONDS, 300.0) + deadline = loop.time() + deadline_seconds + request_id = f"ix:{interaction_id}" + future: asyncio.Future = loop.create_future() + self._pending[request_id] = future + wire = { + "type": "interaction", + "id": request_id, + "interaction_id": interaction_id, + "args": frame, + } + try: + while not future.done(): + remaining = deadline - loop.time() + if remaining <= 0: + raise asyncio.TimeoutError() + try: + await asyncio.wait_for(self._connected.wait(), timeout=remaining) + except asyncio.TimeoutError as exc: + raise asyncio.TimeoutError() from exc + ws = self._pick_target() + if ws is None: + continue + try: + await self._send_to(ws, wire) + except Exception: + await asyncio.sleep(0.1) + continue + drop = asyncio.create_task(self._disconnected.wait()) + try: + await asyncio.wait( + {future, drop}, + timeout=max(0.0, deadline - loop.time()), + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + drop.cancel() + return future.result() + finally: + self._pending.pop(request_id, None) + + def resolve_interaction(self, interaction_id: str, payload: Any) -> None: + request_id = str(interaction_id or "") + if not request_id.startswith("ix:"): + request_id = f"ix:{request_id}" + future = self._pending.get(request_id) + if future is not None and not future.done(): + future.set_result(payload) + self._pending.pop(request_id, None) + bare = request_id[3:] + if isinstance(payload, dict): + self.interaction_broker.resolve( + bare, + status=str(payload.get("status") or "submitted"), + values=payload.get("values") + if isinstance(payload.get("values"), dict) + else {}, + error=payload.get("error"), + meta=payload.get("meta") + if isinstance(payload.get("meta"), dict) + else None, + ) + return + bare = request_id[3:] if request_id.startswith("ix:") else request_id + self.interaction_broker.resolve( + bare, + status=str((payload or {}).get("status") or "submitted") + if isinstance(payload, dict) + else "error", + values=(payload or {}).get("values") + if isinstance(payload, dict) + else {}, + error=(payload or {}).get("error") if isinstance(payload, dict) else None, + meta=(payload or {}).get("meta") if isinstance(payload, dict) else None, + ) + + async def try_answer_interaction(self, text: str) -> bool: + open_id = self.interaction_broker.open_id() + if not open_id: + return False + pending = self._pending.get(f"ix:{open_id}") + if pending is not None and pending.done(): + return False + outcome = self.interaction_broker.answer_text(text) + if not outcome or not outcome.get("handled"): + return False + if outcome.get("status") == "error": + result = { + "status": "cancelled", + "interaction_id": open_id, + "values": {}, + "meta": { + "adapter": "site-chat", + "via": "text_non_answer", + "degraded": True, + }, + "error": None, + } + self.resolve_interaction(open_id, result) + await self._emit( + { + "type": "interaction_closed", + "id": f"ix:{open_id}", + "interaction_id": open_id, + "result": result, + }, + buffer=False, + ) + return False + result = outcome.get("result") or { + "status": outcome.get("status") or "submitted", + "interaction_id": open_id, + "values": {}, + "meta": {"adapter": "site-chat", "via": "text", "degraded": True}, + } + self.resolve_interaction(open_id, result) + await self._emit( + { + "type": "interaction_closed", + "id": f"ix:{open_id}", + "interaction_id": open_id, + "result": result, + }, + buffer=False, + ) + return True + async def _avatar_request(self, action: str, args: dict[str, Any]) -> Any: return await self._browser_request("avatar", action, args) diff --git a/devplacepy/services/devii/text.py b/devplacepy/services/devii/text.py index cc5cf89c..3851ad9d 100644 --- a/devplacepy/services/devii/text.py +++ b/devplacepy/services/devii/text.py @@ -18,6 +18,25 @@ BLANK_LINES = re.compile(r"\n\s*\n\s*\n+") TRAILING_SPACE = re.compile(r"[ \t]+\n") SKIP_HREF_PREFIXES = ("#", "javascript:") +def normalize_newlines(text: str) -> str: + if not text or not isinstance(text, str): + return "" if text is None else str(text) + value = text + while "\\\\n" in value: + value = value.replace("\\\\n", "\\n") + while "\\\\t" in value: + value = value.replace("\\\\t", "\\t") + escaped_n = value.count("\\n") + real_n = value.count("\n") + if escaped_n > real_n: + value = value.replace("\\n", "\n") + escaped_t = value.count("\\t") + real_t = value.count("\t") + if escaped_t > real_t: + value = value.replace("\\t", "\t") + return value + + HIDDEN = "[hidden]" REDACT_FIELD_KEYS = frozenset( { diff --git a/devplacepy/services/devrant/tokens.py b/devplacepy/services/devrant/tokens.py index 02bf2ee7..ddf7684b 100644 --- a/devplacepy/services/devrant/tokens.py +++ b/devplacepy/services/devrant/tokens.py @@ -85,3 +85,6 @@ def revoke_all(user_uid: str) -> None: tokens.update( {"id": token["id"], "deleted_at": stamp, "deleted_by": user_uid}, ["id"] ) + from devplacepy.utils.authcache import clear_user_cache + + clear_user_cache(user_uid) diff --git a/devplacepy/services/gitea/enhance.py b/devplacepy/services/gitea/enhance.py index b631636a..47acbcd1 100644 --- a/devplacepy/services/gitea/enhance.py +++ b/devplacepy/services/gitea/enhance.py @@ -88,7 +88,10 @@ async def enhance_ticket( "max_tokens": MAX_TOKENS, "temperature": 0.2, } - headers = {"Content-Type": "application/json"} + headers = { + "Content-Type": "application/json", + "X-App-Reference": "devplace-gitea-enhance-v-1-0-0", + } if config.ai_key: headers["Authorization"] = f"Bearer {config.ai_key}" diff --git a/devplacepy/services/gitea/planning.py b/devplacepy/services/gitea/planning.py index 16a3b9fb..98a6370e 100644 --- a/devplacepy/services/gitea/planning.py +++ b/devplacepy/services/gitea/planning.py @@ -226,7 +226,10 @@ async def generate_plan( "max_tokens": MAX_TOKENS, "temperature": 0.2, } - headers = {"Content-Type": "application/json"} + headers = { + "Content-Type": "application/json", + "X-App-Reference": "devplace-gitea-planning-v-1-0-0", + } if config.ai_key: headers["Authorization"] = f"Bearer {config.ai_key}" diff --git a/devplacepy/services/jobs/award_service.py b/devplacepy/services/jobs/award_service.py index 12111ce0..085d3bd2 100644 --- a/devplacepy/services/jobs/award_service.py +++ b/devplacepy/services/jobs/award_service.py @@ -169,7 +169,10 @@ class AwardService(JobService): "output_format": "png", } url = f"{INTERNAL_BASE_URL}/openai/v1/images/generations" - headers = {"Content-Type": "application/json"} + headers = { + "Content-Type": "application/json", + "X-App-Reference": "devplace-awards-v-1-0-0", + } if api_key: headers["Authorization"] = f"Bearer {api_key}" with stealth.stealth_sync_client(timeout=AWARD_GENERATION_TIMEOUT_SECONDS) as client: diff --git a/devplacepy/services/jobs/base.py b/devplacepy/services/jobs/base.py index 56fac317..fcfaa5f7 100644 --- a/devplacepy/services/jobs/base.py +++ b/devplacepy/services/jobs/base.py @@ -242,43 +242,30 @@ class JobService(BaseService): return None def collect_metrics(self) -> dict: - rows = queue.list_jobs(kind=self.kind) - by_status = {queue.PENDING: 0, queue.RUNNING: 0, queue.DONE: 0, queue.FAILED: 0} - bytes_in = bytes_out = items = 0 - durations = [] - for row in rows: - by_status[row.get("status", "")] = ( - by_status.get(row.get("status", ""), 0) + 1 - ) - bytes_in += int(row.get("bytes_in") or 0) - bytes_out += int(row.get("bytes_out") or 0) - items += int(row.get("item_count") or 0) - if row.get("status") == queue.DONE and row.get("duration_ms"): - durations.append(int(row["duration_ms"])) - avg_ms = int(sum(durations) / len(durations)) if durations else 0 + metrics = queue.job_metrics(self.kind) + by_status = metrics["by_status"] stats = [ - {"label": "Total jobs", "value": len(rows)}, + {"label": "Total jobs", "value": metrics["total"]}, {"label": "Pending", "value": by_status[queue.PENDING]}, {"label": "Running", "value": by_status[queue.RUNNING]}, {"label": "Done", "value": by_status[queue.DONE]}, {"label": "Failed", "value": by_status[queue.FAILED]}, - {"label": "Items zipped", "value": items}, - {"label": "Bytes in", "value": _human_bytes(bytes_in)}, - {"label": "Bytes out", "value": _human_bytes(bytes_out)}, - {"label": "Avg duration", "value": f"{avg_ms} ms"}, + {"label": "Items zipped", "value": metrics["items"]}, + {"label": "Bytes in", "value": _human_bytes(metrics["bytes_in"])}, + {"label": "Bytes out", "value": _human_bytes(metrics["bytes_out"])}, + {"label": "Avg duration", "value": f"{metrics['avg_ms']} ms"}, ] - recent = sorted(rows, key=lambda r: r.get("uid", ""), reverse=True)[:10] table = { "columns": ["Job", "Status", "Name", "Out", "Items"], "rows": [ [ - row.get("uid", "")[:8], + (row.get("uid") or "")[:8], row.get("status", ""), (row.get("preferred_name") or "")[:32], _human_bytes(int(row.get("bytes_out") or 0)), int(row.get("item_count") or 0), ] - for row in recent + for row in metrics["recent"] ], } return {"stats": stats, "table": table} diff --git a/devplacepy/services/jobs/deepsearch/crawl.py b/devplacepy/services/jobs/deepsearch/crawl.py index 636b87fc..0bba51a1 100644 --- a/devplacepy/services/jobs/deepsearch/crawl.py +++ b/devplacepy/services/jobs/deepsearch/crawl.py @@ -167,29 +167,35 @@ def _snippet_page(candidate: dict, depth: int) -> CrawledPage | None: ) -async def _render_with_playwright(url: str) -> tuple[str, str, int, list[tuple[str, str]]]: +async def _render_with_playwright( + url: str, browser=None +) -> tuple[str, str, int, list[tuple[str, str]]]: from playwright.async_api import async_playwright - async with async_playwright() as pw: + own_browser = browser is None + if own_browser: + pw = await async_playwright().__aenter__() browser = await pw.chromium.launch( headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] ) - try: - context = await browser.new_context(user_agent=USER_AGENT) - page = await context.new_page() - response = await page.goto(url, wait_until="load", timeout=30000) - status = response.status if response else 0 - for hop in [response.url] if response else []: - await guard_public_url(hop) - content = (await page.content())[:MAX_FETCH_BYTES] - await context.close() - extracted = extract_html(content, base_url=url) - return extracted.title, extracted.text, status, extracted.links - finally: + try: + context = await browser.new_context(user_agent=USER_AGENT) + page = await context.new_page() + response = await page.goto(url, wait_until="load", timeout=30000) + status = response.status if response else 0 + for hop in [response.url] if response else []: + await guard_public_url(hop) + content = (await page.content())[:MAX_FETCH_BYTES] + await context.close() + extracted = extract_html(content, base_url=url) + return extracted.title, extracted.text, status, extracted.links + finally: + if own_browser: await browser.close() + await pw.__aexit__(None, None, None) -async def fetch_page(url: str, depth: int) -> CrawledPage | None: +async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None: try: await guard_public_url(url) except BlockedAddressError: @@ -237,7 +243,7 @@ async def fetch_page(url: str, depth: int) -> CrawledPage | None: logger.info("deepsearch decode failed for %s: %s", url, exc) if len(text) < MIN_PAGE_CHARS: try: - r_title, r_text, r_status, r_links = await _render_with_playwright(url) + r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser) if len(r_text) > len(text): title, text, status, source, links = ( r_title or title, @@ -261,12 +267,12 @@ async def fetch_page(url: str, depth: int) -> CrawledPage | None: ) -async def _resolve_candidate(candidate: dict, depth: int) -> CrawledPage | None: +async def _resolve_candidate(candidate: dict, depth: int, browser=None) -> CrawledPage | None: url = candidate["url"] snippet_page = _snippet_page(candidate, depth) if _is_hostile(url): return snippet_page - page = await fetch_page(url, depth) + page = await fetch_page(url, depth, browser) if page and snippet_page: return page if len(page.text) >= len(snippet_page.text) else snippet_page return page or snippet_page @@ -281,92 +287,111 @@ async def crawl( query: str = "", depth: int = 1, ) -> CrawlOutcome: + from playwright.async_api import async_playwright + outcome = CrawlOutcome() fetched = 0 seen_urls = {candidate["url"] for candidate in candidates} level_candidates = list(candidates) total = min(len(level_candidates), max_pages) cancelled = False - for level in range(max(1, depth)): - if cancelled or fetched >= max_pages or not level_candidates: - break - next_candidates: list[dict] = [] - for start in range(0, len(level_candidates), CRAWL_CONCURRENCY): - if fetched >= max_pages: + + pw = None + browser = None + try: + pw = await async_playwright().__aenter__() + browser = await pw.chromium.launch( + headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] + ) + except Exception as exc: + logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc) + + try: + for level in range(max(1, depth)): + if cancelled or fetched >= max_pages or not level_candidates: break - if await should_stop(): - emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"}) - cancelled = True - break - batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched] - for candidate in batch: - emit( - { - "type": "progress", - "done": fetched, - "total": total, - "url": candidate["url"], - "depth": level, - "message": f"Reading {candidate['url']}", - } - ) - if is_cached(candidate["url"]): - emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"}) - fetch_start = time.perf_counter() - results = await asyncio.gather( - *(_resolve_candidate(candidate, level) for candidate in batch), - return_exceptions=True, - ) - elapsed_ms = int((time.perf_counter() - fetch_start) * 1000) - for candidate, page in zip(batch, results): - url = candidate["url"] - if isinstance(page, BaseException): - logger.info("deepsearch fetch crashed for %s: %s", url, page) - page = None - if page is None: - emit( - { - "type": "page_skipped", - "url": url, - "reason": "no readable content", - "elapsed_ms": elapsed_ms, - } - ) - continue + next_candidates: list[dict] = [] + for start in range(0, len(level_candidates), CRAWL_CONCURRENCY): if fetched >= max_pages: break - digest = content_hash(page.text) - if digest in outcome.seen_hashes: + if await should_stop(): + emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"}) + cancelled = True + break + batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched] + for candidate in batch: emit( { - "type": "page_duplicate", - "url": url, - "reason": "duplicate content", - "elapsed_ms": elapsed_ms, + "type": "progress", + "done": fetched, + "total": total, + "url": candidate["url"], + "depth": level, + "message": f"Reading {candidate['url']}", } ) - continue - outcome.seen_hashes.add(digest) - outcome.pages.append(page) - fetched += 1 - emit( - { - "type": "page_loaded", - "url": page.url, - "title": page.title, - "source": page.source, - "depth": level, - "render": page.source == "playwright", - "elapsed_ms": elapsed_ms, - "done": fetched, - "total": total, - } + if is_cached(candidate["url"]): + emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"}) + fetch_start = time.perf_counter() + results = await asyncio.gather( + *(_resolve_candidate(candidate, level, browser) for candidate in batch), + return_exceptions=True, ) - if level + 1 < depth: - for link in relevant_links(page.links, query, LINKS_PER_PAGE): - if link not in seen_urls: - seen_urls.add(link) - next_candidates.append({"url": link}) - level_candidates = next_candidates - total = min(total + len(next_candidates), max_pages) + elapsed_ms = int((time.perf_counter() - fetch_start) * 1000) + for candidate, page in zip(batch, results): + url = candidate["url"] + if isinstance(page, BaseException): + logger.info("deepsearch fetch crashed for %s: %s", url, page) + page = None + if page is None: + emit( + { + "type": "page_skipped", + "url": url, + "reason": "no readable content", + "elapsed_ms": elapsed_ms, + } + ) + continue + if fetched >= max_pages: + break + digest = content_hash(page.text) + if digest in outcome.seen_hashes: + emit( + { + "type": "page_duplicate", + "url": url, + "reason": "duplicate content", + "elapsed_ms": elapsed_ms, + } + ) + continue + outcome.seen_hashes.add(digest) + outcome.pages.append(page) + fetched += 1 + emit( + { + "type": "page_loaded", + "url": page.url, + "title": page.title, + "source": page.source, + "depth": level, + "render": page.source == "playwright", + "elapsed_ms": elapsed_ms, + "done": fetched, + "total": total, + } + ) + if level + 1 < depth: + for link in relevant_links(page.links, query, LINKS_PER_PAGE): + if link not in seen_urls: + seen_urls.add(link) + next_candidates.append({"url": link}) + level_candidates = next_candidates + total = min(total + len(next_candidates), max_pages) + finally: + if browser is not None: + await browser.close() + if pw is not None: + await pw.__aexit__(None, None, None) return outcome diff --git a/devplacepy/services/jobs/deepsearch/enhance.py b/devplacepy/services/jobs/deepsearch/enhance.py index 6a972c1d..19a3a531 100644 --- a/devplacepy/services/jobs/deepsearch/enhance.py +++ b/devplacepy/services/jobs/deepsearch/enhance.py @@ -81,6 +81,7 @@ async def plan_queries( headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "X-App-Reference": "devplace-deepsearch-v-1-0-0", } try: async with stealth.stealth_async_client(timeout=ENHANCE_TIMEOUT_SECONDS) as client: diff --git a/devplacepy/services/jobs/deepsearch/orchestrate.py b/devplacepy/services/jobs/deepsearch/orchestrate.py index e11112ca..73778d5f 100644 --- a/devplacepy/services/jobs/deepsearch/orchestrate.py +++ b/devplacepy/services/jobs/deepsearch/orchestrate.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import json import logging import re @@ -79,10 +80,14 @@ async def _retrieve_chunks(question: str, queries: list[str], store, api_key: st vectors = local_embed(texts).vectors if not vectors or len(vectors[0]) != stored_dim: return [] - per_query = [ - store.hybrid_search(text, vector, top_k=RETRIEVE_TOP_K) - for text, vector in zip(texts, vectors) - ] + per_query = list( + await asyncio.gather( + *( + store.hybrid_search(text, vector, top_k=RETRIEVE_TOP_K) + for text, vector in zip(texts, vectors) + ) + ) + ) merged: list = [] seen: set[str] = set() for tier in zip_longest(*per_query): @@ -354,7 +359,7 @@ async def orchestrate( chunks: list = [] if store is not None: try: - if store.count(): + if await store.count(): chunks = await _retrieve_chunks(question, queries or [], store, api_key) except Exception as exc: logger.warning("deepsearch retrieval failed, using page context: %s", exc) diff --git a/devplacepy/services/jobs/deepsearch/worker.py b/devplacepy/services/jobs/deepsearch/worker.py index 751220d0..d5958e46 100644 --- a/devplacepy/services/jobs/deepsearch/worker.py +++ b/devplacepy/services/jobs/deepsearch/worker.py @@ -149,7 +149,7 @@ async def _index_chunks( if forced_local: collected = local_embed([chunk.text for chunk in chunks]).vectors backend = "local" - store.add(chunks, collected) + await store.add(chunks, collected) final_backend = "local" if forced_local else backend emit({"type": "embed_done", "backend": final_backend, "chunk_count": total}) return total, final_backend diff --git a/devplacepy/services/jobs/isslop/agent/llm.py b/devplacepy/services/jobs/isslop/agent/llm.py index b1e10625..56aee9f4 100644 --- a/devplacepy/services/jobs/isslop/agent/llm.py +++ b/devplacepy/services/jobs/isslop/agent/llm.py @@ -64,7 +64,11 @@ class LlmClient: "messages": messages, "temperature": LLM_TEMPERATURE, } - headers = {"authorization": f"Bearer {self._api_key}", "content-type": "application/json"} + headers = { + "authorization": f"Bearer {self._api_key}", + "content-type": "application/json", + "X-App-Reference": "devplace-isslop-v-1-0-0", + } request_timeout = httpx.Timeout(timeout) if timeout else None last_error = "unknown" for attempt in range(1, LLM_MAX_RETRIES + 1): diff --git a/devplacepy/services/jobs/queue.py b/devplacepy/services/jobs/queue.py index c53d6308..cef08ea3 100644 --- a/devplacepy/services/jobs/queue.py +++ b/devplacepy/services/jobs/queue.py @@ -97,6 +97,49 @@ def touch_job(uid: str, extend_seconds: int) -> None: ) +def job_metrics(kind: str) -> dict: + empty = { + "total": 0, + "by_status": {PENDING: 0, RUNNING: 0, DONE: 0, FAILED: 0}, + "bytes_in": 0, + "bytes_out": 0, + "items": 0, + "avg_ms": 0, + "recent": [], + } + if "jobs" not in db.tables: + return empty + metrics = dict(empty, by_status=dict(empty["by_status"])) + for row in db.query( + "SELECT status, COUNT(*) AS n, " + "COALESCE(SUM(bytes_in), 0) AS bytes_in, " + "COALESCE(SUM(bytes_out), 0) AS bytes_out, " + "COALESCE(SUM(item_count), 0) AS items, " + "COALESCE(AVG(CASE WHEN status = :done AND duration_ms > 0 " + "THEN duration_ms END), 0) AS avg_ms " + "FROM jobs WHERE kind = :kind GROUP BY status", + kind=kind, + done=DONE, + ): + status = row["status"] or "" + count = int(row["n"] or 0) + metrics["total"] += count + metrics["by_status"][status] = metrics["by_status"].get(status, 0) + count + metrics["bytes_in"] += int(row["bytes_in"] or 0) + metrics["bytes_out"] += int(row["bytes_out"] or 0) + metrics["items"] += int(row["items"] or 0) + if status == DONE: + metrics["avg_ms"] = int(row["avg_ms"] or 0) + metrics["recent"] = list( + db.query( + "SELECT uid, status, preferred_name, bytes_out, item_count " + "FROM jobs WHERE kind = :kind ORDER BY id DESC LIMIT 10", + kind=kind, + ) + ) + return metrics + + def list_jobs(kind: str = None, status: str = None, owner: tuple = None) -> list[dict]: if "jobs" not in db.tables: return [] diff --git a/devplacepy/services/messaging/persist.py b/devplacepy/services/messaging/persist.py index 6052f46a..32e9ee42 100644 --- a/devplacepy/services/messaging/persist.py +++ b/devplacepy/services/messaging/persist.py @@ -37,7 +37,8 @@ def _slim_attachment(attachment: dict[str, Any]) -> dict[str, Any]: def message_frame( - message: dict[str, Any], sender_username: str, client_id: Optional[str] = None + message: dict[str, Any], sender_username: str, client_id: Optional[str] = None, + sender_role: Optional[str] = None, ) -> dict[str, Any]: attachments = get_attachments("message", message["uid"]) return { @@ -45,6 +46,7 @@ def message_frame( "uid": message["uid"], "sender_uid": message["sender_uid"], "sender_username": sender_username, + "sender_role": sender_role, "receiver_uid": message["receiver_uid"], "content": message["content"], "created_at": message["created_at"], diff --git a/devplacepy/services/messaging/relay.py b/devplacepy/services/messaging/relay.py index 7114e2f3..459a5d32 100644 --- a/devplacepy/services/messaging/relay.py +++ b/devplacepy/services/messaging/relay.py @@ -72,7 +72,7 @@ class MessageRelay: senders = get_users_by_uids(list(sender_uids)) if sender_uids else {} for row in pending: sender = senders.get(row["sender_uid"]) or {} - frame = message_frame(dict(row), sender.get("username", "")) + frame = message_frame(dict(row), sender.get("username", ""), sender_role=sender.get("role")) message_hub.mark_delivered(row["uid"]) await message_hub.send_to_users( [row["sender_uid"], row["receiver_uid"]], frame diff --git a/devplacepy/services/news/service.py b/devplacepy/services/news/service.py index 7ac5b26e..ff9234a4 100644 --- a/devplacepy/services/news/service.py +++ b/devplacepy/services/news/service.py @@ -594,6 +594,7 @@ class NewsService(BaseService): headers = { "Content-Type": "application/json", + "X-App-Reference": "devplace-news-v-1-0-0", } ai_key = _get_ai_key() if ai_key: @@ -646,7 +647,7 @@ class NewsService(BaseService): "temperature": 0.3, } - headers = {"Content-Type": "application/json"} + headers = {"Content-Type": "application/json", "X-App-Reference": "devplace-news-v-1-0-0"} ai_key = _get_ai_key() if ai_key: headers["Authorization"] = f"Bearer {ai_key}" diff --git a/devplacepy/services/openai_gateway/CLAUDE.md b/devplacepy/services/openai_gateway/CLAUDE.md index db7bf307..6b58f004 100644 --- a/devplacepy/services/openai_gateway/CLAUDE.md +++ b/devplacepy/services/openai_gateway/CLAUDE.md @@ -4,7 +4,7 @@ This file documents the AI gateway subsystem (devplacepy/services/openai_gateway `GatewayService` (`services/openai_gateway/`) is an OpenAI-compatible LLM gateway (the ported `openai5.py`), mounted at `/openai/v1/*` (`routers/openai_gateway.py`, prefix `/openai`). It is a **service that serves an HTTP endpoint** rather than a loop. -- The router is thin: it calls `service_manager.get_service("openai").handle(request, subpath)`. `POST /v1/chat/completions` runs the full gateway (vision augment -> model override -> forward -> optional SSE re-emit via `_fake_stream`); `POST /v1/embeddings` forwards to the configured embeddings upstream (`handle_embeddings`, no vision/streaming); `/v1/{path:path}` is a transparent passthrough to the upstream base. Disabled -> 503, unauthorized -> 401, upstream connection failure -> 502. +- The router is thin: it calls `service_manager.get_service("openai").handle(request, subpath)`. `POST /v1/chat/completions` runs the full gateway (vision augment -> model override -> forward -> optional SSE re-emit via `_fake_stream`); `POST /v1/embeddings` forwards to the configured embeddings upstream (`handle_embeddings`, no vision/streaming); `GET /v1/models` is answered locally from the `gateway_models` chat routes (`_models_response`, publishing the public `molodetz`/`molodetz-pro` names), NOT proxied upstream; `/v1/{path:path}` is a transparent passthrough to the upstream base. **The gateway always forwards NON-streaming upstream** (`payload["stream"] = False`), then re-emits SSE itself when the client asked for `stream`; because `stream_options` is only valid alongside `stream=true`, `handle_chat` strips `stream_options` from the upstream payload (otherwise DeepSeek rejects it with `stream_options should be set along with stream = true`) and, when the client requested `stream_options.include_usage`, `_fake_stream` appends a final `choices: []` usage chunk built from the upstream `usage` before `[DONE]`. Disabled -> 503, unauthorized -> 401, upstream connection failure -> 502. - `main.py` registers it and exempts `/openai` from the rate-limit middleware and the maintenance gate. No new dependency (`httpx` already required). - `GatewayService` is `default_enabled=True` so internal callers work out of the box. @@ -12,6 +12,15 @@ This file documents the AI gateway subsystem (devplacepy/services/openai_gateway Every gateway response (chat, embeddings, passthrough; success and error) carries `X-Gateway-*` headers describing that single call: `Model`, `Backend`, `Prompt-Tokens`, `Completion-Tokens`, `Total-Tokens`, `Cache-Hit-Tokens`, `Cache-Miss-Tokens`, `Reasoning-Tokens`, `Cost-USD`/`Input-Cost-USD`/`Output-Cost-USD` (dollars), `Cost-Native` (1 if the upstream returned a native cost), `Tokens-Per-Second`, `Upstream-Latency-Ms`, and `Context-Window`/`Context-Utilization` when the model's window is known, plus the timing headers `X-Gateway-Upstream-Latency-Ms`/`X-Gateway-Total-Latency-Ms`. `GatewayUsageLedger.record(...)` RETURNS the computed ledger row (or `None` on failure); each handler's `finalize` closure maps it through `usage.usage_response_headers(row)` and attaches it to the returned `Response` (`resp_headers`). The single denied path with no upstream call (embeddings disabled) carries no headers. This is what lets any caller read its own spend - the AI correction worker reads `X-Gateway-Cost-USD`/token headers off its own correction call to accumulate per-user totals (see "AI content correction" in the root `CLAUDE.md`). +## App-reference header (`X-App-Reference`) + +Callers may send an optional `X-App-Reference` header to tag gateway calls by application. The value is validated and stored in the `app_reference` column of `gateway_usage_ledger`, surfaced in analytics and admin reporting. + +- **Validation** (`service._validate_app_reference`): trimmed whitespace, then matched against `^[a-zA-Z0-9_.-]{1,30}$`. Any value failing validation (empty, >30 chars, contains spaces or `@`/`/` etc.) silently falls back to `"default"`. +- **Header name:** `X-App-Reference`. +- **All internal callers** (news, bots, Devii, correction, jobs, deepsearch, dbapi, gitea) should pass a `devplace--v---` value, e.g. `devplace-devii-v-1-0-0`, `devplace-news-v-1-0-0`. +- The column is indexed (`CREATE INDEX IF NOT EXISTS`) for fast per-app queries. + ## Per-worker runtime **Serves in every worker.** Config/enabled come from `site_settings` (via the service's `get_config()`/`is_enabled()`), so any uvicorn worker answers - not just the supervisor worker. Per-worker runtime (`GatewayRuntime`: `httpx.AsyncClient` pool + `asyncio.Semaphore` sized to `gateway_instances`, plus counters and a `VisionCache`) is created lazily on first request and rebuilt when `instances`/`timeout`/cache size change. `gateway_instances` is the **scaling knob** (concurrency per worker); process scaling is uvicorn workers. @@ -83,7 +92,7 @@ Layered ON TOP of the single-provider service config above, which stays THE impl **Resolution is a per-request overlay, not a fork.** At request time `routing.chat_overlay(requested, cfg)` / `routing.embed_overlay(requested, cfg)` / `routing.image_overlay(requested, cfg)` resolve an active route by the requested model name and return a per-request OVERLAY dict of `gateway_*` cfg keys (`gateway_force_model`+`gateway_model`=target, `gateway_upstream_url`/`gateway_api_key` from the provider, the price keys, an augmented `gateway_model_context_map`, and vision overlay keys); `handle_chat`/`handle_embeddings` merge it onto the base cfg (`cfg = {**cfg, **overlay}`) BEFORE everything else, so the existing model-selection / `pricing_from_cfg` / `parse_context_map` / vision / url+key paths transparently use the route's provider, target model, pricing, vision model and context window. `_ensure` (the httpx pool / semaphore / breaker / vision cache) reads only the non-overlaid pool keys, so the connection pool is never churned per request. -**No matching route = `None` overlay = byte-identical legacy behavior** - this is the "nobody feels the transformation" guarantee: `molodetz`/`molodetz~embed`/`molodetz-img-small` and every existing caller are byte-identical when no route matches. A route with a blank provider overlays only model+pricing(+vision), keeping the default upstream url/key. +**No matching route = None overlay = model fallback.** When no route matches, the overlay returns `None` and the handler's model selection logic falls back to the default configured model (`gateway_model` for chat, `gateway_embed_model` for embeddings, `gateway_image_model` for images). The unknown model name is discarded, not forwarded upstream. This means an unknown or misspelled model name never causes a 4xx from the upstream - it is gracefully downgraded to the default. The `force_model` guard and the built-in `molodetz`/`molodetz~embed`/`molodetz-img-small` aliases are still respected before the route check: `force_model` or empty/named-alias model -> default directly; known model -> route resolution; unknown model with no route -> default fallback with a log message. **CRUD.** Admin JSON at `/admin/gateway/{providers,models}` (`routers/admin/gateway_configs.py`, `require_admin`, Pydantic `ProviderIn`/`ModelRouteIn` validation, accepts both JSON from `static/js/GatewayAdmin.js` and form from Devii), audited under `gateway.provider.*`/`gateway.model.*` (category `ai`), included in the admin package with the page at `/admin/gateway` (`templates/admin_gateway.html`, sidebar link, `admin_section="gateway"`). @@ -101,4 +110,4 @@ Real providers sometimes charge more than a flat per-1M rate for one component: - **Overlay + computation.** `chat_overlay`/`embed_overlay` propagate all of these into the per-request cfg dict under `gateway_*` keys exactly like the existing price fields (section above); `usage.pricing_from_cfg` reads them generically (defaulting to `None`/`0`/disabled when absent), so **Layer A (the single global flat Pricing config fields) never gains these dimensions** - only a `gateway_models` route can enable them, preserving the "no matching route = byte-identical legacy behavior" guarantee. `usage.compute_cost` selects tier1 vs tier2 per rate component (`_tiered_rate`) based on whether `norm["prompt"]` (billable input tokens) exceeds the threshold, then applies the off-peak discount (`_effective_rate`/`_off_peak_active`, UTC wraparound-aware) to whichever rate was selected. **The response header format and `compute_cost`'s return shape (`total, input_cost, output_cost, native`) are unchanged** - this is purely an internal rate-selection step before the existing input/output split math runs; a native upstream `cost` (OpenRouter) still overrides the modeled total exactly as before. - **Migration.** New `gateway_models` columns are added via `has_column`/`create_column_by_example` in `routing.ensure_tables()` (the `CREATE TABLE IF NOT EXISTS` DDL string alone would never reach a pre-existing table - see the `database/CLAUDE.md` column-ensure idiom). - **Admin UI.** `/admin/gateway`'s model-route form has a "Tiered / off-peak pricing (optional)" subsection; off-peak start/end render as `` (converted to/from UTC minutes-of-day by `GatewayAdmin.js`), and the routes table shows `tiered`/`off-peak` badges when a route has either dimension configured. -- **DeepSeek's real pricing is already the tier-1 shape, not a new dimension.** DeepSeek's actual API (verified against `api-docs.deepseek.com/quick_start/pricing`) bills three flat per-1M rates - cache-hit input, cache-miss input, output - with no current context-length tier or off-peak window for the V4 models; that shape was already fully modeled by the pre-existing `chat_cache_hit_per_m`/`chat_cache_miss_per_m`/`chat_output_per_m` fields before this section's tier2/off-peak fields existed. `routing.seed_default_deepseek_routes()` (called once from `database.migrate_ai_gateway_settings()` at the end of `init_db()`) idempotently inserts two ready-made routes - `deepseek-v4-flash` (`$0.0028`/`$0.14`/`$0.28` per 1M, 1M context) and `deepseek-v4-pro` (`$0.003625`/`$0.435`/`$0.87` per 1M, 1M context) - only when that `source_model` row does not already exist, so a caller or Devii can request either name explicitly and get correctly-priced, decoupled from whatever the single global `gateway_model` default happens to be set to (switching that global setting between the two real models does NOT retroactively fix the flat Pricing config fields - the seeded routes are the model-agnostic, always-correct way to reference a specific priced model). Neither seeded route sets the tier2/off-peak fields (DeepSeek does not use them today); an admin can add them later on the same row if DeepSeek (or any other provider routed here) introduces such pricing. +- **DeepSeek's real pricing is already the tier-1 shape, not a new dimension.** DeepSeek's actual API (verified against `api-docs.deepseek.com/quick_start/pricing`) bills three flat per-1M rates - cache-hit input, cache-miss input, output - with no current context-length tier or off-peak window for the V4 models; that shape was already fully modeled by the pre-existing `chat_cache_hit_per_m`/`chat_cache_miss_per_m`/`chat_output_per_m` fields before this section's tier2/off-peak fields existed. `routing.seed_default_deepseek_routes()` (called once from `database.migrate_ai_gateway_settings()` at the end of `init_db()`) idempotently inserts four ready-made routes - `deepseek-v4-flash` (`$0.0028`/`$0.14`/`$0.28` per 1M, 1M context), `deepseek-v4-pro` (`$0.003625`/`$0.435`/`$0.87` per 1M, 1M context), and the two public `molodetz` aliases `molodetz` -> `deepseek-v4-flash` (flash rates) and `molodetz-pro` -> `deepseek-v4-pro` (pro rates) - only when that `source_model` row does not already exist, so a caller or Devii can request any name explicitly and get correctly-priced, decoupled from whatever the single global `gateway_model` default happens to be set to (switching that global setting between the two real models does NOT retroactively fix the flat Pricing config fields - the seeded routes are the model-agnostic, always-correct way to reference a specific priced model). The `molodetz`/`molodetz-pro` aliases are the public model names; `GET /v1/models` is served locally from these `source_model` rows (not proxied upstream), so it publishes exactly the models the gateway accepts. Neither seeded route sets the tier2/off-peak fields (DeepSeek does not use them today); an admin can add them later on the same row if DeepSeek (or any other provider routed here) introduces such pricing. diff --git a/devplacepy/services/openai_gateway/gateway.py b/devplacepy/services/openai_gateway/gateway.py index 0b716be8..d8b08758 100644 --- a/devplacepy/services/openai_gateway/gateway.py +++ b/devplacepy/services/openai_gateway/gateway.py @@ -33,7 +33,7 @@ from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCac logger = logging.getLogger(__name__) -def _fake_stream(data: dict, model: str): +def _fake_stream(data: dict, model: str, include_usage: bool = False): chunk_id = data.get("id", f"chatcmpl-{uuid.uuid4().hex[:12]}") created = data.get("created", int(time.time())) out_model = data.get("model", model) @@ -72,6 +72,21 @@ def _fake_stream(data: dict, model: str): for i in range(0, len(content), 50): yield _chunk({"content": content[i : i + 50]}) yield _chunk({}, finish="tool_calls" if tool_calls else "stop") + if include_usage and data.get("usage"): + yield ( + "data: " + + json.dumps( + { + "id": chunk_id, + "object": "chat.completion.chunk", + "created": created, + "model": out_model, + "choices": [], + "usage": data["usage"], + } + ) + + "\n\n" + ) yield "data: [DONE]\n\n" return gen() @@ -217,7 +232,7 @@ class GatewayRuntime: return resp, None, timing async def handle_chat( - self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None + self, body: dict, cfg: dict, owner: tuple, user_agent: str, app_reference: str, log=None ): log = log or (lambda message: None) overlay = chat_overlay(body.get("model"), cfg) @@ -242,6 +257,7 @@ class GatewayRuntime: owner=owner, pricing=pricing, context_map=context_map, + app_reference=app_reference, ) messages = await augmenter.augment_messages(client, messages) self.vision_calls += augmenter.calls @@ -252,14 +268,19 @@ class GatewayRuntime: requested = body.get("model") if cfg["gateway_force_model"] or not requested or requested == "molodetz": model = cfg["gateway_model"] - else: + elif overlay is not None: model = requested + else: + model = cfg["gateway_model"] + log(f"requested model {requested!r} has no route, falling back to {model!r}") stream = bool(body.get("stream")) + include_usage = bool((body.get("stream_options") or {}).get("include_usage")) payload = dict(body) payload["model"] = model payload["messages"] = messages payload["stream"] = False + payload.pop("stream_options", None) headers = {"Content-Type": "application/json"} if cfg["gateway_api_key"]: @@ -287,6 +308,7 @@ class GatewayRuntime: "endpoint": "chat/completions", "model": model, "user_agent": user_agent, + "app_reference": app_reference, **params, **timing, } @@ -372,14 +394,18 @@ class GatewayRuntime: log(f"chat POST -> 200 ({timing['upstream_latency_ms']:.0f}ms)") if stream: return StreamingResponse( - _fake_stream(data, model), + _fake_stream(data, model, include_usage), media_type="text/event-stream", headers=resp_headers, ) - return JSONResponse(content=data, headers=resp_headers) + return Response( + content=resp.content, + media_type="application/json", + headers=resp_headers, + ) async def handle_embeddings( - self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None + self, body: dict, cfg: dict, owner: tuple, user_agent: str, app_reference: str, log=None ): log = log or (lambda message: None) vision_cost = 0.0 @@ -425,8 +451,11 @@ class GatewayRuntime: requested = body.get("model") if cfg["gateway_force_model"] or not requested or requested == "molodetz~embed": model = cfg["gateway_embed_model"] - else: + elif overlay is not None: model = requested + else: + model = cfg["gateway_embed_model"] + log(f"requested embed model {requested!r} has no route, falling back to {model!r}") payload = dict(body) payload["model"] = model @@ -457,6 +486,7 @@ class GatewayRuntime: "endpoint": "embeddings", "model": model, "user_agent": user_agent, + "app_reference": app_reference, **params, **timing, } @@ -544,7 +574,7 @@ class GatewayRuntime: return JSONResponse(content=data, headers=resp_headers) async def handle_images( - self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None + self, body: dict, cfg: dict, owner: tuple, user_agent: str, app_reference: str, log=None ): log = log or (lambda message: None) overlay = image_overlay(body.get("model"), cfg) @@ -595,8 +625,11 @@ class GatewayRuntime: or requested in ("molodetz-img-small", "molodetz-img") ): model = cfg["gateway_image_model"] - else: + elif overlay is not None: model = requested + else: + model = cfg["gateway_image_model"] + log(f"requested image model {requested!r} has no route, falling back to {model!r}") payload = dict(body) payload["model"] = model @@ -627,6 +660,7 @@ class GatewayRuntime: "endpoint": "images/generations", "model": model, "user_agent": user_agent, + "app_reference": app_reference, **params, **timing, } @@ -717,6 +751,7 @@ class GatewayRuntime: cfg: dict, owner: tuple, user_agent: str, + app_reference: str, log=None, ): log = log or (lambda message: None) @@ -746,6 +781,7 @@ class GatewayRuntime: "endpoint": subpath, "model": cfg["gateway_model"], "user_agent": user_agent, + "app_reference": app_reference, **timing, } diff --git a/devplacepy/services/openai_gateway/routing.py b/devplacepy/services/openai_gateway/routing.py index 8a8cb1ab..cc673423 100644 --- a/devplacepy/services/openai_gateway/routing.py +++ b/devplacepy/services/openai_gateway/routing.py @@ -419,6 +419,22 @@ DEEPSEEK_DEFAULT_ROUTES = ( "price_cache_miss_per_m": 0.435, "price_output_per_m": 0.87, }, + { + "source_model": "molodetz", + "target_model": "deepseek-v4-flash", + "context_window": 1_048_576, + "price_cache_hit_per_m": 0.0028, + "price_cache_miss_per_m": 0.14, + "price_output_per_m": 0.28, + }, + { + "source_model": "molodetz-pro", + "target_model": "deepseek-v4-pro", + "context_window": 1_048_576, + "price_cache_hit_per_m": 0.003625, + "price_cache_miss_per_m": 0.435, + "price_output_per_m": 0.87, + }, ) diff --git a/devplacepy/services/openai_gateway/service.py b/devplacepy/services/openai_gateway/service.py index 55cfd59c..1d780086 100644 --- a/devplacepy/services/openai_gateway/service.py +++ b/devplacepy/services/openai_gateway/service.py @@ -3,18 +3,32 @@ import json import logging import os +import re +import time from fastapi import HTTPException, Request +from fastapi.responses import JSONResponse from devplacepy.database import get_int_setting from devplacepy.services.base import BaseService, ConfigField from devplacepy.services.openai_gateway import config from devplacepy.services.openai_gateway.analytics import summary_metrics from devplacepy.services.openai_gateway.gateway import GatewayRuntime +from devplacepy.services.openai_gateway.routing import model_store from devplacepy.utils import get_current_user logger = logging.getLogger(__name__) +APP_REFERENCE_PATTERN = re.compile(r"^[a-zA-Z0-9_.-]{1,30}$") +DEFAULT_APP_REFERENCE = "default" + + +def _validate_app_reference(value: str) -> str: + stripped = (value or "").strip() + if not stripped or not APP_REFERENCE_PATTERN.match(stripped): + return DEFAULT_APP_REFERENCE + return stripped + def _presented_key(request: Request) -> str: key = request.headers.get("X-API-KEY") @@ -449,6 +463,29 @@ class GatewayService(BaseService): return (kind, user.get("uid") or "unknown") return ("anonymous", "anonymous") + def _models_response(self) -> JSONResponse: + created = int(time.time()) + seen: set = set() + data = [] + for row in model_store.list(): + if str(row.get("kind") or "chat") != "chat": + continue + if not row.get("is_active", True): + continue + source = str(row.get("source_model") or "").strip() + if not source or source in seen: + continue + seen.add(source) + data.append( + { + "id": source, + "object": "model", + "created": created, + "owned_by": "molodetz", + } + ) + return JSONResponse({"object": "list", "data": data}) + async def handle(self, request: Request, subpath: str): if not self.is_enabled(): raise HTTPException(status_code=503, detail="Gateway is disabled") @@ -459,6 +496,11 @@ class GatewayService(BaseService): runtime = self.runtime() owner = self.resolve_owner(request) user_agent = request.headers.get("user-agent", "") + app_reference = _validate_app_reference( + request.headers.get("X-App-Reference", DEFAULT_APP_REFERENCE) + ) + if subpath == "models" and request.method == "GET": + return self._models_response() if subpath == "chat/completions" and request.method == "POST": try: body = await request.json() @@ -468,7 +510,7 @@ class GatewayService(BaseService): if not isinstance(body, dict): self.log("Rejected chat request: JSON body was not an object") raise HTTPException(status_code=400, detail="Invalid JSON body") - return await runtime.handle_chat(body, cfg, owner, user_agent, self.log) + return await runtime.handle_chat(body, cfg, owner, user_agent, app_reference, self.log) if subpath == "embeddings" and request.method == "POST": try: body = await request.json() @@ -479,7 +521,7 @@ class GatewayService(BaseService): self.log("Rejected embeddings request: JSON body was not an object") raise HTTPException(status_code=400, detail="Invalid JSON body") return await runtime.handle_embeddings( - body, cfg, owner, user_agent, self.log + body, cfg, owner, user_agent, app_reference, self.log ) if subpath == "images/generations" and request.method == "POST": try: @@ -491,7 +533,7 @@ class GatewayService(BaseService): self.log("Rejected images request: JSON body was not an object") raise HTTPException(status_code=400, detail="Invalid JSON body") return await runtime.handle_images( - body, cfg, owner, user_agent, self.log + body, cfg, owner, user_agent, app_reference, self.log ) body = await request.body() content_type = request.headers.get("content-type", "") @@ -503,6 +545,7 @@ class GatewayService(BaseService): cfg, owner, user_agent, + app_reference, self.log, ) diff --git a/devplacepy/services/openai_gateway/usage.py b/devplacepy/services/openai_gateway/usage.py index b890b1d2..71eed9c5 100644 --- a/devplacepy/services/openai_gateway/usage.py +++ b/devplacepy/services/openai_gateway/usage.py @@ -404,6 +404,7 @@ def usage_response_headers(row: Optional[dict]) -> dict: headers["X-Gateway-Context-Window"] = str(int(row["context_window"])) if row.get("context_utilization") is not None: headers["X-Gateway-Context-Utilization"] = str(row["context_utilization"]) + headers["X-App-Reference"] = str(row.get("app_reference") or "default") return headers @@ -531,6 +532,7 @@ class GatewayUsageLedger: "retry_succeeded": 1 if raw.get("retry_succeeded") else 0, "circuit_open": 1 if raw.get("circuit_open") else 0, "user_agent": (raw.get("user_agent") or "")[:300], + "app_reference": (raw.get("app_reference") or "default")[:30], } get_table(GATEWAY_LEDGER).insert(row) self._audit(raw, norm, cost_usd) @@ -551,6 +553,7 @@ class GatewayUsageLedger: success: bool, status_code: int, latency_ms: float = 0.0, + app_reference: str = "default", ) -> Optional[dict]: try: row = { @@ -591,6 +594,7 @@ class GatewayUsageLedger: "retry_succeeded": 0, "circuit_open": 0, "user_agent": "", + "app_reference": app_reference or "default", } get_table(GATEWAY_LEDGER).insert(row) self._audit_external(row) @@ -695,6 +699,7 @@ def record_rsearch_call( cost_usd=cost, success=success, status_code=status_code, + app_reference="devplace-devii-rsearch-v-1-0-0", ) except Exception as exc: logger.warning("rsearch usage ledger failed: %s", exc) diff --git a/devplacepy/services/openai_gateway/vision.py b/devplacepy/services/openai_gateway/vision.py index c6359d9a..64ed7d5b 100644 --- a/devplacepy/services/openai_gateway/vision.py +++ b/devplacepy/services/openai_gateway/vision.py @@ -94,6 +94,7 @@ class VisionAugmenter: owner: tuple = ("unknown", "unknown"), pricing=None, context_map=None, + app_reference: str = "default", ): self.vision_url = vision_url self.vision_model = vision_model @@ -105,6 +106,7 @@ class VisionAugmenter: self.owner = owner self.pricing = pricing self.context_map = context_map or {} + self.app_reference = app_reference self.calls = 0 self.cost_usd = 0.0 @@ -126,6 +128,7 @@ class VisionAugmenter: "success": success, "error_category": category, "usage": usage, + "app_reference": self.app_reference, }, self.pricing, self.context_map, diff --git a/devplacepy/services/pubsub/hub.py b/devplacepy/services/pubsub/hub.py index 9e5f6b17..c2529403 100644 --- a/devplacepy/services/pubsub/hub.py +++ b/devplacepy/services/pubsub/hub.py @@ -16,28 +16,43 @@ def topic_matches(pattern: str, topic: str) -> bool: return False +def _is_pattern(pattern: str) -> bool: + return pattern == "*" or pattern.endswith(".*") + + class PubSubHub: def __init__(self) -> None: self._subscriptions: dict[str, set] = {} + self._patterns: dict[str, set] = {} + + def _store_for(self, pattern: str) -> dict: + return self._patterns if _is_pattern(pattern) else self._subscriptions def subscribe(self, pattern: str, socket) -> None: - self._subscriptions.setdefault(pattern, set()).add(socket) + self._store_for(pattern).setdefault(pattern, set()).add(socket) def unsubscribe(self, pattern: str, socket) -> None: - sockets = self._subscriptions.get(pattern) + store = self._store_for(pattern) + sockets = store.get(pattern) if not sockets: return sockets.discard(socket) if not sockets: - self._subscriptions.pop(pattern, None) + store.pop(pattern, None) def drop_socket(self, socket) -> None: - for pattern in list(self._subscriptions): - self.unsubscribe(pattern, socket) + for store in (self._subscriptions, self._patterns): + for pattern in list(store): + sockets = store.get(pattern) + if not sockets: + continue + sockets.discard(socket) + if not sockets: + store.pop(pattern, None) def _targets(self, topic: str) -> set: - targets: set = set() - for pattern, sockets in self._subscriptions.items(): + targets: set = set(self._subscriptions.get(topic, ())) + for pattern, sockets in self._patterns.items(): if topic_matches(pattern, topic): targets.update(sockets) return targets @@ -54,9 +69,10 @@ class PubSubHub: return delivered def topics(self) -> list[dict]: + merged = {**self._subscriptions, **self._patterns} return [ {"topic": pattern, "subscribers": len(sockets)} - for pattern, sockets in sorted(self._subscriptions.items()) + for pattern, sockets in sorted(merged.items()) ] diff --git a/devplacepy/services/registry.py b/devplacepy/services/registry.py new file mode 100644 index 00000000..ad47d2a3 --- /dev/null +++ b/devplacepy/services/registry.py @@ -0,0 +1,54 @@ +# retoor + +from __future__ import annotations + +from devplacepy.services.manager import ServiceManager + +_SHELL_MANAGER: ServiceManager | None = None + + +def _instantiate_all(): + from devplacepy.services.audit import AuditService + from devplacepy.services.backup.service import BackupService + from devplacepy.services.bot.service import BotsService + from devplacepy.services.containers.service import ContainerService + from devplacepy.services.devii import DeviiService + from devplacepy.services.gitea.service import IssueTrackerService + from devplacepy.services.news import NewsService + from devplacepy.services.openai_gateway import GatewayService + from devplacepy.services.pubsub import PubSubService + from devplacepy.services.telegram.outbox_service import TelegramOutboxService + from devplacepy.services.telegram.service import TelegramService + from devplacepy.services.xmlrpc import XmlrpcService + + return ( + AuditService(), + BackupService(), + BotsService(), + ContainerService(), + DeviiService(), + IssueTrackerService(), + NewsService(), + GatewayService(), + PubSubService(), + TelegramService(), + TelegramOutboxService(), + XmlrpcService(), + ) + + +def admin_shell_manager() -> ServiceManager: + """Read-only ServiceManager holding one instance of every managed service. + + Never supervised (no set_lock_owner/supervise call) - used only for + describe()/config metadata/key derivation, all of which read or write + through db_client to the shared service_state / site_settings tables. + The owning Tier 3 process is the only one that ever ticks these services. + """ + global _SHELL_MANAGER + if _SHELL_MANAGER is None: + manager = ServiceManager() + for svc in _instantiate_all(): + manager.register(svc) + _SHELL_MANAGER = manager + return _SHELL_MANAGER diff --git a/devplacepy/services/telegram/backend.py b/devplacepy/services/telegram/backend.py index 0819df94..ba428c3c 100644 --- a/devplacepy/services/telegram/backend.py +++ b/devplacepy/services/telegram/backend.py @@ -16,12 +16,26 @@ class TelegramBackend(ABC): @abstractmethod async def send_message( - self, chat_id: int, text: str, parse_mode: str | None + self, + chat_id: int, + text: str, + parse_mode: str | None, + reply_markup: dict[str, Any] | None = None, ) -> dict[str, Any]: ... @abstractmethod async def edit_message_text( - self, chat_id: int, message_id: int, text: str, parse_mode: str | None + self, + chat_id: int, + message_id: int, + text: str, + parse_mode: str | None, + reply_markup: dict[str, Any] | None = None, + ) -> dict[str, Any]: ... + + @abstractmethod + async def answer_callback_query( + self, callback_query_id: str, text: str | None = None ) -> dict[str, Any]: ... @abstractmethod @@ -53,12 +67,20 @@ class HttpTelegramBackend(TelegramBackend): async def get_updates(self, offset: int, timeout: int) -> dict[str, Any]: return await self._call( "getUpdates", - {"offset": offset, "timeout": timeout, "allowed_updates": ["message"]}, + { + "offset": offset, + "timeout": timeout, + "allowed_updates": ["message", "callback_query"], + }, timeout=timeout + 15, ) async def send_message( - self, chat_id: int, text: str, parse_mode: str | None + self, + chat_id: int, + text: str, + parse_mode: str | None, + reply_markup: dict[str, Any] | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { "chat_id": chat_id, @@ -67,10 +89,17 @@ class HttpTelegramBackend(TelegramBackend): } if parse_mode: payload["parse_mode"] = parse_mode + if reply_markup is not None: + payload["reply_markup"] = reply_markup return await self._call("sendMessage", payload, self._timeout) async def edit_message_text( - self, chat_id: int, message_id: int, text: str, parse_mode: str | None + self, + chat_id: int, + message_id: int, + text: str, + parse_mode: str | None, + reply_markup: dict[str, Any] | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { "chat_id": chat_id, @@ -80,8 +109,18 @@ class HttpTelegramBackend(TelegramBackend): } if parse_mode: payload["parse_mode"] = parse_mode + if reply_markup is not None: + payload["reply_markup"] = reply_markup return await self._call("editMessageText", payload, self._timeout) + async def answer_callback_query( + self, callback_query_id: str, text: str | None = None + ) -> dict[str, Any]: + payload: dict[str, Any] = {"callback_query_id": callback_query_id} + if text: + payload["text"] = text + return await self._call("answerCallbackQuery", payload, self._timeout) + async def send_chat_action(self, chat_id: int, action: str) -> dict[str, Any]: return await self._call( "sendChatAction", {"chat_id": chat_id, "action": action}, self._timeout @@ -114,20 +153,50 @@ class FakeTelegramBackend(TelegramBackend): return {"ok": True, "result": ready} async def send_message( - self, chat_id: int, text: str, parse_mode: str | None + self, + chat_id: int, + text: str, + parse_mode: str | None, + reply_markup: dict[str, Any] | None = None, ) -> dict[str, Any]: self._message_id += 1 - self.sent.append({"chat_id": chat_id, "text": text, "parse_mode": parse_mode}) + self.sent.append( + { + "chat_id": chat_id, + "text": text, + "parse_mode": parse_mode, + "reply_markup": reply_markup, + } + ) return {"ok": True, "result": {"message_id": self._message_id}} async def edit_message_text( - self, chat_id: int, message_id: int, text: str, parse_mode: str | None + self, + chat_id: int, + message_id: int, + text: str, + parse_mode: str | None, + reply_markup: dict[str, Any] | None = None, ) -> dict[str, Any]: self.edited.append( - {"chat_id": chat_id, "message_id": message_id, "text": text, "parse_mode": parse_mode} + { + "chat_id": chat_id, + "message_id": message_id, + "text": text, + "parse_mode": parse_mode, + "reply_markup": reply_markup, + } ) return {"ok": True, "result": {"message_id": message_id}} + async def answer_callback_query( + self, callback_query_id: str, text: str | None = None + ) -> dict[str, Any]: + self.actions.append( + {"callback_query_id": callback_query_id, "text": text, "kind": "answer"} + ) + return {"ok": True} + async def send_chat_action(self, chat_id: int, action: str) -> dict[str, Any]: self.actions.append({"chat_id": chat_id, "action": action}) return {"ok": True} diff --git a/devplacepy/services/telegram/bridge.py b/devplacepy/services/telegram/bridge.py index 3f300c0d..6cd34a1d 100644 --- a/devplacepy/services/telegram/bridge.py +++ b/devplacepy/services/telegram/bridge.py @@ -38,11 +38,19 @@ TOO_MANY = "Too many attempts. Wait a few minutes, request a new code, and try a class TelegramConnection: - def __init__(self, service: Any, session: Any, chat_id: int, message_id: int | None) -> None: + def __init__( + self, + service: Any, + session: Any, + chat_id: int, + message_id: int | None, + bridge: "TelegramBridge | None" = None, + ) -> None: self._service = service self._session = session self._chat_id = chat_id self._message_id = message_id + self._bridge = bridge self.done = asyncio.Event() self._finalized = False self._last_status = "" @@ -57,6 +65,8 @@ class TelegramConnection: await self._finalize(html.escape("Error: " + str(payload.get("text", "")))) elif kind in ("status", "trace"): await self._progress(payload) + elif kind == "interaction": + await self._handle_interaction(payload) elif kind in ("avatar", "client"): query_id = payload.get("id") if query_id: @@ -64,6 +74,45 @@ class TelegramConnection: query_id, {"error": "No browser is attached (Telegram session)."} ) + async def _handle_interaction(self, payload: dict[str, Any]) -> None: + request_id = str(payload.get("id") or "") + args = payload.get("args") if isinstance(payload.get("args"), dict) else payload + interaction_id = str( + args.get("interaction_id") or payload.get("interaction_id") or "" + ) + plain = str(args.get("plain") or args.get("markdown") or "") + widgets = args.get("widgets") if isinstance(args.get("widgets"), list) else [] + markup = _telegram_keyboard(interaction_id, widgets, args) + text = html.escape(plain) if plain else "Please choose:" + if markup is not None: + await self._service.send( + self._chat_id, f"
{text}
", reply_markup=markup + ) + else: + await self._service.send(self._chat_id, f"
{text}
") + if self._bridge is None: + self._session.resolve_interaction( + request_id, + { + "status": "error", + "interaction_id": interaction_id, + "error": "Telegram bridge unavailable for interaction reply.", + "values": {}, + }, + ) + return + try: + result = await self._bridge.wait_interaction_reply( + self._chat_id, interaction_id, args + ) + except asyncio.TimeoutError: + result = { + "status": "timeout", + "interaction_id": interaction_id, + "values": {}, + } + self._session.resolve_interaction(request_id, result) + async def _typing_loop(self) -> None: try: while not self.done.is_set(): @@ -122,6 +171,7 @@ class TelegramBridge: self._semaphore = asyncio.Semaphore(max(1, max_concurrent)) self._chat_locks: dict[int, asyncio.Lock] = {} self._attempts: dict[int, list[float]] = {} + self._pending_interactions: dict[int, dict[str, Any]] = {} def _chat_lock(self, chat_id: int) -> asyncio.Lock: lock = self._chat_locks.get(chat_id) @@ -133,11 +183,81 @@ class TelegramBridge: self._chat_locks[chat_id] = lock return lock + async def wait_interaction_reply( + self, chat_id: int, interaction_id: str, args: dict[str, Any] + ) -> dict[str, Any]: + loop = asyncio.get_event_loop() + future: asyncio.Future = loop.create_future() + timeout = float(args.get("timeout_sec") or 0) or 300.0 + self._pending_interactions[chat_id] = { + "interaction_id": interaction_id, + "future": future, + "args": args, + } + try: + return await asyncio.wait_for(future, timeout=timeout) + finally: + current = self._pending_interactions.get(chat_id) + if current and current.get("future") is future: + self._pending_interactions.pop(chat_id, None) + + def _resolve_pending( + self, chat_id: int, result: dict[str, Any] + ) -> bool: + pending = self._pending_interactions.get(chat_id) + if pending is None: + return False + future = pending.get("future") + if future is not None and not future.done(): + future.set_result(result) + return True + return False + async def handle_inbound(self, event: dict[str, Any]) -> None: + kind = event.get("type") or "message" chat_id = int(event["chat_id"]) from_id = int(event["from_id"]) + if kind == "callback": + await self._handle_callback(event) + return text = str(event.get("text", "")).strip() images = [img for img in event.get("images", []) if img] + if chat_id in self._pending_interactions and text: + from devplacepy.services.devii.interaction.parse import parse_plain_reply + from devplacepy.services.devii.interaction.schema import InteractionRequest + + pending = self._pending_interactions[chat_id] + try: + request = InteractionRequest.model_validate( + (pending.get("args") or {}).get("request") + or { + "title": "?", + "widgets": (pending.get("args") or {}).get("widgets") or [], + } + ) + status, values, error = parse_plain_reply(request, text) + self._resolve_pending( + chat_id, + { + "status": status, + "interaction_id": pending.get("interaction_id"), + "values": values, + "error": error, + "meta": {"adapter": "telegram", "degraded": True}, + }, + ) + except Exception: + self._resolve_pending( + chat_id, + { + "status": "error", + "interaction_id": pending.get("interaction_id"), + "values": {}, + "error": "Could not parse reply.", + "meta": {"adapter": "telegram", "degraded": True}, + }, + ) + return link = store.user_for_chat(chat_id) if link is None: await self._handle_unpaired(chat_id, from_id, text) @@ -154,6 +274,55 @@ class TelegramBridge: async with self._semaphore: await self._run_turn(chat_id, user, text, images) + async def _handle_callback(self, event: dict[str, Any]) -> None: + chat_id = int(event["chat_id"]) + data = str(event.get("data") or "") + callback_id = str(event.get("callback_query_id") or "") + if callback_id: + try: + await self._service.answer_callback(callback_id) + except Exception: + pass + pending = self._pending_interactions.get(chat_id) + if pending is None: + return + interaction_id = str(pending.get("interaction_id") or "") + parsed = _parse_callback_data(data) + if not parsed: + return + if parsed.get("i") and parsed["i"] not in interaction_id: + if not interaction_id.endswith(parsed["i"]) and parsed["i"] not in interaction_id: + pass + status = parsed.get("s") or "submitted" + if status == "cancel": + self._resolve_pending( + chat_id, + { + "status": "cancelled", + "interaction_id": interaction_id, + "values": {}, + "meta": {"adapter": "telegram", "degraded": True}, + }, + ) + return + name = parsed.get("k") + value = parsed.get("v") + values: dict[str, Any] = {} + if name is not None: + if value in ("true", "false"): + values[name] = value == "true" + else: + values[name] = value + self._resolve_pending( + chat_id, + { + "status": "submitted", + "interaction_id": interaction_id, + "values": values, + "meta": {"adapter": "telegram", "degraded": True}, + }, + ) + async def _handle_unpaired(self, chat_id: int, from_id: int, text: str) -> None: if text.isdigit() and len(text) == 4: if self._too_many_attempts(chat_id): @@ -247,9 +416,12 @@ class TelegramBridge: ) session.set_timezone(user.get("timezone") or "") placeholder_id = await self._service.send(chat_id, THINKING_PLACEHOLDER) - connection = TelegramConnection(self._service, session, chat_id, placeholder_id) + connection = TelegramConnection( + self._service, session, chat_id, placeholder_id, bridge=self + ) + session.interaction_broker.bind(present_telegram=None) session.attach(connection) - content, audit_text = self._build_content(text, images) + content, audit_text = _build_content(text, images) started = time.monotonic() try: session.spawn_turn(content, audit_text=audit_text) @@ -265,13 +437,74 @@ class TelegramBridge: session.detach(connection) self._service.record_latency(time.monotonic() - started) - @staticmethod - def _build_content(text: str, images: list[str]) -> tuple[Any, str]: - if not images: - return text, text - prompt = text or "Look at the attached image and respond." - content: list[dict[str, Any]] = [{"type": "text", "text": prompt}] - for uri in images: - content.append({"type": "image_url", "image_url": {"url": uri}}) - audit_text = f"{text} [{len(images)} image(s)]".strip() - return content, audit_text + +def _telegram_keyboard( + interaction_id: str, widgets: list[Any], args: dict[str, Any] +) -> dict[str, Any] | None: + short = interaction_id.replace("ix-", "")[:8] + rows: list[list[dict[str, str]]] = [] + inputs = [ + w + for w in widgets + if isinstance(w, dict) and w.get("type") not in ("//", "group", None) + ] + if len(inputs) == 1 and inputs[0].get("type") == "confirm": + name = str(inputs[0].get("name") or "confirm") + submit = str(args.get("submit_label") or "Confirm")[:20] + cancel = str(args.get("cancel_label") or "Cancel")[:20] + rows.append( + [ + { + "text": submit, + "callback_data": f"i={short};k={name};v=true"[:64], + }, + { + "text": cancel, + "callback_data": f"i={short};s=cancel"[:64], + }, + ] + ) + return {"inline_keyboard": rows} + if len(inputs) == 1 and inputs[0].get("type") in ("choice", "select"): + name = str(inputs[0].get("name") or "choice") + options = inputs[0].get("options") or [] + for option in options[:12]: + if not isinstance(option, dict): + continue + value = str(option.get("value") or "")[:20] + label = str(option.get("label") or value)[:40] + rows.append( + [ + { + "text": label, + "callback_data": f"i={short};k={name};v={value}"[:64], + } + ] + ) + if args.get("cancelable", True): + rows.append( + [{"text": "Cancel", "callback_data": f"i={short};s=cancel"[:64]}] + ) + return {"inline_keyboard": rows} if rows else None + return None + + +def _parse_callback_data(data: str) -> dict[str, str]: + out: dict[str, str] = {} + for part in str(data or "").split(";"): + if "=" not in part: + continue + key, value = part.split("=", 1) + out[key.strip()] = value.strip() + return out + + +def _build_content(text: str, images: list[str]) -> tuple[Any, str]: + if not images: + return text, text + prompt = text or "Look at the attached image and respond." + content: list[dict[str, Any]] = [{"type": "text", "text": prompt}] + for uri in images: + content.append({"type": "image_url", "image_url": {"url": uri}}) + audit_text = f"{text} [{len(images)} image(s)]".strip() + return content, audit_text diff --git a/devplacepy/services/telegram/service.py b/devplacepy/services/telegram/service.py index f74eba54..98ac4f27 100644 --- a/devplacepy/services/telegram/service.py +++ b/devplacepy/services/telegram/service.py @@ -229,9 +229,9 @@ class TelegramService(BaseService): self._live_log("worker ready") elif kind == "result": self._resolve(frame) - elif kind == "message": + elif kind in ("message", "callback"): self._stats["in"] += 1 - self._live_log(f"in <- chat {frame.get('chat_id')}") + self._live_log(f"in <- chat {frame.get('chat_id')} ({kind})") if self._bridge is not None: asyncio.create_task(self._dispatch_inbound(frame)) @@ -274,15 +274,28 @@ class TelegramService(BaseService): self._req_seq += 1 return self._req_seq - async def send(self, chat_id: int, text: str, parse_mode: str = "HTML") -> int | None: + async def send( + self, + chat_id: int, + text: str, + parse_mode: str = "HTML", + reply_markup: dict | None = None, + ) -> int | None: req_id = self._next_req() loop = asyncio.get_event_loop() future: asyncio.Future = loop.create_future() self._pending[req_id] = future + command = { + "cmd": "send", + "req_id": req_id, + "chat_id": chat_id, + "text": text, + "parse_mode": parse_mode, + } + if reply_markup is not None: + command["reply_markup"] = reply_markup try: - await self._write( - {"cmd": "send", "req_id": req_id, "chat_id": chat_id, "text": text, "parse_mode": parse_mode} - ) + await self._write(command) except RuntimeError: self._pending.pop(req_id, None) self._stats["errors"] += 1 @@ -299,7 +312,44 @@ class TelegramService(BaseService): return None return result.get("message_id") - async def edit(self, chat_id: int, message_id: int, text: str, parse_mode: str = "HTML") -> bool: + async def edit( + self, + chat_id: int, + message_id: int, + text: str, + parse_mode: str = "HTML", + reply_markup: dict | None = None, + ) -> bool: + req_id = self._next_req() + loop = asyncio.get_event_loop() + future: asyncio.Future = loop.create_future() + self._pending[req_id] = future + command = { + "cmd": "edit", + "req_id": req_id, + "chat_id": chat_id, + "message_id": message_id, + "text": text, + "parse_mode": parse_mode, + } + if reply_markup is not None: + command["reply_markup"] = reply_markup + try: + await self._write(command) + except RuntimeError: + self._pending.pop(req_id, None) + return False + self._stats["edits"] += 1 + try: + result = await asyncio.wait_for(future, timeout=RESULT_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + self._pending.pop(req_id, None) + return False + return bool(result.get("ok")) + + async def answer_callback( + self, callback_query_id: str, text: str | None = None + ) -> bool: req_id = self._next_req() loop = asyncio.get_event_loop() future: asyncio.Future = loop.create_future() @@ -307,18 +357,15 @@ class TelegramService(BaseService): try: await self._write( { - "cmd": "edit", + "cmd": "answer_callback", "req_id": req_id, - "chat_id": chat_id, - "message_id": message_id, + "callback_query_id": callback_query_id, "text": text, - "parse_mode": parse_mode, } ) except RuntimeError: self._pending.pop(req_id, None) return False - self._stats["edits"] += 1 try: result = await asyncio.wait_for(future, timeout=RESULT_TIMEOUT_SECONDS) except asyncio.TimeoutError: diff --git a/devplacepy/services/telegram/worker.py b/devplacepy/services/telegram/worker.py index a291f8a9..88fdd401 100644 --- a/devplacepy/services/telegram/worker.py +++ b/devplacepy/services/telegram/worker.py @@ -91,6 +91,10 @@ class TelegramWorker: return updates async def process_update(self, update: dict[str, Any]) -> None: + callback = update.get("callback_query") + if isinstance(callback, dict): + await self._process_callback(callback) + return message = update.get("message") if not isinstance(message, dict): return @@ -128,6 +132,26 @@ class TelegramWorker: } ) + async def _process_callback(self, callback: dict[str, Any]) -> None: + message = callback.get("message") or {} + chat = message.get("chat") or {} + if chat.get("type") != "private": + return + chat_id = chat.get("id") + from_id = (callback.get("from") or {}).get("id") + if chat_id is None or from_id is None: + return + await self._emit( + { + "type": "callback", + "chat_id": chat_id, + "from_id": from_id, + "data": str(callback.get("data") or ""), + "callback_query_id": str(callback.get("id") or ""), + "message_id": message.get("message_id"), + } + ) + async def _fetch_image(self, file_id: str | None, mime: str | None = None) -> str: if not file_id: return "" @@ -156,6 +180,8 @@ class TelegramWorker: await self._edit(command) elif kind == "chat_action": await self._chat_action(command) + elif kind == "answer_callback": + await self._answer_callback(command) except Exception as exc: # noqa: BLE001 - one bad command must not kill the worker await self._log(f"command {kind} failed: {exc}") req_id = command.get("req_id") @@ -166,10 +192,17 @@ class TelegramWorker: chat_id = command["chat_id"] text = str(command.get("text", "")) parse_mode = command.get("parse_mode") + reply_markup = command.get("reply_markup") await self._throttle() - resp = await self._backend.send_message(chat_id, text, parse_mode) + resp = await self._backend.send_message( + chat_id, text, parse_mode, reply_markup=reply_markup + ) resp = await self._retry_if_needed( - resp, lambda mode: self._backend.send_message(chat_id, text, mode), parse_mode + resp, + lambda mode: self._backend.send_message( + chat_id, text, mode, reply_markup=reply_markup + ), + parse_mode, ) await self._emit_result(command.get("req_id"), resp) @@ -178,15 +211,27 @@ class TelegramWorker: message_id = command["message_id"] text = str(command.get("text", "")) parse_mode = command.get("parse_mode") + reply_markup = command.get("reply_markup") await self._throttle() - resp = await self._backend.edit_message_text(chat_id, message_id, text, parse_mode) + resp = await self._backend.edit_message_text( + chat_id, message_id, text, parse_mode, reply_markup=reply_markup + ) resp = await self._retry_if_needed( resp, - lambda mode: self._backend.edit_message_text(chat_id, message_id, text, mode), + lambda mode: self._backend.edit_message_text( + chat_id, message_id, text, mode, reply_markup=reply_markup + ), parse_mode, ) await self._emit_result(command.get("req_id"), resp) + async def _answer_callback(self, command: dict[str, Any]) -> None: + callback_query_id = str(command.get("callback_query_id") or "") + text = command.get("text") + await self._throttle() + resp = await self._backend.answer_callback_query(callback_query_id, text) + await self._emit_result(command.get("req_id"), resp) + async def _chat_action(self, command: dict[str, Any]) -> None: await self._throttle() await self._backend.send_chat_action(command["chat_id"], command.get("action", "typing")) diff --git a/devplacepy/services/weblock.py b/devplacepy/services/weblock.py new file mode 100644 index 00000000..215b1a35 --- /dev/null +++ b/devplacepy/services/weblock.py @@ -0,0 +1,28 @@ +# retoor + +from __future__ import annotations + +import fcntl +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +_LOCK_FD = None + + +def acquire_web_lock(lock_path: Path) -> bool: + """Non-blocking exclusive lock so exactly one `web` worker owns + in-process-hub-dependent work (DbApiJobService, its query WS). + Web is the one service that can still run N>1 real OS workers + (workers="auto"), unlike every other Tier 3 service (workers=1).""" + global _LOCK_FD + lock_path.parent.mkdir(parents=True, exist_ok=True) + fd = open(lock_path, "w") + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + fd.close() + return False + _LOCK_FD = fd + return True diff --git a/devplacepy/static/css/components/ai-choice.css b/devplacepy/static/css/components/ai-choice.css new file mode 100644 index 00000000..6f35af79 --- /dev/null +++ b/devplacepy/static/css/components/ai-choice.css @@ -0,0 +1,60 @@ +/* retoor */ + +.ai-choice, +.ai-choice-multi { + display: block; + margin: 0.75rem 0; +} + +.ai-choice-options { + display: flex; + flex-direction: column; + gap: 0.4rem; + max-height: min(50vh, 22rem); + overflow-y: auto; + padding-right: 0.15rem; +} + +.ai-choice-option { + text-align: left; + border: 1px solid var(--devii-border, rgba(255, 255, 255, 0.12)); + background: var(--devii-bar-bg, #0b0b0b); + color: inherit; + border-radius: 8px; + padding: 0.5rem 0.75rem; + cursor: pointer; + font: inherit; +} + +.ai-choice-option.selected { + border-color: var(--devii-accent, #3fb950); +} + +.ai-choice-option-desc, +.ai-choice-check small { + display: block; + color: var(--devii-dim, #6a737d); + font-size: 0.85em; + margin-top: 0.15rem; +} + +.ai-choice-select { + width: 100%; + max-width: 28rem; + background: var(--devii-bar-bg, #0b0b0b); + color: inherit; + border: 1px solid var(--devii-border, rgba(255, 255, 255, 0.12)); + border-radius: 8px; + padding: 0.45rem 0.6rem; + font: inherit; +} + +.ai-choice-check { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + gap: 0.45rem; + border: 1px solid var(--devii-border, rgba(255, 255, 255, 0.12)); + border-radius: 8px; + padding: 0.45rem 0.6rem; +} diff --git a/devplacepy/static/css/components/ai-confirm.css b/devplacepy/static/css/components/ai-confirm.css new file mode 100644 index 00000000..31769059 --- /dev/null +++ b/devplacepy/static/css/components/ai-confirm.css @@ -0,0 +1,27 @@ +/* retoor */ + +.ai-confirm { + display: block; + margin: 0.75rem 0; +} + +.ai-confirm-options { + display: flex; + gap: 0.5rem; + flex-wrap: wrap; +} + +.ai-confirm-option { + border: 1px solid var(--devii-border, rgba(255, 255, 255, 0.12)); + background: var(--devii-bar-bg, #0b0b0b); + color: inherit; + border-radius: 8px; + padding: 0.45rem 0.85rem; + cursor: pointer; + font: inherit; +} + +.ai-confirm-option.selected { + border-color: var(--devii-accent, #3fb950); + box-shadow: 0 0 0 1px var(--devii-accent, #3fb950); +} diff --git a/devplacepy/static/css/components/ai-field.css b/devplacepy/static/css/components/ai-field.css new file mode 100644 index 00000000..183246f4 --- /dev/null +++ b/devplacepy/static/css/components/ai-field.css @@ -0,0 +1,28 @@ +/* retoor */ + +.ai-field { + display: block; + margin: 0.75rem 0; +} + +.ai-field-wrap { + display: flex; + flex-direction: column; + gap: 0.35rem; +} + +.ai-field-input { + width: 100%; + max-width: 28rem; + background: var(--devii-bar-bg, #0b0b0b); + color: inherit; + border: 1px solid var(--devii-border, rgba(255, 255, 255, 0.12)); + border-radius: 8px; + padding: 0.5rem 0.65rem; + font: inherit; +} + +.ai-field-input:focus-visible { + outline: none; + box-shadow: 0 0 0 2px rgba(63, 185, 80, 0.35); +} diff --git a/devplacepy/static/css/components/ai-interaction.css b/devplacepy/static/css/components/ai-interaction.css new file mode 100644 index 00000000..a761e907 --- /dev/null +++ b/devplacepy/static/css/components/ai-interaction.css @@ -0,0 +1,148 @@ +/* retoor */ + +.ai-interaction { + --ai-space-1: 0.25rem; + --ai-space-2: 0.5rem; + --ai-space-3: 0.75rem; + --ai-space-4: 1rem; + --ai-radius-md: 10px; + --ai-color-fg: var(--devii-agent, var(--text-primary, #e6edf3)); + --ai-color-muted: var(--devii-dim, var(--text-muted, #7a6a90)); + --ai-color-accent: var(--devii-accent, var(--accent, #ff6b35)); + --ai-color-border: var(--devii-border, var(--border, rgba(255, 255, 255, 0.12))); + --ai-color-surface: var(--devii-code-bg, var(--bg-card, #1a1030)); + --ai-color-surface-2: var(--devii-bar-bg, var(--bg-secondary, #120821)); + --ai-focus-ring: 0 0 0 2px rgba(63, 185, 80, 0.35); + + display: block; + box-sizing: border-box; + width: 100%; + max-width: 100%; + margin: var(--ai-space-3) 0; + padding: var(--ai-space-3); + border: 1px solid var(--ai-color-border); + border-radius: var(--ai-radius-md); + background: var(--ai-color-surface); + color: var(--ai-color-fg); + overflow: visible; +} + +.ai-interaction-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--ai-space-2); + margin-bottom: var(--ai-space-3); +} + +.ai-interaction-title { + margin: 0; + font-size: 1rem; + font-weight: 600; +} + +.ai-status { + font-size: 0.75rem; + color: var(--ai-color-muted); + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.ai-status[data-state="submitted"] { + color: var(--ai-color-accent); +} + +.ai-status[data-state="error"], +.ai-status[data-state="cancelled"], +.ai-status[data-state="timeout"] { + color: var(--devii-error, var(--danger, #e53935)); +} + +.ai-widget-label { + display: block; + font-weight: 600; + margin-bottom: var(--ai-space-1); +} + +.ai-widget-help, +.ai-help { + display: block; + color: var(--ai-color-muted); + font-size: 0.85rem; + margin-bottom: var(--ai-space-2); +} + +.ai-actions { + display: flex; + gap: var(--ai-space-2); + margin-top: var(--ai-space-3); + flex-wrap: wrap; + position: sticky; + bottom: 0; + z-index: 2; + padding: var(--ai-space-2) 0 var(--ai-space-1); + background: linear-gradient( + to bottom, + rgba(0, 0, 0, 0) 0%, + var(--ai-color-surface) 28%, + var(--ai-color-surface) 100% + ); +} + +.ai-actions.align-end { + justify-content: flex-end; +} + +.ai-btn { + border: 1px solid var(--ai-color-border); + border-radius: 8px; + padding: 0.45rem 0.9rem; + background: var(--ai-color-surface-2); + color: var(--ai-color-fg); + cursor: pointer; + font: inherit; +} + +.ai-btn:focus-visible { + outline: none; + box-shadow: var(--ai-focus-ring); +} + +.ai-btn-primary { + background: var(--ai-color-accent); + border-color: transparent; + color: #041008; + font-weight: 600; +} + +.ai-btn-secondary { + opacity: 0.92; +} + +.ai-group { + margin: var(--ai-space-3) 0; + padding: var(--ai-space-2); + border-left: 2px solid var(--ai-color-border); +} + +.ai-group-label { + font-weight: 600; + margin-bottom: var(--ai-space-2); +} + +.ai-markdown-fallback pre { + white-space: pre-wrap; + margin: 0; + font: inherit; + color: var(--ai-color-muted); +} + +@media (max-width: 520px) { + .ai-actions { + flex-direction: column; + } + + .ai-btn { + width: 100%; + } +} diff --git a/devplacepy/static/css/devii.css b/devplacepy/static/css/devii.css index 9ab7b960..1bbe11f5 100644 --- a/devplacepy/static/css/devii.css +++ b/devplacepy/static/css/devii.css @@ -256,6 +256,21 @@ devii-terminal .devii-line { white-space: pre-wrap; } +devii-terminal .devii-interaction-line { + white-space: normal; + overflow: visible; + max-width: 100%; + padding-bottom: 8px; +} + +devii-terminal .devii-interaction-line ai-interaction, +devii-terminal .devii-interaction-line .ai-interaction { + display: block; + width: 100%; + max-width: 100%; + box-sizing: border-box; +} + devii-terminal .devii-user { color: var(--devii-user); } diff --git a/devplacepy/static/css/gists.css b/devplacepy/static/css/gists.css index 96e2f273..7efd0da6 100644 --- a/devplacepy/static/css/gists.css +++ b/devplacepy/static/css/gists.css @@ -184,6 +184,10 @@ margin-bottom: 1.5rem; } +.gist-rendered-content { + margin-bottom: 1.5rem; +} + .gist-code-block { margin-bottom: 1.5rem; border: 1px solid var(--border); diff --git a/devplacepy/static/css/landing.css b/devplacepy/static/css/landing.css index a47c11a4..76d6a928 100644 --- a/devplacepy/static/css/landing.css +++ b/devplacepy/static/css/landing.css @@ -1,5 +1,413 @@ /* retoor */ +.dashboard { + max-width: 1200px; + margin: 0 auto; + padding: 1.5rem 1.25rem 0; + width: 100%; + position: relative; + z-index: 1; +} + +.dashboard-top { + display: flex; + align-items: center; + justify-content: space-between; + flex-wrap: wrap; + gap: 1rem; + padding: 1.25rem 1.5rem; + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: var(--radius-xl); + margin-bottom: 1.25rem; +} + +.dashboard-actions { + display: flex; + align-items: center; + gap: 1.25rem; + flex-wrap: wrap; +} + +.dashboard-welcome { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.dashboard-greeting { + font-size: 1.15rem; + color: var(--text-primary); + white-space: nowrap; +} + +.dashboard-greeting strong { + color: var(--accent); +} + +.dashboard-buttons { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; +} + +.dashboard-btn { + padding: 0.55rem 1.1rem; + border-radius: var(--radius-lg); + border: 1px solid var(--border); + background: var(--bg-card); + color: var(--text-primary); + font-size: 0.875rem; + font-weight: 600; + white-space: nowrap; + transition: border-color 0.15s ease, transform 0.15s ease; +} + +.dashboard-btn:hover { + border-color: var(--accent); + transform: translateY(-1px); +} + +.dashboard-btn-primary { + background: var(--accent); + color: var(--white); + border-color: var(--accent); + font-weight: 700; + padding: 0.55rem 1.5rem; +} + +.dashboard-btn-primary:hover { + background: var(--accent-hover); + box-shadow: 0 4px 16px rgba(255, 107, 53, 0.25); +} + +.dashboard-stats { + display: flex; + gap: 2rem; +} + +.dashboard-stat { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.2rem; +} + +.dashboard-stat-value { + font-size: 1.5rem; + font-weight: 700; + color: var(--accent); + line-height: 1; +} + +.dashboard-stat-label { + font-size: 0.6875rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--text-secondary); +} + +.dashboard-stat-level { + min-width: 140px; +} + +.dashboard-xp-bar { + width: 100%; + max-width: 120px; + height: 5px; + background: var(--border); + border-radius: 3px; + overflow: hidden; + margin-top: 0.15rem; +} + +.dashboard-xp-fill { + height: 100%; + background: var(--accent-gradient, linear-gradient(90deg, var(--accent), #f0932b)); + border-radius: 3px; + transition: width 0.4s ease; +} + +.dashboard-xp-label { + font-size: 0.625rem; + color: var(--text-muted); + margin-top: 0.1rem; +} + +.dashboard-main { + display: grid; + grid-template-columns: 1fr 280px; + gap: 1.25rem; + align-items: start; +} + +.dashboard-feed-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 0.875rem; +} + +.dashboard-feed-header h2 { + font-size: 1.15rem; + font-weight: 700; + color: var(--text-primary); + margin: 0; +} + +.dashboard-feed-link { + font-size: 0.8125rem; + font-weight: 600; + color: var(--accent); +} + +.dashboard-feed-link:hover { + color: var(--accent-hover); +} + +.dashboard-posts-grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 0.875rem; +} + +.dashboard-post-card { + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: var(--radius-xl); + padding: 1rem 1.1rem; + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.dashboard-post-card:hover { + transform: translateY(-1px); + box-shadow: var(--shadow-lg); +} + +.dashboard-post-header { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.dashboard-post-author-wrap { + display: flex; + flex-direction: column; + flex: 1; + min-width: 0; +} + +.dashboard-post-author { + font-size: 0.75rem; + font-weight: 600; + color: var(--text-primary); +} + +.dashboard-post-time { + font-size: 0.65rem; + color: var(--text-muted); +} + +.dashboard-post-title { + font-size: 0.9375rem; + font-weight: 700; + line-height: 1.35; + margin: 0; +} + +.dashboard-post-title a { + color: var(--text-primary); +} + +.dashboard-post-title a:hover { + color: var(--accent); + text-decoration: none; +} + +.dashboard-post-content { + font-size: 0.8125rem; + color: var(--text-secondary); + line-height: 1.45; + display: -webkit-box; + -webkit-line-clamp: 3; + -webkit-box-orient: vertical; + overflow: hidden; +} + +.dashboard-post-footer { + display: flex; + align-items: center; + gap: 0.625rem; + padding-top: 0.5rem; + border-top: 1px solid var(--border); + margin-top: auto; +} + +.dashboard-post-stat { + font-size: 0.6875rem; + color: var(--text-muted); + font-weight: 600; +} + +.dashboard-post-open { + font-size: 0.75rem; + font-weight: 600; + color: var(--accent); + margin-left: auto; +} + +.dashboard-post-open:hover { + color: var(--accent-hover); +} + +.dashboard-sidebar { + display: flex; + flex-direction: column; + gap: 1rem; + position: sticky; + top: 5rem; +} + +.dashboard-sidebar-card { + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: var(--radius-xl); + padding: 1rem 1.1rem; +} + +.dashboard-sidebar-heading { + font-size: 0.8125rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--text-secondary); + margin: 0 0 0.75rem; +} + +.dashboard-sidebar-link { + display: inline-block; + margin-top: 0.625rem; + font-size: 0.75rem; + font-weight: 600; + color: var(--accent); +} + +.dashboard-sidebar-link:hover { + color: var(--accent-hover); +} + +.dashboard-contributor-list { + list-style: none; + padding: 0; + margin: 0; + display: flex; + flex-direction: column; + gap: 0.625rem; +} + +.dashboard-contributor { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.dashboard-contributor-meta { + display: flex; + flex-direction: column; + flex: 1; + min-width: 0; +} + +.dashboard-contributor-name { + font-size: 0.8125rem; + font-weight: 600; + color: var(--text-primary); +} + +.dashboard-contributor-stars { + font-size: 0.6875rem; + color: var(--text-muted); +} + +.dashboard-tag-cloud { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; +} + +.dashboard-tag { + display: flex; + align-items: center; + gap: 0.35rem; + padding: 0.3rem 0.55rem; + border-radius: var(--radius-lg); + border: 1px solid var(--border); + background: var(--bg-card); + transition: border-color 0.15s ease; +} + +.dashboard-tag:hover { + border-color: var(--accent); +} + +.dashboard-tag-count { + font-size: 0.6875rem; + font-weight: 600; + color: var(--text-muted); +} + +@media (max-width: 900px) { + .dashboard-main { + grid-template-columns: 1fr; + } + .dashboard-sidebar { + position: static; + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 0.875rem; + } + .dashboard-top { + flex-direction: column; + align-items: flex-start; + } + .dashboard-posts-grid { + grid-template-columns: 1fr; + } +} + +@media (max-width: 480px) { + .dashboard { + padding: 1rem 0.75rem 0; + } + .dashboard-top { + padding: 1rem; + } + .dashboard-actions { + flex-direction: column; + align-items: flex-start; + width: 100%; + } + .dashboard-buttons { + width: 100%; + } + .dashboard-btn { + flex: 1; + text-align: center; + } + .dashboard-stats { + width: 100%; + justify-content: space-around; + } + .dashboard-sidebar { + grid-template-columns: 1fr; + } + .dashboard-greeting { + white-space: normal; + } +} + .landing { min-height: 100vh; background: var(--bg-gradient); @@ -104,7 +512,7 @@ } .landing-hero-user { - gap: 1.5rem; + gap: 1.25rem; } .landing-welcome { @@ -119,18 +527,56 @@ .landing-welcome-eyebrow { display: block; - font-size: 1rem; - color: var(--text-secondary); - font-weight: 500; -} - -.landing-welcome-name { - font-size: 2.5rem; + font-size: 1.5rem; font-weight: 800; - line-height: 1.1; + line-height: 1.2; letter-spacing: -0.03em; color: var(--text-primary); - margin: 0; +} + +.landing-welcome-context { + display: block; + font-size: 0.875rem; + color: var(--text-secondary); + margin-top: 0.25rem; +} + +.landing-actions { + display: flex; + flex-wrap: wrap; + justify-content: center; + gap: 0.75rem; +} + +.landing-action-primary { + padding: 0.7rem 2rem; + border-radius: var(--radius-lg); + background: var(--accent); + color: var(--white); + font-size: 0.9375rem; + font-weight: 700; +} + +.landing-action-primary:hover { + background: var(--accent-hover); + transform: translateY(-2px); + box-shadow: 0 8px 24px rgba(255, 107, 53, 0.3); +} + +.landing-action-secondary { + padding: 0.7rem 1.25rem; + border-radius: var(--radius-lg); + border: 1px solid var(--border); + background: var(--bg-card); + color: var(--text-primary); + font-size: 0.9375rem; + font-weight: 600; + transition: border-color 0.15s ease, transform 0.15s ease; +} + +.landing-action-secondary:hover { + border-color: var(--accent); + transform: translateY(-2px); } .landing-stats { @@ -160,6 +606,33 @@ color: var(--text-secondary); } +.landing-stat-level { + min-width: 160px; +} + +.landing-xp-bar { + width: 100%; + max-width: 140px; + height: 6px; + background: var(--border); + border-radius: 3px; + overflow: hidden; + margin-top: 0.25rem; +} + +.landing-xp-fill { + height: 100%; + background: var(--accent-gradient, linear-gradient(90deg, var(--accent), #f0932b)); + border-radius: 3px; + transition: width 0.4s ease; +} + +.landing-xp-label { + font-size: 0.6875rem; + color: var(--text-muted); + margin-top: 0.15rem; +} + .landing-quicklinks { display: flex; flex-wrap: wrap; @@ -184,8 +657,8 @@ } @media (max-width: 480px) { - .landing-welcome-name { - font-size: 1.75rem; + .landing-welcome-eyebrow { + font-size: 1.25rem; } .landing-stats { gap: 1.5rem; diff --git a/devplacepy/static/css/messages.css b/devplacepy/static/css/messages.css index 71dd2edf..343cbfac 100644 --- a/devplacepy/static/css/messages.css +++ b/devplacepy/static/css/messages.css @@ -303,10 +303,30 @@ background: var(--bg-card); } -.messages-input-area input[type="text"] { +.messages-input-area textarea { flex: 1; min-width: 0; margin: 0; + resize: none; + overflow-y: auto; + max-height: 160px; + line-height: 1.5; + padding: 0.5rem 0.75rem; + font-family: inherit; + font-size: 0.875rem; + border-radius: var(--radius); + background: var(--bg-secondary); + border: 1px solid var(--border); + color: var(--text-primary); + outline: none; +} + +.messages-input-area textarea:focus { + border-color: var(--accent); +} + +.messages-input-area textarea::placeholder { + color: var(--text-muted); } .messages-input-area dp-upload { @@ -449,7 +469,7 @@ justify-content: center; } - .messages-input-area input[type="text"] { + .messages-input-area textarea { font-size: 16px; } } @@ -467,7 +487,7 @@ min-height: 48px; } - .messages-input-area input[type="text"] { + .messages-input-area textarea { font-size: 16px; } diff --git a/devplacepy/static/js/Application.js b/devplacepy/static/js/Application.js index de9d1d24..cc43bb8c 100644 --- a/devplacepy/static/js/Application.js +++ b/devplacepy/static/js/Application.js @@ -23,6 +23,7 @@ import { CustomizationToggle } from "./CustomizationToggle.js"; import { NotificationPrefs } from "./NotificationPrefs.js"; import { AiCorrection } from "./AiCorrection.js"; import { AiModifier } from "./AiModifier.js"; +import { InteractionsPref } from "./InteractionsPref.js"; import { TelegramPairing } from "./TelegramPairing.js"; import { AdminNotificationDefaults } from "./AdminNotificationDefaults.js"; import { UserAiUsage } from "./UserAiUsage.js"; @@ -45,10 +46,25 @@ import { ScrollMemory } from "./ScrollMemory.js"; import { GameFarm } from "./GameFarm.js"; import { Accessibility } from "./Accessibility.js"; import { OverflowTabs } from "./OverflowTabs.js"; +import { AiAutoload } from "./autoload/AiAutoload.js"; +import "./components/AiInteraction.js"; +import "./components/AiStatus.js"; +import "./components/AiActions.js"; +import "./components/AiHelp.js"; +import "./components/AiOption.js"; +import "./components/AiConfirm.js"; +import "./components/AiChoice.js"; +import "./components/AiChoiceMulti.js"; +import "./components/AiField.js"; +import "./components/AiSelect.js"; +import "./components/AiGroup.js"; +import "./components/AiMarkdownFallback.js"; class Application { constructor() { this.accessibility = new Accessibility(); + this.aiAutoload = new AiAutoload(document); + this.aiAutoload.start(); this.dialog = document.createElement("dp-dialog"); this.contextMenu = document.createElement("dp-context-menu"); this.toast = document.createElement("dp-toast"); @@ -77,6 +93,7 @@ class Application { this.notificationPrefs = new NotificationPrefs(this.pubsub); this.aiCorrection = new AiCorrection(); this.aiModifier = new AiModifier(); + this.interactionsPref = new InteractionsPref(); this.telegramPairing = new TelegramPairing(this.pubsub); this.adminNotificationDefaults = new AdminNotificationDefaults(); this.userAiUsage = new UserAiUsage(); diff --git a/devplacepy/static/js/ContentRenderer.js b/devplacepy/static/js/ContentRenderer.js index 99887563..67282015 100644 --- a/devplacepy/static/js/ContentRenderer.js +++ b/devplacepy/static/js/ContentRenderer.js @@ -20,8 +20,32 @@ export class ContentRenderer { return text.replace(/\u2014/g, "-"); } - render(text) { + render(text, authorAdmin = false) { if (!text) return ""; + + let widgets = []; + const widgetRe = /([\s\S]*?)<\/dp-widget>/gi; + if (authorAdmin && widgetRe.test(text)) { + widgetRe.lastIndex = 0; + let m; + let idx = 0; + let parts = []; + let last = 0; + while ((m = widgetRe.exec(text)) !== null) { + if (m.index > last) { + parts.push(text.substring(last, m.index)); + } + parts.push("\x00WIDGET_" + idx + "\x00"); + widgets.push(m[1]); + idx++; + last = m.index + m[0].length; + } + if (last < text.length) { + parts.push(text.substring(last)); + } + text = parts.join(""); + } + text = this.normalizeDashes(text); text = this.replaceShortcodes(text); @@ -37,6 +61,10 @@ export class ContentRenderer { } html = DOMPurify.sanitize(html); + for (let i = 0; i < widgets.length; i++) { + html = html.replace("\x00WIDGET_" + i + "\x00", widgets[i]); + } + html = this.processMedia(html); return html; @@ -67,8 +95,32 @@ export class ContentRenderer { return div.innerHTML; } - renderInline(text) { + renderInline(text, authorAdmin = false) { if (!text) return ""; + + let widgets = []; + const widgetRe = /([\s\S]*?)<\/dp-widget>/gi; + if (authorAdmin && widgetRe.test(text)) { + widgetRe.lastIndex = 0; + let m; + let idx = 0; + let parts = []; + let last = 0; + while ((m = widgetRe.exec(text)) !== null) { + if (m.index > last) { + parts.push(text.substring(last, m.index)); + } + parts.push("\x00WIDGET_" + idx + "\x00"); + widgets.push(m[1]); + idx++; + last = m.index + m[0].length; + } + if (last < text.length) { + parts.push(text.substring(last)); + } + text = parts.join(""); + } + text = this.normalizeDashes(text); text = this.replaceShortcodes(text); @@ -83,10 +135,16 @@ export class ContentRenderer { throw new Error("DOMPurify not loaded; refusing to render untrusted HTML"); } - return DOMPurify.sanitize(html, { + html = DOMPurify.sanitize(html, { ALLOWED_TAGS: ["b", "strong", "i", "em", "code", "del", "s", "mark", "sub", "sup", "span", "br"], ALLOWED_ATTR: [], }); + + for (let i = 0; i < widgets.length; i++) { + html = html.replace("\x00WIDGET_" + i + "\x00", widgets[i]); + } + + return html; } processMedia(html) { @@ -241,8 +299,9 @@ export class ContentRenderer { applyTo(element) { if (!element) return; + const authorAdmin = element.hasAttribute("data-author-admin"); const text = element.textContent || element.innerText || ""; - const rendered = this.render(text); + const rendered = this.render(text, authorAdmin); element.innerHTML = rendered; } diff --git a/devplacepy/static/js/InteractionsPref.js b/devplacepy/static/js/InteractionsPref.js new file mode 100644 index 00000000..c8bda48b --- /dev/null +++ b/devplacepy/static/js/InteractionsPref.js @@ -0,0 +1,67 @@ +// retoor + +import { Http } from "./Http.js"; + +class InteractionsPref { + constructor() { + this.root = document.querySelector("[data-interactions]"); + if (!this.root) return; + this.username = this.root.dataset.username; + this.toggle = this.root.querySelector("[data-interactions-toggle]"); + this.status = this.root.querySelector("[data-interactions-status]"); + this.save = this.root.querySelector("[data-interactions-save]"); + this.reset = this.root.querySelector("[data-interactions-reset]"); + if (this.save) this.save.addEventListener("click", () => this.submit(false)); + if (this.reset) this.reset.addEventListener("click", () => this.submit(true)); + } + + async submit(reset) { + if (!this.toggle) return; + if (this.save) this.save.disabled = true; + if (this.reset) this.reset.disabled = true; + this.setStatus("Saving...", ""); + try { + const body = reset + ? { reset: "true", enabled: "false" } + : { + enabled: this.toggle.checked ? "true" : "false", + reset: "false", + }; + const result = await Http.send( + `/profile/${this.username}/interactions`, + body + ); + if (result && result.data) { + this.toggle.checked = !!result.data.enabled; + } + this.setStatus(reset ? "Using site default" : "Saved", "success"); + this.flash( + reset + ? "Interactive widgets follow the site default" + : "Interactive widget preference saved", + "success" + ); + } catch (error) { + this.setStatus("Could not save", "error"); + this.flash("Could not save interactive widget preference", "error"); + } finally { + if (this.save) this.save.disabled = false; + if (this.reset) this.reset.disabled = false; + } + } + + setStatus(message, type) { + if (!this.status) return; + this.status.textContent = message; + this.status.dataset.state = type; + } + + flash(message, type) { + if (window.app && window.app.toast) { + window.app.toast.show(message, { type }); + } + } +} + +window.InteractionsPref = InteractionsPref; +export { InteractionsPref }; diff --git a/devplacepy/static/js/MessagesLayout.js b/devplacepy/static/js/MessagesLayout.js index 2c9bd428..4b0ba035 100644 --- a/devplacepy/static/js/MessagesLayout.js +++ b/devplacepy/static/js/MessagesLayout.js @@ -5,6 +5,8 @@ import { contentRenderer } from "./ContentRenderer.js"; const TYPING_THROTTLE_MS = 1500; const TYPING_HIDE_MS = 4000; +const AUTO_SCROLL_MARGIN_PX = 100; +const STABILIZE_MAX_FRAMES = 300; export class MessagesLayout { constructor() { @@ -27,6 +29,12 @@ export class MessagesLayout { this._lastTypingSent = 0; this._typingHideTimer = null; this._socketReady = false; + this._userAtBottom = true; + this._stabilizeFrames = 0; + this._stabilizePending = false; + + // Expose for diagnostics + window.__messagesLayout = this; this.scrollThreadToEnd(); if (this.input) { @@ -37,9 +45,83 @@ export class MessagesLayout { this.bindForm(); this.bindTyping(); this.bindViewport(); + this.bindAutoScroll(); + this._startScrollWatcher(); this.markRead(); } + _startScrollWatcher() { + if (!this.thread) return; + + const onAnyChange = () => { + if (!this._userAtBottom) return; + if (this._stabilizePending) return; + this._stabilizePending = true; + requestAnimationFrame(() => { + this._stabilizePending = false; + this._stabilizeScroll(); + }); + }; + + this._scrollWatcher = new MutationObserver(onAnyChange); + this._scrollWatcher.observe(this.thread, { + childList: true, + subtree: true, + }); + + this.thread.addEventListener("load", (e) => { + if (e.target.tagName === "IMG" && this._userAtBottom) { + onAnyChange(); + } + }, true); + + // Kick initial stabilization + onAnyChange(); + } + + _stabilizeScroll() { + if (!this.thread) { + this._stabilizeFrames = 0; + return; + } + if (!this._userAtBottom) { + this._stabilizeFrames = 0; + return; + } + if (this._stabilizeFrames >= STABILIZE_MAX_FRAMES) { + this._stabilizeFrames = 0; + return; + } + this._stabilizeFrames++; + this.thread.scrollTop = this.thread.scrollHeight; + const atBottom = this.thread.scrollHeight + - this.thread.scrollTop + - this.thread.clientHeight < 10; + if (!atBottom) { + this._stabilizePending = true; + requestAnimationFrame(() => { + this._stabilizePending = false; + this._stabilizeScroll(); + }); + } else { + this._stabilizeFrames = 0; + } + } + + bindAutoScroll() { + if (!this.thread) return; + this.thread.addEventListener("scroll", () => { + if (this._stabilizePending) return; + const atBottom = this.thread.scrollHeight + - this.thread.scrollTop + - this.thread.clientHeight < AUTO_SCROLL_MARGIN_PX; + this._userAtBottom = atBottom; + if (atBottom) { + this._stabilizeScroll(); + } + }, { passive: true }); + } + bindViewport() { const page = document.querySelector(".page-messages"); if (!page || !this.input || !this.form) return; @@ -55,7 +137,8 @@ export class MessagesLayout { if (newInset !== currentInset) { page.style.setProperty("--kb-inset", `${Math.round(newInset)}px`); - requestAnimationFrame(() => this.scrollThreadToEnd()); + this._userAtBottom = true; + this.scrollThreadToEnd(); } }; @@ -83,8 +166,21 @@ export class MessagesLayout { this._viewportIO = io; } + this.input.addEventListener("keydown", (event) => { + if (event.key === "Enter" && !event.shiftKey) { + event.preventDefault(); + this.form.requestSubmit(); + } + }); + + this.input.addEventListener("input", () => { + this.input.style.height = "auto"; + this.input.style.height = Math.min(this.input.scrollHeight, 160) + "px"; + }); + this.input.addEventListener("focus", () => { - requestAnimationFrame(() => { + this._userAtBottom = true; + const doScroll = () => { this.scrollThreadToEnd(); if (window.innerWidth < 768) { const retry = (delay) => setTimeout(() => { @@ -95,7 +191,8 @@ export class MessagesLayout { retry(350); retry(600); } - }); + }; + requestAnimationFrame(doScroll); }); this.input.addEventListener("blur", () => { @@ -240,12 +337,11 @@ export class MessagesLayout { pending.dataset.msgUid = frame.uid; const oldBody = pending.querySelector(".rendered-content"); if (oldBody && frame.content !== undefined) { - const body = document.createElement("div"); - body.className = "rendered-content"; - body.setAttribute("data-render", ""); - body.textContent = frame.content || ""; - oldBody.replaceWith(body); - this.renderBody(body); + const content = document.createElement("dp-content"); + content.setAttribute("no-copy", ""); + if (frame.sender_role === "Admin") content.setAttribute("data-author-admin", ""); + content.textContent = frame.content || ""; + oldBody.replaceWith(content); } const time = pending.querySelector(".message-time"); if (time && frame.created_at) { @@ -277,6 +373,7 @@ export class MessagesLayout { time: frame.time_ago, iso: frame.created_at, uid: frame.uid, + senderRole: frame.sender_role, attachments: frame.attachments, }); this.insertBubble(bubble); @@ -287,17 +384,17 @@ export class MessagesLayout { this.bumpConversation(frame, frame.sender_uid === this.selfUid); } - buildBubble({ content, mine, time, iso, uid, clientId, attachments, attachmentCount }) { + buildBubble({ content, mine, time, iso, uid, clientId, senderRole, attachments, attachmentCount }) { const bubble = document.createElement("div"); bubble.className = "message-bubble " + (mine ? "mine" : "theirs"); if (uid) bubble.dataset.msgUid = uid; if (clientId) bubble.dataset.clientId = clientId; - const body = document.createElement("div"); - body.className = "rendered-content"; - body.setAttribute("data-render", ""); - body.textContent = content || ""; - bubble.appendChild(body); + const el = document.createElement("dp-content"); + el.setAttribute("no-copy", ""); + if (senderRole === "Admin") el.setAttribute("data-author-admin", ""); + el.textContent = content || ""; + bubble.appendChild(el); const gallery = this.renderAttachments(attachments); if (gallery) { @@ -332,20 +429,9 @@ export class MessagesLayout { bubble.appendChild(receipt); } - this.renderBody(body); return bubble; } - renderBody(el) { - contentRenderer.ready.then(() => { - contentRenderer.applyTo(el); - el.querySelectorAll("img:not(.avatar-img)").forEach((img) => { - img.dataset.lightbox = ""; - }); - contentRenderer.highlightAll(); - }); - } - renderAttachments(attachments) { if (!attachments || !attachments.length) return null; const gallery = document.createElement("div"); @@ -434,8 +520,8 @@ export class MessagesLayout { } scrollThreadToEnd() { - if (this.thread) { - this.thread.scrollTop = this.thread.scrollHeight; - } + if (!this.thread) return; + if (!this._userAtBottom) return; + this.thread.scrollTop = this.thread.scrollHeight; } } diff --git a/devplacepy/static/js/autoload/AiAutoload.js b/devplacepy/static/js/autoload/AiAutoload.js new file mode 100644 index 00000000..0e44159b --- /dev/null +++ b/devplacepy/static/js/autoload/AiAutoload.js @@ -0,0 +1,114 @@ +// retoor + +import { assetUrl } from "../assetVersion.js"; + +const TAG_RE = /^ai-[a-z0-9-]+$/; + +const ALLOWLIST = { + "ai-interaction": "AiInteraction.js", + "ai-confirm": "AiConfirm.js", + "ai-choice": "AiChoice.js", + "ai-choice-multi": "AiChoiceMulti.js", + "ai-field": "AiField.js", + "ai-select": "AiSelect.js", + "ai-option": "AiOption.js", + "ai-group": "AiGroup.js", + "ai-help": "AiHelp.js", + "ai-actions": "AiActions.js", + "ai-status": "AiStatus.js", + "ai-markdown-fallback": "AiMarkdownFallback.js", +}; + +export class AiAutoload { + constructor(root = document) { + this.root = root; + this.loading = new Map(); + this.failed = new Set(); + this._observer = null; + } + + start() { + this.scan(this.root); + if (this._observer) return; + this._observer = new MutationObserver((records) => { + for (const record of records) { + for (const node of record.addedNodes) { + if (node.nodeType !== 1) continue; + this.scan(node); + } + } + }); + this._observer.observe(this.root === document ? document.documentElement : this.root, { + childList: true, + subtree: true, + }); + } + + stop() { + if (this._observer) { + this._observer.disconnect(); + this._observer = null; + } + } + + scan(root) { + if (!root || root.nodeType !== 1) return; + const nodes = []; + if (TAG_RE.test(root.localName || "")) nodes.push(root); + if (root.querySelectorAll) { + nodes.push(...root.querySelectorAll("*")); + } + for (const el of nodes) { + const name = el.localName || ""; + if (!TAG_RE.test(name)) continue; + this.ensure(name, el).catch(() => null); + } + } + + ensure(tag, el = null) { + if (customElements.get(tag)) return Promise.resolve(); + if (this.loading.has(tag)) { + return this.loading.get(tag); + } + const file = ALLOWLIST[tag]; + if (!file) { + this.failed.add(tag); + this._markFailed(el, tag); + return Promise.reject(new Error(`ai widget not allowlisted: ${tag}`)); + } + const url = assetUrl(`/static/js/components/${file}`); + const task = import(url) + .then(() => { + this.loading.delete(tag); + this.failed.delete(tag); + if (!customElements.get(tag)) { + return customElements.whenDefined(tag); + } + return undefined; + }) + .catch((error) => { + this.loading.delete(tag); + this.failed.add(tag); + console.error("AiAutoload failed", tag, url, error); + this._markFailed(el, tag); + window.setTimeout(() => this.failed.delete(tag), 3000); + throw error; + }); + this.loading.set(tag, task); + return task; + } + + _markFailed(el, tag) { + if (!el || !el.isConnected || !el.replaceWith) return; + if (el.getAttribute("data-ai-load-failed") === tag) return; + el.setAttribute("data-ai-load-failed", tag); + const status = document.createElement("div"); + status.className = "ai-status"; + status.dataset.state = "error"; + status.setAttribute("aria-live", "polite"); + status.textContent = `Widget failed to load (${tag})`; + el.replaceWith(status); + } +} + +export default AiAutoload; diff --git a/devplacepy/static/js/components/AiActions.js b/devplacepy/static/js/components/AiActions.js new file mode 100644 index 00000000..b3101f8a --- /dev/null +++ b/devplacepy/static/js/components/AiActions.js @@ -0,0 +1,52 @@ +// retoor + +import { Component } from "./Component.js"; +import { assetUrl } from "../assetVersion.js"; + +const CSS_ID = "ai-actions-css"; + +export class AiActions extends Component { + static get observedAttributes() { + return ["align", "sticky", "submit-label", "cancel-label", "cancelable"]; + } + + _ensureCss() { + if (document.getElementById(CSS_ID)) return; + const link = document.createElement("link"); + link.id = CSS_ID; + link.rel = "stylesheet"; + link.href = assetUrl("/static/css/components/ai-interaction.css"); + document.head.appendChild(link); + } + + connectedCallback() { + this._ensureCss(); + if (this._built) return; + this._built = true; + this.classList.add("ai-actions"); + if (this.attr("align") === "end") this.classList.add("align-end"); + this.innerHTML = ""; + if (this.hasAttribute("cancelable")) { + const cancel = document.createElement("button"); + cancel.type = "button"; + cancel.className = "ai-btn ai-btn-secondary"; + cancel.dataset.aiCancel = "1"; + cancel.textContent = this.attr("cancel-label", "Cancel"); + this.appendChild(cancel); + } + if (!this.hasAttribute("submit-hidden")) { + const submit = document.createElement("button"); + submit.type = "button"; + submit.className = "ai-btn ai-btn-primary"; + submit.dataset.aiSubmit = "1"; + submit.textContent = this.attr("submit-label", "Confirm"); + this.appendChild(submit); + } + } +} + +if (!customElements.get("ai-actions")) { + customElements.define("ai-actions", AiActions); +} + +export default AiActions; diff --git a/devplacepy/static/js/components/AiChoice.js b/devplacepy/static/js/components/AiChoice.js new file mode 100644 index 00000000..ae7e6cb6 --- /dev/null +++ b/devplacepy/static/js/components/AiChoice.js @@ -0,0 +1,137 @@ +// retoor + +import { Component } from "./Component.js"; +import { assetUrl } from "../assetVersion.js"; + +const CSS_ID = "ai-choice-css"; + +export class AiChoice extends Component { + static get observedAttributes() { + return ["name", "label", "help", "display", "required", "disabled", "value"]; + } + + constructor() { + super(); + this._value = ""; + } + + _ensureCss() { + if (document.getElementById(CSS_ID)) return; + const link = document.createElement("link"); + link.id = CSS_ID; + link.rel = "stylesheet"; + link.href = assetUrl("/static/css/components/ai-choice.css"); + document.head.appendChild(link); + } + + connectedCallback() { + this._ensureCss(); + if (this._built) return; + this._built = true; + this.classList.add("ai-choice"); + this._value = this.attr("value", ""); + const name = this.attr("name", "choice"); + const label = this.attr("label", "Choice"); + const display = this.attr("display", "auto"); + const options = [...this.querySelectorAll("ai-option")].map((el) => ({ + value: el.getAttribute("value") || "", + label: el.getAttribute("label") || el.textContent || "", + description: el.getAttribute("description") || "", + disabled: el.hasAttribute("disabled"), + selected: el.hasAttribute("selected"), + })); + this.innerHTML = ""; + const title = document.createElement("div"); + title.className = "ai-widget-label"; + title.textContent = label; + this.appendChild(title); + if (this.attr("help")) { + const help = document.createElement("div"); + help.className = "ai-widget-help"; + help.textContent = this.attr("help"); + this.appendChild(help); + } + if (display === "select" || (display === "auto" && options.length > 6)) { + const select = document.createElement("select"); + select.className = "ai-choice-select"; + select.name = name; + const blank = document.createElement("option"); + blank.value = ""; + blank.textContent = "Select…"; + select.appendChild(blank); + for (const option of options) { + const el = document.createElement("option"); + el.value = option.value; + el.textContent = option.label; + el.disabled = option.disabled; + if (option.selected || option.value === this._value) el.selected = true; + select.appendChild(el); + } + select.addEventListener("change", () => { + this._value = select.value; + this._emit(name); + }); + this.appendChild(select); + this._value = select.value; + return; + } + const group = document.createElement("div"); + group.className = "ai-choice-options"; + group.setAttribute("role", "radiogroup"); + group.setAttribute("aria-label", label); + for (const option of options) { + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "ai-choice-option"; + btn.disabled = option.disabled; + btn.dataset.value = option.value; + btn.innerHTML = ``; + btn.querySelector(".ai-choice-option-label").textContent = option.label; + if (option.description) { + const desc = document.createElement("span"); + desc.className = "ai-choice-option-desc"; + desc.textContent = option.description; + btn.appendChild(desc); + } + if (option.selected || option.value === this._value) { + btn.classList.add("selected"); + this._value = option.value; + } + btn.addEventListener("click", () => { + this._value = option.value; + group.querySelectorAll(".ai-choice-option").forEach((el) => el.classList.remove("selected")); + btn.classList.add("selected"); + this._emit(name); + }); + group.appendChild(btn); + } + this.appendChild(group); + } + + _emit(name) { + this.dispatchEvent( + new CustomEvent("ai:change", { + bubbles: true, + composed: true, + detail: { name, value: this._value }, + }) + ); + } + + getValue() { + return this._value; + } + + validate() { + if (this.boolAttr("required") !== false && !this._value) { + return { valid: false, message: `${this.attr("label", "Choice")} is required` }; + } + return { valid: true }; + } +} + +if (!customElements.get("ai-choice")) { + customElements.define("ai-choice", AiChoice); +} + +export default AiChoice; diff --git a/devplacepy/static/js/components/AiChoiceMulti.js b/devplacepy/static/js/components/AiChoiceMulti.js new file mode 100644 index 00000000..49c7cf4d --- /dev/null +++ b/devplacepy/static/js/components/AiChoiceMulti.js @@ -0,0 +1,111 @@ +// retoor + +import { Component } from "./Component.js"; +import { assetUrl } from "../assetVersion.js"; + +const CSS_ID = "ai-choice-multi-css"; + +export class AiChoiceMulti extends Component { + static get observedAttributes() { + return ["name", "label", "help", "required", "disabled", "min-selected", "max-selected"]; + } + + constructor() { + super(); + this._values = new Set(); + } + + _ensureCss() { + if (document.getElementById(CSS_ID)) return; + const link = document.createElement("link"); + link.id = CSS_ID; + link.rel = "stylesheet"; + link.href = assetUrl("/static/css/components/ai-choice.css"); + document.head.appendChild(link); + } + + connectedCallback() { + this._ensureCss(); + if (this._built) return; + this._built = true; + this.classList.add("ai-choice-multi"); + const name = this.attr("name", "choices"); + const label = this.attr("label", "Choices"); + const options = [...this.querySelectorAll("ai-option")].map((el) => ({ + value: el.getAttribute("value") || "", + label: el.getAttribute("label") || el.textContent || "", + description: el.getAttribute("description") || "", + disabled: el.hasAttribute("disabled"), + selected: el.hasAttribute("selected"), + })); + this.innerHTML = ""; + const title = document.createElement("div"); + title.className = "ai-widget-label"; + title.textContent = label; + this.appendChild(title); + if (this.attr("help")) { + const help = document.createElement("div"); + help.className = "ai-widget-help"; + help.textContent = this.attr("help"); + this.appendChild(help); + } + const group = document.createElement("div"); + group.className = "ai-choice-options multi"; + for (const option of options) { + const labelEl = document.createElement("label"); + labelEl.className = "ai-choice-check"; + const input = document.createElement("input"); + input.type = "checkbox"; + input.value = option.value; + input.disabled = option.disabled; + if (option.selected) { + input.checked = true; + this._values.add(option.value); + } + input.addEventListener("change", () => { + if (input.checked) this._values.add(option.value); + else this._values.delete(option.value); + this.dispatchEvent( + new CustomEvent("ai:change", { + bubbles: true, + composed: true, + detail: { name, value: this.getValue() }, + }) + ); + }); + const span = document.createElement("span"); + span.textContent = option.label; + labelEl.append(input, span); + if (option.description) { + const desc = document.createElement("small"); + desc.textContent = option.description; + labelEl.appendChild(desc); + } + group.appendChild(labelEl); + } + this.appendChild(group); + } + + getValue() { + return [...this._values]; + } + + validate() { + const min = this.intAttr("min-selected", this.boolAttr("required") ? 1 : 0); + const max = this.intAttr("max-selected", 0); + const count = this._values.size; + if (count < min) { + return { valid: false, message: `${this.attr("label", "Choices")}: select at least ${min}` }; + } + if (max > 0 && count > max) { + return { valid: false, message: `${this.attr("label", "Choices")}: select at most ${max}` }; + } + return { valid: true }; + } +} + +if (!customElements.get("ai-choice-multi")) { + customElements.define("ai-choice-multi", AiChoiceMulti); +} + +export default AiChoiceMulti; diff --git a/devplacepy/static/js/components/AiConfirm.js b/devplacepy/static/js/components/AiConfirm.js new file mode 100644 index 00000000..f0e121a7 --- /dev/null +++ b/devplacepy/static/js/components/AiConfirm.js @@ -0,0 +1,103 @@ +// retoor + +import { Component } from "./Component.js"; +import { assetUrl } from "../assetVersion.js"; + +const CSS_ID = "ai-confirm-css"; + +export class AiConfirm extends Component { + static get observedAttributes() { + return ["name", "label", "help", "confirm-label", "reject-label", "default", "required", "disabled"]; + } + + constructor() { + super(); + this._value = null; + } + + _ensureCss() { + if (document.getElementById(CSS_ID)) return; + const link = document.createElement("link"); + link.id = CSS_ID; + link.rel = "stylesheet"; + link.href = assetUrl("/static/css/components/ai-confirm.css"); + document.head.appendChild(link); + } + + connectedCallback() { + this._ensureCss(); + if (this._built) return; + this._built = true; + this.classList.add("ai-confirm"); + const name = this.attr("name", "confirm"); + const label = this.attr("label", "Confirm"); + const yes = this.attr("confirm-label", "Yes"); + const no = this.attr("reject-label", "No"); + const def = this.getAttribute("default"); + this._value = def === "true" ? true : def === "false" ? false : null; + this.innerHTML = ""; + const title = document.createElement("div"); + title.className = "ai-widget-label"; + title.textContent = label; + this.appendChild(title); + if (this.attr("help")) { + const help = document.createElement("div"); + help.className = "ai-widget-help"; + help.textContent = this.attr("help"); + this.appendChild(help); + } + const group = document.createElement("div"); + group.className = "ai-confirm-options"; + group.setAttribute("role", "radiogroup"); + group.setAttribute("aria-label", label); + for (const [val, text] of [ + [true, yes], + [false, no], + ]) { + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "ai-confirm-option"; + btn.textContent = text; + btn.dataset.value = String(val); + if (this._value === val) btn.classList.add("selected"); + btn.addEventListener("click", () => { + this._value = val; + group.querySelectorAll(".ai-confirm-option").forEach((el) => el.classList.remove("selected")); + btn.classList.add("selected"); + this.dispatchEvent( + new CustomEvent("ai:change", { + bubbles: true, + composed: true, + detail: { name, value: val }, + }) + ); + const shell = this.closest("ai-interaction"); + if (shell && typeof shell.submit === "function") { + const fields = shell.querySelectorAll( + "ai-confirm, ai-choice, ai-choice-multi, ai-field, ai-select" + ); + if (fields.length === 1) shell.submit(); + } + }); + group.appendChild(btn); + } + this.appendChild(group); + } + + getValue() { + return this._value; + } + + validate() { + if (this.boolAttr("required") && this._value === null) { + return { valid: false, message: `${this.attr("label", "Confirm")} is required` }; + } + return { valid: true }; + } +} + +if (!customElements.get("ai-confirm")) { + customElements.define("ai-confirm", AiConfirm); +} + +export default AiConfirm; diff --git a/devplacepy/static/js/components/AiField.js b/devplacepy/static/js/components/AiField.js new file mode 100644 index 00000000..fc7417b0 --- /dev/null +++ b/devplacepy/static/js/components/AiField.js @@ -0,0 +1,207 @@ +// retoor + +import { Component } from "./Component.js"; +import { assetUrl } from "../assetVersion.js"; + +const CSS_ID = "ai-field-css"; +const EUROPEAN_DATE = /^(\d{1,2})[./-](\d{1,2})[./-](\d{4})$/; +const ISO_DATE = /^(\d{4})-(\d{2})-(\d{2})$/; + +function toEuropeanDate(value) { + const text = String(value || "").trim(); + if (!text) return ""; + const iso = text.match(ISO_DATE); + if (iso) return `${iso[3]}/${iso[2]}/${iso[1]}`; + const eu = text.match(EUROPEAN_DATE); + if (eu) { + const day = eu[1].padStart(2, "0"); + const month = eu[2].padStart(2, "0"); + return `${day}/${month}/${eu[3]}`; + } + return text; +} + +function parseEuropeanDate(value) { + const text = String(value || "").trim(); + if (!text) return null; + let day; + let month; + let year; + const iso = text.match(ISO_DATE); + if (iso) { + year = Number(iso[1]); + month = Number(iso[2]); + day = Number(iso[3]); + } else { + const eu = text.match(EUROPEAN_DATE); + if (!eu) return null; + day = Number(eu[1]); + month = Number(eu[2]); + year = Number(eu[3]); + } + if (month < 1 || month > 12 || day < 1 || day > 31) return null; + const date = new Date(Date.UTC(year, month - 1, day)); + if ( + date.getUTCFullYear() !== year || + date.getUTCMonth() !== month - 1 || + date.getUTCDate() !== day + ) { + return null; + } + const dd = String(day).padStart(2, "0"); + const mm = String(month).padStart(2, "0"); + return { + european: `${dd}/${mm}/${year}`, + iso: `${year}-${mm}-${dd}`, + }; +} + +export class AiField extends Component { + static get observedAttributes() { + return [ + "name", + "type", + "label", + "help", + "placeholder", + "min", + "max", + "step", + "maxlength", + "pattern", + "required", + "disabled", + "value", + ]; + } + + _ensureCss() { + if (document.getElementById(CSS_ID)) return; + const link = document.createElement("link"); + link.id = CSS_ID; + link.rel = "stylesheet"; + link.href = assetUrl("/static/css/components/ai-field.css"); + document.head.appendChild(link); + } + + connectedCallback() { + this._ensureCss(); + if (this._built) return; + this._built = true; + this.classList.add("ai-field"); + const type = this.attr("type", "text"); + this._fieldType = type; + const name = this.attr("name", "field"); + const label = this.attr("label", name); + const wrap = document.createElement("label"); + wrap.className = "ai-field-wrap"; + const title = document.createElement("span"); + title.className = "ai-widget-label"; + title.textContent = label; + wrap.appendChild(title); + const helpText = + this.attr("help") || + (type === "date" ? "Format: DD/MM/YYYY" : ""); + if (helpText) { + const help = document.createElement("span"); + help.className = "ai-widget-help"; + help.textContent = helpText; + wrap.appendChild(help); + } + const input = document.createElement("input"); + input.className = "ai-field-input"; + input.name = name; + if (type === "number") { + input.type = "number"; + } else { + input.type = "text"; + } + if (type === "date") { + input.inputMode = "numeric"; + input.autocomplete = "off"; + input.placeholder = this.attr("placeholder", "DD/MM/YYYY"); + input.maxLength = 10; + input.pattern = "\\d{1,2}[/.-]\\d{1,2}[/.-]\\d{4}"; + input.title = "DD/MM/YYYY"; + input.value = toEuropeanDate(this.attr("value", "")); + } else { + input.placeholder = this.attr("placeholder", ""); + input.value = this.attr("value", ""); + } + if (this.hasAttribute("min") && type === "number") input.min = this.attr("min"); + if (this.hasAttribute("max") && type === "number") input.max = this.attr("max"); + if (this.hasAttribute("step") && type === "number") input.step = this.attr("step"); + if (this.hasAttribute("maxlength") && type !== "date") { + input.maxLength = this.intAttr("maxlength", 0) || 524288; + } + if (this.attr("pattern") && type !== "date") input.pattern = this.attr("pattern"); + if (this.boolAttr("required")) input.required = true; + if (this.boolAttr("disabled")) input.disabled = true; + input.addEventListener("input", () => { + if (this._fieldType === "date") { + const cleaned = input.value.replace(/[^\d/.\-]/g, "").slice(0, 10); + if (cleaned !== input.value) input.value = cleaned; + } + this.dispatchEvent( + new CustomEvent("ai:change", { + bubbles: true, + composed: true, + detail: { name, value: this.getValue() }, + }) + ); + }); + input.addEventListener("blur", () => { + if (this._fieldType !== "date") return; + const parsed = parseEuropeanDate(input.value); + if (parsed) input.value = parsed.european; + }); + wrap.appendChild(input); + this.innerHTML = ""; + this.appendChild(wrap); + this._input = input; + } + + getValue() { + if (!this._input) return this.attr("value", ""); + if (this._fieldType === "number" || this._input.type === "number") { + const n = this._input.valueAsNumber; + return Number.isNaN(n) ? null : n; + } + if (this._fieldType === "date") { + const parsed = parseEuropeanDate(this._input.value); + return parsed ? parsed.european : this._input.value.trim(); + } + return this._input.value; + } + + validate() { + if (!this._input) return { valid: true }; + if (this._fieldType === "date") { + const raw = this._input.value.trim(); + if (!raw) { + if (this.boolAttr("required")) { + return { valid: false, message: "Enter a date as DD/MM/YYYY" }; + } + return { valid: true }; + } + if (!parseEuropeanDate(raw)) { + return { valid: false, message: "Date must be DD/MM/YYYY" }; + } + return { valid: true }; + } + if (!this._input.checkValidity()) { + return { valid: false, message: this._input.validationMessage || "Invalid value" }; + } + return { valid: true }; + } + + focus() { + if (this._input) this._input.focus(); + } +} + +if (!customElements.get("ai-field")) { + customElements.define("ai-field", AiField); +} + +export default AiField; diff --git a/devplacepy/static/js/components/AiGroup.js b/devplacepy/static/js/components/AiGroup.js new file mode 100644 index 00000000..c601f23b --- /dev/null +++ b/devplacepy/static/js/components/AiGroup.js @@ -0,0 +1,38 @@ +// retoor + +import { Component } from "./Component.js"; +import { assetUrl } from "../assetVersion.js"; + +const CSS_ID = "ai-group-css"; + +export class AiGroup extends Component { + static get observedAttributes() { + return ["label", "collapsed", "compact"]; + } + + _ensureCss() { + if (document.getElementById(CSS_ID)) return; + const link = document.createElement("link"); + link.id = CSS_ID; + link.rel = "stylesheet"; + link.href = assetUrl("/static/css/components/ai-interaction.css"); + document.head.appendChild(link); + } + + connectedCallback() { + this._ensureCss(); + if (this._built) return; + this._built = true; + this.classList.add("ai-group"); + const legend = document.createElement("div"); + legend.className = "ai-group-label"; + legend.textContent = this.attr("label", "Group"); + this.insertBefore(legend, this.firstChild); + } +} + +if (!customElements.get("ai-group")) { + customElements.define("ai-group", AiGroup); +} + +export default AiGroup; diff --git a/devplacepy/static/js/components/AiHelp.js b/devplacepy/static/js/components/AiHelp.js new file mode 100644 index 00000000..00b53303 --- /dev/null +++ b/devplacepy/static/js/components/AiHelp.js @@ -0,0 +1,22 @@ +// retoor + +import { Component } from "./Component.js"; + +export class AiHelp extends Component { + static get observedAttributes() { + return ["text"]; + } + + connectedCallback() { + this.classList.add("ai-help"); + if (!this.textContent.trim()) { + this.textContent = this.attr("text", ""); + } + } +} + +if (!customElements.get("ai-help")) { + customElements.define("ai-help", AiHelp); +} + +export default AiHelp; diff --git a/devplacepy/static/js/components/AiInteraction.js b/devplacepy/static/js/components/AiInteraction.js new file mode 100644 index 00000000..52e3bd49 --- /dev/null +++ b/devplacepy/static/js/components/AiInteraction.js @@ -0,0 +1,217 @@ +// retoor + +import { Component } from "./Component.js"; +import { assetUrl } from "../assetVersion.js"; + +const CSS_ID = "ai-interaction-css"; + +export class AiInteraction extends Component { + static get observedAttributes() { + return [ + "id", + "channel", + "state", + "mode", + "title", + "cancelable", + "timeout-sec", + "submit-label", + "cancel-label", + ]; + } + + constructor() { + super(); + this._built = false; + this._timer = null; + this._closed = false; + } + + _ensureCss() { + if (document.getElementById(CSS_ID)) return; + const link = document.createElement("link"); + link.id = CSS_ID; + link.rel = "stylesheet"; + link.href = assetUrl("/static/css/components/ai-interaction.css"); + document.head.appendChild(link); + } + + connectedCallback() { + this._ensureCss(); + if (!this._built) { + this._built = true; + this._build(); + } + this._bind(); + this._startTimeout(); + } + + disconnectedCallback() { + if (this._timer) { + clearTimeout(this._timer); + this._timer = null; + } + } + + _build() { + this.classList.add("ai-interaction"); + if (!this.querySelector(".ai-interaction-head")) { + const head = document.createElement("div"); + head.className = "ai-interaction-head"; + const title = document.createElement("h3"); + title.className = "ai-interaction-title"; + title.textContent = this.attr("title", "Question"); + head.appendChild(title); + const status = document.createElement("ai-status"); + status.setAttribute("state", this.attr("state", "open")); + status.className = "ai-interaction-status"; + head.appendChild(status); + this.insertBefore(head, this.firstChild); + } + if (!this.querySelector("ai-actions")) { + const onlyConfirm = + this.querySelectorAll("ai-confirm").length === 1 && + this.querySelectorAll("ai-choice, ai-choice-multi, ai-field, ai-select").length === 0; + const actions = document.createElement("ai-actions"); + actions.setAttribute("align", "end"); + actions.setAttribute("submit-label", this.attr("submit-label", "Confirm")); + actions.setAttribute("cancel-label", this.attr("cancel-label", "Cancel")); + if (this.boolAttr("cancelable") || this.getAttribute("cancelable") === null) { + actions.setAttribute("cancelable", ""); + } + if (onlyConfirm) { + actions.setAttribute("submit-hidden", ""); + } + this.appendChild(actions); + } + } + + _bind() { + if (this._bound) return; + this._bound = true; + this.addEventListener("click", (event) => { + const submit = event.target.closest("[data-ai-submit]"); + const cancel = event.target.closest("[data-ai-cancel]"); + if (submit) { + event.preventDefault(); + this.submit(); + } else if (cancel) { + event.preventDefault(); + this.cancel("user"); + } + }); + this.addEventListener("keydown", (event) => { + if (event.key === "Escape" && this.boolAttr("cancelable") !== false) { + event.preventDefault(); + this.cancel("escape"); + } + }); + } + + _startTimeout() { + const sec = this.intAttr("timeout-sec", 0); + if (!sec || sec <= 0) return; + this._timer = setTimeout(() => this.timeout(), sec * 1000); + } + + collectValues() { + const values = {}; + const fields = this.querySelectorAll( + "ai-confirm, ai-choice, ai-choice-multi, ai-field, ai-select" + ); + for (const field of fields) { + if (typeof field.getValue !== "function") continue; + const name = field.getAttribute("name"); + if (!name) continue; + values[name] = field.getValue(); + } + return values; + } + + validate() { + const fields = this.querySelectorAll( + "ai-confirm, ai-choice, ai-choice-multi, ai-field, ai-select" + ); + for (const field of fields) { + if (typeof field.validate === "function") { + const result = field.validate(); + if (result && result.valid === false) { + if (typeof field.focus === "function") field.focus(); + return result; + } + } + } + return { valid: true }; + } + + submit() { + if (this._closed) return; + const validity = this.validate(); + if (!validity.valid) { + this._setStatus("error", validity.message || "Invalid input"); + return; + } + const values = this.collectValues(); + this._close("submitted"); + this.dispatchEvent( + new CustomEvent("ai:submit", { + bubbles: true, + composed: true, + detail: { values, interaction_id: this.id }, + }) + ); + } + + cancel(reason = "user") { + if (this._closed) return; + this._close("cancelled"); + this.dispatchEvent( + new CustomEvent("ai:cancel", { + bubbles: true, + composed: true, + detail: { reason, interaction_id: this.id }, + }) + ); + } + + timeout() { + if (this._closed) return; + this._close("timeout"); + this.dispatchEvent( + new CustomEvent("ai:timeout", { + bubbles: true, + composed: true, + detail: { + timeout_sec: this.intAttr("timeout-sec", 0), + interaction_id: this.id, + }, + }) + ); + } + + _close(state) { + this._closed = true; + this.setAttribute("state", state); + this._setStatus(state); + this.querySelectorAll("button, input, select, textarea").forEach((el) => { + el.disabled = true; + }); + if (this._timer) { + clearTimeout(this._timer); + this._timer = null; + } + } + + _setStatus(state, message = "") { + const status = this.querySelector("ai-status"); + if (!status) return; + status.setAttribute("state", state); + if (message) status.setAttribute("message", message); + } +} + +if (!customElements.get("ai-interaction")) { + customElements.define("ai-interaction", AiInteraction); +} + +export default AiInteraction; diff --git a/devplacepy/static/js/components/AiMarkdownFallback.js b/devplacepy/static/js/components/AiMarkdownFallback.js new file mode 100644 index 00000000..e1829cba --- /dev/null +++ b/devplacepy/static/js/components/AiMarkdownFallback.js @@ -0,0 +1,23 @@ +// retoor + +import { Component } from "./Component.js"; + +export class AiMarkdownFallback extends Component { + static get observedAttributes() { + return ["source"]; + } + + connectedCallback() { + this.classList.add("ai-markdown-fallback"); + const pre = document.createElement("pre"); + pre.textContent = this.attr("source", this.textContent || ""); + this.innerHTML = ""; + this.appendChild(pre); + } +} + +if (!customElements.get("ai-markdown-fallback")) { + customElements.define("ai-markdown-fallback", AiMarkdownFallback); +} + +export default AiMarkdownFallback; diff --git a/devplacepy/static/js/components/AiOption.js b/devplacepy/static/js/components/AiOption.js new file mode 100644 index 00000000..a4e6a58e --- /dev/null +++ b/devplacepy/static/js/components/AiOption.js @@ -0,0 +1,15 @@ +// retoor + +import { Component } from "./Component.js"; + +export class AiOption extends Component { + connectedCallback() { + this.hidden = true; + } +} + +if (!customElements.get("ai-option")) { + customElements.define("ai-option", AiOption); +} + +export default AiOption; diff --git a/devplacepy/static/js/components/AiSelect.js b/devplacepy/static/js/components/AiSelect.js new file mode 100644 index 00000000..81883a2f --- /dev/null +++ b/devplacepy/static/js/components/AiSelect.js @@ -0,0 +1,16 @@ +// retoor + +import { AiChoice } from "./AiChoice.js"; + +export class AiSelect extends AiChoice { + connectedCallback() { + if (!this.hasAttribute("display")) this.setAttribute("display", "select"); + super.connectedCallback(); + } +} + +if (!customElements.get("ai-select")) { + customElements.define("ai-select", AiSelect); +} + +export default AiSelect; diff --git a/devplacepy/static/js/components/AiStatus.js b/devplacepy/static/js/components/AiStatus.js new file mode 100644 index 00000000..0be8fec1 --- /dev/null +++ b/devplacepy/static/js/components/AiStatus.js @@ -0,0 +1,32 @@ +// retoor + +import { Component } from "./Component.js"; + +export class AiStatus extends Component { + static get observedAttributes() { + return ["state", "message"]; + } + + connectedCallback() { + this.classList.add("ai-status"); + this.setAttribute("aria-live", "polite"); + this._render(); + } + + attributeChangedCallback() { + this._render(); + } + + _render() { + const state = this.attr("state", "open"); + const message = this.attr("message", state); + this.dataset.state = state; + this.textContent = message; + } +} + +if (!customElements.get("ai-status")) { + customElements.define("ai-status", AiStatus); +} + +export default AiStatus; diff --git a/devplacepy/static/js/components/AppTitle.js b/devplacepy/static/js/components/AppTitle.js index d0fff3ce..d80c666e 100644 --- a/devplacepy/static/js/components/AppTitle.js +++ b/devplacepy/static/js/components/AppTitle.js @@ -19,7 +19,8 @@ export class AppTitle extends Component { this._rendered = true; this.classList.add("rendered-title"); const source = (this.textContent || "").trim(); - this.innerHTML = contentRenderer.renderInline(source); + const authorAdmin = this.hasAttribute("data-author-admin"); + this.innerHTML = contentRenderer.renderInline(source, authorAdmin); } } diff --git a/devplacepy/static/js/devii/devii-terminal.js b/devplacepy/static/js/devii/devii-terminal.js index 0cdd3133..58b43e47 100644 --- a/devplacepy/static/js/devii/devii-terminal.js +++ b/devplacepy/static/js/devii/devii-terminal.js @@ -602,6 +602,10 @@ export default class DeviiTerminalElement extends FloatingWindow { await this._avatarRequest(message); } else if (message.type === "client") { await this._clientRequest(message); + } else if (message.type === "interaction") { + await this._interactionRequest(message); + } else if (message.type === "interaction_closed") { + this._closeOpenInteraction(message.result || message); } else if (message.type === "auth") { this._onAuth(message); } @@ -675,6 +679,275 @@ export default class DeviiTerminalElement extends FloatingWindow { this.socket.send({ type: "client_result", id: message.id, result }); } + async _interactionRequest(message) { + const args = message.args || {}; + const interactionId = args.interaction_id || message.interaction_id || ""; + try { + const result = await this._mountInteraction(args, interactionId); + this.socket.send({ + type: "interaction_result", + id: message.id, + interaction_id: interactionId, + result, + }); + } catch (error) { + this.socket.send({ + type: "interaction_result", + id: message.id, + interaction_id: interactionId, + result: { + status: "error", + interaction_id: interactionId, + values: {}, + error: String(error && error.message ? error.message : error), + }, + }); + } + } + + async _mountInteraction(args, interactionId) { + const root = document.createElement("ai-interaction"); + root.id = interactionId || `ix-${Date.now()}`; + root.setAttribute("channel", args.channel || "site-chat"); + root.setAttribute("state", "open"); + root.setAttribute("title", args.title || "Question"); + root.setAttribute("submit-label", args.submit_label || "Confirm"); + root.setAttribute("cancel-label", args.cancel_label || "Cancel"); + if (args.cancelable !== false) root.setAttribute("cancelable", ""); + if (args.timeout_sec) root.setAttribute("timeout-sec", String(args.timeout_sec)); + + if (args.description) { + const help = document.createElement("ai-help"); + help.setAttribute("text", args.description); + root.appendChild(help); + } + + for (const widget of args.widgets || []) { + const el = this._widgetElement(widget); + if (el) root.appendChild(el); + } + + const line = document.createElement("div"); + line.className = "devii-line devii-agent devii-interaction-line"; + line.appendChild(root); + this._append(line); + + const tags = new Set( + Array.from(root.querySelectorAll("*")) + .map((el) => el.localName) + .filter((name) => name && name.startsWith("ai-")) + ); + tags.add("ai-interaction"); + if (window.app && window.app.aiAutoload) { + window.app.aiAutoload.scan(root); + await Promise.all( + Array.from(tags).map((tag) => + window.app.aiAutoload.ensure(tag).catch(() => null) + ) + ); + } + await Promise.all( + Array.from(tags).map((tag) => { + if (customElements.get(tag)) return Promise.resolve(); + return Promise.race([ + customElements.whenDefined(tag), + new Promise((resolve) => window.setTimeout(resolve, 2500)), + ]).catch(() => null); + }) + ); + const missing = Array.from(tags).filter((tag) => !customElements.get(tag)); + if (missing.length) { + console.error("Devii interaction widgets not defined:", missing.join(", ")); + } + await new Promise((resolve) => { + window.requestAnimationFrame(() => window.requestAnimationFrame(resolve)); + }); + this._revealInteraction(root); + + return new Promise((resolve) => { + let settled = false; + const finish = (status, values, error = null, meta = null) => { + if (settled) return; + settled = true; + if (this._interactionReveal) { + this._interactionReveal.disconnect(); + this._interactionReveal = null; + } + if (this._openInteraction && this._openInteraction.id === interactionId) { + this._openInteraction = null; + } + if (root && typeof root._close === "function") { + root._close(status === "submitted" ? "submitted" : status); + } else if (root) { + root.setAttribute("state", status); + } + resolve({ + status, + interaction_id: interactionId, + values: values || {}, + error, + meta: meta || { adapter: "site-chat", degraded: false }, + }); + }; + + this._openInteraction = { + id: interactionId, + requestId: `ix:${interactionId}`, + root, + finish, + args, + }; + + root.addEventListener("ai:submit", (event) => { + finish("submitted", (event.detail && event.detail.values) || {}); + }); + root.addEventListener("ai:cancel", () => { + finish("cancelled", {}); + }); + root.addEventListener("ai:timeout", () => { + finish("timeout", {}); + }); + }); + } + + _closeOpenInteraction(result) { + const open = this._openInteraction; + if (!open || typeof open.finish !== "function") return; + const payload = result && typeof result === "object" ? result : {}; + open.finish( + payload.status || "submitted", + payload.values || {}, + payload.error || null, + payload.meta || { adapter: "site-chat", via: "text", degraded: true } + ); + } + + _revealInteraction(root) { + if (!root || !this.output) return; + const scrollToEnd = () => { + const actions = root.querySelector("ai-actions"); + const target = actions || root; + try { + target.scrollIntoView({ block: "end", inline: "nearest", behavior: "instant" }); + } catch (error) { + this.output.scrollTop = this.output.scrollHeight; + } + const rect = root.getBoundingClientRect(); + const box = this.output.getBoundingClientRect(); + if (rect.bottom > box.bottom - 8) { + this.output.scrollTop += rect.bottom - box.bottom + 16; + } + if (rect.top < box.top + 8 && root.offsetHeight <= this.output.clientHeight - 24) { + this.output.scrollTop -= box.top + 8 - rect.top; + } + }; + scrollToEnd(); + if (typeof ResizeObserver === "undefined") return; + if (this._interactionReveal) this._interactionReveal.disconnect(); + let frames = 0; + this._interactionReveal = new ResizeObserver(() => { + frames += 1; + scrollToEnd(); + if (frames >= 6 && this._interactionReveal) { + this._interactionReveal.disconnect(); + this._interactionReveal = null; + } + }); + this._interactionReveal.observe(root); + } + + _widgetElement(widget) { + if (!widget || typeof widget !== "object") return null; + const type = widget.type; + if (type === "//") { + const help = document.createElement("ai-help"); + help.setAttribute("text", widget.text || widget.label || ""); + return help; + } + if (type === "group") { + const group = document.createElement("ai-group"); + group.setAttribute("label", widget.label || "Group"); + for (const child of widget.widgets || []) { + const el = this._widgetElement(child); + if (el) group.appendChild(el); + } + return group; + } + if (type === "confirm") { + const el = document.createElement("ai-confirm"); + el.setAttribute("name", widget.name || "confirm"); + el.setAttribute("label", widget.label || "Confirm"); + if (widget.help) el.setAttribute("help", widget.help); + if (widget.default === true || widget.default === false) { + el.setAttribute("default", String(widget.default)); + } + if (widget.required !== false) el.setAttribute("required", ""); + return el; + } + if (type === "choice" || type === "select") { + const el = document.createElement(type === "select" ? "ai-select" : "ai-choice"); + el.setAttribute("name", widget.name || "choice"); + el.setAttribute("label", widget.label || "Choice"); + if (widget.help) el.setAttribute("help", widget.help); + if (widget.display) el.setAttribute("display", widget.display); + if (widget.default != null) el.setAttribute("value", String(widget.default)); + if (widget.required !== false) el.setAttribute("required", ""); + for (const option of widget.options || []) { + const opt = document.createElement("ai-option"); + opt.setAttribute("value", option.value || ""); + opt.setAttribute("label", option.label || option.value || ""); + if (option.description) opt.setAttribute("description", option.description); + if (option.disabled) opt.setAttribute("disabled", ""); + if (widget.default != null && String(widget.default) === String(option.value)) { + opt.setAttribute("selected", ""); + } + el.appendChild(opt); + } + return el; + } + if (type === "choice_multi") { + const el = document.createElement("ai-choice-multi"); + el.setAttribute("name", widget.name || "choices"); + el.setAttribute("label", widget.label || "Choices"); + if (widget.help) el.setAttribute("help", widget.help); + if (widget.required !== false) el.setAttribute("required", ""); + const defaults = Array.isArray(widget.default) ? widget.default.map(String) : []; + for (const option of widget.options || []) { + const opt = document.createElement("ai-option"); + opt.setAttribute("value", option.value || ""); + opt.setAttribute("label", option.label || option.value || ""); + if (option.description) opt.setAttribute("description", option.description); + if (defaults.includes(String(option.value))) opt.setAttribute("selected", ""); + el.appendChild(opt); + } + return el; + } + if (type === "text" || type === "number" || type === "date") { + const el = document.createElement("ai-field"); + el.setAttribute("name", widget.name || "field"); + el.setAttribute("type", type); + el.setAttribute("label", widget.label || widget.name || "Field"); + if (type === "date") { + el.setAttribute("help", widget.help || "Format: DD/MM/YYYY"); + el.setAttribute("placeholder", widget.placeholder || "DD/MM/YYYY"); + } else if (widget.help) { + el.setAttribute("help", widget.help); + } + if (widget.placeholder && type !== "date") { + el.setAttribute("placeholder", widget.placeholder); + } + if (widget.max_length != null) el.setAttribute("maxlength", String(widget.max_length)); + if (widget.min != null) el.setAttribute("min", String(widget.min)); + if (widget.max != null) el.setAttribute("max", String(widget.max)); + if (widget.step != null) el.setAttribute("step", String(widget.step)); + if (widget.pattern) el.setAttribute("pattern", widget.pattern); + if (widget.default != null) el.setAttribute("value", String(widget.default)); + if (widget.required !== false) el.setAttribute("required", ""); + return el; + } + return null; + } + async _avatarRequest(message) { const avatar = this._avatar(); let result = { error: "no avatar attached" }; @@ -800,19 +1073,43 @@ export default class DeviiTerminalElement extends FloatingWindow { _renderHistory(messages) { this.output.innerHTML = ""; (messages || []).forEach((message) => { + const content = this._normalizeText(message && message.content); + if (!content) return; if (message.role === "user") { - this._userLine(message.content); - } else if (message.role === "assistant" && message.content) { - this._agent(message.content); + this._userLine(content); + } else if (message.role === "assistant") { + this._agent(content); } }); } + _normalizeText(text) { + let value = String(text == null ? "" : text); + while (value.indexOf("\\\\n") !== -1) { + value = value.split("\\\\n").join("\\n"); + } + while (value.indexOf("\\\\t") !== -1) { + value = value.split("\\\\t").join("\\t"); + } + const escapedN = (value.match(/\\n/g) || []).length; + const realN = (value.match(/\n/g) || []).length; + if (escapedN > realN) { + value = value.split("\\n").join("\n"); + } + const escapedT = (value.match(/\\t/g) || []).length; + const realT = (value.match(/\t/g) || []).length; + if (escapedT > realT) { + value = value.split("\\t").join("\t"); + } + return value; + } + _agent(text) { + const source = this._normalizeText(text); const block = document.createElement("div"); block.className = "devii-line devii-agent"; - block.innerHTML = this.markdown.render(text); - this._attachCopy(block, text); + block.innerHTML = this.markdown.render(source); + this._attachCopy(block, source); this._append(block); } diff --git a/devplacepy/templates/_comment.html b/devplacepy/templates/_comment.html index 27642610..7cb3810d 100644 --- a/devplacepy/templates/_comment.html +++ b/devplacepy/templates/_comment.html @@ -19,7 +19,7 @@ {% if parent_author %}↳ @{{ parent_author.get('username', '') }}{% endif %} {{ dt_ago(item.comment.get('created_at')) if item.comment.get('created_at') else item.time_ago }} -
{{ render_content(item.comment['content']) }}
+
{{ render_content(item.comment['content'], author_is_admin=is_admin(item.author)) }}
{% set attachments = item.get('attachments', []) %} {% if attachments %} {% include "_attachment_display.html" %} diff --git a/devplacepy/templates/_comment_form.html b/devplacepy/templates/_comment_form.html index 2af8a2eb..cbbdfde6 100644 --- a/devplacepy/templates/_comment_form.html +++ b/devplacepy/templates/_comment_form.html @@ -3,7 +3,7 @@ {% set _user = user %}{% set _size = 32 %}{% set _size_class = "sm" %}{% include "_avatar_link.html" %} - +
-

{{ render_title(item.post['title']) }}

+

{{ render_title(item.post['title'], author_is_admin=is_admin(item.author)) }}

{% endif %} -
{{ render_content(item.post['content'][:300] ~ ('...' if item.post['content']|length > 300 else '')) }}
+
{{ render_content(item.post['content'][:300] ~ ('...' if item.post['content']|length > 300 else ''), author_is_admin=is_admin(item.author)) }}
{% if item.attachments %} {% include "_attachment_display.html" %} diff --git a/devplacepy/templates/docs/ai-correction.html b/devplacepy/templates/docs/ai-correction.html index f625eda8..9a10ba5d 100644 --- a/devplacepy/templates/docs/ai-correction.html +++ b/devplacepy/templates/docs/ai-correction.html @@ -65,7 +65,7 @@ instruction if you like, and press **Save**. The default instruction is: > Leave literary as is, only do punctuation and casing -Replace it with any instruction up to 2000 characters, for example "fix spelling and grammar but keep +Replace it with any instruction up to 20000 characters, for example "fix spelling and grammar but keep my tone" or "translate to formal English". The setting is private: only you (and administrators) can see or change it. diff --git a/devplacepy/templates/docs/ai-modifier.html b/devplacepy/templates/docs/ai-modifier.html index d0dd3ba7..a2806ee7 100644 --- a/devplacepy/templates/docs/ai-modifier.html +++ b/devplacepy/templates/docs/ai-modifier.html @@ -97,7 +97,7 @@ like, and press **Save**. The default prompt is: > Execute what is behind `@ai` (the prompt) and replace that part including `@ai` -Replace it with any instruction up to 2000 characters. The setting is private: only you (and +Replace it with any instruction up to 20000 characters. The setting is private: only you (and administrators) can see or change it. ## Ask Devii diff --git a/devplacepy/templates/docs/devii-tools.html b/devplacepy/templates/docs/devii-tools.html index c450459d..9995614d 100644 --- a/devplacepy/templates/docs/devii-tools.html +++ b/devplacepy/templates/docs/devii-tools.html @@ -52,6 +52,7 @@ oldest Admin account only (the database API tools), gated the same way. | `notification` | preferences | `notification_list` / `notification_set` / `notification_reset` read and write the owner's per-type per-channel notification preferences. | | `ai_correction` | content correction | `ai_correction_get` / `ai_correction_set` read and write the owner's AI content-correction settings. | | `ai_modifier` | content modifier | `ai_modifier_get` / `ai_modifier_set` read and write the owner's `@ai` modifier settings. | +| `interaction` | interactive widgets | `ui_prompt` / `ui_cancel` / `ui_notify` present channel-aware asks; `interactions_get` / `interactions_set` read and write the owner's preference (admin default on the Devii service; users may override or reset to inherit). | | `telegram` | Telegram | `telegram_send` pushes a message to the owner's paired Telegram from a turn or scheduled task. | ## Complete action reference diff --git a/devplacepy/templates/gist_detail.html b/devplacepy/templates/gist_detail.html index f6c1377c..80889f36 100644 --- a/devplacepy/templates/gist_detail.html +++ b/devplacepy/templates/gist_detail.html @@ -12,7 +12,7 @@
-

{{ render_title(gist['title']) }}

+

{{ render_title(gist['title'], author_is_admin=is_admin(author)) }}

📝 {{ language_name(gist['language']) }}
@@ -26,13 +26,16 @@
{% if gist.get('description') %} -
{{ render_content(gist['description']) }}
+
{{ render_content(gist['description'], author_is_admin=is_admin(author)) }}
{% endif %} {% if attachments %} {% include "_attachment_display.html" %} {% endif %} + {% if gist['language'] == 'markdown_rendered' %} +
{{ render_content(gist['source_code'], author_is_admin=is_admin(author)) }}
+ {% else %}
{{ language_name(gist['language']) }} @@ -40,6 +43,7 @@ 
{{ gist['source_code'] }}
+ {% endif %}
diff --git a/devplacepy/templates/gists.html b/devplacepy/templates/gists.html index e1dd9699..3e7e980b 100644 --- a/devplacepy/templates/gists.html +++ b/devplacepy/templates/gists.html @@ -58,7 +58,7 @@ {% set _label = item.gist['title'] %} {% include "_card_link.html" %}
-

{{ render_title(item.gist['title']) }}

+

{{ render_title(item.gist['title'], author_is_admin=is_admin(item.author)) }}

{% set _type = "gist" %}{% set _uid = item.gist['uid'] %}{% set _my_vote = item.my_vote %}{% set _count = item.gist.get('stars', 0) %}{% set _btn_class = "gist-card-star" %}{% set _stop = True %}{% include "_star_vote.html" %}
@@ -67,7 +67,7 @@
{% if item.gist.get('description') %} -
{{ render_content(item.gist['description'][:200]) }}
+
{{ render_content(item.gist['description'][:200], author_is_admin=is_admin(item.author)) }}
{% endif %} {% if item.recent_comments %} diff --git a/devplacepy/templates/landing.html b/devplacepy/templates/landing.html index 05fb4fca..dedf7fe8 100644 --- a/devplacepy/templates/landing.html +++ b/devplacepy/templates/landing.html @@ -7,39 +7,113 @@ {% block content %}
{% if user %} -
-
- {% set _user = user %}{% set _size = 80 %}{% set _size_class = "lg" %}{% include "_avatar_link.html" %} -
- Welcome back, -

{{ user['username'] }}

+
+
+
+
+ {% set _user = user %}{% set _size = 48 %}{% set _size_class = "md" %}{% include "_avatar_link.html" %} + Welcome back, {{ user['username'] }} +
+
+ New Post + Code Farm + Projects + Gists +
+
+
+
+ {{ user_post_count }} + Posts +
+
+ {{ user_stars }} + Stars +
+
+ Lv {{ user_level }} + Level +
+
+
+ {{ xp_progress_pct }}% to Lv {{ user_level + 1 }} +
- 📝 Go to your feed → - -
-
- {{ user_post_count }} - Posts -
-
- {{ user_stars }} - Stars -
-
- Lv {{ user.get('level', 1) }} - Level +
+
+ {% if landing_posts %} +
+

Latest Posts

+ Browse all → +
+
+ {% for item in landing_posts %} +
+ {% set _href = "/posts/" ~ item.slug %} + {% set _label = item.post.get('title') or 'View post' %} + {% include "_card_link.html" %} +
+ {% set _user = item.author %} + {% set _size = 28 %} + {% set _size_class = "sm" %} + {% include "_avatar_link.html" %} +
+ {% set _user = item.author %}{% set _class = "dashboard-post-author" %}{% include "_user_link.html" %} + {{ dt_ago(item.created_at) if item.created_at else item.time_ago }} +
+ {{ item.post['topic'] }} +
+ {% if item.post.get('title') %} +

+ {{ render_title(item.post['title'], author_is_admin=is_admin(item.author)) }} +

+ {% endif %} +
{{ render_content(item.post['content'][:200] ~ ('...' if item.post['content']|length > 200 else ''), author_is_admin=is_admin(item.author)) }}
+
+ +{{ item.stars }} + 💬 {{ item.comment_count }} + Open → +
+
+ {% endfor %} +
+ {% endif %}
+
- -
- New post - Projects - Gists - Code Farm - Notifications -
-
+
{% else %}

Devplace.net - The Developer Social Network

@@ -105,7 +179,7 @@
- {% if landing_posts %} + {% if landing_posts and not user %}

Latest Posts

@@ -130,10 +204,10 @@
{% if item.post.get('title') %}

- {{ render_title(item.post['title']) }} + {{ render_title(item.post['title'], author_is_admin=is_admin(item.author)) }}

{% endif %} -
{{ render_content(item.post['content'][:200] ~ ('...' if item.post['content']|length > 200 else '')) }}
+
{{ render_content(item.post['content'][:200] ~ ('...' if item.post['content']|length > 200 else ''), author_is_admin=is_admin(item.author)) }}
+{{ item.stars }} 💬 {{ item.comment_count }} @@ -237,4 +311,4 @@ {% include "_footer_links.html" %}
-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/devplacepy/templates/messages.html b/devplacepy/templates/messages.html index e470694f..77e81a53 100644 --- a/devplacepy/templates/messages.html +++ b/devplacepy/templates/messages.html @@ -43,7 +43,7 @@
{% for msg in messages %}
-
{{ render_content(msg.message['content']) }}
+ {{ msg.message['content'] }} {% if msg.attachments %} {% with attachments=msg.attachments %} {% include "_attachment_display.html" %} @@ -58,7 +58,7 @@
- + {% if post.get('title') %} -

{{ render_title(post['title']) }}

+

{{ render_title(post['title'], author_is_admin=is_admin(author)) }}

{% endif %} -
{{ render_content(post['content']) }}
+
{{ render_content(post['content'], author_is_admin=is_admin(author)) }}
{% if attachments %} {% include "_attachment_display.html" %} @@ -97,7 +97,7 @@
{% for item in related_posts %} - {{ render_title(item.post['title']) if item.post.get('title') else content_preview(item.post['content'], 60) }} + {{ render_title(item.post['title'], author_is_admin=is_admin(item.author)) if item.post.get('title') else content_preview(item.post['content'], 60) }} {{ item.author['username'] }} · {{ dt_ago(item.created_at) if item.created_at else item.time_ago }} {% endfor %} diff --git a/devplacepy/templates/profile.html b/devplacepy/templates/profile.html index 196192c1..d7b9359f 100644 --- a/devplacepy/templates/profile.html +++ b/devplacepy/templates/profile.html @@ -102,7 +102,7 @@
Bio
-
{{ render_content(profile_user.get('bio', '')) or 'Not set' }}
+
{{ render_content(profile_user.get('bio', ''), author_is_admin=is_admin(profile_user)) or 'Not set' }}
@@ -150,7 +150,7 @@
Bio -
{{ render_content(profile_user.get('bio', '')) or 'Not set' }}
+
{{ render_content(profile_user.get('bio', ''), author_is_admin=is_admin(profile_user)) or 'Not set' }}
Location @@ -226,7 +226,7 @@
- +
@@ -251,7 +251,7 @@
- +
@@ -259,6 +259,24 @@

Runs only when your text contains an inline `@ai instruction` directive: it executes that instruction and removes the `@ai` marker, using your API key. Applies to posts, comments, projects, gists, direct messages and your bio. Code and source files are never touched.

+
+
+ Devii interactive widgets +
+
+ Enable interactive prompts + +
+
+ + + +
+

When enabled, Devii may ask you questions with buttons and forms (ui_prompt) in the site chat, Telegram, or CLI. The administrator sets the site default (currently {{ 'on' if interactions_default else 'off' }}); you may override it here or ask Devii via interactions_set. Source: {{ interactions_source or 'default' }}.

+
+
Telegram @@ -535,7 +553,7 @@ {% for p in projects %}
-

{{ render_title(p['title']) }}

+

{{ render_title(p['title'], author_is_admin=is_admin(profile_user)) }}

@@ -543,7 +561,7 @@ {{ p.get('project_type', 'software')|replace('_', ' ')|capitalize }}
-
{{ render_content(p.get('description', '')[:200]) }}
+
{{ render_content(p.get('description', '')[:200], author_is_admin=is_admin(profile_user)) }}
{% if p.get('platforms') %}
{% for plat in p['platforms'].split(',') %} @@ -560,14 +578,14 @@ {% set g = entry.gist %}
-

{{ render_title(g['title']) }}

+

{{ render_title(g['title'], author_is_admin=is_admin(profile_user)) }}

☆ {{ g.get('stars', 0) }}
📝 {{ language_name(g['language']) }}
{% if g.get('description') %} -
{{ render_content(g['description'][:200]) }}
+
{{ render_content(g['description'][:200], author_is_admin=is_admin(profile_user)) }}
{% endif %}
{{ dt_ago(entry.gist.created_at) if entry.gist.created_at else entry.time_ago }} @@ -596,7 +614,7 @@ {{ person['username'] }}{% set _user = person %}{% include "_presence_dot.html" %}{% include "_award_badge.html" %} {% if user and not person['is_self'] %} @@ -696,7 +714,7 @@ {{ dt_ago(act['created_at']) if act['created_at'] else act['time_ago'] }}
-
{{ render_content(act['content']) }}
+
{{ render_content(act['content'], author_is_admin=is_admin(profile_user)) }}
{% else %}
No activity yet.
diff --git a/devplacepy/templates/project_detail.html b/devplacepy/templates/project_detail.html index 82e16711..2b57ad19 100644 --- a/devplacepy/templates/project_detail.html +++ b/devplacepy/templates/project_detail.html @@ -10,7 +10,7 @@
-

{{ render_title(project['title']) }}

+

{{ render_title(project['title'], author_is_admin=is_admin(author)) }}

● {{ project.get('status', 'In Development') }}
@@ -47,7 +47,7 @@
-
{{ render_content(project.get('description', '')) }}
+
{{ render_content(project.get('description', ''), author_is_admin=is_admin(author)) }}
{% if attachments %} {% include "_attachment_display.html" %} diff --git a/devplacepy/templating.py b/devplacepy/templating.py index 19eb4f25..4e328b68 100644 --- a/devplacepy/templating.py +++ b/devplacepy/templating.py @@ -18,25 +18,7 @@ from devplacepy.content import is_owner as _owns from devplacepy.customization import custom_css_tag, custom_js_tag, page_type_for from devplacepy.services import presence -EM_DASH_FORMS = ("\u2014", "—", "—", "—", "—") - - -def normalize_em_dash(text: str) -> str: - for form in EM_DASH_FORMS: - text = text.replace(form, "-") - return text - - -class _Template(jinja2.Template): - def render(self, *args, **kwargs) -> str: - return normalize_em_dash(super().render(*args, **kwargs)) - - async def render_async(self, *args, **kwargs) -> str: - return normalize_em_dash(await super().render_async(*args, **kwargs)) - - templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) -templates.env.template_class = _Template templates.env.auto_reload = TEMPLATE_AUTO_RELOAD @@ -217,6 +199,7 @@ _LANGUAGE_NAMES = { "yaml": "YAML", "json": "JSON", "markdown": "Markdown", + "markdown_rendered": "Markdown Rendered", "swift": "Swift", "php": "PHP", "ruby": "Ruby", diff --git a/devplacepy/utils/accounts.py b/devplacepy/utils/accounts.py index 93c956e8..e9c53c2f 100644 --- a/devplacepy/utils/accounts.py +++ b/devplacepy/utils/accounts.py @@ -35,6 +35,7 @@ def _create_account(username: str, email: str, password_hash: str) -> tuple[str, "ai_modifier_enabled": 1, "ai_modifier_prompt": DEFAULT_MODIFIER_PROMPT, "ai_modifier_sync": 1, + "interactions_enabled": -1, "avatar_seed": None, "created_at": datetime.now(timezone.utc).isoformat(), } diff --git a/devplacepy/utils/auth.py b/devplacepy/utils/auth.py index 0590cfb3..63761981 100644 --- a/devplacepy/utils/auth.py +++ b/devplacepy/utils/auth.py @@ -132,17 +132,33 @@ def _resolve_user(request: Request): def _user_from_devrant_token(key: str): if not key or len(key) != 40 or not all(c in "0123456789abcdef" for c in key.lower()): return None + _sync_auth_cache() + cache_key = f"d:{key}" + cached = _user_cache.get(cache_key) + if cached is not None: + return cached from devplacepy.services.devrant.tokens import resolve_user_by_key - return resolve_user_by_key(key) + user = resolve_user_by_key(key) + if user: + _user_cache.set(cache_key, user) + return user def _user_from_access_token(key: str): if not key or len(key) != 64: return None + _sync_auth_cache() + cache_key = f"t:{key}" + cached = _user_cache.get(cache_key) + if cached is not None: + return cached from devplacepy.services.access_tokens import resolve_token - return resolve_token(key) + user = resolve_token(key) + if user: + _user_cache.set(cache_key, user) + return user def get_current_user(request: Request): diff --git a/devplacepy/utils/authcache.py b/devplacepy/utils/authcache.py index 04b99b1d..273181ee 100644 --- a/devplacepy/utils/authcache.py +++ b/devplacepy/utils/authcache.py @@ -10,11 +10,12 @@ def _sync_auth_cache() -> None: sync_local_cache("auth", _user_cache) -def clear_user_cache(user_uid: str) -> None: +def clear_user_cache(user_uid: str, *, propagate: bool = True) -> None: for token, user in _user_cache.items(): if user.get("uid") == user_uid: _user_cache.pop(token) - bump_cache_version("auth") + if propagate: + bump_cache_version("auth") def clear_session_cache(token: str) -> None: diff --git a/devplacepy/utils/rewards.py b/devplacepy/utils/rewards.py index 8675c624..e383a30c 100644 --- a/devplacepy/utils/rewards.py +++ b/devplacepy/utils/rewards.py @@ -39,7 +39,7 @@ def award_xp(user_uid: str, amount: int) -> dict: new_xp = max(0, current_xp + amount) new_level = level_for_xp(new_xp) users.update({"uid": user_uid, "xp": new_xp, "level": new_level}, ["uid"]) - clear_user_cache(user_uid) + clear_user_cache(user_uid, propagate=False) leveled_up = new_level > current_level from devplacepy.services.audit import record as audit diff --git a/tests/api/admin/awards.py b/tests/api/admin/awards.py new file mode 100644 index 00000000..a213410b --- /dev/null +++ b/tests/api/admin/awards.py @@ -0,0 +1,110 @@ +# retoor + +import io +import requests +from datetime import datetime, timezone +from PIL import Image +from tests.conftest import BASE_URL +from devplacepy.attachments import store_attachment +from devplacepy.db_client import get_table +from devplacepy.utils import generate_uid, make_combined_slug + +JSON = {"Accept": "application/json"} + + +def _login(seeded_db, who): + s = requests.Session() + creds = seeded_db[who] + s.post( + f"{BASE_URL}/auth/login", + data={"email": creds["email"], "password": creds["password"]}, + allow_redirects=True, + ) + return s + + +def _png(): + buf = io.BytesIO() + Image.new("RGBA", (24, 24), (5, 5, 5, 255)).save(buf, format="PNG") + return buf.getvalue() + + +def _seed_published(receiver_username, giver_username): + receiver = get_table("users").find_one(username=receiver_username) + giver = get_table("users").find_one(username=giver_username) + uid = generate_uid() + slug = make_combined_slug("Admin revoke", uid) + now = datetime.now(timezone.utc).isoformat() + att512 = store_attachment(_png(), "award-512.png", receiver["uid"]) + att256 = store_attachment(_png(), "award-256.png", giver["uid"]) + att64 = store_attachment(_png(), "award-64.png", giver["uid"]) + get_table("awards").insert( + { + "uid": uid, + "slug": slug, + "description": "Admin revoke", + "giver_uid": giver["uid"], + "receiver_uid": receiver["uid"], + "attachment_uid_512": att512["uid"], + "attachment_uid_256": att256["uid"], + "attachment_uid_64": att64["uid"], + "generated_at": now, + "created_at": now, + "job_uid": "", + "deleted_at": None, + "deleted_by": None, + } + ) + get_table("users").update( + { + "uid": receiver["uid"], + "award_count": 1, + "last_award_at": now, + "last_award_slug": slug, + "last_award_uid": uid, + }, + ["uid"], + ) + return uid, slug, att512["uid"] + + +def test_member_cannot_revoke_award(seeded_db): + uid, _, _ = _seed_published("bob_test", "alice_test") + bob = _login(seeded_db, "bob") + r = bob.post(f"{BASE_URL}/admin/awards/{uid}/revoke", allow_redirects=False) + assert r.status_code in (302, 303) + admin = _login(seeded_db, "alice") + admin.post(f"{BASE_URL}/admin/awards/{uid}/revoke", headers=JSON) + + +def test_admin_revoke_soft_deletes_and_recomputes(seeded_db): + uid, slug, att512 = _seed_published("bob_test", "alice_test") + admin = _login(seeded_db, "alice") + r = admin.post(f"{BASE_URL}/admin/awards/{uid}/revoke", headers=JSON) + assert r.status_code == 200 + row = get_table("awards").find_one(uid=uid) + assert row.get("deleted_at") + assert get_table("attachments").find_one(uid=att512).get("deleted_at") + bob = get_table("users").find_one(username="bob_test") + assert bob.get("award_count") == 0 + assert requests.get(f"{BASE_URL}/awards/{slug}/64").status_code == 404 + + +def test_media_gallery_hides_revoked_attachment(seeded_db): + uid, _, att512 = _seed_published("bob_test", "alice_test") + admin = _login(seeded_db, "alice") + admin.post(f"{BASE_URL}/admin/awards/{uid}/revoke", headers=JSON) + r = requests.get(f"{BASE_URL}/profile/bob_test?tab=media", headers=JSON) + assert r.status_code == 200 + media = r.json().get("media", []) + assert all(item.get("uid") != att512 for item in media) + + +def test_admin_revoke_audit_recorded(seeded_db): + uid, _, _ = _seed_published("bob_test", "alice_test") + admin = _login(seeded_db, "alice") + admin.post(f"{BASE_URL}/admin/awards/{uid}/revoke", headers=JSON) + data = admin.get( + f"{BASE_URL}/admin/audit-log", headers=JSON, params={"event_key": "award.revoke"} + ).json() + assert any(entry.get("target_uid") == uid for entry in data["entries"]) \ No newline at end of file diff --git a/tests/api/awards/index.py b/tests/api/awards/index.py new file mode 100644 index 00000000..be9eee70 --- /dev/null +++ b/tests/api/awards/index.py @@ -0,0 +1,116 @@ +# retoor + +import io +import requests +from datetime import datetime, timezone +from PIL import Image +from tests.conftest import BASE_URL +from devplacepy.attachments import store_attachment +from devplacepy.db_client import get_table +from devplacepy.utils import generate_uid, make_combined_slug + + +def _png(): + buf = io.BytesIO() + Image.new("RGBA", (32, 32), (20, 40, 60, 255)).save(buf, format="PNG") + return buf.getvalue() + + +def _seed_published(receiver_uid, giver_uid, description="Published award"): + uid = generate_uid() + slug = make_combined_slug(description, uid) + now = datetime.now(timezone.utc).isoformat() + att = store_attachment(_png(), "award-256.png", receiver_uid) + get_table("awards").insert( + { + "uid": uid, + "slug": slug, + "description": description, + "giver_uid": giver_uid, + "receiver_uid": receiver_uid, + "attachment_uid_512": att["uid"], + "attachment_uid_256": att["uid"], + "attachment_uid_64": att["uid"], + "generated_at": now, + "created_at": now, + "job_uid": "", + "deleted_at": None, + "deleted_by": None, + } + ) + return uid, slug, att["url"] + + +def _user(): + uid = generate_uid() + get_table("users").insert( + { + "uid": uid, + "username": f"aw_{uid[:8]}", + "email": f"{uid[:8]}@t.dev", + "created_at": datetime.now(timezone.utc).isoformat(), + } + ) + return uid + + +def test_published_award_redirects(app_server): + giver = _user() + receiver = _user() + _, slug, url = _seed_published(receiver, giver) + r = requests.get(f"{BASE_URL}/awards/{slug}/256", allow_redirects=False) + assert r.status_code == 302 + assert r.headers["Location"] == url + assert "immutable" in r.headers.get("Cache-Control", "") + + +def test_pending_award_returns_404(app_server): + uid = generate_uid() + slug = make_combined_slug("pending", uid) + now = datetime.now(timezone.utc).isoformat() + get_table("awards").insert( + { + "uid": uid, + "slug": slug, + "description": "pending", + "giver_uid": _user(), + "receiver_uid": _user(), + "attachment_uid_512": "", + "attachment_uid_256": "", + "attachment_uid_64": "", + "generated_at": None, + "created_at": now, + "job_uid": "", + "deleted_at": None, + "deleted_by": None, + } + ) + assert requests.get(f"{BASE_URL}/awards/{slug}/256").status_code == 404 + + +def test_revoked_award_returns_404(app_server): + giver = _user() + receiver = _user() + uid, slug, _ = _seed_published(receiver, giver, "revoked") + get_table("awards").update( + {"uid": uid, "deleted_at": datetime.now(timezone.utc).isoformat(), "deleted_by": giver}, + ["uid"], + ) + assert requests.get(f"{BASE_URL}/awards/{slug}/64").status_code == 404 + + +def test_lookup_by_bare_uid_works(app_server): + giver = _user() + receiver = _user() + uid, slug, url = _seed_published(receiver, giver, "uid lookup") + r = requests.get(f"{BASE_URL}/awards/{uid}/256", allow_redirects=False) + assert r.status_code == 302 + assert r.headers["Location"] == url + assert slug + + +def test_invalid_size_returns_404(app_server): + giver = _user() + receiver = _user() + _, slug, _ = _seed_published(receiver, giver) + assert requests.get(f"{BASE_URL}/awards/{slug}/128").status_code == 404 \ No newline at end of file diff --git a/tests/api/profile/award.py b/tests/api/profile/award.py new file mode 100644 index 00000000..14263335 --- /dev/null +++ b/tests/api/profile/award.py @@ -0,0 +1,190 @@ +# retoor + +import time +import requests +from datetime import datetime, timezone +from tests.conftest import BASE_URL +from devplacepy.db_client import get_table +from devplacepy.utils import generate_uid, make_combined_slug + +JSON = {"Accept": "application/json"} +_counter = [0] + + +def _signup(): + _counter[0] += 1 + name = f"award{int(time.time() * 1000)}{_counter[0]}" + session = requests.Session() + session.post( + f"{BASE_URL}/auth/signup", + data={ + "username": name, + "email": f"{name}@t.dev", + "password": "secret123", + "confirm_password": "secret123", + }, + allow_redirects=True, + ) + return session, name + + +def _login(seeded_db, who): + s = requests.Session() + creds = seeded_db[who] + s.post( + f"{BASE_URL}/auth/login", + data={"email": creds["email"], "password": creds["password"]}, + allow_redirects=True, + ) + return s + + +def _uid(username): + return get_table("users").find_one(username=username)["uid"] + + +def _audit_find(admin, event_key, target_uid): + data = admin.get( + f"{BASE_URL}/admin/audit-log", headers=JSON, params={"event_key": event_key} + ).json() + for entry in data["entries"]: + if entry.get("target_uid") == target_uid: + return entry + return None + + +def test_give_award_json_shape(app_server): + giver, _ = _signup() + _, receiver_name = _signup() + r = giver.post( + f"{BASE_URL}/profile/{receiver_name}/award", + headers=JSON, + json={"description": "Great debugging"}, + ) + assert r.status_code == 200, r.text + data = r.json() + assert data["ok"] is True + assert data["data"]["award_uid"] + assert data["data"]["award_slug"] + + +def test_guest_cannot_give_award(seeded_db): + r = requests.post( + f"{BASE_URL}/profile/bob_test/award", + headers=JSON, + json={"description": "Nope"}, + allow_redirects=False, + ) + assert r.status_code in (302, 303, 401, 403) + + +def test_self_award_rejected(seeded_db): + bob = _login(seeded_db, "bob") + r = bob.post( + f"{BASE_URL}/profile/bob_test/award", + headers=JSON, + json={"description": "Self"}, + ) + assert r.status_code == 400 + + +def test_blocked_pair_rejects_award(seeded_db): + alice = _login(seeded_db, "alice") + bob = _login(seeded_db, "bob") + alice.post(f"{BASE_URL}/block/bob_test", allow_redirects=False) + try: + r = alice.post( + f"{BASE_URL}/profile/bob_test/award", + headers=JSON, + json={"description": "Blocked"}, + ) + assert r.status_code == 400 + r2 = bob.post( + f"{BASE_URL}/profile/alice_test/award", + headers=JSON, + json={"description": "Reverse blocked"}, + ) + assert r2.status_code == 400 + finally: + alice.post(f"{BASE_URL}/block/unblock/bob_test", allow_redirects=False) + + +def test_giver_cooldown_rejects_second_award(app_server): + giver, giver_name = _signup() + _, target_a = _signup() + _, target_b = _signup() + first = giver.post( + f"{BASE_URL}/profile/{target_a}/award", + headers=JSON, + json={"description": "First award"}, + ) + assert first.status_code == 200 + second = giver.post( + f"{BASE_URL}/profile/{target_b}/award", + headers=JSON, + json={"description": "Second award"}, + ) + assert second.status_code == 400 + assert giver_name + + +def test_receiver_cooldown_rejects_second_award(app_server): + giver_a, _ = _signup() + giver_b, _ = _signup() + _, receiver = _signup() + first = giver_a.post( + f"{BASE_URL}/profile/{receiver}/award", + headers=JSON, + json={"description": "From giver a"}, + ) + assert first.status_code == 200 + second = giver_b.post( + f"{BASE_URL}/profile/{receiver}/award", + headers=JSON, + json={"description": "From giver b"}, + ) + assert second.status_code == 400 + + +def test_muted_receiver_still_accepts_award(app_server): + giver, giver_name = _signup() + receiver_session, receiver_name = _signup() + receiver_session.post(f"{BASE_URL}/mute/{giver_name}", allow_redirects=False) + try: + r = giver.post( + f"{BASE_URL}/profile/{receiver_name}/award", + headers=JSON, + json={"description": "Muted ok"}, + ) + assert r.status_code == 200 + finally: + receiver_session.post(f"{BASE_URL}/mute/unmute/{giver_name}", allow_redirects=False) + + +def test_valid_post_creates_pending_row_and_job(app_server): + giver, _ = _signup() + _, receiver_name = _signup() + r = giver.post( + f"{BASE_URL}/profile/{receiver_name}/award", + headers=JSON, + json={"description": "Fresh pending"}, + ) + assert r.status_code == 200 + award_uid = r.json()["data"]["award_uid"] + row = get_table("awards").find_one(uid=award_uid) + assert row.get("generated_at") is None + job = get_table("jobs").find_one(uid=row.get("job_uid")) + assert job and job.get("kind") == "award" + + +def test_award_give_audit_recorded(app_server): + giver, _ = _signup() + _, receiver_name = _signup() + r = giver.post( + f"{BASE_URL}/profile/{receiver_name}/award", + headers=JSON, + json={"description": "Audit me"}, + ) + assert r.status_code == 200 + entry = _audit_find(giver, "award.give", _uid(receiver_name)) + assert entry is not None \ No newline at end of file diff --git a/tests/api/profile/awards_tab.py b/tests/api/profile/awards_tab.py new file mode 100644 index 00000000..c6fb5ae8 --- /dev/null +++ b/tests/api/profile/awards_tab.py @@ -0,0 +1,93 @@ +# retoor + +import io +import requests +from datetime import datetime, timezone +from PIL import Image +from tests.conftest import BASE_URL +from devplacepy.attachments import store_attachment +from devplacepy.db_client import get_table +from devplacepy.utils import generate_uid, make_combined_slug + +JSON = {"Accept": "application/json"} + + +def _png(): + buf = io.BytesIO() + Image.new("RGBA", (16, 16), (90, 10, 10, 255)).save(buf, format="PNG") + return buf.getvalue() + + +def _seed_published(receiver_username, giver_username, description): + receiver = get_table("users").find_one(username=receiver_username) + giver = get_table("users").find_one(username=giver_username) + uid = generate_uid() + slug = make_combined_slug(description, uid) + now = datetime.now(timezone.utc).isoformat() + att = store_attachment(_png(), "award-256.png", receiver["uid"]) + get_table("awards").insert( + { + "uid": uid, + "slug": slug, + "description": description, + "giver_uid": giver["uid"], + "receiver_uid": receiver["uid"], + "attachment_uid_512": att["uid"], + "attachment_uid_256": att["uid"], + "attachment_uid_64": att["uid"], + "generated_at": now, + "created_at": now, + "job_uid": "", + "deleted_at": None, + "deleted_by": None, + } + ) + get_table("users").update( + { + "uid": receiver["uid"], + "award_count": 1, + "last_award_at": now, + "last_award_slug": slug, + "last_award_uid": uid, + }, + ["uid"], + ) + return slug + + +def test_awards_tab_html_contains_grid_and_slug(seeded_db): + slug = _seed_published("bob_test", "alice_test", "Tab award") + r = requests.get(f"{BASE_URL}/profile/bob_test?tab=awards") + assert r.status_code == 200 + assert "awards-grid" in r.text + assert slug in r.text + + +def test_guest_can_read_awards_tab(seeded_db): + _seed_published("bob_test", "alice_test", "Guest read") + r = requests.get(f"{BASE_URL}/profile/bob_test?tab=awards") + assert r.status_code == 200 + assert "Give Award" not in r.text + + +def test_awards_tab_json_includes_list(seeded_db): + slug = _seed_published("bob_test", "alice_test", "JSON tab") + r = requests.get( + f"{BASE_URL}/profile/bob_test?tab=awards", + headers=JSON, + ) + assert r.status_code == 200 + data = r.json() + assert data["awards_count"] >= 1 + assert any(item.get("slug") == slug for item in data.get("awards", [])) + + +def test_awards_tab_absent_when_count_zero(seeded_db): + bob = get_table("users").find_one(username="bob_test") + get_table("users").update( + {"uid": bob["uid"], "award_count": 0, "last_award_uid": None, "last_award_slug": None}, + ["uid"], + ) + r = requests.get(f"{BASE_URL}/profile/bob_test") + assert r.status_code == 200 + assert "tab=awards" not in r.text \ No newline at end of file diff --git a/tests/api/profile/interactions.py b/tests/api/profile/interactions.py new file mode 100644 index 00000000..f1d0170f --- /dev/null +++ b/tests/api/profile/interactions.py @@ -0,0 +1,109 @@ +# retoor + +import time +import pytest +import requests +from tests.conftest import BASE_URL +from devplacepy.database import get_table, refresh_snapshot, set_setting + +JSON = {"Accept": "application/json"} +_counter = [0] + + +@pytest.fixture(scope="module", autouse=True) +def _settings(app_server): + set_setting("rate_limit_per_minute", "1000000") + set_setting("registration_open", "1") + set_setting("devii_interactions_default", "1") + yield + + +def _member(): + _counter[0] += 1 + name = f"ixp{int(time.time() * 1000)}{_counter[0]}" + requests.post( + f"{BASE_URL}/auth/signup", + data={ + "username": name, + "email": f"{name}@t.dev", + "password": "secret123", + "confirm_password": "secret123", + }, + allow_redirects=True, + ) + refresh_snapshot() + user = get_table("users").find_one(username=name) + return name, user["uid"], user["api_key"] + + +def _admin_key(seeded_db): + refresh_snapshot() + return get_table("users").find_one(username="alice_test")["api_key"] + + +def test_owner_disables_interactions(app_server): + name, uid, key = _member() + r = requests.post( + f"{BASE_URL}/profile/{name}/interactions", + headers={"X-API-KEY": key, **JSON}, + data={"enabled": "false"}, + ) + assert r.status_code == 200, r.text[:200] + body = r.json()["data"] + assert body["enabled"] is False + assert body["source"] == "user" + refresh_snapshot() + assert get_table("users").find_one(uid=uid)["interactions_enabled"] == 0 + + +def test_owner_resets_to_default(app_server): + name, uid, key = _member() + requests.post( + f"{BASE_URL}/profile/{name}/interactions", + headers={"X-API-KEY": key, **JSON}, + data={"enabled": "false"}, + ) + r = requests.post( + f"{BASE_URL}/profile/{name}/interactions", + headers={"X-API-KEY": key, **JSON}, + data={"reset": "true"}, + ) + assert r.status_code == 200 + body = r.json()["data"] + assert body["source"] == "default" + refresh_snapshot() + assert get_table("users").find_one(uid=uid)["interactions_enabled"] == -1 + + +def test_admin_sets_for_other_user(seeded_db): + name, uid, key = _member() + r = requests.post( + f"{BASE_URL}/profile/{name}/interactions", + headers={"X-API-KEY": _admin_key(seeded_db), **JSON}, + data={"enabled": "false"}, + ) + assert r.status_code == 200 + refresh_snapshot() + assert get_table("users").find_one(uid=uid)["interactions_enabled"] == 0 + + +def test_non_owner_forbidden(app_server): + name, uid, key = _member() + _, _, other_key = _member() + r = requests.post( + f"{BASE_URL}/profile/{name}/interactions", + headers={"X-API-KEY": other_key, **JSON}, + data={"enabled": "false"}, + ) + assert r.status_code == 403 + + +def test_guest_rejected(app_server): + name, uid, key = _member() + r = requests.post( + f"{BASE_URL}/profile/{name}/interactions", + headers=JSON, + data={"enabled": "false"}, + allow_redirects=False, + ) + assert r.status_code in (401, 303) diff --git a/tests/unit/awards/images.py b/tests/unit/awards/images.py new file mode 100644 index 00000000..7b871d33 --- /dev/null +++ b/tests/unit/awards/images.py @@ -0,0 +1,40 @@ +# retoor + +import io + +from PIL import Image +from devplacepy.awards.images import enforce_rgba_png, resize_award_png + + +def _rgba_png(size=512, color=(10, 20, 30, 200)): + buf = io.BytesIO() + Image.new("RGBA", (size, size), color).save(buf, format="PNG") + return buf.getvalue() + + +def test_resize_award_png_outputs_exact_square_dimensions(): + source = _rgba_png(512) + for size in (512, 256, 64): + out = resize_award_png(source, size) + img = Image.open(io.BytesIO(out)) + assert img.size == (size, size) + + +def test_resize_award_png_preserves_rgba_mode(): + out = resize_award_png(_rgba_png(512), 256) + img = Image.open(io.BytesIO(out)) + assert img.mode == "RGBA" + + +def test_resize_award_png_downscale_is_idempotent_in_size(): + first = resize_award_png(_rgba_png(512), 64) + second = resize_award_png(first, 64) + assert Image.open(io.BytesIO(first)).size == (64, 64) + assert Image.open(io.BytesIO(second)).size == (64, 64) + + +def test_enforce_rgba_png_returns_valid_png(): + out = enforce_rgba_png(_rgba_png(32)) + img = Image.open(io.BytesIO(out)) + assert img.mode == "RGBA" + assert img.size == (32, 32) \ No newline at end of file diff --git a/tests/unit/database/awards.py b/tests/unit/database/awards.py new file mode 100644 index 00000000..07ab1093 --- /dev/null +++ b/tests/unit/database/awards.py @@ -0,0 +1,198 @@ +# retoor + +from datetime import datetime, timedelta, timezone + +import pytest +from devplacepy.db_client import get_table, init_db +from devplacepy.db_client import ( + can_give_award, + can_receive_award, + has_giver_cooldown, + has_receiver_cooldown, + recompute_user_award_stats, + revoke_award, +) +from devplacepy.utils import generate_uid, make_combined_slug + + +@pytest.fixture(autouse=True) +def _db(local_db): + init_db() + yield + + +def _user(prefix="award"): + uid = generate_uid() + get_table("users").insert( + { + "uid": uid, + "username": f"{prefix}_{uid[:8]}", + "email": f"{uid[:8]}@t.dev", + "created_at": datetime.now(timezone.utc).isoformat(), + "award_count": 0, + "last_award_at": None, + "last_award_slug": None, + "last_award_uid": None, + } + ) + return uid + + +def _award(giver_uid, receiver_uid, description="Great work", generated_at=None): + uid = generate_uid() + slug = make_combined_slug(description, uid) + now = datetime.now(timezone.utc).isoformat() + get_table("awards").insert( + { + "uid": uid, + "slug": slug, + "description": description, + "giver_uid": giver_uid, + "receiver_uid": receiver_uid, + "attachment_uid_512": "", + "attachment_uid_256": "", + "attachment_uid_64": "", + "generated_at": generated_at, + "created_at": now, + "job_uid": "", + "deleted_at": None, + "deleted_by": None, + } + ) + return uid, slug + + +def _attachment(user_uid): + uid = generate_uid() + get_table("attachments").insert( + { + "uid": uid, + "target_type": "award", + "target_uid": "", + "user_uid": user_uid, + "original_filename": "award.png", + "stored_name": f"{uid}.png", + "directory": "aa/bb", + "file_size": 10, + "mime_type": "image/png", + "created_at": datetime.now(timezone.utc).isoformat(), + "deleted_at": None, + "deleted_by": None, + } + ) + return uid + + +def test_slug_matches_make_combined_slug(): + uid = generate_uid() + slug = make_combined_slug("Ship it", uid) + assert slug.endswith("-ship-it") or slug == uid.replace("-", "")[-12:] + + +def test_can_give_award_respects_giver_cooldown(monkeypatch): + giver = _user("giver") + receiver = _user("recv") + _award(giver, receiver) + monkeypatch.setattr( + "devplacepy.database.awards.get_int_setting", + lambda key, default: 24 if "cooldown" in key else default, + ) + assert has_giver_cooldown(giver) is True + assert can_give_award(giver, receiver) is False + + +def test_can_receive_award_respects_receiver_cooldown(monkeypatch): + giver = _user("giver2") + receiver = _user("recv2") + _award(giver, receiver) + monkeypatch.setattr( + "devplacepy.database.awards.get_int_setting", + lambda key, default: 24 if "receive" in key else default, + ) + assert has_receiver_cooldown(receiver) is True + assert can_receive_award(receiver) is False + + +def test_cooldown_ignores_soft_deleted_rows(): + giver = _user("giver3") + receiver = _user("recv3") + uid, _ = _award(giver, receiver) + get_table("awards").update( + {"uid": uid, "deleted_at": datetime.now(timezone.utc).isoformat(), "deleted_by": giver}, + ["uid"], + ) + assert has_giver_cooldown(giver) is False + assert has_receiver_cooldown(receiver) is False + + +def test_recompute_user_award_stats_sets_latest_generated(): + receiver = _user("recv4") + giver = _user("giver4") + older = datetime.now(timezone.utc) - timedelta(hours=2) + newer = datetime.now(timezone.utc) - timedelta(hours=1) + uid_old, slug_old = _award(giver, receiver, "older", older.isoformat()) + uid_new, slug_new = _award(giver, receiver, "newer", newer.isoformat()) + recompute_user_award_stats(receiver) + row = get_table("users").find_one(uid=receiver) + assert row["award_count"] == 2 + assert row["last_award_uid"] == uid_new + assert row["last_award_slug"] == slug_new + assert uid_old != uid_new + assert slug_old != slug_new + + +def test_revoke_latest_recomputes_to_previous(): + receiver = _user("recv5") + giver = _user("giver5") + older = datetime.now(timezone.utc) - timedelta(hours=2) + newer = datetime.now(timezone.utc) - timedelta(hours=1) + uid_old, slug_old = _award(giver, receiver, "keep", older.isoformat()) + uid_new, _ = _award(giver, receiver, "gone", newer.isoformat()) + recompute_user_award_stats(receiver) + revoke_award(uid_new, giver) + row = get_table("users").find_one(uid=receiver) + assert row["award_count"] == 1 + assert row["last_award_uid"] == uid_old + assert row["last_award_slug"] == slug_old + + +def test_revoke_clears_stats_when_count_zero(): + receiver = _user("recv6") + giver = _user("giver6") + now = datetime.now(timezone.utc).isoformat() + uid, _ = _award(giver, receiver, "only", now) + recompute_user_award_stats(receiver) + revoke_award(uid, giver) + row = get_table("users").find_one(uid=receiver) + assert row["award_count"] == 0 + assert not row.get("last_award_uid") + + +def test_revoke_soft_deletes_attachments_under_one_stamp(): + receiver = _user("recv7") + giver = _user("giver7") + admin = _user("admin7") + now = datetime.now(timezone.utc).isoformat() + uid, _ = _award(giver, receiver, "with files", now) + a512 = _attachment(receiver) + a256 = _attachment(giver) + a64 = _attachment(giver) + get_table("awards").update( + { + "uid": uid, + "attachment_uid_512": a512, + "attachment_uid_256": a256, + "attachment_uid_64": a64, + }, + ["uid"], + ) + revoke_award(uid, admin) + award = get_table("awards").find_one(uid=uid) + assert award.get("deleted_at") + stamps = { + get_table("attachments").find_one(uid=a512).get("deleted_at"), + get_table("attachments").find_one(uid=a256).get("deleted_at"), + get_table("attachments").find_one(uid=a64).get("deleted_at"), + } + assert len(stamps) == 1 + assert award.get("deleted_at") in stamps \ No newline at end of file diff --git a/tests/unit/services/containers.py.bak b/tests/unit/services/containers.py.bak deleted file mode 100644 index f6a6c491..00000000 --- a/tests/unit/services/containers.py.bak +++ /dev/null @@ -1,334 +0,0 @@ -# retoor - -from datetime import timedelta -import pytest -import requests -from tests.conftest import BASE_URL, run_async -from devplacepy.database import db, get_table, init_db, refresh_snapshot -from devplacepy.utils import generate_uid -from devplacepy import config, project_files -from devplacepy.services.containers import api, store, runtime -from devplacepy.services.containers.backend.base import Mount, PortMapping, RunSpec -from devplacepy.services.containers.backend.docker_cli import build_run_argv, parse_size -from devplacepy.services.containers.backend.fake import FakeBackend -from devplacepy.services.containers.service import ContainerService -from devplacepy.services.containers.api import INSTANCE_LABEL -from devplacepy.services.devii.tasks.schedule import Schedule, now_utc -_CONTAINER_TABLES = ( - "instances", - "instance_events", - "instance_metrics", - "instance_schedules", -) -@pytest.fixture(autouse=True) -def _init_db_containers(): - init_db() - yield -@pytest.fixture -def env(tmp_path, monkeypatch): - fake = FakeBackend() - runtime.set_backend(fake) - monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path / "ws") - pid = "ctest-p1" - project = {"uid": pid, "slug": "ctest", "title": "C", "user_uid": "ctest-u1"} - user = {"uid": "ctest-u1", "username": "ctestadmin"} - get_table("users").insert( - { - "uid": user["uid"], - "username": user["username"], - "email": "ctestadmin@example.com", - "api_key": generate_uid(), - "password_hash": "", - "role": "Member", - "is_active": True, - "level": 1, - "xp": 0, - "stars": 0, - "deleted_at": None, - "deleted_by": None, - } - ) - refresh_snapshot() - project_files.write_text_file(pid, user, "app.py", "print(1)\n") - yield {"fake": fake, "project": project, "user": user} - runtime.set_backend(None) - get_table("users").delete(uid=user["uid"]) - for table in _CONTAINER_TABLES: - if table in db.tables: - for row in [r for r in get_table(table).find()]: - if str(row.get("project_uid", "")).startswith("ctest"): - get_table(table).delete(uid=row["uid"]) - for row in list(get_table("project_files").find()): - if str(row.get("project_uid", "")).startswith("ctest"): - get_table("project_files").delete(uid=row["uid"]) -def _ready_instance(env, **kwargs): - return run_async( - api.create_instance(env["project"], name=kwargs.pop("name", "inst"), **kwargs) - ) -def _promote_admin(username: str) -> None: - users = get_table("users") - user = users.find_one(username=username) - if user: - users.update({"uid": user["uid"], "role": "Admin"}, ["uid"]) -def _api_key(username: str) -> str: - refresh_snapshot() - return get_table("users").find_one(username=username)["api_key"] - - -def test_build_run_argv_exact(): - spec = RunSpec( - image="ppy:latest", - name="inst", - labels={INSTANCE_LABEL: "u1"}, - env={"A": "B"}, - cpu_limit="1.5", - mem_limit="512m", - ports=[PortMapping(8080, 80)], - mounts=[Mount("/ws", "/app")], - restart_policy="on-failure", - command=["python", "app.py"], - ) - argv = build_run_argv(spec) - assert argv[:5] == ["docker", "run", "-d", "--name", "inst"] - assert "--label" in argv and f"{INSTANCE_LABEL}=u1" in argv - assert "--cpus" in argv and "1.5" in argv - assert "-p" in argv and "8080:80/tcp" in argv - assert "-v" in argv and "/ws:/app:rw" in argv - assert "--restart" in argv and "on-failure" in argv - assert argv[-3:] == ["ppy:latest", "python", "app.py"] - - -def test_never_policy_not_passed_to_docker(): - spec = RunSpec(image="ppy:latest", name="n", restart_policy="never") - assert "--restart" not in build_run_argv(spec) - - -def test_parse_size(): - assert parse_size("1.0GiB") == 1024**3 - assert parse_size("512MB") == 512 * 1024**2 - - -def test_create_instance_uses_shared_image(env): - inst = run_async( - api.create_instance( - env["project"], name="inst", actor=("user", env["user"]["uid"]) - ) - ) - assert inst["name"] == "inst" - assert inst["owner_uid"] == "ctest-u1" - spec = api.run_spec_for(inst, config.CONTAINER_IMAGE) - assert spec.image == config.CONTAINER_IMAGE - assert any(m.container == "/app" for m in spec.mounts) - - -def test_create_instance_requires_built_image(env): - async def no_image(ref): - return False - - env["fake"].image_exists = no_image - with pytest.raises(api.ContainerError): - run_async(api.create_instance(env["project"], name="inst")) - - -def test_reconcile_launches_and_stops(env): - inst = _ready_instance(env, restart_policy="never", autostart=True) - assert inst["desired_state"] == store.DESIRED_RUNNING - service = ContainerService() - run_async(service.run_once()) - refresh_snapshot() - inst = store.get_instance(inst["uid"]) - assert inst["status"] == store.ST_RUNNING and inst["container_id"] - assert [r.name for r in run_async(env["fake"].ps())] == [inst["slug"]] - api.set_desired_state(inst, store.DESIRED_STOPPED) - run_async(service.run_once()) - refresh_snapshot() - assert store.get_instance(inst["uid"])["status"] == store.ST_STOPPED - - -def test_manual_start_relaunches_after_never_policy_exit(env): - fake = env["fake"] - inst = _ready_instance(env, restart_policy="never", autostart=True) - service = ContainerService() - run_async(service.run_once()) - refresh_snapshot() - inst = store.get_instance(inst["uid"]) - cid = inst["container_id"] - assert inst["status"] == store.ST_RUNNING and cid - - fake._set_state(cid, "exited", 0) - run_async(service.run_once()) - refresh_snapshot() - inst = store.get_instance(inst["uid"]) - assert inst["status"] == store.ST_STOPPED - assert inst["desired_state"] == store.DESIRED_STOPPED - - api.set_desired_state(inst, store.DESIRED_RUNNING) - refresh_snapshot() - run_async(service.run_once()) - refresh_snapshot() - inst = store.get_instance(inst["uid"]) - assert inst["status"] == store.ST_RUNNING - assert inst["desired_state"] == store.DESIRED_RUNNING - assert cid in fake.removed - assert inst["container_id"] and inst["container_id"] != cid - - -def test_reconcile_reaps_orphan(env): - fake = env["fake"] - run_async( - fake.run( - RunSpec( - image="ppy:latest", name="ghost", labels={INSTANCE_LABEL: "missing-uid"} - ) - ) - ) - service = ContainerService() - run_async(service.run_once()) - assert not run_async(fake.ps()) - assert fake.removed - - -def test_reconcile_removes_marked_instance(env): - inst = _ready_instance(env, autostart=True) - service = ContainerService() - run_async(service.run_once()) - refresh_snapshot() - inst = store.get_instance(inst["uid"]) - api.mark_for_removal(inst) - run_async(service.run_once()) - refresh_snapshot() - assert store.get_instance(inst["uid"]) is None - assert not run_async(env["fake"].ps()) - - -def test_schedule_fires(env): - inst = _ready_instance(env, autostart=False) - assert inst["desired_state"] == store.DESIRED_STOPPED - past = Schedule(kind="once", run_at=now_utc() - timedelta(hours=1)) - api.add_schedule(inst, "start", past) - service = ContainerService() - run_async(service._fire_schedules()) - refresh_snapshot() - assert store.get_instance(inst["uid"])["desired_state"] == store.DESIRED_RUNNING - - -def test_ingress_validation(env): - from devplacepy.services.containers.backend.base import PortMapping - - assert api.validate_ingress("zwoeks", 8899, [PortMapping(8899, 8899)]) == ( - "zwoeks", - 8899, - ) - assert api.validate_ingress("", None, []) == ("", 0) - with pytest.raises(api.ContainerError): - api.validate_ingress("BAD SLUG", None, [PortMapping(80, 80)]) - with pytest.raises(api.ContainerError): - api.validate_ingress("x", 9999, [PortMapping(80, 80)]) - store.create_instance( - { - "uid": "z", - "project_uid": "ctest-p1", - "name": "z", - "ports_json": "[]", - "ingress_slug": "taken", - } - ) - with pytest.raises(api.ContainerError): - api.validate_ingress("taken", None, [PortMapping(80, 80)]) - - -def test_validate_boot_languages(): - assert api.validate_boot("none", "ignored") == ("none", "") - assert api.validate_boot("python", "print(1)") == ("python", "print(1)") - with pytest.raises(api.ContainerError): - api.validate_boot("ruby", "puts 1") - with pytest.raises(api.ContainerError): - api.validate_boot("bash", " ") - - -def test_boot_script_precedence_in_run_spec(env): - inst = _ready_instance( - env, - boot_language="python", - boot_script="print('boot')\n", - boot_command="python other.py", - autostart=False, - ) - spec = api.run_spec_for(inst, config.CONTAINER_IMAGE) - assert spec.command == ["python", "/app/.devplace_boot.py"] - inst2 = _ready_instance( - env, name="i2", boot_command="python serve.py", autostart=False - ) - spec2 = api.run_spec_for(inst2, config.CONTAINER_IMAGE) - assert spec2.command == ["/bin/sh", "-c", "python serve.py"] - - -def test_materialize_boot_script_writes_file(env, tmp_path): - workspace = tmp_path / "boot-ws" - inst = _ready_instance( - env, name="boot", boot_language="bash", boot_script="echo hi\n", autostart=False - ) - store.update_instance(inst["uid"], {"workspace_dir": str(workspace)}) - inst = store.get_instance(inst["uid"]) - api.materialize_boot_script(inst) - written = workspace / ".devplace_boot.sh" - assert written.is_file() - assert written.read_text() == "echo hi\n" - - -def test_run_as_uid_drives_pravda_env(env): - _promote_admin(env["user"]["username"]) - key = _api_key(env["user"]["username"]) - inst = _ready_instance( - env, name="runas", run_as_uid=env["user"]["uid"], autostart=False - ) - pravda = api.pravda_env(inst) - assert pravda["PRAVDA_API_KEY"] == key - assert pravda["PRAVDA_USER_UID"] == env["user"]["uid"] - - -def test_run_as_uid_rejects_unknown_user(env): - with pytest.raises(api.ContainerError): - run_async( - api.create_instance( - env["project"], name="bad", run_as_uid="nope-uid", autostart=False - ) - ) - - -def test_start_on_boot_pass_forces_running(env): - inst = _ready_instance(env, name="boot", start_on_boot=True, autostart=False) - assert inst["desired_state"] == store.DESIRED_STOPPED - service = ContainerService() - service._boot_pass(store.all_instances()) - refresh_snapshot() - assert store.get_instance(inst["uid"])["desired_state"] == store.DESIRED_RUNNING - - -def test_status_change_records_event(env): - inst = _ready_instance(env, name="hist", restart_policy="never", autostart=True) - service = ContainerService() - run_async(service.run_once()) - refresh_snapshot() - inst = store.get_instance(inst["uid"]) - api.set_desired_state(inst, store.DESIRED_STOPPED) - run_async(service.run_once()) - refresh_snapshot() - events = [e["event"] for e in store.list_events(inst["uid"])] - assert "status_change" in events - - -def test_bidirectional_sync_newer_wins(env, tmp_path): - workspace = tmp_path / "sync-ws" - workspace.mkdir() - pid = env["project"]["uid"] - user = env["user"] - project_files.write_text_file(pid, user, "shared.txt", "from project\n") - fs_only = workspace / "fromfs.txt" - fs_only.write_text("from fs\n") - counts = project_files.sync_dir_bidirectional(pid, str(workspace), user) - assert counts["exported"] >= 1 - assert counts["imported"] >= 1 - assert (workspace / "shared.txt").read_text() == "from project\n" - imported = project_files.read_file(pid, "fromfs.txt") - assert imported["content"] == "from fs\n" diff --git a/tests/unit/services/devii/actions/catalog.py b/tests/unit/services/devii/actions/catalog.py index 6ccb9a03..47282bfd 100644 --- a/tests/unit/services/devii/actions/catalog.py +++ b/tests/unit/services/devii/actions/catalog.py @@ -48,6 +48,20 @@ def test_seo_report_is_public_read_only_http_action(): assert "uid" in names +def test_ui_prompt_is_local_interaction_action(): + from devplacepy.services.devii.registry import CATALOG + + by_name = CATALOG.by_name() + action = by_name["ui_prompt"] + assert action.handler == "interaction" + assert action.requires_auth is False + names = {p.name for p in action.params} + assert "title" in names + assert "widgets" in names + assert "ui_cancel" in by_name + assert "ui_notify" in by_name + + def test_block_mute_tools_exist_as_http_actions(): expected = { "block_user": "/block/{username}", diff --git a/tests/unit/services/devii/interaction/__init__.py b/tests/unit/services/devii/interaction/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/tests/unit/services/devii/interaction/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/tests/unit/services/devii/interaction/answer_text.py b/tests/unit/services/devii/interaction/answer_text.py new file mode 100644 index 00000000..80cc4276 --- /dev/null +++ b/tests/unit/services/devii/interaction/answer_text.py @@ -0,0 +1,48 @@ +# retoor + +from tests.conftest import run_async +from devplacepy.services.devii.interaction.broker import InteractionBroker + + +def test_answer_text_completes_open_prompt(): + captured = {} + + async def wait_site(interaction_id, frame, timeout): + captured["id"] = interaction_id + outcome = broker.answer_text("13/01/1990") + assert outcome["handled"] is True + assert outcome["status"] == "submitted" + return outcome["result"] + + broker = InteractionBroker("main", wait_site=wait_site) + result = run_async( + broker.prompt( + { + "title": "When?", + "widgets": [{"type": "date", "name": "day", "label": "Day"}], + } + ) + ) + assert result.status == "submitted" + assert result.values == {"day": "13/01/1990"} + assert result.meta.get("via") == "text" + + +def test_answer_text_rejects_free_chat_as_date(): + async def wait_site(interaction_id, frame, timeout): + bad = broker.answer_text("Show me number input then.") + assert bad["status"] == "error" + good = broker.answer_text("13/01/1990") + return good["result"] + + broker = InteractionBroker("main", wait_site=wait_site) + result = run_async( + broker.prompt( + { + "title": "When?", + "widgets": [{"type": "date", "name": "day", "label": "Day"}], + } + ) + ) + assert result.status == "submitted" + assert result.values == {"day": "13/01/1990"} diff --git a/tests/unit/services/devii/interaction/broker.py b/tests/unit/services/devii/interaction/broker.py new file mode 100644 index 00000000..1f07dfb1 --- /dev/null +++ b/tests/unit/services/devii/interaction/broker.py @@ -0,0 +1,93 @@ +# retoor + +import asyncio + +from tests.conftest import run_async +from devplacepy.services.devii.interaction.broker import InteractionBroker + + +def test_broker_site_prompt_returns_submitted_values(): + async def wait_site(interaction_id, frame, timeout): + assert frame["title"] == "Deploy?" + assert frame["widgets"][0]["type"] == "confirm" + return { + "status": "submitted", + "interaction_id": interaction_id, + "values": {"delete": True}, + } + + broker = InteractionBroker("main", wait_site=wait_site) + + async def run(): + return await broker.prompt( + { + "title": "Deploy?", + "widgets": [ + { + "type": "confirm", + "name": "delete", + "label": "Delete", + "default": False, + } + ], + } + ) + + result = run_async(run()) + assert result.status == "submitted" + assert result.values == {"delete": True} + assert result.meta["adapter"] == "site-chat" + assert result.meta["degraded"] is False + assert broker.open_id() is None + + +def test_broker_plain_adapter_parses_reply(): + async def present_plain(menu: str) -> str: + assert "yes" in menu.lower() or "Delete" in menu + return "yes" + + broker = InteractionBroker("cli", present_plain=present_plain) + result = run_async( + broker.prompt( + { + "title": "Delete?", + "widgets": [ + {"type": "confirm", "name": "ok", "label": "OK", "default": False} + ], + } + ) + ) + assert result.status == "submitted" + assert result.values == {"ok": True} + assert result.meta["degraded"] is True + + +def test_broker_invalid_args_raise(): + from devplacepy.services.devii.errors import ToolInputError + + broker = InteractionBroker("main") + try: + run_async(broker.prompt({"title": "x", "widgets": []})) + assert False, "expected ToolInputError" + except ToolInputError: + pass + + +def test_broker_timeout_status(): + async def wait_site(interaction_id, frame, timeout): + raise asyncio.TimeoutError() + + broker = InteractionBroker("main", wait_site=wait_site) + result = run_async( + broker.prompt( + { + "title": "T?", + "timeout_sec": 1, + "widgets": [ + {"type": "confirm", "name": "ok", "label": "OK"} + ], + } + ) + ) + assert result.status == "timeout" + assert result.values == {} diff --git a/tests/unit/services/devii/interaction/capabilities.py b/tests/unit/services/devii/interaction/capabilities.py new file mode 100644 index 00000000..f51b099d --- /dev/null +++ b/tests/unit/services/devii/interaction/capabilities.py @@ -0,0 +1,61 @@ +# retoor + +from devplacepy.services.devii.interaction.capabilities import ( + CHANNEL_CLI, + CHANNEL_SITE_CHAT, + CHANNEL_TELEGRAM, + CHANNEL_UNKNOWN, + channel_id_for_session, + context_for, + fragment_for, + interactions_enabled, +) + + +def test_session_channel_mapping(): + assert channel_id_for_session("main") == CHANNEL_SITE_CHAT + assert channel_id_for_session("docs") == CHANNEL_SITE_CHAT + assert channel_id_for_session("telegram") == CHANNEL_TELEGRAM + assert channel_id_for_session("cli") == CHANNEL_CLI + assert channel_id_for_session("nope") == CHANNEL_UNKNOWN + + +def test_site_context_exposes_ui_prompt(local_db): + ctx = context_for(CHANNEL_SITE_CHAT, owner_kind="user", owner_id="u1") + assert ctx.capabilities["rich_widgets"] is True + assert "ui_prompt" in ctx.tools + assert "ui_cancel" in ctx.tools + assert "ui_notify" in ctx.tools + assert "interactions_get" in ctx.tools + assert "interactions_set" in ctx.tools + + +def test_open_interaction_gates_to_cancel_only(local_db): + ctx = context_for( + CHANNEL_SITE_CHAT, + open_interaction_id="ix-abc", + owner_kind="user", + owner_id="u1", + ) + assert "ui_cancel" in ctx.tools + assert "ui_prompt" not in ctx.tools + assert "interactions_get" in ctx.tools + + +def test_unknown_channel_has_no_ui_prompt(local_db): + ctx = context_for(CHANNEL_UNKNOWN, owner_kind="user", owner_id="u1") + assert "ui_prompt" not in ctx.tools + assert ctx.capabilities["confirm"] is False + assert "interactions_get" in ctx.tools + + +def test_fragment_is_compact(local_db): + text = fragment_for(context_for(CHANNEL_TELEGRAM, owner_kind="guest")) + assert "CHANNEL: telegram" in text + assert "TOOLS: ui_prompt" in text + assert "INTERACTION: none" in text + assert len(text) < 600 + + +def test_interactions_enabled_default(local_db): + assert interactions_enabled("guest", "") is True diff --git a/tests/unit/services/devii/interaction/controller.py b/tests/unit/services/devii/interaction/controller.py new file mode 100644 index 00000000..055170ae --- /dev/null +++ b/tests/unit/services/devii/interaction/controller.py @@ -0,0 +1,55 @@ +# retoor + +import json + +from tests.conftest import run_async +from devplacepy.services.devii.interaction.broker import InteractionBroker +from devplacepy.services.devii.interaction.controller import InteractionController + + +def test_ui_prompt_dispatch(): + async def wait_site(interaction_id, frame, timeout): + return { + "status": "submitted", + "interaction_id": interaction_id, + "values": {"env": "staging"}, + } + + broker = InteractionBroker("main", wait_site=wait_site) + controller = InteractionController(broker) + raw = run_async( + controller.dispatch( + "ui_prompt", + { + "title": "Env?", + "widgets": [ + { + "type": "choice", + "name": "env", + "label": "Env", + "options": [ + {"value": "staging", "label": "Staging"}, + {"value": "prod", "label": "Prod"}, + ], + } + ], + }, + ) + ) + data = json.loads(raw) + assert data["status"] == "submitted" + assert data["values"]["env"] == "staging" + + +def test_ui_notify_on_site(): + events = [] + + async def emit(payload): + events.append(payload) + + broker = InteractionBroker("main", emit=emit) + controller = InteractionController(broker) + raw = run_async(controller.dispatch("ui_notify", {"text": "Working..."})) + data = json.loads(raw) + assert data["status"] == "ok" + assert events and events[0]["type"] == "status" diff --git a/tests/unit/services/devii/interaction/markdown.py b/tests/unit/services/devii/interaction/markdown.py new file mode 100644 index 00000000..92ea9c07 --- /dev/null +++ b/tests/unit/services/devii/interaction/markdown.py @@ -0,0 +1,57 @@ +# retoor + +from devplacepy.services.devii.interaction.markdown import ( + project_markdown, + project_plain_menu, +) +from devplacepy.services.devii.interaction.schema import validate_prompt_args + + +def _request(): + return validate_prompt_args( + { + "title": "Deploy to staging?", + "description": "Ship the current build.", + "widgets": [ + { + "type": "choice", + "name": "env", + "label": "Target environment", + "default": "staging", + "options": [ + {"value": "production", "label": "Production"}, + {"value": "staging", "label": "Staging"}, + {"value": "dev", "label": "Development"}, + ], + }, + { + "type": "text", + "name": "notes", + "label": "Ship notes", + "required": False, + "max_length": 200, + }, + ], + } + ) + + +def test_markdown_projection_has_title_and_id(): + md = project_markdown(_request(), "ix-a1b2") + assert "### Deploy to staging?" in md + assert "`staging`" in md + assert "" in md + assert "[Confirm]" in md + + +def test_plain_menu_for_confirm(): + req = validate_prompt_args( + { + "title": "Delete?", + "widgets": [ + {"type": "confirm", "name": "delete", "label": "Delete", "default": False} + ], + } + ) + text = project_plain_menu(req) + assert "yes | no" in text.lower() or "Reply: yes" in text diff --git a/tests/unit/services/devii/interaction/parse.py b/tests/unit/services/devii/interaction/parse.py new file mode 100644 index 00000000..19763483 --- /dev/null +++ b/tests/unit/services/devii/interaction/parse.py @@ -0,0 +1,117 @@ +# retoor + +from devplacepy.services.devii.interaction.parse import parse_plain_reply +from devplacepy.services.devii.interaction.schema import validate_prompt_args + + +def test_parse_confirm_yes(): + req = validate_prompt_args( + { + "title": "Delete?", + "widgets": [{"type": "confirm", "name": "delete", "label": "Delete"}], + } + ) + status, values, err = parse_plain_reply(req, "yes") + assert status == "submitted" + assert values == {"delete": True} + assert err is None + + +def test_parse_choice_by_index(): + req = validate_prompt_args( + { + "title": "Env?", + "widgets": [ + { + "type": "choice", + "name": "env", + "label": "Env", + "options": [ + {"value": "production", "label": "Production"}, + {"value": "staging", "label": "Staging"}, + ], + } + ], + } + ) + status, values, err = parse_plain_reply(req, "2") + assert status == "submitted" + assert values == {"env": "staging"} + + +def test_parse_cancel(): + req = validate_prompt_args( + { + "title": "Env?", + "widgets": [ + { + "type": "choice", + "name": "env", + "label": "Env", + "options": [{"value": "a", "label": "A"}], + } + ], + } + ) + status, values, err = parse_plain_reply(req, "cancel") + assert status == "cancelled" + assert values == {} + + +def test_parse_date_european(): + req = validate_prompt_args( + { + "title": "When?", + "widgets": [{"type": "date", "name": "day", "label": "Day"}], + } + ) + status, values, err = parse_plain_reply(req, "13/01/1990") + assert status == "submitted" + assert values == {"day": "13/01/1990"} + assert err is None + + +def test_parse_date_iso_normalized_to_european(): + req = validate_prompt_args( + { + "title": "When?", + "widgets": [{"type": "date", "name": "day", "label": "Day"}], + } + ) + status, values, err = parse_plain_reply(req, "1990-01-13") + assert status == "submitted" + assert values == {"day": "13/01/1990"} + + +def test_parse_confirm_dutch_ja(): + req = validate_prompt_args( + { + "title": "Ok?", + "widgets": [{"type": "confirm", "name": "ok", "label": "Ok"}], + } + ) + status, values, err = parse_plain_reply(req, "ja") + assert status == "submitted" + assert values == {"ok": True} + + +def test_parse_choice_by_label(): + req = validate_prompt_args( + { + "title": "Lang?", + "widgets": [ + { + "type": "choice", + "name": "lang", + "label": "Language", + "options": [ + {"value": "python", "label": "Python"}, + {"value": "rust", "label": "Rust"}, + ], + } + ], + } + ) + status, values, err = parse_plain_reply(req, "Rust") + assert status == "submitted" + assert values == {"lang": "rust"} diff --git a/tests/unit/services/devii/interaction/prefs.py b/tests/unit/services/devii/interaction/prefs.py new file mode 100644 index 00000000..68240559 --- /dev/null +++ b/tests/unit/services/devii/interaction/prefs.py @@ -0,0 +1,61 @@ +# retoor + +from devplacepy.database import get_table, set_setting +from devplacepy.services.devii.interaction import prefs +from devplacepy.utils.accounts import register_account + + +def _user(username="ixpref1"): + try: + register_account(username, f"{username}@t.dev", "secret123") + except Exception: + pass + return get_table("users").find_one(username=username) + + +def test_admin_default_applies_when_user_inherits(local_db): + set_setting("devii_interactions_default", "1") + user = _user("ixpref_inherit") + get_table("users").update( + {"uid": user["uid"], "interactions_enabled": -1}, ["uid"] + ) + assert prefs.effective_for("user", user["uid"]) is True + snap = prefs.snapshot("user", user["uid"]) + assert snap["source"] == "default" + assert snap["override"] is None + + +def test_user_override_off(local_db): + set_setting("devii_interactions_default", "1") + user = _user("ixpref_off") + prefs.set_user_pref(user["uid"], False) + assert prefs.effective_for("user", user["uid"]) is False + snap = prefs.snapshot("user", user["uid"]) + assert snap["source"] == "user" + assert snap["override"] is False + assert snap["default"] is True + + +def test_user_override_on_when_default_off(local_db): + set_setting("devii_interactions_default", "0") + user = _user("ixpref_on") + prefs.set_user_pref(user["uid"], True) + assert prefs.effective_for("user", user["uid"]) is True + assert prefs.admin_default() is False + + +def test_reset_returns_to_default(local_db): + set_setting("devii_interactions_default", "0") + user = _user("ixpref_reset") + prefs.set_user_pref(user["uid"], True) + snap = prefs.set_user_pref(user["uid"], None) + assert snap["source"] == "default" + assert snap["enabled"] is False + assert get_table("users").find_one(uid=user["uid"])["interactions_enabled"] == -1 + + +def test_guest_uses_admin_default(local_db): + set_setting("devii_interactions_default", "0") + assert prefs.effective_for("guest", "cookie") is False + set_setting("devii_interactions_default", "1") + assert prefs.effective_for("guest", "cookie") is True diff --git a/tests/unit/services/devii/interaction/schema.py b/tests/unit/services/devii/interaction/schema.py new file mode 100644 index 00000000..1f6f170a --- /dev/null +++ b/tests/unit/services/devii/interaction/schema.py @@ -0,0 +1,96 @@ +# retoor + +import pytest + +from devplacepy.services.devii.errors import ToolInputError +from devplacepy.services.devii.interaction.schema import ( + validate_prompt_args, +) + + +def test_valid_confirm_prompt(): + req = validate_prompt_args( + { + "title": "Delete archive row?", + "description": "Remote files are untouched.", + "widgets": [ + { + "type": "confirm", + "name": "delete", + "label": "Delete archive row", + "default": False, + } + ], + "submit_label": "Delete", + "cancel_label": "Keep", + } + ) + assert req.title.startswith("Delete") + assert req.widgets[0].type == "confirm" + assert req.widgets[0].name == "delete" + + +def test_choice_requires_options(): + with pytest.raises(ToolInputError): + validate_prompt_args( + { + "title": "Env?", + "widgets": [{"type": "choice", "name": "env", "label": "Env"}], + } + ) + + +def test_rejects_empty_widgets(): + with pytest.raises(ToolInputError): + validate_prompt_args({"title": "x", "widgets": []}) + + +def test_rejects_bad_option_value(): + with pytest.raises(ToolInputError): + validate_prompt_args( + { + "title": "Pick", + "widgets": [ + { + "type": "choice", + "name": "x", + "label": "X", + "options": [{"value": "BAD VALUE!", "label": "Bad"}], + } + ], + } + ) + + +def test_group_nesting_limit(): + leaf = { + "type": "text", + "name": "a", + "label": "A", + "required": False, + } + deep = { + "type": "group", + "label": "G1", + "widgets": [ + { + "type": "group", + "label": "G2", + "widgets": [ + { + "type": "group", + "label": "G3", + "widgets": [ + { + "type": "group", + "label": "G4", + "widgets": [leaf], + } + ], + } + ], + } + ], + } + with pytest.raises(ToolInputError): + validate_prompt_args({"title": "Deep", "widgets": [deep]}) diff --git a/tests/unit/services/devii/session.py b/tests/unit/services/devii/session.py index 41bf2233..d5e58bdb 100644 --- a/tests/unit/services/devii/session.py +++ b/tests/unit/services/devii/session.py @@ -41,8 +41,34 @@ def test_main_channel_keeps_full_catalog_and_behavior(local_db): main = _session("main", "sess-main") names = _tool_names(main) assert "search_docs" in names + assert "ui_prompt" in names + assert "ui_cancel" in names assert len(names) > 1 - assert "TRUTH RULES AND BEHAVIOR" in main._compose_system_prompt() + prompt = main._compose_system_prompt() + assert "TRUTH RULES AND BEHAVIOR" in prompt + assert "CHANNEL: site-chat" in prompt + assert "Interactive asks (CA-IWP)" in prompt + + +def test_docs_channel_excludes_ui_prompt(local_db): + docs = _session("docs", "sess-docs-ui") + assert "ui_prompt" not in _tool_names(docs) + + +def test_telegram_channel_includes_ui_prompt(local_db): + tg = _session("telegram", "sess-tg-ui") + names = _tool_names(tg) + assert "ui_prompt" in names + prompt = tg._compose_system_prompt() + assert "CHANNEL: telegram" in prompt + + +def test_guest_main_has_ui_prompt_without_pref_tools(local_db): + main = _session("main", "sess-guest-ui") + names = _tool_names(main) + assert "ui_prompt" in names + assert "interactions_get" not in names + assert "interactions_set" not in names def test_docs_channel_does_not_start_scheduler(local_db): diff --git a/tests/unit/services/devii/text_normalize.py b/tests/unit/services/devii/text_normalize.py new file mode 100644 index 00000000..82c60ec7 --- /dev/null +++ b/tests/unit/services/devii/text_normalize.py @@ -0,0 +1,23 @@ +# retoor + +from devplacepy.services.devii.text import normalize_newlines + + +def test_normalize_literal_backslash_n(): + raw = "[compacted earlier turns]\n\nHello\\n\\n## Title\\n\\n| A | B |" + out = normalize_newlines(raw) + assert "\\n" not in out or out.count("\n") > raw.count("\n") + assert "\n\n## Title\n\n" in out + assert out.startswith("[compacted earlier turns]\n\nHello\n") + + +def test_normalize_leaves_real_newlines(): + raw = "line one\nline two\nline three" + assert normalize_newlines(raw) == raw + + +def test_normalize_double_escaped(): + raw = "a\\\\n\\\\nb" + out = normalize_newlines(raw) + assert "\n" in out + assert out == "a\n\nb" diff --git a/tests/unit/services/jobs/award_service.py b/tests/unit/services/jobs/award_service.py new file mode 100644 index 00000000..512589b9 --- /dev/null +++ b/tests/unit/services/jobs/award_service.py @@ -0,0 +1,174 @@ +# retoor + +import base64 +import io +from datetime import datetime, timezone + +import pytest +from PIL import Image +from devplacepy.db_client import get_award_usage, get_table, init_db +from devplacepy.db_client import recompute_user_award_stats +from devplacepy.services.jobs.award_service import AwardService +from devplacepy.utils import generate_uid, make_combined_slug +from tests.conftest import run_async + + +@pytest.fixture(autouse=True) +def _db(local_db, tmp_path, monkeypatch): + init_db() + monkeypatch.setattr("devplacepy.attachments.ATTACHMENTS_DIR", tmp_path / "attachments") + yield + + +def _png_bytes(): + buf = io.BytesIO() + Image.new("RGBA", (64, 64), (40, 80, 120, 255)).save(buf, format="PNG") + return buf.getvalue() + + +class _FakeResponse: + def __init__(self, payload, headers=None): + self._payload = payload + self.headers = headers or {} + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +class _FakeClient: + def __init__(self, *args, **kwargs): + self._png = _png_bytes() + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def post(self, url, json=None, headers=None): + encoded = base64.b64encode(self._png).decode("ascii") + return _FakeResponse( + {"data": [{"b64_json": encoded}]}, + headers={ + "X-Gateway-Cost-USD": "0.001", + "X-Gateway-Prompt-Tokens": "10", + "X-Gateway-Completion-Tokens": "0", + "X-Gateway-Total-Tokens": "10", + "X-Gateway-Upstream-Latency-Ms": "100", + "X-Gateway-Total-Latency-Ms": "120", + }, + ) + + +def _seed_pending(giver_uid, receiver_uid, description="Nice job"): + uid = generate_uid() + slug = make_combined_slug(description, uid) + get_table("awards").insert( + { + "uid": uid, + "slug": slug, + "description": description, + "giver_uid": giver_uid, + "receiver_uid": receiver_uid, + "attachment_uid_512": "", + "attachment_uid_256": "", + "attachment_uid_64": "", + "generated_at": None, + "created_at": datetime.now(timezone.utc).isoformat(), + "job_uid": "", + "deleted_at": None, + "deleted_by": None, + } + ) + return uid, slug + + +def _user(prefix): + uid = generate_uid() + get_table("users").insert( + { + "uid": uid, + "username": f"{prefix}_{uid[:8]}", + "email": f"{uid[:8]}@t.dev", + "api_key": generate_uid(), + "created_at": datetime.now(timezone.utc).isoformat(), + } + ) + return uid + + +def _job(award_uid, giver_uid, receiver_uid, api_key): + return { + "uid": generate_uid(), + "payload": { + "award_uid": award_uid, + "giver_uid": giver_uid, + "receiver_uid": receiver_uid, + "description": "Nice job", + "api_key": api_key, + }, + } + + +def test_process_finalizes_award(monkeypatch): + monkeypatch.setattr("devplacepy.services.jobs.award_service.stealth.stealth_sync_client", _FakeClient) + giver = _user("giver") + receiver = _user("recv") + api_key = get_table("users").find_one(uid=giver)["api_key"] + award_uid, _ = _seed_pending(giver, receiver) + run_async(AwardService().process(_job(award_uid, giver, receiver, api_key))) + row = get_table("awards").find_one(uid=award_uid) + assert row.get("generated_at") + assert row.get("attachment_uid_512") + assert row.get("attachment_uid_256") + assert row.get("attachment_uid_64") + recompute_user_award_stats(receiver) + user = get_table("users").find_one(uid=receiver) + assert user.get("award_count") == 1 + + +def test_process_is_idempotent(monkeypatch): + monkeypatch.setattr("devplacepy.services.jobs.award_service.stealth.stealth_sync_client", _FakeClient) + giver = _user("giver2") + receiver = _user("recv2") + api_key = get_table("users").find_one(uid=giver)["api_key"] + award_uid, _ = _seed_pending(giver, receiver) + job = _job(award_uid, giver, receiver, api_key) + svc = AwardService() + run_async(svc.process(job)) + first = get_table("awards").find_one(uid=award_uid) + run_async(svc.process(job)) + second = get_table("awards").find_one(uid=award_uid) + assert first["generated_at"] == second["generated_at"] + assert first["attachment_uid_512"] == second["attachment_uid_512"] + + +def test_process_skips_soft_deleted_award(monkeypatch): + monkeypatch.setattr("devplacepy.services.jobs.award_service.stealth.stealth_sync_client", _FakeClient) + giver = _user("giver3") + receiver = _user("recv3") + api_key = get_table("users").find_one(uid=giver)["api_key"] + award_uid, _ = _seed_pending(giver, receiver) + get_table("awards").update( + {"uid": award_uid, "deleted_at": datetime.now(timezone.utc).isoformat(), "deleted_by": giver}, + ["uid"], + ) + result = run_async(AwardService().process(_job(award_uid, giver, receiver, api_key))) + assert result.get("skipped") is True + assert not get_table("awards").find_one(uid=award_uid).get("generated_at") + + +def test_award_usage_accumulates_from_gateway_headers(monkeypatch): + monkeypatch.setattr("devplacepy.services.jobs.award_service.stealth.stealth_sync_client", _FakeClient) + get_table("award_usage").delete() + giver = _user("giver4") + receiver = _user("recv4") + api_key = get_table("users").find_one(uid=giver)["api_key"] + award_uid, _ = _seed_pending(giver, receiver) + run_async(AwardService().process(_job(award_uid, giver, receiver, api_key))) + usage = get_award_usage() + assert usage["calls"] >= 1 + assert usage["cost_usd"] > 0 \ No newline at end of file diff --git a/tests/unit/services/openai_gateway/gateway.py b/tests/unit/services/openai_gateway/gateway.py index 9239795c..59d787c6 100644 --- a/tests/unit/services/openai_gateway/gateway.py +++ b/tests/unit/services/openai_gateway/gateway.py @@ -107,6 +107,7 @@ def test_model_is_forced(local_db, monkeypatch): cfg, ("guest", "test"), "test", + "default", ) ) assert rt._client.calls[-1][1]["model"] == "deepseek-chat" @@ -143,6 +144,7 @@ def test_model_route_overrides_upstream(local_db, monkeypatch): cfg, ("guest", "test"), "test", + "default", ) ) url, body = rt._client.calls[-1] @@ -169,7 +171,7 @@ def test_vision_rewrites_image_to_text(local_db, monkeypatch): ], } ] - run_async(rt.handle_chat({"messages": msgs}, cfg, ("guest", "test"), "test")) + run_async(rt.handle_chat({"messages": msgs}, cfg, ("guest", "test"), "test", "default")) sent = rt._client.calls[-1][1]["messages"][0]["content"] assert isinstance(sent, str) and "vision" in sent.lower() @@ -185,6 +187,7 @@ def test_streaming_emits_sse(local_db, monkeypatch): cfg, ("guest", "test"), "test", + "default", ) ) @@ -282,6 +285,7 @@ def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch): cfg, ("guest", "test"), "test", + "default", ) ) assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b" @@ -302,6 +306,7 @@ def test_embeddings_success_records_ledger(local_db, monkeypatch): cfg, ("guest", "ledger_probe"), "test", + "default", ) ) assert resp.status_code == 200 @@ -325,7 +330,7 @@ def test_embeddings_disabled_returns_503(local_db, monkeypatch): rt = svc.runtime() resp = run_async( rt.handle_embeddings( - {"input": "hello"}, cfg, ("guest", "test"), "test" + {"input": "hello"}, cfg, ("guest", "test"), "test", "default" ) ) assert resp.status_code == 503 @@ -372,6 +377,7 @@ def test_images_remaps_alias_to_configured_model(local_db, monkeypatch): cfg, ("guest", "test"), "test", + "default", ) ) assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro" @@ -411,6 +417,7 @@ def test_image_route_overrides_upstream(local_db, monkeypatch): cfg, ("guest", "test"), "test", + "default", ) ) url, body = rt._client.calls[-1] @@ -435,6 +442,7 @@ def test_images_success_records_ledger(local_db, monkeypatch): cfg, ("guest", "img_ledger"), "test", + "default", ) ) assert resp.status_code == 200 @@ -459,7 +467,7 @@ def test_images_disabled_returns_503(local_db, monkeypatch): rt = svc.runtime() resp = run_async( rt.handle_images( - {"prompt": "badge"}, cfg, ("guest", "test"), "test" + {"prompt": "badge"}, cfg, ("guest", "test"), "test", "default" ) ) assert resp.status_code == 503 @@ -489,3 +497,271 @@ def test_compute_cost_embed_branch(): assert output_cost == 0.0 assert abs(total - 0.01) < 1e-9 assert abs(input_cost - 0.01) < 1e-9 + + +def test_app_reference_stored_in_ledger_for_chat(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_force_model"] = True + cfg["gateway_model"] = "deepseek-chat" + rt = svc.runtime() + run_async( + rt.handle_chat( + {"messages": [{"role": "user", "content": "ref_test"}]}, + cfg, + ("guest", "ref_probe"), + "test-ua", + "my-custom-app", + ) + ) + row = get_table("gateway_usage_ledger").find_one(owner_id="ref_probe") + assert row is not None + assert row["app_reference"] == "my-custom-app" + assert row["backend"] == "chat" + + +def test_app_reference_defaults_when_not_passed(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_force_model"] = True + cfg["gateway_model"] = "deepseek-chat" + rt = svc.runtime() + run_async( + rt.handle_chat( + {"messages": [{"role": "user", "content": "default_test"}]}, + cfg, + ("guest", "default_probe"), + "test-ua", + "default", + ) + ) + row = get_table("gateway_usage_ledger").find_one(owner_id="default_probe") + assert row is not None + assert row["app_reference"] == "default" + + +def test_app_reference_stored_in_ledger_for_embeddings(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_embed_enabled"] = True + cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b" + rt = svc.runtime() + run_async( + rt.handle_embeddings( + {"input": "hello"}, + cfg, + ("guest", "embed_ref"), + "test-ua", + "devplace-embed-test-v-1-0-0", + ) + ) + row = get_table("gateway_usage_ledger").find_one(owner_id="embed_ref") + assert row is not None + assert row["app_reference"] == "devplace-embed-test-v-1-0-0" + assert row["backend"] == "embed" + + +def test_app_reference_stored_in_ledger_for_images(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_image_enabled"] = True + cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro" + rt = svc.runtime() + run_async( + rt.handle_images( + {"prompt": "ref badge"}, + cfg, + ("guest", "img_ref"), + "test-ua", + "devplace-image-test-v-1-0-0", + ) + ) + row = get_table("gateway_usage_ledger").find_one(owner_id="img_ref") + assert row is not None + assert row["app_reference"] == "devplace-image-test-v-1-0-0" + assert row["backend"] == "image" + + +def test_app_reference_column_exists(local_db): + db = get_table("gateway_usage_ledger").db + if "gateway_usage_ledger" not in db.tables: + return + rows = list(db.query("PRAGMA table_info('gateway_usage_ledger')")) + column_names = [r["name"] for r in rows] + assert "app_reference" in column_names + + +def test_chat_unknown_model_falls_back_to_default(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_force_model"] = False + cfg["gateway_model"] = "deepseek-v4-flash" + rt = svc.runtime() + run_async( + rt.handle_chat( + {"model": "nonexistent-model-v99", "messages": [{"role": "user", "content": "hi"}]}, + cfg, + ("guest", "fallback_test_chat"), + "test", + "default", + ) + ) + assert rt._client.calls[-1][1]["model"] == "deepseek-v4-flash" + + +def test_embeddings_unknown_model_falls_back_to_default(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_force_model"] = False + cfg["gateway_embed_enabled"] = True + cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b" + rt = svc.runtime() + run_async( + rt.handle_embeddings( + {"model": "nonexistent-embed-model", "input": "hello"}, + cfg, + ("guest", "fallback_test_embed"), + "test", + "default", + ) + ) + assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b" + + +def test_images_unknown_model_falls_back_to_default(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_force_model"] = False + cfg["gateway_image_enabled"] = True + cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro" + rt = svc.runtime() + run_async( + rt.handle_images( + {"model": "nonexistent-image-model", "prompt": "test badge"}, + cfg, + ("guest", "fallback_test_img"), + "test", + "default", + ) + ) + assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro" + + +class FakeClientWithUsage_openai_gateway(FakeClient_openai_gateway): + async def send(self, request): + self.calls.append((request.url, request.json_body)) + body = request.json_body or {} + return FakeResp_openai_gateway( + payload={ + "id": "x", + "model": body.get("model"), + "choices": [{"message": {"content": "hi there"}}], + "usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10}, + } + ) + + +def test_stream_options_stripped_from_upstream(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + rt = svc.runtime() + run_async( + rt.handle_chat( + { + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + cfg, + ("guest", "test"), + "test", + "default", + ) + ) + upstream_payload = rt._client.calls[-1][1] + assert "stream_options" not in upstream_payload + assert upstream_payload["stream"] is False + + +def test_include_usage_emits_usage_chunk(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + rt = svc.runtime() + resp = run_async( + rt.handle_chat( + { + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + cfg, + ("guest", "test"), + "test", + "default", + ) + ) + + async def drain(): + out = [] + async for chunk in resp.body_iterator: + out.append(chunk if isinstance(chunk, str) else chunk.decode()) + return "".join(out) + + body = run_async(drain()) + assert '"usage"' in body + assert '"total_tokens": 10' in body + assert "[DONE]" in body + + +def test_no_usage_chunk_without_include_usage(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + rt = svc.runtime() + resp = run_async( + rt.handle_chat( + {"messages": [{"role": "user", "content": "hi"}], "stream": True}, + cfg, + ("guest", "test"), + "test", + "default", + ) + ) + + async def drain(): + out = [] + async for chunk in resp.body_iterator: + out.append(chunk if isinstance(chunk, str) else chunk.decode()) + return "".join(out) + + body = run_async(drain()) + assert '"usage"' not in body + assert "[DONE]" in body + + +def test_models_endpoint_publishes_molodetz(local_db): + from devplacepy.services.openai_gateway import routing + + routing.seed_default_deepseek_routes() + try: + svc = GatewayService() + resp = svc._models_response() + payload = json.loads(resp.body.decode()) + ids = {m["id"] for m in payload["data"]} + assert payload["object"] == "list" + assert {"molodetz", "molodetz-pro"} <= ids + for model in payload["data"]: + assert model["object"] == "model" + assert model["owned_by"] == "molodetz" + finally: + routing.model_store.remove("molodetz") + routing.model_store.remove("molodetz-pro") diff --git a/tests/unit/services/openai_gateway/routing.py b/tests/unit/services/openai_gateway/routing.py index 700ca82b..8c7f5bf7 100644 --- a/tests/unit/services/openai_gateway/routing.py +++ b/tests/unit/services/openai_gateway/routing.py @@ -280,3 +280,19 @@ def test_seed_default_deepseek_routes_is_idempotent_and_preserves_customization( context_window=1_048_576, ) ) + + +def test_seed_publishes_molodetz_aliases(local_db): + r.seed_default_deepseek_routes() + try: + molodetz = r.model_store.resolve("molodetz", "chat") + molodetz_pro = r.model_store.resolve("molodetz-pro", "chat") + assert molodetz is not None + assert molodetz.target_model == "deepseek-v4-flash" + assert molodetz_pro is not None + assert molodetz_pro.target_model == "deepseek-v4-pro" + sources = {row["source_model"] for row in r.model_store.list()} + assert {"molodetz", "molodetz-pro"} <= sources + finally: + r.model_store.remove("molodetz") + r.model_store.remove("molodetz-pro") diff --git a/tests/unit/services/openai_gateway/usage.py b/tests/unit/services/openai_gateway/usage.py index 9a82800c..22fe9ae0 100644 --- a/tests/unit/services/openai_gateway/usage.py +++ b/tests/unit/services/openai_gateway/usage.py @@ -219,3 +219,32 @@ def test_compute_cost_native_upstream_cost_unaffected_by_tiering(): total, _, _, native = compute_cost({"cost": 0.05}, norm, tiered, "chat") assert native is True assert total == 0.05 + + +def test_validate_app_reference_accepts_valid_slugs(): + from devplacepy.services.openai_gateway.service import _validate_app_reference + + assert _validate_app_reference("devplace-devii-v-1-0-0") == "devplace-devii-v-1-0-0" + assert _validate_app_reference("my-app") == "my-app" + assert _validate_app_reference("a") == "a" + assert _validate_app_reference("abcdefghijklmnopqrstuvwxyz0123") == "abcdefghijklmnopqrstuvwxyz0123" + assert _validate_app_reference("test_app.release-2") == "test_app.release-2" + + +def test_validate_app_reference_rejects_invalid(): + from devplacepy.services.openai_gateway.service import _validate_app_reference + + assert _validate_app_reference("") == "default" + assert _validate_app_reference(" ") == "default" + assert _validate_app_reference("hello world") == "default" + assert _validate_app_reference("name@domain") == "default" + assert _validate_app_reference("name/domain") == "default" + assert _validate_app_reference(None) == "default" + assert _validate_app_reference("abcdefghijklmnopqrstuvwxyz01234") == "default" + + +def test_validate_app_reference_strips_whitespace(): + from devplacepy.services.openai_gateway.service import _validate_app_reference + + assert _validate_app_reference(" my-app ") == "my-app" + assert _validate_app_reference("\tdevplace\t") == "devplace" diff --git a/tests/unit/services/telegram/worker.py b/tests/unit/services/telegram/worker.py index de190f19..0ea03546 100644 --- a/tests/unit/services/telegram/worker.py +++ b/tests/unit/services/telegram/worker.py @@ -38,6 +38,32 @@ def test_private_text_message_emitted(): assert messages[0]["chat_id"] == 42 and messages[0]["text"] == "hello" +def test_callback_query_emitted(): + backend = FakeTelegramBackend() + worker, frames = _worker(backend) + _run( + worker.process_update( + { + "update_id": 2, + "callback_query": { + "id": "cb1", + "from": {"id": 7}, + "data": "i=del1;k=env;v=stg", + "message": { + "message_id": 9, + "chat": {"id": 42, "type": "private"}, + }, + }, + } + ) + ) + callbacks = [f for f in frames if f["type"] == "callback"] + assert len(callbacks) == 1 + assert callbacks[0]["chat_id"] == 42 + assert callbacks[0]["data"] == "i=del1;k=env;v=stg" + assert callbacks[0]["callback_query_id"] == "cb1" + + def test_group_message_ignored(): backend = FakeTelegramBackend() worker, frames = _worker(backend) @@ -78,14 +104,14 @@ def test_photo_becomes_data_uri(): def test_html_send_falls_back_to_plain_on_entity_error(): class EntityBackend(FakeTelegramBackend): - async def send_message(self, chat_id, text, parse_mode): + async def send_message(self, chat_id, text, parse_mode, reply_markup=None): if parse_mode: return { "ok": False, "error_code": 400, "description": "Bad Request: can't parse entities", } - return await super().send_message(chat_id, text, None) + return await super().send_message(chat_id, text, None, reply_markup=reply_markup) backend = EntityBackend() worker, frames = _worker(backend) diff --git a/tests/unit/services/test_compounds.py b/tests/unit/services/test_compounds.py new file mode 100644 index 00000000..d0c28d81 --- /dev/null +++ b/tests/unit/services/test_compounds.py @@ -0,0 +1,212 @@ +# retoor + +from datetime import datetime, timezone + +import pytest + +from devplacepy.db_client import get_table +from devplacepy.utils import generate_uid + + +def _now(): + return datetime.now(timezone.utc).isoformat() + + +def _make_user(local_db, **overrides): + uid = generate_uid() + row = { + "uid": uid, + "username": f"cpd_{uid[:8]}", + "email": f"cpd_{uid[:8]}@t.dev", + "api_key": generate_uid(), + "role": "Member", + "is_active": True, + "created_at": _now(), + "bio": "", + } + row.update(overrides) + get_table("users").insert(row) + return row + + +def _make_post(local_db, user, **overrides): + from devplacepy.utils import make_combined_slug + + uid = generate_uid() + row = { + "uid": uid, + "user_uid": user["uid"], + "title": "Compound test post", + "content": "Body of the compound test post.", + "topic": "devlog", + "slug": make_combined_slug("Compound test post", uid), + "stars": 0, + "created_at": _now(), + "deleted_at": None, + "deleted_by": None, + } + row.update(overrides) + get_table("posts").insert(row) + return row + + +def _make_project(local_db, user, **overrides): + from devplacepy.utils import make_combined_slug + + uid = generate_uid() + row = { + "uid": uid, + "user_uid": user["uid"], + "title": "Compound test project", + "description": "", + "slug": make_combined_slug("Compound test project", uid), + "stars": 0, + "is_private": 0, + "created_at": _now(), + "deleted_at": None, + "deleted_by": None, + } + row.update(overrides) + get_table("projects").insert(row) + return row + + +def _make_gist(local_db, user, **overrides): + from devplacepy.utils import make_combined_slug + + uid = generate_uid() + row = { + "uid": uid, + "user_uid": user["uid"], + "title": "Compound test gist", + "description": "", + "source_code": "print('hi')", + "language": "python", + "slug": make_combined_slug("Compound test gist", uid), + "stars": 0, + "created_at": _now(), + "deleted_at": None, + "deleted_by": None, + } + row.update(overrides) + get_table("gists").insert(row) + return row + + +# Section 5.3 page compounds actually wired to a caller (db_client.build_*) +# and consumed by a real web route. Each assertion checks the template +# context keys the corresponding route/schema depends on. + + +def test_build_feed_page_shape(local_db): + from devplacepy_services.database.compounds_page import build_feed_page + + user = _make_user(local_db) + post = _make_post(local_db, user) + + result = build_feed_page(user=user) + + for key in ("posts", "next_cursor", "stats", "top_authors", "daily_topic", "current_tab"): + assert key in result + uids = [item["post"]["uid"] for item in result["posts"]] + assert post["uid"] in uids + item = next(i for i in result["posts"] if i["post"]["uid"] == post["uid"]) + for key in ("attachments", "recent_comments", "reactions", "bookmarked", "poll"): + assert key in item + + +def test_build_post_detail_shape(local_db): + from devplacepy_services.database.compounds_page import build_post_detail + + user = _make_user(local_db) + post = _make_post(local_db, user) + + result = build_post_detail(post["uid"], user=user) + + assert result["post"]["uid"] == post["uid"] + assert result["author"]["uid"] == user["uid"] + assert isinstance(result["comments"], list) + assert isinstance(result["attachments"], list) + + +def test_build_post_detail_missing_post(local_db): + from devplacepy_services.database.compounds_page import build_post_detail + + result = build_post_detail("does-not-exist") + assert result["post"] is None + + +def test_build_profile_bundle_shape(local_db): + from devplacepy_services.database.compounds_page import build_profile_bundle + + user = _make_user(local_db) + + result = build_profile_bundle(user["uid"]) + + assert result["user"]["uid"] == user["uid"] + assert "followers" in result["follow_counts"] + assert "following" in result["follow_counts"] + + +def test_build_leaderboard_page_shape(local_db): + from devplacepy_services.database.compounds_page import build_leaderboard_page + + user = _make_user(local_db) + + result = build_leaderboard_page(viewer_uid=user["uid"]) + + assert "leaderboard" in result + assert isinstance(result["leaderboard"], list) + + +def test_build_project_detail_shape(local_db): + from devplacepy_services.database.compounds_page import build_project_detail + + user = _make_user(local_db) + project = _make_project(local_db, user) + + result = build_project_detail(project["uid"]) + + assert result["project"]["uid"] == project["uid"] + assert result["owner"]["uid"] == user["uid"] + + +def test_build_gist_detail_shape(local_db): + from devplacepy_services.database.compounds_page import build_gist_detail + + user = _make_user(local_db) + gist = _make_gist(local_db, user) + + result = build_gist_detail(gist["uid"], user=user) + + assert result["gist"]["uid"] == gist["uid"] + assert result["author"]["uid"] == user["uid"] + assert isinstance(result["comments"], list) + + +# Section 5.3 rows registered as HTTP endpoints (devplacepy_services/database/routes.py) +# but not yet called by any web route (routers/messages.py, notifications.py, +# game/, admin/index.py all still do their own direct db_client calls) - these +# four builders are unfinished stubs, tracked here so a real implementation +# lands as a visible test change rather than silently. + + +@pytest.mark.parametrize( + "fn_name,expected_keys", + [ + ("build_messages_page", {"threads", "unread", "partners"}), + ("build_notifications_page", {"notifications", "actors", "unread_count"}), + ("build_game_state", {"store", "leaderboard"}), + ("build_admin_dashboard", {"stats", "service_states"}), + ], +) +def test_stub_compounds_documented(local_db, fn_name, expected_keys): + import devplacepy_services.database.compounds_page as compounds_page + + fn = getattr(compounds_page, fn_name) + result = fn() if fn_name != "build_admin_dashboard" else fn() + assert set(result.keys()) == expected_keys, ( + f"{fn_name} shape changed - if this is a real implementation now, " + "replace this stub-tracking test with a real assertion and update " + "splitup.md Section 17.2" + ) diff --git a/tests/unit/services/test_conformance.py b/tests/unit/services/test_conformance.py new file mode 100644 index 00000000..a84e0fb5 --- /dev/null +++ b/tests/unit/services/test_conformance.py @@ -0,0 +1,56 @@ +# retoor + +import os +import subprocess +import sys + +import pytest + +from devplacepy_services.base.config import PROFILES + +SERVICE_MODULES = ( + "devplacepy_services.database.main", + "devplacepy_services.pubsub.main", + "devplacepy_services.web.main", + "devplacepy_services.gateway.main", + "devplacepy_services.jobs.main", + "devplacepy_services.devii.main", + "devplacepy_services.bot.main", + "devplacepy_services.backup.main", + "devplacepy_services.containers.main", + "devplacepy_services.telegram.main", + "devplacepy_services.email.main", + "devplacepy_services.news.main", + "devplacepy_services.gitea.main", + "devplacepy_services.audit.main", + "devplacepy_services.xmlrpc.main", +) + + +@pytest.mark.parametrize("module_name", SERVICE_MODULES) +def test_service_imports(module_name): + env = os.environ.copy() + env["DEVPLACE_DISABLE_SERVICES"] = "1" + env["DEVPLACE_DATABASE_URL"] = "sqlite:///:memory:" + env.pop("DEVPLACE_REMOTE_DB", None) + env.pop("DEVPLACE_DB_SERVICE", None) + script = ( + f"import importlib; m = importlib.import_module('{module_name}'); " + "assert hasattr(m, 'app')" + ) + result = subprocess.run( + [sys.executable, "-c", script], + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_port_profiles_complete(): + for profile in ("micro-dev", "micro-test", "micro-prod"): + ports = PROFILES[profile] + assert "web" in ports + assert ports["web"] in (10500, 20500) \ No newline at end of file