docs: document server-side rendering pipeline, response timing middleware, and Telegram pairing API

- Add comprehensive documentation for backend content rendering in AGENTS.md, detailing the new `render_content` and `render_title` Jinja globals built on mistune with media processing, emoji shortcodes, and XSS protection
- Document the `X-Response-Time` header and bottom-left render time indicator in README.md
- Update bot token pricing documentation to clarify fallback vs gateway cost headers
- Add `email_accounts` to soft-delete tables and `idx_users_role` composite index in database schema
- Implement `telegram_pairings` and `telegram_links` table creation with column migration and indexes
- Add `/profile/{username}/telegram` endpoint to docs API with request/unpair actions
- Register `TelegramService` in main.py lifespan and add `response_timing` middleware emitting `X-Response-Time` header
- Introduce `TelegramPairForm` model and `guard_public_host_sync` synchronous host validation function
This commit is contained in:
2026-06-18 22:09:34 +00:00
parent 95dca73291
commit 6ceca3d0d4
146 changed files with 6079 additions and 392 deletions
@@ -0,0 +1 @@
# retoor <retoor@molodetz.nl>
@@ -0,0 +1,213 @@
# retoor <retoor@molodetz.nl>
import dataclasses
import json
import pytest
from devplacepy import database
from devplacepy.services.devii.config import load_settings
from devplacepy.services.devii.email.controller import EmailController
from devplacepy.services.devii.errors import AuthRequiredError, ToolInputError
from devplacepy.utils import generate_uid
from tests.conftest import run_async
def _settings(**overrides):
base = load_settings()
if overrides:
return dataclasses.replace(base, **overrides)
return base
def _owner():
return f"email-owner-{generate_uid()[-12:]}"
def test_guest_is_denied(local_db):
controller = EmailController(_settings(), owner_kind="guest", owner_id="g1")
with pytest.raises(AuthRequiredError):
run_async(controller.dispatch("email_accounts_list", {}))
def test_disabled_flag_blocks_user(local_db):
controller = EmailController(
_settings(email_enabled=False), owner_kind="user", owner_id="u1"
)
with pytest.raises(ToolInputError):
run_async(controller.dispatch("email_accounts_list", {}))
def test_unknown_tool_raises(local_db):
controller = EmailController(_settings(), owner_kind="user", owner_id=_owner())
with pytest.raises(ToolInputError):
run_async(controller.dispatch("email_no_such_tool", {}))
def test_account_set_masks_password(local_db):
owner = _owner()
controller = EmailController(_settings(), owner_kind="user", owner_id=owner)
out = run_async(
controller.dispatch(
"email_account_set",
{
"account": "work",
"imap_host": "mail.example.com",
"smtp_host": "smtp.example.com",
"username": "me@example.com",
"password": "topsecret",
},
)
)
account = json.loads(out)["account"]
assert "password" not in account
assert account["password_set"] is True
assert account["username"] == "me@example.com"
def test_account_set_applies_default_ports(local_db):
owner = _owner()
controller = EmailController(_settings(), owner_kind="user", owner_id=owner)
out = run_async(
controller.dispatch(
"email_account_set",
{"account": "work", "imap_host": "mail.example.com"},
)
)
account = json.loads(out)["account"]
assert account["imap_port"] == 993
assert account["smtp_port"] == 587
def test_account_set_coerces_booleans_and_ports(local_db):
owner = _owner()
controller = EmailController(_settings(), owner_kind="user", owner_id=owner)
out = run_async(
controller.dispatch(
"email_account_set",
{
"account": "work",
"imap_ssl": "true",
"smtp_starttls": "0",
"imap_port": "1993",
},
)
)
account = json.loads(out)["account"]
assert account["imap_ssl"] is True
assert account["smtp_starttls"] is False
assert account["imap_port"] == 1993
def test_account_set_rejects_non_integer_port(local_db):
owner = _owner()
controller = EmailController(_settings(), owner_kind="user", owner_id=owner)
with pytest.raises(ToolInputError):
run_async(
controller.dispatch(
"email_account_set",
{"account": "work", "imap_port": "not-a-port"},
)
)
def test_account_get_unknown_label_raises(local_db):
controller = EmailController(_settings(), owner_kind="user", owner_id=_owner())
with pytest.raises(ToolInputError):
run_async(controller.dispatch("email_account_get", {"account": "missing"}))
def test_account_requires_label(local_db):
controller = EmailController(_settings(), owner_kind="user", owner_id=_owner())
with pytest.raises(ToolInputError):
run_async(controller.dispatch("email_account_get", {}))
def test_account_delete_soft_deletes(local_db):
owner = _owner()
controller = EmailController(_settings(), owner_kind="user", owner_id=owner)
run_async(
controller.dispatch(
"email_account_set",
{"account": "work", "imap_host": "mail.example.com"},
)
)
out = run_async(controller.dispatch("email_account_delete", {"account": "work"}))
assert json.loads(out)["removed"] == 1
assert database.get_email_account("user", owner, "work") is None
def test_accounts_list_excludes_deleted(local_db):
owner = _owner()
controller = EmailController(_settings(), owner_kind="user", owner_id=owner)
run_async(
controller.dispatch(
"email_account_set", {"account": "keep", "imap_host": "a.example.com"}
)
)
run_async(
controller.dispatch(
"email_account_set", {"account": "drop", "imap_host": "b.example.com"}
)
)
run_async(controller.dispatch("email_account_delete", {"account": "drop"}))
out = run_async(controller.dispatch("email_accounts_list", {}))
labels = [account["label"] for account in json.loads(out)["accounts"]]
assert "keep" in labels
assert "drop" not in labels
def test_protocol_tool_without_account_raises(local_db):
controller = EmailController(_settings(), owner_kind="user", owner_id=_owner())
with pytest.raises(ToolInputError):
run_async(controller.dispatch("email_list_folders", {"account": "absent"}))
def test_criteria_builds_imap_search_tokens(local_db):
controller = EmailController(_settings(), owner_kind="user", owner_id=_owner())
criteria = controller._criteria(
{"unseen": True, "from": "alice@x.dev", "subject": "hi"}
)
assert "UNSEEN" in criteria
assert "FROM" in criteria
assert "SUBJECT" in criteria
def test_limit_is_clamped(local_db):
controller = EmailController(_settings(), owner_kind="user", owner_id=_owner())
assert controller._limit({"limit": 9999}) == 100
assert controller._limit({"limit": 0}) == 1
assert controller._limit({"limit": "bad"}) == 25
def test_mark_rejects_invalid_state(local_db):
owner = _owner()
controller = EmailController(_settings(), owner_kind="user", owner_id=owner)
run_async(
controller.dispatch(
"email_account_set", {"account": "work", "imap_host": "mail.example.com"}
)
)
with pytest.raises(ToolInputError):
run_async(
controller.dispatch(
"email_mark", {"account": "work", "uid": "5", "state": "bogus"}
)
)
def test_send_requires_recipient(local_db):
owner = _owner()
controller = EmailController(_settings(), owner_kind="user", owner_id=owner)
run_async(
controller.dispatch(
"email_account_set", {"account": "work", "smtp_host": "smtp.example.com"}
)
)
with pytest.raises(ToolInputError):
run_async(
controller.dispatch(
"email_send", {"account": "work", "subject": "hi", "body": "b"}
)
)
+1
View File
@@ -0,0 +1 @@
# retoor <retoor@molodetz.nl>
+113
View File
@@ -0,0 +1,113 @@
# retoor <retoor@molodetz.nl>
from email.message import EmailMessage
import pytest
from devplacepy.services.email.client import (
EmailClient,
_decode_date,
_header_addresses,
imap_quote,
)
from devplacepy.services.email.config import EmailAccount
from devplacepy.services.email.errors import EmailError
def _account(**overrides):
base = {
"label": "work",
"imap_host": "127.0.0.1",
"imap_port": 993,
"imap_ssl": 1,
"smtp_host": "127.0.0.1",
"smtp_port": 587,
"smtp_starttls": 1,
"username": "me@example.com",
"password": "secret",
}
base.update(overrides)
return EmailAccount.from_row(base)
def test_imap_quote_escapes_quotes_and_backslashes():
assert imap_quote('a"b\\c') == '"a\\"b\\\\c"'
def test_header_addresses_extracts_only_addresses():
parsed = _header_addresses("Alice <a@x.dev>, b@y.dev")
assert parsed == ["Alice <a@x.dev>", "b@y.dev"]
def test_header_addresses_empty_is_empty_list():
assert _header_addresses("") == []
def test_decode_date_parses_rfc2822():
iso = _decode_date("Wed, 18 Jun 2025 10:30:00 +0000")
assert iso is not None and iso.startswith("2025-06-18T10:30:00")
def test_decode_date_returns_raw_on_unparseable():
assert _decode_date("not a date") == "not a date"
def test_decode_date_empty_is_none():
assert _decode_date("") is None
def test_imap_without_host_raises_config_error():
client = EmailClient(_account(imap_host=""))
with pytest.raises(EmailError) as exc:
client.list_folders()
assert exc.value.kind == "config"
def test_smtp_without_host_raises_config_error():
client = EmailClient(_account(smtp_host=""))
with pytest.raises(EmailError) as exc:
client.send_message(to=["x@y.dev"], subject="hi", body="b")
assert exc.value.kind == "config"
def test_imap_loopback_host_is_blocked():
client = EmailClient(_account(imap_host="127.0.0.1"))
with pytest.raises(EmailError) as exc:
client.list_folders()
assert exc.value.kind == "blocked"
def test_smtp_loopback_host_is_blocked():
client = EmailClient(_account(smtp_host="127.0.0.1"))
with pytest.raises(EmailError) as exc:
client.send_message(to=["x@y.dev"], subject="hi", body="b")
assert exc.value.kind == "blocked"
def test_render_message_truncates_oversized_body():
client = EmailClient(_account())
message = EmailMessage()
message["From"] = "Alice <a@x.dev>"
message["To"] = "b@y.dev"
message["Subject"] = "topic"
message.set_content("x" * 200_000)
rendered = client._render_message("17", "INBOX", ["\\Seen"], message)
assert rendered["body_truncated"] is True
assert len(rendered["body"]) == 80_000
assert rendered["uid"] == "17"
assert rendered["to"] == ["b@y.dev"]
def test_render_message_lists_attachments():
client = EmailClient(_account())
message = EmailMessage()
message["From"] = "a@x.dev"
message["Subject"] = "with attachment"
message.set_content("body")
message.add_attachment(
b"payload", maintype="application", subtype="pdf", filename="report.pdf"
)
rendered = client._render_message("3", "INBOX", [], message)
assert rendered["body"] == "body"
assert rendered["attachments"][0]["filename"] == "report.pdf"
assert rendered["attachments"][0]["content_type"] == "application/pdf"
+59
View File
@@ -0,0 +1,59 @@
# retoor <retoor@molodetz.nl>
from devplacepy.services.email.config import EmailAccount
def test_defaults_apply_imap_993_and_smtp_587():
account = EmailAccount.from_row({"username": "me@example.com"})
assert account.imap_port == 993
assert account.smtp_port == 587
def test_from_address_falls_back_to_username():
account = EmailAccount.from_row({"username": "me@example.com"})
assert account.from_address == "me@example.com"
def test_explicit_from_address_wins():
account = EmailAccount.from_row(
{"username": "login@example.com", "from_address": "public@example.com"}
)
assert account.from_address == "public@example.com"
def test_boolean_coercion_from_strings():
account = EmailAccount.from_row(
{
"imap_ssl": "true",
"imap_starttls": "0",
"smtp_ssl": "yes",
"smtp_starttls": "on",
}
)
assert account.imap_ssl is True
assert account.imap_starttls is False
assert account.smtp_ssl is True
assert account.smtp_starttls is True
def test_boolean_coercion_from_integers():
account = EmailAccount.from_row({"imap_ssl": 1, "smtp_ssl": 0})
assert account.imap_ssl is True
assert account.smtp_ssl is False
def test_port_coercion_falls_back_on_garbage():
account = EmailAccount.from_row({"imap_port": "not-a-port", "smtp_port": "2525"})
assert account.imap_port == 993
assert account.smtp_port == 2525
def test_missing_fields_become_empty_strings():
account = EmailAccount.from_row({})
assert account.label == ""
assert account.imap_host == ""
assert account.smtp_host == ""
assert account.username == ""
assert account.password == ""
assert account.from_address == ""
assert account.from_name == ""
@@ -17,7 +17,7 @@ def _patch_pipeline(monkeypatch, pages):
async def fake_plan(query, api_key):
return [query, f"{query} overview"]
async def fake_search(queries):
async def fake_search(queries, emit=lambda frame: None):
return [{"url": page.url, "title": page.title, "description": ""} for page in pages]
async def fake_crawl(candidates, max_pages, emit, is_cached, should_stop):
+1
View File
@@ -0,0 +1 @@
# retoor <retoor@molodetz.nl>
+53
View File
@@ -0,0 +1,53 @@
# retoor <retoor@molodetz.nl>
from devplacepy.services.telegram.format import (
markdown_to_telegram_html,
split_for_telegram,
)
def test_markdown_basic_formatting():
out = markdown_to_telegram_html("Hello **bold** and _em_ and `code`")
assert "<b>bold</b>" in out
assert "<i>em</i>" in out
assert "<code>code</code>" in out
def test_markdown_link_kept():
out = markdown_to_telegram_html("see [docs](https://example.com/x)")
assert '<a href="https://example.com/x">docs</a>' in out
def test_markdown_escapes_special_characters():
out = markdown_to_telegram_html("a < b & c > d")
assert "&lt;" in out and "&amp;" in out and "&gt;" in out
def test_markdown_drops_unsupported_tags():
out = markdown_to_telegram_html("# Heading\n\ntext")
assert "<h1>" not in out
assert "<b>Heading</b>" in out
def test_split_respects_limit():
chunks = split_for_telegram("a" * 5000, limit=3900)
assert all(len(chunk) <= 3900 for chunk in chunks)
assert "".join(chunks) == "a" * 5000
def test_split_short_text_single_chunk():
assert split_for_telegram("hello") == ["hello"]
def test_split_breaks_single_oversized_line():
chunks = split_for_telegram("b" * 9000, limit=4000)
assert all(len(chunk) <= 4000 for chunk in chunks)
assert "".join(chunks) == "b" * 9000
def test_split_empty_text_returns_single_empty_chunk():
assert split_for_telegram("") == [""]
def test_markdown_empty_input_is_empty():
assert markdown_to_telegram_html("") == ""
+78
View File
@@ -0,0 +1,78 @@
# retoor <retoor@molodetz.nl>
from datetime import datetime, timedelta, timezone
from devplacepy.database import get_table
from devplacepy.services.telegram import store
def _seed_user(uid: str) -> None:
users = get_table("users")
if not users.find_one(uid=uid):
users.insert({"uid": uid, "username": uid, "api_key": f"key-{uid}"})
def test_issue_code_is_four_digits(local_db):
_seed_user("tg-user-1")
issued = store.issue_code("tg-user-1")
assert issued["code"].isdigit() and len(issued["code"]) == 4
assert issued["ttl_minutes"] >= 1
def test_verify_binds_and_is_single_use(local_db):
_seed_user("tg-user-2")
code = store.issue_code("tg-user-2")["code"]
user = store.verify_code(code, chat_id=555, from_id=777)
assert user is not None and user["uid"] == "tg-user-2"
assert store.is_paired("tg-user-2")
assert store.user_for_chat(555)["user_uid"] == "tg-user-2"
assert store.verify_code(code, chat_id=555, from_id=777) is None
def test_reissue_invalidates_previous_code(local_db):
_seed_user("tg-user-3")
first = store.issue_code("tg-user-3")["code"]
store.issue_code("tg-user-3")
assert store.verify_code(first, chat_id=10, from_id=11) is None
def test_expired_code_rejected(local_db):
_seed_user("tg-user-4")
code = store.issue_code("tg-user-4")["code"]
pairings = get_table("telegram_pairings")
row = pairings.find_one(user_uid="tg-user-4", used=0)
past = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat()
pairings.update({"id": row["id"], "expires_at": past}, ["id"])
assert store.verify_code(code, chat_id=1, from_id=2) is None
def test_unpair_removes_link(local_db):
_seed_user("tg-user-5")
code = store.issue_code("tg-user-5")["code"]
store.verify_code(code, chat_id=88, from_id=99)
assert store.unpair("tg-user-5") == 1
assert not store.is_paired("tg-user-5")
def test_verify_rejects_non_four_digit_code(local_db):
assert store.verify_code("abcd", chat_id=1, from_id=2) is None
assert store.verify_code("12345", chat_id=1, from_id=2) is None
assert store.verify_code("12", chat_id=1, from_id=2) is None
def test_user_for_chat_unknown_is_none(local_db):
assert store.user_for_chat(999_999) is None
assert store.link_for_user("tg-no-such-user") is None
def test_unpair_unknown_user_removes_nothing(local_db):
assert store.unpair("tg-never-paired") == 0
def test_rebind_moves_chat_to_latest_user(local_db):
_seed_user("tg-user-6")
_seed_user("tg-user-7")
store.bind("tg-user-6", chat_id=4242, from_id=1)
store.bind("tg-user-7", chat_id=4242, from_id=1)
assert store.user_for_chat(4242)["user_uid"] == "tg-user-7"
assert not store.is_paired("tg-user-6")
+100
View File
@@ -0,0 +1,100 @@
# retoor <retoor@molodetz.nl>
import asyncio
from devplacepy.services.telegram.backend import FakeTelegramBackend
from devplacepy.services.telegram.worker import TelegramWorker
def _run(coro):
return asyncio.new_event_loop().run_until_complete(coro)
def _worker(backend):
frames = []
async def emit(frame):
frames.append(frame)
return TelegramWorker(backend, emit), frames
def test_private_text_message_emitted():
backend = FakeTelegramBackend()
worker, frames = _worker(backend)
_run(
worker.process_update(
{
"update_id": 1,
"message": {
"chat": {"id": 42, "type": "private"},
"from": {"id": 7},
"text": "hello",
},
}
)
)
messages = [f for f in frames if f["type"] == "message"]
assert len(messages) == 1
assert messages[0]["chat_id"] == 42 and messages[0]["text"] == "hello"
def test_group_message_ignored():
backend = FakeTelegramBackend()
worker, frames = _worker(backend)
_run(
worker.process_update(
{
"update_id": 1,
"message": {
"chat": {"id": 9, "type": "group"},
"from": {"id": 7},
"text": "x",
},
}
)
)
assert not [f for f in frames if f["type"] == "message"]
def test_photo_becomes_data_uri():
backend = FakeTelegramBackend()
backend.files["abc"] = b"\xff\xd8\xff" + b"0" * 50
worker, frames = _worker(backend)
_run(
worker.process_update(
{
"update_id": 1,
"message": {
"chat": {"id": 42, "type": "private"},
"from": {"id": 7},
"photo": [{"file_id": "abc", "file_size": 53}],
},
}
)
)
message = [f for f in frames if f["type"] == "message"][0]
assert message["images"][0].startswith("data:image/jpeg;base64,")
def test_html_send_falls_back_to_plain_on_entity_error():
class EntityBackend(FakeTelegramBackend):
async def send_message(self, chat_id, text, parse_mode):
if parse_mode:
return {
"ok": False,
"error_code": 400,
"description": "Bad Request: can't parse entities",
}
return await super().send_message(chat_id, text, None)
backend = EntityBackend()
worker, frames = _worker(backend)
_run(
worker.handle_command(
{"cmd": "send", "req_id": "r1", "chat_id": 42, "text": "<b>x</b>", "parse_mode": "HTML"}
)
)
assert backend.sent[-1]["parse_mode"] is None
result = [f for f in frames if f["type"] == "result"][0]
assert result["ok"] and result["req_id"] == "r1"