feat: add TTLCache for get_cache_version and TEMPLATE_AUTO_RELOAD config with Makefile worker count variables

devplacepy/services/background.py

@@ -2,18 +2,30 @@
 
 import asyncio
 import logging
+import queue
+import threading
 from typing import Any, Callable, Optional
 
 logger = logging.getLogger(__name__)
 
 QUEUE_MAXSIZE = 10000
-DRAIN_BATCH = 64
+JOIN_TIMEOUT_SECONDS = 10
+GET_TIMEOUT_SECONDS = 1
+
+
+class _Stop:
+    pass
+
+
+_STOP = _Stop()
 
 
 class BackgroundQueue:
     def __init__(self, maxsize: int = QUEUE_MAXSIZE) -> None:
-        self._queue: asyncio.Queue = asyncio.Queue(maxsize=maxsize)
-        self._task: Optional[asyncio.Task] = None
+        self._queue: "queue.Queue" = queue.Queue(maxsize=maxsize)
+        self._thread: Optional[threading.Thread] = None
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._counters_lock = threading.Lock()
         self._submitted = 0
         self._processed = 0
         self._failed = 0
@@ -21,44 +33,55 @@ class BackgroundQueue:
 
     @property
     def running(self) -> bool:
-        return self._task is not None and not self._task.done()
+        return self._thread is not None and self._thread.is_alive()
+
+    @property
+    def loop(self) -> Optional[asyncio.AbstractEventLoop]:
+        return self._loop
 
     def submit(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None:
-        self._submitted += 1
+        with self._counters_lock:
+            self._submitted += 1
         if not self.running:
             self._execute(fn, args, kwargs, inline=True)
             return
         try:
             self._queue.put_nowait((fn, args, kwargs))
-        except asyncio.QueueFull:
+        except queue.Full:
             logger.warning("background queue full; running task inline")
             self._execute(fn, args, kwargs, inline=True)
 
     def _execute(self, fn: Callable[..., Any], args: tuple, kwargs: dict, inline: bool = False) -> None:
-        if inline:
-            self._inline += 1
         try:
             fn(*args, **kwargs)
-            self._processed += 1
+            with self._counters_lock:
+                self._processed += 1
+                if inline:
+                    self._inline += 1
         except Exception as exc:
-            self._failed += 1
+            with self._counters_lock:
+                self._failed += 1
+                if inline:
+                    self._inline += 1
             logger.warning("background task %s failed: %s", getattr(fn, "__name__", fn), exc)
 
     async def start(self) -> None:
         if self.running:
             return
-        self._task = asyncio.create_task(self._drain())
-        logger.info("background task queue started")
+        self._loop = asyncio.get_running_loop()
+        self._thread = threading.Thread(
+            target=self._worker, name="background-queue", daemon=True
+        )
+        self._thread.start()
+        logger.info("background task queue worker thread started")
 
     async def stop(self) -> None:
-        task = self._task
-        self._task = None
-        if task is not None:
-            task.cancel()
-            try:
-                await task
-            except asyncio.CancelledError:
-                pass
+        thread = self._thread
+        self._thread = None
+        self._loop = None
+        if thread is not None:
+            self._queue.put(_STOP)
+            await asyncio.to_thread(thread.join, JOIN_TIMEOUT_SECONDS)
         self._flush_remaining()
         logger.info(
             "background task queue stopped processed=%s failed=%s inline=%s",
@@ -70,24 +93,24 @@ class BackgroundQueue:
     def _flush_remaining(self) -> None:
         while True:
             try:
-                fn, args, kwargs = self._queue.get_nowait()
-            except asyncio.QueueEmpty:
+                item = self._queue.get_nowait()
+            except queue.Empty:
                 break
+            if item is _STOP:
+                continue
+            fn, args, kwargs = item
             self._execute(fn, args, kwargs)
-            self._queue.task_done()
 
-    async def _drain(self) -> None:
+    def _worker(self) -> None:
         while True:
-            fn, args, kwargs = await self._queue.get()
+            try:
+                item = self._queue.get(timeout=GET_TIMEOUT_SECONDS)
+            except queue.Empty:
+                continue
+            if item is _STOP:
+                return
+            fn, args, kwargs = item
             self._execute(fn, args, kwargs)
-            self._queue.task_done()
-            for _ in range(DRAIN_BATCH - 1):
-                try:
-                    fn, args, kwargs = self._queue.get_nowait()
-                except asyncio.QueueEmpty:
-                    break
-                self._execute(fn, args, kwargs)
-                self._queue.task_done()
 
     def stats(self) -> dict:
         return {

devplacepy/services/bot/bot.py

@@ -46,11 +46,12 @@ logger = logging.getLogger(__name__)
 
 
 class DevPlaceBot:
-    def __init__(self, cfg: BotRuntimeConfig, on_event=None):
+    def __init__(self, cfg: BotRuntimeConfig, on_event=None, slot: int = 0):
         self.cfg = cfg
         self.base_url = cfg.base_url
         self.state_path = cfg.state_path
         self.on_event = on_event
+        self._slot = slot
         self.state = BotState.load(cfg.state_path)
         if not self.state.persona:
             self.state.persona = random.choice(PERSONAS)
@@ -139,6 +140,15 @@ class DevPlaceBot:
             self.state.log = self.state.log[-250:]
         logger.info("[%s] %s (%s)", self._identity(), line, self._cost_tag())
         self._notify(f"[{self._identity()}] {line}")
+        self.b.note(action=line)
+        asyncio.create_task(self.b.capture(tag.lower(), force=True))
+
+    def _bind_monitor(self) -> None:
+        self.b.bind_monitor(
+            self._slot,
+            username=self.state.username,
+            persona=self.state.persona,
+        )
 
     async def _generate(self, fn: Any, *args: Any) -> Any:
         try:
@@ -2773,6 +2783,8 @@ class DevPlaceBot:
                     await asyncio.sleep(60)
                     continue
 
+                self._bind_monitor()
+
                 if self._ai_decisions:
                     await self._ensure_identity()
                 use_ai = self._ai_decisions and bool(self.state.identity)

devplacepy/services/bot/browser.py

@@ -3,10 +3,18 @@
 import asyncio
 import logging
 import random
+import time
 from typing import Optional
 
 from playwright.async_api import Page, Playwright, async_playwright
 
+from devplacepy.services.bot.config import (
+    MONITOR_JPEG_QUALITY,
+    MONITOR_MIN_INTERVAL_SECONDS,
+    MONITOR_SCALE,
+)
+from devplacepy.services.bot.monitor import monitor
+
 logger = logging.getLogger(__name__)
 
 DEFAULT_USER_AGENT = (
@@ -30,6 +38,8 @@ class BotBrowser:
         self._browser = None
         self._context = None
         self._page: Optional[Page] = None
+        self._monitor_slot: Optional[int] = None
+        self._last_capture: float = 0.0
 
     async def launch(self) -> None:
         self._playwright = await async_playwright().start()
@@ -70,6 +80,44 @@ class BotBrowser:
     def page(self) -> Page:
         return self._page
 
+    def bind_monitor(
+        self, slot: int, *, username: str = "", persona: str = ""
+    ) -> None:
+        self._monitor_slot = slot
+        monitor.update_meta(slot, username=username, persona=persona)
+
+    def note(self, *, action: str = "", status: str = "") -> None:
+        if self._monitor_slot is None:
+            return
+        monitor.update_meta(self._monitor_slot, action=action, status=status)
+
+    async def capture(self, reason: str = "", *, force: bool = False) -> None:
+        if self._monitor_slot is None or self._page is None:
+            return
+        now = time.time()
+        if not force and now - self._last_capture < MONITOR_MIN_INTERVAL_SECONDS:
+            return
+        self._last_capture = now
+        try:
+            current_url = ""
+            try:
+                current_url = self._page.url
+            except Exception:
+                current_url = ""
+            monitor.update_meta(
+                self._monitor_slot, status=reason, url=current_url
+            )
+            image = await self._page.screenshot(
+                type="jpeg",
+                quality=MONITOR_JPEG_QUALITY,
+                scale=MONITOR_SCALE,
+                full_page=False,
+                timeout=4000,
+            )
+            monitor.store_image(self._monitor_slot, image)
+        except Exception as e:
+            logger.debug("monitor capture (%s) failed: %s", reason, e)
+
     async def goto(self, url: str) -> None:
         try:
             await self._page.goto(url, wait_until="domcontentloaded", timeout=15000)
@@ -80,6 +128,7 @@ class BotBrowser:
             except Exception as e2:
                 logger.warning("goto failed for %s: %s", url[:80], e2)
         await self._idle(0.5, 1.5)
+        await self.capture("page load", force=True)
 
     async def _idle(self, lo: float = 0.5, hi: float = 2.0) -> None:
         await asyncio.sleep(random.uniform(lo, hi))
@@ -92,6 +141,7 @@ class BotBrowser:
         except Exception as e:
             logger.debug("scroll failed: %s", e)
         await self._idle(0.2, 0.6)
+        await self.capture("scroll")
 
     async def scroll_to_bottom(self) -> None:
         try:
@@ -112,6 +162,7 @@ class BotBrowser:
                 await self._page.keyboard.type(ch, delay=random.randint(20, 60))
                 if ch == " ":
                     await asyncio.sleep(random.uniform(0.02, 0.08))
+            await self.capture("field input")
             return True
         except Exception as e:
             logger.debug("fill failed for '%s': %s", sel, e)
@@ -175,5 +226,6 @@ class BotBrowser:
         try:
             await self._page.reload(timeout=15000)
             await self._idle()
+            await self.capture("reload", force=True)
         except Exception as e:
             logger.debug("reload failed: %s", e)

devplacepy/services/bot/config.py

@@ -14,6 +14,10 @@ OUTPUT_COST_PER_1M_DEFAULT = 1.10
 COST_WINDOW_SECONDS = 600
 COST_WARMUP_SECONDS = 120
 
+MONITOR_JPEG_QUALITY = 35
+MONITOR_SCALE = "css"
+MONITOR_MIN_INTERVAL_SECONDS = 1.0
+
 STATE_DIR = BOT_DIR
 ARTICLE_REGISTRY_PATH = BOT_DIR / "article_registry.json"
 

devplacepy/services/bot/monitor.py

@@ -0,0 +1,122 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from devplacepy.config import BOT_DIR
+
+logger = logging.getLogger(__name__)
+
+MONITOR_DIR: Path = BOT_DIR / "monitor"
+FRAME_SUFFIX = ".jpg"
+ACTIVE_WINDOW_SECONDS = 90
+
+
+@dataclass
+class BotFrame:
+    slot: int
+    username: str = ""
+    persona: str = ""
+    action: str = ""
+    status: str = ""
+    url: str = ""
+    captured_at: float = 0.0
+    has_image: bool = False
+
+    def label(self) -> str:
+        return self.username or f"bot{self.slot}"
+
+    def age_seconds(self) -> float:
+        return max(0.0, time.time() - self.captured_at) if self.captured_at else 0.0
+
+    def is_active(self) -> bool:
+        return self.has_image and self.age_seconds() <= ACTIVE_WINDOW_SECONDS
+
+    def as_dict(self) -> dict:
+        return {
+            "slot": self.slot,
+            "username": self.username,
+            "persona": self.persona,
+            "action": self.action,
+            "status": self.status,
+            "url": self.url,
+            "label": self.label(),
+            "captured_at": int(self.captured_at) if self.captured_at else 0,
+            "age_seconds": int(self.age_seconds()),
+            "has_image": self.has_image,
+            "active": self.is_active(),
+        }
+
+
+@dataclass
+class BotMonitor:
+    frames: dict[int, BotFrame] = field(default_factory=dict)
+
+    def frame_path(self, slot: int) -> Path:
+        return MONITOR_DIR / f"slot{slot}{FRAME_SUFFIX}"
+
+    def update_meta(
+        self,
+        slot: int,
+        *,
+        username: str = "",
+        persona: str = "",
+        action: str = "",
+        status: str = "",
+        url: str = "",
+    ) -> None:
+        frame = self.frames.get(slot) or BotFrame(slot=slot)
+        if username:
+            frame.username = username
+        if persona:
+            frame.persona = persona
+        if action:
+            frame.action = action
+        if status:
+            frame.status = status
+        if url:
+            frame.url = url
+        self.frames[slot] = frame
+
+    def store_image(self, slot: int, image: bytes) -> None:
+        if not image:
+            return
+        frame = self.frames.get(slot) or BotFrame(slot=slot)
+        try:
+            MONITOR_DIR.mkdir(parents=True, exist_ok=True)
+            path = self.frame_path(slot)
+            tmp = path.with_suffix(path.suffix + ".tmp")
+            tmp.write_bytes(image)
+            tmp.replace(path)
+        except OSError as e:
+            logger.debug("monitor store_image slot %s failed: %s", slot, e)
+            return
+        frame.has_image = True
+        frame.captured_at = time.time()
+        self.frames[slot] = frame
+
+    def drop(self, slot: int) -> None:
+        self.frames.pop(slot, None)
+        try:
+            self.frame_path(slot).unlink(missing_ok=True)
+        except OSError as e:
+            logger.debug("monitor drop slot %s failed: %s", slot, e)
+
+    def read_image(self, slot: int) -> Optional[bytes]:
+        path = self.frame_path(slot)
+        if not path.exists():
+            return None
+        try:
+            return path.read_bytes()
+        except OSError as e:
+            logger.debug("monitor read_image slot %s failed: %s", slot, e)
+            return None
+
+    def snapshot(self) -> list[dict]:
+        return [self.frames[slot].as_dict() for slot in sorted(self.frames)]
+
+
+monitor = BotMonitor()

devplacepy/services/bot/service.py

@@ -8,6 +8,7 @@ from datetime import datetime, timedelta, timezone
 
 from devplacepy.services.base import BaseService, ConfigField
 from devplacepy.services.bot import config
+from devplacepy.services.bot.monitor import monitor
 
 logger = logging.getLogger(__name__)
 
@@ -283,7 +284,7 @@ class BotsService(BaseService):
             ai_decisions=cfg["bot_ai_decisions"],
             decision_temperature=cfg["bot_decision_temperature"],
         )
-        bot = DevPlaceBot(rc, on_event=self.log)
+        bot = DevPlaceBot(rc, on_event=self.log, slot=slot)
         task = asyncio.create_task(self._run_slot(slot, bot, cfg["bot_max_actions"]))
         self._fleet[slot] = {"bot": bot, "task": task}
         self.log(f"Launched bot{slot}")
@@ -319,6 +320,7 @@ class BotsService(BaseService):
         except Exception as e:
             self.log(f"bot{slot} stop error: {e}")
         await self._close_bot(slot, entry.get("bot"))
+        monitor.drop(slot)
         self.log(f"Stopped bot{slot}")
 
     async def _close_bot(self, slot: int, bot) -> None:
@@ -401,7 +403,7 @@ class BotsService(BaseService):
             ],
             "rows": rows,
         }
-        return {"stats": stats, "table": table}
+        return {"stats": stats, "table": table, "frames": monitor.snapshot()}
 
     def _sample_cost(
         self, now: datetime, total_cost: float

devplacepy/services/containers/api.py

@@ -30,6 +30,10 @@ INSTANCE_LABEL = "devplace.instance"
 PROJECT_LABEL = "devplace.project"
 HOST_PORT_MIN = 20001
 HOST_PORT_MAX = 65535
+BOOT_LANGUAGES = ("none", "python", "bash")
+BOOT_SCRIPT_FILES = {"python": ".devplace_boot.py", "bash": ".devplace_boot.sh"}
+BOOT_SCRIPT_RUNNERS = {"python": "python", "bash": "bash"}
+MAX_BOOT_SCRIPT_CHARS = 100_000
 
 
 class ContainerError(ValueError):
@@ -43,6 +47,36 @@ def _validate_limits(cpu_limit: str, mem_limit: str) -> None:
         raise ContainerError("memory limit must look like 512m, 1g, or a byte count")
 
 
+def validate_run_as(run_as_uid) -> str:
+    uid = str(run_as_uid or "").strip()
+    if not uid:
+        return ""
+    from devplacepy import database
+
+    user = database.get_users_by_uids([uid]).get(uid)
+    if not user:
+        raise ContainerError(f"run-as user not found: {uid}")
+    return uid
+
+
+def validate_boot(boot_language, boot_script) -> tuple:
+    language = str(boot_language or "none").strip().lower() or "none"
+    if language not in BOOT_LANGUAGES:
+        raise ContainerError(
+            f"boot language must be one of {', '.join(BOOT_LANGUAGES)}"
+        )
+    script = str(boot_script or "")
+    if language == "none":
+        script = ""
+    if len(script) > MAX_BOOT_SCRIPT_CHARS:
+        raise ContainerError(
+            f"boot script exceeds the {MAX_BOOT_SCRIPT_CHARS}-character limit"
+        )
+    if language != "none" and not script.strip():
+        raise ContainerError("boot script is required when a boot language is set")
+    return language, script
+
+
 def parse_ports(value) -> list:
     ports = []
     if not value:
@@ -164,6 +198,10 @@ async def create_instance(
     *,
     name: str,
     boot_command: str = "",
+    boot_language: str = "none",
+    boot_script: str = "",
+    run_as_uid: str = "",
+    start_on_boot: bool = False,
     env="",
     cpu_limit: str = "",
     mem_limit: str = "",
@@ -184,6 +222,8 @@ async def create_instance(
             f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}"
         )
     _validate_limits(cpu_limit, mem_limit)
+    run_as_uid = validate_run_as(run_as_uid)
+    boot_language, boot_script = validate_boot(boot_language, boot_script)
     port_list = assign_host_ports(parse_ports(ports))
     env_map = parse_env(env)
     name = (name or "").strip()
@@ -200,8 +240,12 @@ async def create_instance(
         "project_uid": project["uid"],
         "created_by": actor[1] if actor and actor[0] == "user" else "",
         "owner_uid": project.get("user_uid", ""),
+        "run_as_uid": run_as_uid,
         "name": name,
         "boot_command": boot_command or "",
+        "boot_language": boot_language,
+        "boot_script": boot_script,
+        "start_on_boot": 1 if start_on_boot else 0,
         "env_json": json.dumps(env_map),
         "cpu_limit": str(cpu_limit or ""),
         "mem_limit": str(mem_limit or ""),
@@ -259,12 +303,105 @@ def mark_for_removal(instance: dict, *, actor=("system", "system")) -> None:
     store.record_event(instance, "remove", actor[0], actor[1])
 
 
+def update_instance_config(
+    instance: dict,
+    *,
+    run_as_uid=None,
+    boot_language=None,
+    boot_script=None,
+    boot_command=None,
+    restart_policy=None,
+    start_on_boot=None,
+    cpu_limit=None,
+    mem_limit=None,
+    actor=("system", "system"),
+) -> dict:
+    changes: dict = {}
+    if run_as_uid is not None:
+        changes["run_as_uid"] = validate_run_as(run_as_uid)
+    if boot_language is not None or boot_script is not None:
+        language = (
+            boot_language
+            if boot_language is not None
+            else instance.get("boot_language", "none")
+        )
+        script = (
+            boot_script if boot_script is not None else instance.get("boot_script", "")
+        )
+        language, script = validate_boot(language, script)
+        changes["boot_language"] = language
+        changes["boot_script"] = script
+    if boot_command is not None:
+        changes["boot_command"] = str(boot_command or "")[:500]
+    if restart_policy is not None:
+        if restart_policy not in store.RESTART_POLICIES:
+            raise ContainerError(
+                f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}"
+            )
+        changes["restart_policy"] = restart_policy
+    if start_on_boot is not None:
+        changes["start_on_boot"] = 1 if start_on_boot else 0
+    if cpu_limit is not None or mem_limit is not None:
+        cpu = cpu_limit if cpu_limit is not None else instance.get("cpu_limit", "")
+        mem = mem_limit if mem_limit is not None else instance.get("mem_limit", "")
+        _validate_limits(cpu, mem)
+        changes["cpu_limit"] = str(cpu or "")
+        changes["mem_limit"] = str(mem or "")
+    if not changes:
+        return store.get_instance(instance["uid"])
+    store.update_instance(instance["uid"], changes)
+    store.record_event(
+        instance, "configure", actor[0], actor[1], {"fields": sorted(changes)}
+    )
+    return store.get_instance(instance["uid"])
+
+
+def set_start_on_boot(
+    instance: dict, enabled: bool, *, actor=("system", "system")
+) -> dict:
+    store.update_instance(instance["uid"], {"start_on_boot": 1 if enabled else 0})
+    store.record_event(
+        instance, "start_on_boot", actor[0], actor[1], {"enabled": bool(enabled)}
+    )
+    return store.get_instance(instance["uid"])
+
+
+def materialize_boot_script(instance: dict) -> None:
+    language = (instance.get("boot_language") or "none").strip().lower()
+    workspace = instance.get("workspace_dir")
+    if not workspace:
+        return
+    for filename in BOOT_SCRIPT_FILES.values():
+        stale = Path(workspace) / filename
+        if stale.is_file():
+            try:
+                stale.unlink()
+            except OSError:
+                pass
+    if language not in BOOT_SCRIPT_FILES:
+        return
+    script = instance.get("boot_script") or ""
+    if not script.strip():
+        return
+    target = Path(workspace) / BOOT_SCRIPT_FILES[language]
+    try:
+        Path(workspace).mkdir(parents=True, exist_ok=True)
+        target.write_text(script, encoding="utf-8")
+    except OSError:
+        pass
+
+
 def pravda_env(instance: dict) -> dict:
     from devplacepy import database, seo
 
     base_url = seo.public_base_url()
     api_key = ""
-    for uid in (instance.get("created_by"), instance.get("owner_uid")):
+    user_uid = instance.get("owner_uid") or ""
+    for uid in (
+        instance.get("run_as_uid"),
+        instance.get("created_by"),
+        instance.get("owner_uid"),
+    ):
         if not uid:
             continue
         user = database.get_users_by_uids([uid]).get(uid)
@@ -277,7 +414,7 @@ def pravda_env(instance: dict) -> dict:
         "PRAVDA_BASE_URL": base_url,
         "PRAVDA_OPENAI_URL": f"{base_url}/openai/v1" if base_url else "",
         "PRAVDA_API_KEY": api_key,
-        "PRAVDA_USER_UID": instance.get("owner_uid") or "",
+        "PRAVDA_USER_UID": instance.get("run_as_uid") or user_uid,
         "PRAVDA_CONTAINER_NAME": instance.get("name") or "",
         "PRAVDA_CONTAINER_UID": instance.get("uid") or "",
         "PRAVDA_INGRESS_URL": ingress_url,
@@ -296,8 +433,15 @@ def run_spec_for(instance: dict, image_tag: str) -> RunSpec:
             mounts.append(
                 Mount(extra["host"], extra["container"], extra.get("mode", "rw"))
             )
+    language = (instance.get("boot_language") or "none").strip().lower()
     boot = (instance.get("boot_command") or "").strip()
-    command = ["/bin/sh", "-c", boot] if boot else ["sleep", "infinity"]
+    if language in BOOT_SCRIPT_FILES and (instance.get("boot_script") or "").strip():
+        script_path = f"{WORKSPACE_MOUNT}/{BOOT_SCRIPT_FILES[language]}"
+        command = [BOOT_SCRIPT_RUNNERS[language], script_path]
+    elif boot:
+        command = ["/bin/sh", "-c", boot]
+    else:
+        command = ["sleep", "infinity"]
     return RunSpec(
         image=image_tag,
         name=instance["slug"],
@@ -315,15 +459,39 @@ def run_spec_for(instance: dict, image_tag: str) -> RunSpec:
     )
 
 
-async def sync_workspace(instance: dict, user: dict) -> int:
+async def sync_workspace(instance: dict, user: dict) -> dict:
     workspace = instance.get("workspace_dir")
     if not workspace:
         raise ContainerError("instance has no workspace")
-    count = await asyncio.to_thread(
-        project_files.import_from_dir, instance["project_uid"], workspace, user
+    counts = await asyncio.to_thread(
+        project_files.sync_dir_bidirectional, instance["project_uid"], workspace, user
     )
-    store.record_event(instance, "sync", "user", user["uid"], {"imported": count})
-    return count
+    store.record_event(
+        instance,
+        "sync",
+        "user",
+        user["uid"],
+        {"exported": counts["exported"], "imported": counts["imported"]},
+    )
+    return counts
+
+
+def sync_bidirectional_sync(instance: dict, user: dict) -> dict:
+    workspace = instance.get("workspace_dir")
+    if not workspace:
+        return {"exported": 0, "imported": 0}
+    counts = project_files.sync_dir_bidirectional(
+        instance["project_uid"], workspace, user
+    )
+    if counts["exported"] or counts["imported"]:
+        store.record_event(
+            instance,
+            "sync",
+            "service",
+            "system",
+            {"exported": counts["exported"], "imported": counts["imported"]},
+        )
+    return counts
 
 
 def add_schedule(instance: dict, action: str, schedule: Schedule) -> dict:

devplacepy/services/containers/service.py

@@ -1,6 +1,8 @@
 # retoor <retoor@molodetz.nl>
 
+import asyncio
 import json
+import time
 
 from devplacepy import config
 from devplacepy.services.base import BaseService, ConfigField
@@ -10,6 +12,34 @@ from devplacepy.services.devii.tasks.schedule import next_run, now_utc, to_iso
 from devplacepy.services.audit import record as audit
 
 AUTO_RESTART = ("always", "on-failure", "unless-stopped")
+SYNC_EVERY_SECONDS = 60
+
+
+def _set_status(inst, changes: dict, *, actor_kind: str = "service", reason: str = "") -> dict:
+    uid = inst["uid"]
+    old_status = inst.get("status")
+    new_status = changes.get("status", old_status)
+    store.update_instance(uid, changes)
+    inst.update(changes)
+    if new_status != old_status:
+        detail = {"from": old_status, "to": new_status}
+        if reason:
+            detail["reason"] = reason
+        store.record_event(inst, "status_change", actor_kind, "", detail)
+        audit.record_system(
+            "container.instance.status",
+            actor_kind="service",
+            origin="scheduler",
+            target_type="instance",
+            target_uid=uid,
+            target_label=inst.get("name"),
+            old_value=old_status,
+            new_value=new_status,
+            metadata={"reason": reason} if reason else None,
+            summary=f"instance {inst.get('name')} status {old_status} -> {new_status}",
+            links=[audit.instance(uid, inst.get("name"))],
+        )
+    return inst
 
 
 def _audit_reconcile(inst, action, summary, metadata=None):
@@ -51,6 +81,8 @@ class ContainerService(BaseService):
     def __init__(self):
         super().__init__(name="containers", interval_seconds=5)
         self._metric_tick = 0
+        self._booted = False
+        self._last_sync_at = 0.0
 
     async def run_once(self) -> None:
         backend = get_backend()
@@ -69,6 +101,12 @@ class ContainerService(BaseService):
         instances = store.all_instances()
         known = {inst["uid"] for inst in instances}
 
+        if not self._booted:
+            self._boot_pass(instances)
+            self._booted = True
+
+        await self._periodic_sync(instances, by_uid)
+
         for uid, row in by_uid.items():
             if uid not in known:
                 try:
@@ -125,13 +163,13 @@ class ContainerService(BaseService):
                 if gateway and gateway != (inst.get("container_gateway") or ""):
                     changes["container_gateway"] = gateway
                 if changes:
-                    store.update_instance(uid, changes)
+                    _set_status(inst, changes)
             elif ps.state == "paused":
                 await backend.unpause(ps.container_id)
-                store.update_instance(uid, {"status": store.ST_RUNNING})
+                _set_status(inst, {"status": store.ST_RUNNING}, reason="unpause")
             elif ps.state == "created":
                 await backend.start(ps.container_id)
-                store.update_instance(uid, {"status": store.ST_RUNNING})
+                _set_status(inst, {"status": store.ST_RUNNING}, reason="start")
             else:
                 await self._handle_exit(backend, inst, ps)
 
@@ -139,21 +177,29 @@ class ContainerService(BaseService):
             if ps is not None and ps.state in ("running", "restarting", "paused"):
                 await backend.stop(ps.container_id)
             if status != store.ST_STOPPED:
-                store.update_instance(
-                    uid, {"status": store.ST_STOPPED, "stopped_at": store.now()}
+                _set_status(
+                    inst,
+                    {"status": store.ST_STOPPED, "stopped_at": store.now()},
+                    reason="stopped",
                 )
 
         elif desired == store.DESIRED_PAUSED:
             if ps is not None and ps.state == "running":
                 await backend.pause(ps.container_id)
-            store.update_instance(uid, {"status": store.ST_PAUSED})
+            if status != store.ST_PAUSED:
+                _set_status(inst, {"status": store.ST_PAUSED}, reason="paused")
 
     async def _launch(self, backend, inst) -> None:
+        await self._sync_one(inst)
+        try:
+            await asyncio.to_thread(api.materialize_boot_script, inst)
+        except Exception as exc:
+            self.log(f"boot script materialize {inst['name']} failed: {exc}")
         spec = api.run_spec_for(inst, config.CONTAINER_IMAGE)
         try:
             cid = await backend.run(spec)
         except Exception as exc:
-            store.update_instance(inst["uid"], {"status": store.ST_CRASHED})
+            _set_status(inst, {"status": store.ST_CRASHED}, reason="launch_failed")
             store.record_event(
                 inst, "launch_failed", "reconciler", "", {"reason": str(exc)}
             )
@@ -169,11 +215,56 @@ class ContainerService(BaseService):
             changes["container_ip"] = ip
         if gateway:
             changes["container_gateway"] = gateway
-        store.update_instance(inst["uid"], changes)
+        _set_status(inst, changes, reason="launch")
         store.record_event(inst, "start", "reconciler", "")
         self.log(f"launched instance {inst['name']}")
         _audit_reconcile(inst, "start", f"reconciler started instance {inst['name']}")
 
+    def _boot_pass(self, instances) -> None:
+        for inst in instances:
+            if int(inst.get("start_on_boot") or 0) != 1:
+                continue
+            if inst.get("desired_state") == store.DESIRED_RUNNING:
+                continue
+            store.update_instance(inst["uid"], {"desired_state": store.DESIRED_RUNNING})
+            inst["desired_state"] = store.DESIRED_RUNNING
+            store.record_event(inst, "start_on_boot", "reconciler", "")
+            self.log(f"start_on_boot forced running for {inst['name']}")
+
+    def _sync_user(self, inst) -> dict:
+        from devplacepy import database
+
+        for uid in (
+            inst.get("run_as_uid"),
+            inst.get("created_by"),
+            inst.get("owner_uid"),
+        ):
+            if not uid:
+                continue
+            user = database.get_users_by_uids([uid]).get(uid)
+            if user:
+                return user
+        return {"uid": "system", "username": "system"}
+
+    async def _sync_one(self, inst) -> None:
+        try:
+            await asyncio.to_thread(
+                api.sync_bidirectional_sync, inst, self._sync_user(inst)
+            )
+        except Exception as exc:
+            self.log(f"sync {inst.get('name')} failed: {exc}")
+
+    async def _periodic_sync(self, instances, by_uid) -> None:
+        now = time.monotonic()
+        if now - self._last_sync_at < SYNC_EVERY_SECONDS:
+            return
+        self._last_sync_at = now
+        for inst in instances:
+            ps = by_uid.get(inst["uid"])
+            if ps is None or ps.state != "running":
+                continue
+            await self._sync_one(inst)
+
     async def _capture_net(self, backend, cid: str) -> tuple:
         try:
             data = await backend.inspect(cid)
@@ -204,12 +295,13 @@ class ContainerService(BaseService):
         if policy in AUTO_RESTART and not (policy == "on-failure" and exit_code == 0):
             try:
                 await backend.start(ps.container_id)
-                store.update_instance(
-                    uid,
+                _set_status(
+                    inst,
                     {
                         "status": store.ST_RUNNING,
                         "restart_count": int(inst.get("restart_count") or 0) + 1,
                     },
+                    reason="policy_restart",
                 )
                 store.record_event(
                     inst,
@@ -228,14 +320,15 @@ class ContainerService(BaseService):
             except Exception as exc:
                 self.log(f"policy restart {inst['name']} failed: {exc}")
         terminal = store.ST_CRASHED if exit_code != 0 else store.ST_STOPPED
-        store.update_instance(
-            uid,
+        _set_status(
+            inst,
             {
                 "status": terminal,
                 "desired_state": store.DESIRED_STOPPED,
                 "exit_code": exit_code,
                 "stopped_at": store.now(),
             },
+            reason="exit",
         )
         if terminal == store.ST_CRASHED:
             store.record_event(

devplacepy/services/containers/store.py

@@ -48,6 +48,10 @@ def create_instance(row: dict) -> dict:
         "restart_count": 0,
         "ingress_slug": "",
         "ingress_port": 0,
+        "run_as_uid": "",
+        "boot_language": "none",
+        "boot_script": "",
+        "start_on_boot": 0,
         "started_at": "",
         "stopped_at": "",
         "created_at": now(),

devplacepy/services/devii/actions/catalog.py

@@ -1160,6 +1160,19 @@ ACTIONS: tuple[Action, ...] = (
         params=(path("uid", "Audit event uid."),),
         requires_admin=True,
     ),
+    Action(
+        name="bot_monitor",
+        method="GET",
+        path="/admin/bots/data",
+        summary="Live screenshot monitor of every running bot persona (admin only)",
+        description=(
+            "Returns JSON: one entry per running bot slot with its username, persona, current "
+            "action and status text, last page url, frame age in seconds, whether it is active, "
+            "and a frame_url to the latest low-quality screenshot. Use this to see what the bot "
+            "fleet is doing right now. Frames live only in the worker running the Bots service."
+        ),
+        requires_admin=True,
+    ),
     Action(
         name="admin_list_users",
         method="GET",

devplacepy/services/devii/actions/container_actions.py

@@ -45,6 +45,22 @@ CONTAINER_ACTIONS: tuple[Action, ...] = (
             arg(
                 "boot_command", "Optional command to run on boot, e.g. 'python app.py'."
             ),
+            arg(
+                "boot_language",
+                "Optional boot source language: 'none', 'python', or 'bash'. When set with boot_script, the script is materialized into /app and run on launch (takes precedence over boot_command).",
+            ),
+            arg(
+                "boot_script",
+                "Optional boot source code (the body of the python or bash script) run on launch when boot_language is python or bash.",
+            ),
+            arg(
+                "run_as_uid",
+                "Optional DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID). Does NOT change the container OS user, which is always pravda (uid 1000).",
+            ),
+            arg(
+                "start_on_boot",
+                "Force this instance to running whenever the container service starts ('true' or 'false', default false).",
+            ),
             arg("restart_policy", "never, always, on-failure, or unless-stopped."),
             arg("env", "Optional env vars as KEY=VALUE lines."),
             arg(
@@ -90,6 +106,32 @@ CONTAINER_ACTIONS: tuple[Action, ...] = (
             ),
         ),
     ),
+    Action(
+        name="container_configure_instance",
+        method="LOCAL",
+        path="",
+        handler="container",
+        requires_admin=True,
+        summary="Update an instance's run-as user, boot language/script/command, restart policy, start-on-boot flag, and resource limits",
+        params=(
+            SLUG,
+            arg("instance", "Instance name, slug, or uid.", required=True),
+            arg(
+                "run_as_uid",
+                "DevPlace user uid whose identity and API key are injected (PRAVDA_API_KEY, PRAVDA_USER_UID); pass empty to clear. Does NOT change the container OS user (always pravda, uid 1000).",
+            ),
+            arg("boot_language", "Boot source language: 'none', 'python', or 'bash'."),
+            arg("boot_script", "Boot source code body run on launch."),
+            arg("boot_command", "Fallback boot command used when no boot_script is set."),
+            arg("restart_policy", "never, always, on-failure, or unless-stopped."),
+            arg(
+                "start_on_boot",
+                "Force running on container-service start ('true' or 'false').",
+            ),
+            arg("cpu_limit", "CPU limit, e.g. 1 or 1.5."),
+            arg("mem_limit", "Memory limit, e.g. 512m or 1g."),
+        ),
+    ),
     Action(
         name="container_logs",
         method="LOCAL",

devplacepy/services/devii/actions/dispatcher.py

@@ -103,6 +103,7 @@ _DEVII_MECHANIC_EVENTS = {
 
 _DEVII_CONTAINER_EVENTS = {
     "container_create_instance": "container.instance.create",
+    "container_configure_instance": "container.instance.configure",
     "container_exec": "container.instance.exec",
     "container_schedule": "container.schedule.create",
 }

devplacepy/services/devii/actions/spec.py

@@ -42,6 +42,10 @@ class Action:
         "chunks",
         "rsearch",
         "container",
+        "customization",
+        "notification",
+        "behavior",
+        "virtual_tool",
     ] = "http"
     freeform_body: bool = False
     ajax: bool = False

devplacepy/services/devii/container/controller.py

@@ -79,10 +79,20 @@ class ContainerController:
             "no",
             "off",
         )
+        start_on_boot = str(arguments.get("start_on_boot", "false")).lower() in (
+            "true",
+            "1",
+            "yes",
+            "on",
+        )
         inst = await api.create_instance(
             project,
             name=str(arguments.get("name", "")),
             boot_command=str(arguments.get("boot_command", "")),
+            boot_language=str(arguments.get("boot_language", "none")),
+            boot_script=str(arguments.get("boot_script", "")),
+            run_as_uid=str(arguments.get("run_as_uid", "")),
+            start_on_boot=start_on_boot,
             env=arguments.get("env", ""),
             cpu_limit=str(arguments.get("cpu_limit", "")),
             mem_limit=str(arguments.get("mem_limit", "")),
@@ -105,8 +115,8 @@ class ContainerController:
         if action == "delete":
             api.mark_for_removal(inst, actor=actor)
         elif action == "sync":
-            count = await api.sync_workspace(inst, self._actor_user())
-            return json.dumps({"status": "synced", "imported": count})
+            counts = await api.sync_workspace(inst, self._actor_user())
+            return json.dumps({"status": "synced", **counts})
         elif action == "restart":
             api.request_restart(inst, actor=actor)
         elif action in ("start", "resume"):
@@ -119,6 +129,32 @@ class ContainerController:
             raise ToolInputError(f"unknown action: {action}")
         return json.dumps({"status": "ok", "action": action, "instance": inst["uid"]})
 
+    async def _configure_instance(self, arguments) -> str:
+        project = self._project(arguments)
+        inst = self._instance(project, str(arguments.get("instance", "")))
+        actor = ("user", self._actor_user()["uid"])
+        kwargs: dict = {}
+        for key in (
+            "run_as_uid",
+            "boot_language",
+            "boot_script",
+            "boot_command",
+            "restart_policy",
+            "cpu_limit",
+            "mem_limit",
+        ):
+            if key in arguments and arguments.get(key) is not None:
+                kwargs[key] = str(arguments.get(key))
+        if "start_on_boot" in arguments and arguments.get("start_on_boot") is not None:
+            kwargs["start_on_boot"] = str(arguments.get("start_on_boot")).lower() in (
+                "true",
+                "1",
+                "yes",
+                "on",
+            )
+        updated = api.update_instance_config(inst, actor=actor, **kwargs)
+        return json.dumps({"status": "configured", "instance": updated}, default=str)
+
     async def _logs(self, arguments) -> str:
         project = self._project(arguments)
         inst = self._instance(project, str(arguments.get("instance", "")))

devplacepy/services/messaging/__init__.py

@@ -0,0 +1,7 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.messaging.hub import message_hub
+from devplacepy.services.messaging.persist import message_frame, persist_message
+from devplacepy.services.messaging.relay import message_relay
+
+__all__ = ["message_frame", "message_hub", "message_relay", "persist_message"]

devplacepy/services/messaging/hub.py

@@ -0,0 +1,84 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from collections import OrderedDict
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+from fastapi import WebSocket
+
+logger = logging.getLogger("messaging.hub")
+
+DELIVERED_CAP = 4000
+
+
+class ConnectionManager:
+    def __init__(self) -> None:
+        self._connections: dict[str, set[WebSocket]] = {}
+        self._last_seen: dict[str, str] = {}
+        self._delivered: "OrderedDict[str, bool]" = OrderedDict()
+
+    def register(self, user_uid: str, websocket: WebSocket) -> bool:
+        sockets = self._connections.setdefault(user_uid, set())
+        was_offline = len(sockets) == 0
+        sockets.add(websocket)
+        self._last_seen.pop(user_uid, None)
+        logger.info(
+            "messaging socket registered for %s (sockets=%d)", user_uid, len(sockets)
+        )
+        return was_offline
+
+    def unregister(self, user_uid: str, websocket: WebSocket) -> bool:
+        sockets = self._connections.get(user_uid)
+        if not sockets:
+            return False
+        sockets.discard(websocket)
+        if not sockets:
+            self._connections.pop(user_uid, None)
+            self._last_seen[user_uid] = datetime.now(timezone.utc).isoformat()
+            logger.info("messaging socket removed for %s (now offline)", user_uid)
+            return True
+        logger.debug(
+            "messaging socket removed for %s (sockets=%d)", user_uid, len(sockets)
+        )
+        return False
+
+    def is_online(self, user_uid: str) -> bool:
+        return bool(self._connections.get(user_uid))
+
+    def has_connections(self) -> bool:
+        return bool(self._connections)
+
+    def connected_user_uids(self) -> set[str]:
+        return set(self._connections.keys())
+
+    def mark_delivered(self, message_uid: str) -> None:
+        self._delivered[message_uid] = True
+        self._delivered.move_to_end(message_uid)
+        while len(self._delivered) > DELIVERED_CAP:
+            self._delivered.popitem(last=False)
+
+    def was_delivered(self, message_uid: str) -> bool:
+        return message_uid in self._delivered
+
+    def last_seen(self, user_uid: str) -> Optional[str]:
+        return self._last_seen.get(user_uid)
+
+    def sockets_for(self, user_uid: str) -> list[WebSocket]:
+        return list(self._connections.get(user_uid, ()))
+
+    async def send_to_user(self, user_uid: str, frame: dict[str, Any]) -> None:
+        for websocket in self.sockets_for(user_uid):
+            try:
+                await websocket.send_json(frame)
+            except Exception:  # noqa: BLE001
+                logger.debug("dropping frame to stale socket for %s", user_uid)
+
+    async def send_to_users(
+        self, user_uids: list[str], frame: dict[str, Any]
+    ) -> None:
+        for user_uid in dict.fromkeys(user_uids):
+            await self.send_to_user(user_uid, frame)
+
+
+message_hub = ConnectionManager()

devplacepy/services/messaging/persist.py

@@ -0,0 +1,154 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+from devplacepy.attachments import get_attachments, link_attachments
+from devplacepy.database import get_table
+from devplacepy.templating import clear_messages_cache
+from devplacepy.utils import (
+    create_mention_notifications,
+    create_notification,
+    generate_uid,
+    time_ago,
+)
+from devplacepy.services.audit import record as audit
+
+logger = logging.getLogger("messaging.persist")
+
+MAX_CONTENT_LENGTH = 2000
+
+
+def _slim_attachment(attachment: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "uid": attachment["uid"],
+        "url": attachment["url"],
+        "thumbnail_url": attachment.get("thumbnail_url"),
+        "is_image": bool(attachment.get("is_image")),
+        "is_video": bool(attachment.get("is_video")),
+        "original_filename": attachment.get("original_filename", ""),
+        "file_size": attachment.get("file_size", 0),
+        "mime_type": attachment.get("mime_type", ""),
+    }
+
+
+def message_frame(
+    message: dict[str, Any], sender_username: str, client_id: Optional[str] = None
+) -> dict[str, Any]:
+    attachments = get_attachments("message", message["uid"])
+    return {
+        "type": "message",
+        "uid": message["uid"],
+        "sender_uid": message["sender_uid"],
+        "sender_username": sender_username,
+        "receiver_uid": message["receiver_uid"],
+        "content": message["content"],
+        "created_at": message["created_at"],
+        "time_ago": time_ago(message["created_at"]),
+        "client_id": client_id,
+        "attachments": [_slim_attachment(a) for a in attachments],
+    }
+
+
+def persist_message(
+    sender: dict[str, Any],
+    receiver_uid: str,
+    content: str,
+    attachment_uids: Optional[list[str]] = None,
+    *,
+    request: Any = None,
+    origin: str = "web",
+) -> Optional[dict[str, Any]]:
+    content = (content or "").strip()[:MAX_CONTENT_LENGTH]
+    attachment_uids = attachment_uids or []
+    if not content and not attachment_uids:
+        return None
+
+    receiver = get_table("users").find_one(uid=receiver_uid)
+    if not receiver:
+        return None
+
+    sender_uid = sender["uid"]
+    sender_username = sender.get("username", "")
+    messages_table = get_table("messages")
+    msg_uid = generate_uid()
+    created_at = datetime.now(timezone.utc).isoformat()
+    messages_table.insert(
+        {
+            "uid": msg_uid,
+            "sender_uid": sender_uid,
+            "receiver_uid": receiver_uid,
+            "content": content,
+            "read": False,
+            "created_at": created_at,
+        }
+    )
+
+    link_attachments(attachment_uids, "message", msg_uid)
+
+    if sender_uid != receiver_uid:
+        create_notification(
+            receiver_uid,
+            "message",
+            f"{sender_username} sent you a message",
+            sender_uid,
+            f"/messages?with_uid={sender_uid}",
+        )
+        clear_messages_cache(receiver_uid)
+
+    create_mention_notifications(
+        content, sender_uid, f"/messages?with_uid={receiver_uid}"
+    )
+
+    logger.info(
+        "Message %s sent from %s to %s via %s",
+        msg_uid,
+        sender_username,
+        receiver_uid,
+        origin,
+    )
+    logger.debug("message %s content length=%d", msg_uid, len(content))
+
+    summary = (
+        f"{sender_username} sent a message to "
+        f"{receiver.get('username') or receiver_uid}: {content}"
+    )
+    links = [
+        audit.target("message", msg_uid),
+        audit.recipient(receiver_uid, receiver.get("username")),
+    ]
+    if request is not None:
+        audit.record(
+            request,
+            "message.send",
+            user=sender,
+            target_type="message",
+            target_uid=msg_uid,
+            summary=summary,
+            metadata={"origin": origin},
+            links=links,
+        )
+    else:
+        audit.record_system(
+            "message.send",
+            actor_kind="user",
+            actor_uid=sender_uid,
+            actor_username=sender_username,
+            actor_role="admin" if sender.get("role") == "Admin" else "member",
+            origin=origin,
+            target_type="message",
+            target_uid=msg_uid,
+            summary=summary,
+            metadata={"origin": origin},
+            links=links,
+        )
+
+    return {
+        "uid": msg_uid,
+        "sender_uid": sender_uid,
+        "receiver_uid": receiver_uid,
+        "content": content,
+        "read": False,
+        "created_at": created_at,
+    }

devplacepy/services/messaging/relay.py

@@ -0,0 +1,89 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import logging
+from typing import Optional
+
+from devplacepy.database import db, get_users_by_uids
+from devplacepy.services.messaging.hub import message_hub
+from devplacepy.services.messaging.persist import message_frame
+
+logger = logging.getLogger("messaging.relay")
+
+POLL_INTERVAL_SECONDS = 1.0
+BATCH_LIMIT = 500
+
+
+class MessageRelay:
+    def __init__(self) -> None:
+        self._task: Optional[asyncio.Task] = None
+        self._watermark: int = 0
+        self._primed: bool = False
+
+    def start(self) -> None:
+        if self._task is not None and not self._task.done():
+            return
+        self._task = asyncio.create_task(self._run())
+        logger.info("message relay started")
+
+    def _max_id(self) -> int:
+        if "messages" not in db.tables:
+            return 0
+        rows = list(db.query("SELECT MAX(id) AS max_id FROM messages"))
+        value = rows[0]["max_id"] if rows else None
+        return int(value or 0)
+
+    async def _run(self) -> None:
+        try:
+            if not self._primed:
+                self._watermark = self._max_id()
+                self._primed = True
+                logger.debug("message relay primed at watermark %d", self._watermark)
+            while message_hub.has_connections():
+                try:
+                    await self._tick()
+                except Exception:  # noqa: BLE001
+                    logger.exception("message relay tick failed")
+                await asyncio.sleep(POLL_INTERVAL_SECONDS)
+        finally:
+            self._task = None
+            logger.info("message relay stopped (no active connections)")
+
+    async def _tick(self) -> None:
+        if "messages" not in db.tables:
+            return
+        rows = list(
+            db.query(
+                "SELECT * FROM messages WHERE id > :wm ORDER BY id ASC LIMIT :lim",
+                wm=self._watermark,
+                lim=BATCH_LIMIT,
+            )
+        )
+        if not rows:
+            return
+        connected = message_hub.connected_user_uids()
+        pending = [
+            row
+            for row in rows
+            if (row["sender_uid"] in connected or row["receiver_uid"] in connected)
+            and not message_hub.was_delivered(row["uid"])
+        ]
+        sender_uids = {row["sender_uid"] for row in pending}
+        senders = get_users_by_uids(list(sender_uids)) if sender_uids else {}
+        for row in pending:
+            sender = senders.get(row["sender_uid"]) or {}
+            frame = message_frame(dict(row), sender.get("username", ""))
+            message_hub.mark_delivered(row["uid"])
+            await message_hub.send_to_users(
+                [row["sender_uid"], row["receiver_uid"]], frame
+            )
+        self._watermark = max(self._watermark, max(row["id"] for row in rows))
+        logger.debug(
+            "message relay delivered %d of %d new rows, watermark %d",
+            len(pending),
+            len(rows),
+            self._watermark,
+        )
+
+
+message_relay = MessageRelay()