blindxfish/devplacepy
1
0
Fork 0
forked from retoor/devplacepy
Code Pull Requests Activity

fix: correct audit event names, attachment deletion, redirect safety, and add account deactivation check

Browse Source 
- Fix audit event names in CLI prune/clear commands from `cli.seo.*` to `cli.seo_meta.*`
- Refactor `_delete_attachment_file` to accept full attachment dict instead of storage_path string, using directory and stored_name fields with ATTACHMENTS_DIR
- Add `safe_next` validation for referer header in validation error redirect and media redirect
- Add `is_active` check in login router to reject deactivated accounts with "Account is deactivated" error
- Replace raw `request.headers.get("Referer")` with `redirect_back()` utility in bookmarks, polls, reactions, and votes routers
- Move `mark_conversation_read` call from `get_conversation_messages` to `messages_page` to avoid side effects during message retrieval
- Fix poll audit link to use `option.get("label")` instead of `option.get("text")`
- Add `VOTABLE` set validation in votes router to reject invalid target types with 400 response
- Strip control characters (0x00-0x20) from URLs in `_safe_url` instead of simple strip
- Add `__getattr__` fallback in services `__init__.py` for dynamic attribute access
This commit is contained in:
retoor
2026-06-21 16:46:27 +00:00
parent a3dd747d07
commit 19cc85f409
31 changed files with 202 additions and 344 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devplacepy/services/__init__.py
+12 -3
View File
@@ -1,6 +1,15 @@
# retoor <retoor@molodetz.nl>
from devplacepy.services.base import BaseService
from devplacepy.services.manager import service_manager
__all__ = ["BaseService", "service_manager"]
def __getattr__(name):
if name == "BaseService":
from devplacepy.services.base import BaseService
return BaseService
if name == "service_manager":
from devplacepy.services.manager import service_manager
return service_manager
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
devplacepy/services/ai_modifier.py
+1 -1
View File
@@ -88,7 +88,7 @@ def _run_modification(
modified, usage = modify_text(api_key, prompt, original, context)
if usage:
for key in totals:
totals[key] += usage[key]
totals[key] += usage.get(key, 0)
if modified and modified != original:
updates[field] = modified
if updates:
devplacepy/services/audit/store.py
+12 -11
View File
@@ -158,22 +158,23 @@ def sweep(cutoff: str, batch_size: int = 5000, max_batches: int = 100) -> tuple[
if not uids:
break
placeholders, params = _in_clause(uids)
if LINKS_TABLE in db.tables:
link_rows = list(
with db:
if LINKS_TABLE in db.tables:
link_rows = list(
db.query(
f"SELECT COUNT(*) AS c FROM {LINKS_TABLE} "
f"WHERE audit_uid IN ({placeholders})",
**params,
)
)
removed_links += int(link_rows[0]["c"]) if link_rows else 0
db.query(
f"SELECT COUNT(*) AS c FROM {LINKS_TABLE} "
f"WHERE audit_uid IN ({placeholders})",
f"DELETE FROM {LINKS_TABLE} WHERE audit_uid IN ({placeholders})",
**params,
)
)
removed_links += int(link_rows[0]["c"]) if link_rows else 0
db.query(
f"DELETE FROM {LINKS_TABLE} WHERE audit_uid IN ({placeholders})",
**params,
f"DELETE FROM {AUDIT_TABLE} WHERE uid IN ({placeholders})", **params
)
db.query(
f"DELETE FROM {AUDIT_TABLE} WHERE uid IN ({placeholders})", **params
)
removed_events += len(uids)
if len(uids) < batch_size:
break
devplacepy/services/backup/service.py
+7 -1
View File
@@ -185,7 +185,13 @@ class BackupService(JobService):
f"backup worker exited {proc.returncode}: "
f"{err.decode('utf-8', 'replace')[:500]}"
)
return json.loads(out.decode("utf-8"))
payload = out.decode("utf-8").strip()
if not payload:
raise RuntimeError(
f"backup worker produced no output: "
f"{err.decode('utf-8', 'replace')[:500]}"
)
return json.loads(payload)
def _archive_name(self, target: str, backup_uid: str) -> str:
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
devplacepy/services/containers/terminal.py
-247
View File
@@ -1,247 +0,0 @@
# retoor <retoor@molodetz.nl>
import asyncio
import fcntl
import logging
import os
import pty
import re
import signal
import struct
import termios
from dataclasses import dataclass
from datetime import datetime, timezone
from devplacepy.services.pubsub.hub import pubsub
from devplacepy.services.containers import store
from devplacepy.services.containers.backend.base import WORKSPACE_MOUNT
from devplacepy.services.audit import record as audit
logger = logging.getLogger(__name__)
CONTROL_PATTERN = "container.*.exec.control.*"
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _frame(topic: str, data) -> dict:
return {"type": "message", "topic": topic, "data": data, "ts": _now()}
def _status(text: str) -> str:
return f"\r\n\x1b[33m[devplace] {text}\x1b[0m\r\n"
@dataclass
class _TerminalSession:
container_uid: str
session: str
inst: dict
master: int | None = None
proc: asyncio.subprocess.Process | None = None
stdin_callback: object | None = None
resize_callback: object | None = None
@property
def stdout_topic(self) -> str:
return f"container.{self.container_uid}.exec.stdout.{self.session}"
def _stdin_topic(self) -> str:
return f"container.{self.container_uid}.exec.stdin.{self.session}"
def _resize_topic(self) -> str:
return f"container.{self.container_uid}.exec.resize.{self.session}"
async def start(self) -> None:
inst = self.inst
container_id = inst.get("container_id")
if inst.get("status") != "running" or not container_id:
await self._publish(_status("Container is not running."))
raise RuntimeError("Container is not running or missing container_id")
master, slave = pty.openpty()
self.master = master
session = self.session
if session and re.match(r"^[A-Za-z0-9_-]{1,64}$", session):
shell_cmd = (
f"if command -v tmux >/dev/null 2>&1; then exec tmux new-session -A -s {session}; "
"elif command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi"
)
else:
shell_cmd = (
"if command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi"
)
proc = await asyncio.create_subprocess_exec(
"docker",
"exec",
"-it",
"-w",
WORKSPACE_MOUNT,
container_id,
"/bin/sh",
"-c",
shell_cmd,
stdin=slave,
stdout=slave,
stderr=slave,
start_new_session=True,
)
os.close(slave)
self.proc = proc
self.stdin_callback = self._on_stdin
self.resize_callback = self._on_resize
pubsub.subscribe_callback(self._stdin_topic(), self.stdin_callback)
pubsub.subscribe_callback(self._resize_topic(), self.resize_callback)
asyncio.create_task(self._pump_out())
async def _pump_out(self) -> None:
loop = asyncio.get_event_loop()
try:
while True:
data = await loop.run_in_executor(None, os.read, self.master, 4096)
if not data:
break
await self._publish(data.decode("utf-8", "replace"))
except OSError:
pass
except Exception:
pass
async def _publish(self, text: str) -> None:
await pubsub.publish(self.stdout_topic, _frame(self.stdout_topic, text))
async def _on_stdin(self, topic: str, frame: dict) -> None:
data = frame.get("data", "")
if not data or self.master is None:
return
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, os.write, self.master, data.encode("utf-8"))
async def _on_resize(self, topic: str, frame: dict) -> None:
data = frame.get("data", {})
if not isinstance(data, dict):
return
rows = int(data.get("rows") or 0)
cols = int(data.get("cols") or 0)
if rows <= 0 or cols <= 0 or self.master is None:
return
try:
fcntl.ioctl(
self.master, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0)
)
except OSError:
pass
if self.proc and self.proc.returncode is None:
try:
self.proc.send_signal(signal.SIGWINCH)
except (ProcessLookupError, OSError, ValueError):
pass
async def cleanup(self) -> None:
if self.stdin_callback:
pubsub.unsubscribe_callback(self._stdin_topic(), self.stdin_callback)
self.stdin_callback = None
if self.resize_callback:
pubsub.unsubscribe_callback(self._resize_topic(), self.resize_callback)
self.resize_callback = None
if self.proc and self.proc.returncode is None:
try:
self.proc.kill()
except ProcessLookupError:
pass
self.proc = None
if self.master is not None:
try:
os.close(self.master)
except OSError:
pass
self.master = None
class ContainerTerminalService:
def __init__(self) -> None:
self._sessions: dict[str, _TerminalSession] = {}
def _key(self, container_uid: str, session: str) -> str:
return f"{container_uid}:{session}"
async def start(self) -> None:
await pubsub.subscribe_callback(CONTROL_PATTERN, self._on_control)
async def stop(self) -> None:
pubsub.unsubscribe_callback(CONTROL_PATTERN, self._on_control)
for key in list(self._sessions):
await self._cleanup_key(key)
async def _on_control(self, topic: str, frame: dict) -> None:
parts = topic.split(".")
if len(parts) < 5:
return
container_uid = parts[1]
session = parts[4]
payload = frame.get("data", {})
if not isinstance(payload, dict):
return
command = payload.get("command", "")
if command == "open":
await self._open(container_uid, session)
elif command == "close":
await self._cleanup(container_uid, session)
async def _open(self, container_uid: str, session: str) -> None:
key = self._key(container_uid, session)
if key in self._sessions:
existing = self._sessions[key]
await existing._publish(_status("re-attached to session"))
return
inst = store.get_instance(container_uid)
if not inst or inst.get("status") != "running" or not inst.get("container_id"):
status = inst.get("status", "not found") if inst else "not found"
stdout_topic = f"container.{container_uid}.exec.stdout.{session}"
await pubsub.publish(
stdout_topic,
_frame(stdout_topic, _status(f"Terminal unavailable: container {status}")),
)
return
session_obj = _TerminalSession(container_uid, session, inst)
try:
await session_obj.start()
except Exception as exc:
await session_obj.cleanup()
stdout_topic = f"container.{container_uid}.exec.stdout.{session}"
await pubsub.publish(
stdout_topic,
_frame(stdout_topic, _status(f"Failed to start terminal: {exc}")),
)
return
self._sessions[key] = session_obj
asyncio.create_task(self._monitor(key, session_obj))
async def _monitor(self, key: str, session_obj: _TerminalSession) -> None:
try:
await session_obj.proc.wait()
except Exception:
pass
finally:
await self._cleanup_key(key)
async def _cleanup(self, container_uid: str, session: str) -> None:
await self._cleanup_key(self._key(container_uid, session))
async def _cleanup_key(self, key: str) -> None:
session_obj = self._sessions.pop(key, None)
if session_obj:
await session_obj.cleanup()
terminal_service = ContainerTerminalService()
devplacepy/services/correction.py
+1 -1
View File
@@ -173,7 +173,7 @@ def _run_correction(
corrected, usage = correct_text(api_key, prompt, original)
if usage:
for key in totals:
totals[key] += usage[key]
totals[key] += usage.get(key, 0)
if corrected and corrected != original:
updates[field] = corrected
if updates:
devplacepy/services/deepsearch/store.py
+4 -1
View File
@@ -7,6 +7,7 @@ import math
import re
from collections import Counter
from dataclasses import dataclass, field
from urllib.parse import urlparse
from devplacepy.config import DEEPSEARCH_CHROMA_DIR
@@ -237,7 +238,9 @@ class VectorStore:
chunks = self.all_chunks()
if not chunks:
return {"chunks": 0, "domains": 0, "sources": 0, "avg_chunk_chars": 0}
domains = {chunk.metadata.get("url", "") for chunk in chunks}
domains = {
urlparse(chunk.metadata.get("url", "")).netloc for chunk in chunks
}
domains.discard("")
sources = {chunk.source for chunk in chunks}
sources.discard("")
devplacepy/services/devii/virtual_tools/store.py
+13 -6
View File
@@ -81,7 +81,12 @@ class VirtualToolStore:
"created_at": now,
"updated_at": now,
}
self._table.insert(record)
existing = self._table.find_one(name=record["name"], **self._scope)
if existing:
record["id"] = existing["id"]
self._table.update(record, ["id"])
else:
self._table.insert(record)
logger.info(
"Virtual tool created name=%s owner=%s/%s",
record.get("name"),
@@ -91,8 +96,11 @@ class VirtualToolStore:
return record
def update(self, name: str, changes: dict[str, Any]) -> None:
changes = {**changes, "name": name, **self._scope, "updated_at": _now_iso()}
self._table.update(changes, ["name", "owner_kind", "owner_id"])
row = self._table.find_one(name=name, deleted_at=None, **self._scope)
if not row:
return
changes = {**changes, "id": row["id"], "updated_at": _now_iso()}
self._table.update(changes, ["id"])
logger.debug("Virtual tool updated name=%s changes=%s", name, list(changes))
def delete(self, name: str) -> bool:
@@ -101,12 +109,11 @@ class VirtualToolStore:
return False
self._table.update(
{
"name": name,
"id": row["id"],
"deleted_at": _now_iso(),
"deleted_by": f"{self._owner_kind}:{self._owner_id}",
**self._scope,
},
["name", "owner_kind", "owner_id"],
["id"],
)
logger.info("Virtual tool soft-deleted name=%s", name)
return True
devplacepy/services/jobs/deepsearch/service.py
+30 -17
View File
@@ -77,6 +77,7 @@ class DeepsearchService(JobService):
metadata={"query": query, "error": str(exc)[:200]},
links=[audit.job(uid)],
)
shutil.rmtree(output_dir, ignore_errors=True)
raise
report = self._load_report(output_dir)
@@ -196,23 +197,35 @@ class DeepsearchService(JobService):
)
summary: dict = {}
worker_error = ""
while True:
line = await proc.stdout.readline()
if not line:
break
try:
frame = json.loads(line.decode("utf-8", "replace"))
except (ValueError, TypeError):
continue
hub.publish(uid, frame)
if frame.get("type") == "report_ready":
summary = frame
elif frame.get("type") == "rsearch":
self._ledger_rsearch(owner_kind, owner_id, frame)
elif frame.get("type") == "error":
worker_error = frame.get("message", "worker error")
err = (await proc.stderr.read()).decode("utf-8", "replace")
await proc.wait()
err = ""
stderr_task = asyncio.create_task(proc.stderr.read())
try:
while True:
line = await proc.stdout.readline()
if not line:
break
try:
frame = json.loads(line.decode("utf-8", "replace"))
except (ValueError, TypeError):
continue
hub.publish(uid, frame)
if frame.get("type") == "report_ready":
summary = frame
elif frame.get("type") == "rsearch":
self._ledger_rsearch(owner_kind, owner_id, frame)
elif frame.get("type") == "error":
worker_error = frame.get("message", "worker error")
err = (await stderr_task).decode("utf-8", "replace")
await proc.wait()
finally:
if proc.returncode is None:
try:
proc.kill()
except ProcessLookupError:
pass
await proc.wait()
if not stderr_task.done():
stderr_task.cancel()
if proc.returncode != 0 or not summary:
raise RuntimeError(
worker_error or err[:500] or f"deepsearch worker exited {proc.returncode}"
devplacepy/services/jobs/deepsearch/worker.py
+6 -1
View File
@@ -109,6 +109,7 @@ async def _index_chunks(
backend = "gateway"
forced_local = False
done = 0
collected: list[list[float]] = []
for batch_no, start in enumerate(range(0, total, EMBED_BATCH), start=1):
batch = chunks[start : start + EMBED_BATCH]
emit(
@@ -131,7 +132,7 @@ async def _index_chunks(
forced_local = True
result = local_embed([chunk.text for chunk in batch])
backend = result.backend
store.add(batch, result.vectors)
collected.extend(result.vectors)
done += len(batch)
emit(
{
@@ -145,6 +146,10 @@ async def _index_chunks(
"message": f"Embedded batch {batch_no}/{total_batches} ({backend})",
}
)
if forced_local:
collected = local_embed([chunk.text for chunk in chunks]).vectors
backend = "local"
store.add(chunks, collected)
final_backend = "local" if forced_local else backend
emit({"type": "embed_done", "backend": final_backend, "chunk_count": total})
return total, final_backend
devplacepy/services/jobs/seo/service.py
+28 -15
View File
@@ -63,6 +63,7 @@ class SeoService(JobService):
metadata={"target": target, "error": str(exc)[:200]},
links=[audit.job(uid)],
)
shutil.rmtree(output_dir, ignore_errors=True)
raise
report = self._load_report(output_dir)
@@ -120,21 +121,33 @@ class SeoService(JobService):
)
summary: dict = {}
worker_error = ""
while True:
line = await proc.stdout.readline()
if not line:
break
try:
frame = json.loads(line.decode("utf-8", "replace"))
except (ValueError, TypeError):
continue
hub.publish(uid, frame)
if frame.get("type") == "report_ready":
summary = frame
elif frame.get("type") == "error":
worker_error = frame.get("message", "worker error")
err = (await proc.stderr.read()).decode("utf-8", "replace")
await proc.wait()
err = ""
stderr_task = asyncio.create_task(proc.stderr.read())
try:
while True:
line = await proc.stdout.readline()
if not line:
break
try:
frame = json.loads(line.decode("utf-8", "replace"))
except (ValueError, TypeError):
continue
hub.publish(uid, frame)
if frame.get("type") == "report_ready":
summary = frame
elif frame.get("type") == "error":
worker_error = frame.get("message", "worker error")
err = (await stderr_task).decode("utf-8", "replace")
await proc.wait()
finally:
if proc.returncode is None:
try:
proc.kill()
except ProcessLookupError:
pass
await proc.wait()
if not stderr_task.done():
stderr_task.cancel()
if proc.returncode != 0 or not summary:
raise RuntimeError(
worker_error or err[:500] or f"seo worker exited {proc.returncode}"
devplacepy/services/jobs/seo_meta_service.py
+1 -1
View File
@@ -164,7 +164,7 @@ class SeoMetaService(JobService):
)
if usage:
for key in totals:
totals[key] += usage[key]
totals[key] += usage.get(key, 0)
parsed = self._parse(raw)
if not parsed:
return {**defaults, "source": "plain"}
devplacepy/services/news.py
+4
View File
@@ -290,6 +290,10 @@ def _decode_phash(data: bytes) -> tuple[str, int, int] | None:
async def _get_article_images(url: str, client: httpx.AsyncClient) -> list[dict]:
try:
await net_guard.guard_public_url(url)
except Exception:
return []
try:
resp = await client.get(url, timeout=10.0)
resp.raise_for_status()
devplacepy/services/telegram/bridge.py
+7 -1
View File
@@ -126,6 +126,9 @@ class TelegramBridge:
def _chat_lock(self, chat_id: int) -> asyncio.Lock:
lock = self._chat_locks.get(chat_id)
if lock is None:
if len(self._chat_locks) > 1024:
for key in [k for k, v in self._chat_locks.items() if not v.locked()]:
del self._chat_locks[key]
lock = asyncio.Lock()
self._chat_locks[chat_id] = lock
return lock
@@ -186,7 +189,10 @@ class TelegramBridge:
def _too_many_attempts(self, chat_id: int) -> bool:
now = time.monotonic()
recent = [t for t in self._attempts.get(chat_id, []) if now - t < ATTEMPT_WINDOW_SECONDS]
self._attempts[chat_id] = recent
if recent:
self._attempts[chat_id] = recent
else:
self._attempts.pop(chat_id, None)
return len(recent) >= ATTEMPT_LIMIT
def _register_attempt(self, chat_id: int) -> None:
devplacepy/services/xmlrpc/bridge.py
+2 -2
View File
@@ -121,9 +121,9 @@ def call(method: Method, params: dict[str, Any], auth: dict[str, str]) -> Any:
data=form or None,
)
except Exception as exc:
logger.warning("Bridge transport error for %s: %s", method.name, exc)
logger.warning("Bridge transport error for %s: %r", method.name, exc)
raise xmlrpc.client.Fault(
FAULT_TRANSPORT, f"Transport error calling {method.name}: {exc}"
FAULT_TRANSPORT, f"Transport error calling {method.name}"
)
body = _decode(response)
devplacepy/services/xmlrpc/server.py
+9
View File
@@ -6,7 +6,11 @@ import logging
from socketserver import ForkingMixIn
from xmlrpc.server import SimpleXMLRPCDispatcher, SimpleXMLRPCRequestHandler, SimpleXMLRPCServer
from defusedxml.xmlrpc import monkey_patch as _harden_xmlrpc
from devplacepy.config import XMLRPC_BIND, XMLRPC_PORT
_harden_xmlrpc()
from . import bridge
from .registry import Method, build_index, method_help, method_signature
@@ -42,6 +46,11 @@ class _RequestHandler(SimpleXMLRPCRequestHandler):
rpc_paths = RPC_PATHS
def decode_request_content(self, data):
if b"<!DOCTYPE" in data or b"<!ENTITY" in data:
self.send_response(400)
self.send_header("Content-length", "0")
self.end_headers()
return None
self.server.current_auth = _auth_from_headers(self.headers)
return super().decode_request_content(data)