Compare commits

..
Author SHA1 Message Date
Typosaurus 54a7805a90 ticket #85 attempt 1 2026-07-23 01:31:16 +00:00
Typosaurus 1d1efc0dfc ticket #85 attempt 1 2026-07-23 01:19:19 +00:00
9 changed files with 103 additions and 583 deletions
File diff suppressed because one or more lines are too long
+21
View File
@@ -30,6 +30,26 @@ def migrate_bug_tables_to_issue_tables() -> None:
logger.info("Dropped table %s after migration", source_name) logger.info("Dropped table %s after migration", source_name)
def _add_message_unique_index() -> None:
if "messages" not in db.tables:
return
with db:
db.query(
"""
DELETE FROM messages WHERE id NOT IN (
SELECT MIN(id) FROM messages GROUP BY sender_uid, receiver_uid, content
)
"""
)
_index(
db,
"messages",
"idx_messages_unique_sender_receiver_content",
["sender_uid", "receiver_uid", "content"],
unique=True,
)
def init_db(): def init_db():
tables = db.tables tables = db.tables
_index(db, "users", "idx_users_username", ["username"]) _index(db, "users", "idx_users_username", ["username"])
@@ -131,6 +151,7 @@ def init_db():
"idx_messages_conversation_rev", "idx_messages_conversation_rev",
["receiver_uid", "sender_uid"], ["receiver_uid", "sender_uid"],
) )
_add_message_unique_index()
_index(db, "notifications", "idx_notifications_user", ["user_uid"]) _index(db, "notifications", "idx_notifications_user", ["user_uid"])
_index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"]) _index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
_index(db, "push_registration", "idx_push_registration_user", ["user_uid"]) _index(db, "push_registration", "idx_push_registration_user", ["user_uid"])
+1 -1
View File
@@ -88,7 +88,7 @@ def gateway_complete(
{"role": "system", "content": system}, {"role": "system", "content": system},
{"role": "user", "content": text}, {"role": "user", "content": text},
], ],
"temperature": 0.1, "temperature": 0.0,
} }
headers = { headers = {
"Content-Type": "application/json", "Content-Type": "application/json",
+59 -9
View File
@@ -1,6 +1,9 @@
# retoor <retoor@molodetz.nl> # retoor <retoor@molodetz.nl>
import hashlib
import logging import logging
import time
from collections import OrderedDict
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Any, Optional from typing import Any, Optional
@@ -15,12 +18,16 @@ from devplacepy.utils import (
track_action, track_action,
) )
from devplacepy.services.audit import record as audit from devplacepy.services.audit import record as audit
from sqlalchemy.exc import IntegrityError
from devplacepy.services.correction import schedule_correction from devplacepy.services.correction import schedule_correction
from devplacepy.services.ai_modifier import schedule_modification from devplacepy.services.ai_modifier import schedule_modification
logger = logging.getLogger("messaging.persist") logger = logging.getLogger("messaging.persist")
MAX_CONTENT_LENGTH = 2000 MAX_CONTENT_LENGTH = 2000
DEDUP_WINDOW_SECONDS = 3
_content_cache: dict[str, tuple[float, str]] = OrderedDict()
def _slim_attachment(attachment: dict[str, Any]) -> dict[str, Any]: def _slim_attachment(attachment: dict[str, Any]) -> dict[str, Any]:
@@ -81,19 +88,60 @@ def persist_message(
sender_uid = sender["uid"] sender_uid = sender["uid"]
sender_username = sender.get("username", "") sender_username = sender.get("username", "")
content_hash = hashlib.sha256(
f"{sender_uid}:{receiver_uid}:{content}".encode()
).hexdigest()[:16]
now = time.time()
last_seen, cached_uid = _content_cache.get(content_hash, (0.0, None))
if now - last_seen < DEDUP_WINDOW_SECONDS and cached_uid is not None:
logger.debug(
"Dedup hit for message hash %s (original uid %s)", content_hash, cached_uid
)
cached = get_table("messages").find_one(uid=cached_uid)
if cached:
return {
"uid": cached["uid"],
"sender_uid": cached["sender_uid"],
"receiver_uid": cached["receiver_uid"],
"content": cached["content"],
"read": cached.get("read", False),
"created_at": cached["created_at"],
}
messages_table = get_table("messages") messages_table = get_table("messages")
msg_uid = generate_uid() msg_uid = generate_uid()
created_at = datetime.now(timezone.utc).isoformat() created_at = datetime.now(timezone.utc).isoformat()
messages_table.insert(
{ try:
"uid": msg_uid, messages_table.insert(
"sender_uid": sender_uid, {
"receiver_uid": receiver_uid, "uid": msg_uid,
"content": content, "sender_uid": sender_uid,
"read": False, "receiver_uid": receiver_uid,
"created_at": created_at, "content": content,
"read": False,
"created_at": created_at,
}
)
except IntegrityError:
existing = messages_table.find_one(
sender_uid=sender_uid, receiver_uid=receiver_uid, content=content
)
if not existing:
raise
logger.debug(
"Dedup via unique constraint for message (uid %s)", existing["uid"]
)
_content_cache[content_hash] = (time.time(), existing["uid"])
return {
"uid": existing["uid"],
"sender_uid": existing["sender_uid"],
"receiver_uid": existing["receiver_uid"],
"content": existing["content"],
"read": existing.get("read", False),
"created_at": existing["created_at"],
} }
)
link_attachments(attachment_uids, "message", msg_uid) link_attachments(attachment_uids, "message", msg_uid)
schedule_correction(sender, "messages", msg_uid, request) schedule_correction(sender, "messages", msg_uid, request)
@@ -114,6 +162,8 @@ def persist_message(
) )
track_action(sender_uid, "message") track_action(sender_uid, "message")
_content_cache[content_hash] = (time.time(), msg_uid)
logger.info( logger.info(
"Message %s sent from %s to %s via %s", "Message %s sent from %s to %s via %s",
msg_uid, msg_uid,
-6
View File
@@ -236,12 +236,6 @@ dp-chat[mode="embed"] {
border-bottom-right-radius: 4px; border-bottom-right-radius: 4px;
} }
.message-bubble.mine a,
.message-bubble.mine .rendered-content a {
color: var(--white);
text-decoration: underline;
}
.message-bubble.theirs { .message-bubble.theirs {
align-self: flex-start; align-self: flex-start;
background: var(--bg-card-hover); background: var(--bg-card-hover);
-525
View File
@@ -1,525 +0,0 @@
// retoor <retoor@molodetz.nl>
import { MessagesSocket } from "./MessagesSocket.js";
import { contentRenderer } from "./ContentRenderer.js";
const TYPING_THROTTLE_MS = 1500;
const TYPING_HIDE_MS = 4000;
const AUTO_SCROLL_MARGIN_PX = 100;
const STABILIZE_MAX_FRAMES = 300;
export class MessagesLayout {
constructor() {
this.layout = document.querySelector(".messages-layout");
if (!this.layout) {
return;
}
this.thread = document.querySelector(".messages-thread");
this.form = document.querySelector(".messages-input-area");
this.input = this.form ? this.form.querySelector('input[name="content"]') : null;
this.upload = this.form ? this.form.querySelector("dp-upload") : null;
this.sendBtn = this.form ? this.form.querySelector(".messages-send-btn") : null;
this._uploading = false;
this._pendingSends = new Set();
this.typingEl = document.getElementById("typing-indicator");
this.selfUid = this.layout.dataset.selfUid || "";
this.otherUid = this.layout.dataset.otherUid || "";
this._lastTypingSent = 0;
this._typingHideTimer = null;
this._socketReady = false;
this._userAtBottom = true;
this._stabilizeFrames = 0;
this._stabilizePending = false;
// Expose for diagnostics
window.__messagesLayout = this;
this.scrollThreadToEnd();
if (this.input) {
this.input.focus({ preventScroll: true });
}
this.connect();
this.bindForm();
this.bindTyping();
this.bindViewport();
this.bindAutoScroll();
this._startScrollWatcher();
this.markRead();
}
_startScrollWatcher() {
if (!this.thread) return;
const onAnyChange = () => {
if (!this._userAtBottom) return;
if (this._stabilizePending) return;
this._stabilizePending = true;
requestAnimationFrame(() => {
this._stabilizePending = false;
this._stabilizeScroll();
});
};
this._scrollWatcher = new MutationObserver(onAnyChange);
this._scrollWatcher.observe(this.thread, {
childList: true,
subtree: true,
});
this.thread.addEventListener("load", (e) => {
if (e.target.tagName === "IMG" && this._userAtBottom) {
onAnyChange();
}
}, true);
// Kick initial stabilization
onAnyChange();
}
_stabilizeScroll() {
if (!this.thread) {
this._stabilizeFrames = 0;
return;
}
if (!this._userAtBottom) {
this._stabilizeFrames = 0;
return;
}
if (this._stabilizeFrames >= STABILIZE_MAX_FRAMES) {
this._stabilizeFrames = 0;
return;
}
this._stabilizeFrames++;
this.thread.scrollTop = this.thread.scrollHeight;
const atBottom = this.thread.scrollHeight
- this.thread.scrollTop
- this.thread.clientHeight < 10;
if (!atBottom) {
this._stabilizePending = true;
requestAnimationFrame(() => {
this._stabilizePending = false;
this._stabilizeScroll();
});
} else {
this._stabilizeFrames = 0;
}
}
bindAutoScroll() {
if (!this.thread) return;
this.thread.addEventListener("scroll", () => {
if (this._stabilizePending) return;
const atBottom = this.thread.scrollHeight
- this.thread.scrollTop
- this.thread.clientHeight < AUTO_SCROLL_MARGIN_PX;
this._userAtBottom = atBottom;
if (atBottom) {
this._stabilizeScroll();
}
}, { passive: true });
}
bindViewport() {
const page = document.querySelector(".page-messages");
if (!page || !this.input || !this.form) return;
const viewport = window.visualViewport;
const ensureInputVisible = () => {
const vh = viewport ? viewport.height : window.innerHeight;
const formRect = this.form.getBoundingClientRect();
const currentInset = parseInt(page.style.getPropertyValue("--kb-inset")) || 0;
const delta = formRect.bottom - vh;
const newInset = Math.max(0, currentInset + delta);
if (newInset !== currentInset) {
page.style.setProperty("--kb-inset", `${Math.round(newInset)}px`);
this._userAtBottom = true;
this.scrollThreadToEnd();
}
};
if (viewport) {
let insetTimer = null;
const onViewportChange = () => {
cancelAnimationFrame(insetTimer);
insetTimer = requestAnimationFrame(ensureInputVisible);
};
viewport.addEventListener("resize", onViewportChange);
viewport.addEventListener("scroll", onViewportChange);
}
const ro = new ResizeObserver(() => ensureInputVisible());
ro.observe(page);
if (this.layout) ro.observe(this.layout);
if (this.form) {
const io = new IntersectionObserver((entries) => {
for (const entry of entries) {
if (!entry.isIntersecting) ensureInputVisible();
}
}, { root: null, threshold: [0, 0.5, 1] });
io.observe(this.form);
this._viewportIO = io;
}
this.input.addEventListener("keydown", (event) => {
if (event.key === "Enter" && !event.shiftKey) {
event.preventDefault();
this.form.requestSubmit();
}
});
this.input.addEventListener("input", () => {
this.input.style.height = "auto";
this.input.style.height = Math.min(this.input.scrollHeight, 160) + "px";
});
this.input.addEventListener("focus", () => {
this._userAtBottom = true;
const doScroll = () => {
this.scrollThreadToEnd();
if (window.innerWidth < 768) {
const retry = (delay) => setTimeout(() => {
ensureInputVisible();
this.scrollThreadToEnd();
}, delay);
retry(100);
retry(350);
retry(600);
}
};
requestAnimationFrame(doScroll);
});
this.input.addEventListener("blur", () => {
page.style.setProperty("--kb-inset", "0px");
});
ensureInputVisible();
}
connect() {
this.socket = new MessagesSocket({
onReady: () => this.onReady(),
onMessage: (frame) => this.onFrame(frame),
onClose: () => { this._socketReady = false; },
});
this.socket.connect();
}
onReady() {
this._socketReady = true;
if (this.otherUid) {
this.markRead();
}
}
onFrame(frame) {
switch (frame.type) {
case "message":
this.handleIncoming(frame);
break;
case "typing":
if (frame.from_uid === this.otherUid) this.showTyping();
break;
case "read":
if (frame.by_uid === this.otherUid) this.markReceipts();
break;
default:
break;
}
}
bindForm() {
if (!this.form || !this.input) return;
this.form.addEventListener("submit", (event) => {
if (!this._socketReady || !this.socket.isOpen()) {
return;
}
event.preventDefault();
this.sendViaSocket();
});
if (this.upload) {
this.upload.addEventListener("dp-upload:busy", (event) => {
this._uploading = !!(event.detail && event.detail.busy);
this.refreshSendButton();
});
}
}
refreshSendButton() {
if (!this.sendBtn) return;
const busy = this._uploading || this._pendingSends.size > 0;
this.sendBtn.disabled = busy;
this.sendBtn.classList.toggle("is-sending", busy);
}
sendViaSocket() {
const content = (this.input.value || "").trim();
const attachmentUids = this.collectAttachments();
if (!content && attachmentUids.length === 0) return;
const clientId = "c" + Date.now() + Math.random().toString(36).slice(2, 8);
this.appendOptimistic(content, clientId, attachmentUids.length);
const ok = this.socket.send({
type: "send",
receiver_uid: this.otherUid,
content,
attachment_uids: attachmentUids,
client_id: clientId,
});
if (!ok) {
this.form.submit();
return;
}
this._pendingSends.add(clientId);
this.refreshSendButton();
window.setTimeout(() => this.clearPendingSend(clientId), 8000);
this.input.value = "";
if (this.upload && typeof this.upload.clear === "function") {
this.upload.clear();
}
this.input.focus({ preventScroll: true });
}
clearPendingSend(clientId) {
if (this._pendingSends.delete(clientId)) {
this.refreshSendButton();
}
}
collectAttachments() {
const hidden = this.form.querySelector('input[name="attachment_uids"]');
if (!hidden || !hidden.value) return [];
return hidden.value.split(",").map((v) => v.trim()).filter(Boolean);
}
bindTyping() {
if (!this.input) return;
this.input.addEventListener("input", () => {
if (!this._socketReady || !this.otherUid) return;
const now = Date.now();
if (now - this._lastTypingSent < TYPING_THROTTLE_MS) return;
this._lastTypingSent = now;
this.socket.send({ type: "typing", receiver_uid: this.otherUid });
});
}
appendOptimistic(content, clientId, attachmentCount) {
const bubble = this.buildBubble({
content,
mine: true,
clientId,
time: "now",
iso: new Date().toISOString(),
attachmentCount,
});
bubble.classList.add("pending");
this.insertBubble(bubble);
}
handleIncoming(frame) {
if (frame.uid && this.thread &&
this.thread.querySelector(`.message-bubble[data-msg-uid="${frame.uid}"]`)) {
if (frame.client_id) this.clearPendingSend(frame.client_id);
return;
}
if (frame.sender_uid === this.selfUid && frame.client_id) {
this.clearPendingSend(frame.client_id);
const pending = this.thread.querySelector(`.message-bubble[data-client-id="${frame.client_id}"]`);
if (pending) {
pending.classList.remove("pending");
pending.dataset.msgUid = frame.uid;
const oldBody = pending.querySelector(".rendered-content");
if (oldBody && frame.content !== undefined) {
const content = document.createElement("dp-content");
content.setAttribute("no-copy", "");
if (frame.sender_role === "Admin") content.setAttribute("data-author-admin", "");
content.textContent = frame.content || "";
oldBody.replaceWith(content);
}
const time = pending.querySelector(".message-time");
if (time && frame.created_at) {
time.setAttribute("datetime", frame.created_at);
time.dataset.dt = "";
time.dataset.dtMode = "ago";
if (window.app && window.app.localTime) window.app.localTime.apply(time);
else time.textContent = frame.time_ago;
} else if (time) {
time.textContent = frame.time_ago;
}
const placeholder = pending.querySelector(".attachment-pending");
if (placeholder) placeholder.remove();
const gallery = this.renderAttachments(frame.attachments);
if (gallery) pending.insertBefore(gallery, time || null);
this.bumpConversation(frame, true);
return;
}
}
const partnerUid = frame.sender_uid === this.selfUid ? frame.receiver_uid : frame.sender_uid;
const inThisThread = this.otherUid && partnerUid === this.otherUid;
if (inThisThread) {
const mine = frame.sender_uid === this.selfUid;
const bubble = this.buildBubble({
content: frame.content,
mine,
time: frame.time_ago,
iso: frame.created_at,
uid: frame.uid,
senderRole: frame.sender_role,
attachments: frame.attachments,
});
this.insertBubble(bubble);
if (!mine && this._socketReady) {
this.socket.send({ type: "read", with_uid: this.otherUid });
}
}
this.bumpConversation(frame, frame.sender_uid === this.selfUid);
}
buildBubble({ content, mine, time, iso, uid, clientId, senderRole, attachments, attachmentCount }) {
const bubble = document.createElement("div");
bubble.className = "message-bubble " + (mine ? "mine" : "theirs");
if (uid) bubble.dataset.msgUid = uid;
if (clientId) bubble.dataset.clientId = clientId;
const el = document.createElement("dp-content");
el.setAttribute("no-copy", "");
if (senderRole === "Admin") el.setAttribute("data-author-admin", "");
el.textContent = content || "";
bubble.appendChild(el);
const gallery = this.renderAttachments(attachments);
if (gallery) {
bubble.appendChild(gallery);
} else if (attachmentCount > 0) {
const placeholder = document.createElement("div");
placeholder.className = "attachment-pending";
placeholder.textContent = attachmentCount === 1
? "Uploading attachment..."
: `Uploading ${attachmentCount} attachments...`;
bubble.appendChild(placeholder);
}
const timeEl = document.createElement(iso ? "time" : "span");
timeEl.className = "message-time";
if (iso) {
timeEl.setAttribute("datetime", iso);
timeEl.dataset.dt = "";
timeEl.dataset.dtMode = "ago";
if (window.app && window.app.localTime) window.app.localTime.apply(timeEl);
else timeEl.textContent = time || "";
} else {
timeEl.textContent = time || "";
}
bubble.appendChild(timeEl);
if (mine) {
const receipt = document.createElement("span");
receipt.className = "message-receipt";
receipt.hidden = true;
receipt.innerHTML = "&#x2713;&#x2713;";
bubble.appendChild(receipt);
}
return bubble;
}
renderAttachments(attachments) {
if (!attachments || !attachments.length) return null;
const gallery = document.createElement("div");
gallery.className = "attachment-gallery";
for (const att of attachments) {
const item = document.createElement("div");
item.className = "attachment-gallery-item";
if (att.is_image) {
const img = document.createElement("img");
img.src = att.thumbnail_url || att.url;
img.alt = att.original_filename || "";
img.loading = "lazy";
img.className = "gallery-thumb";
img.dataset.lightbox = "";
img.dataset.full = att.url;
if (att.mime_type) img.dataset.mime = att.mime_type;
item.appendChild(img);
} else if (att.is_video) {
const video = document.createElement("video");
video.src = att.url;
video.controls = true;
video.preload = "metadata";
video.className = "gallery-video";
item.appendChild(video);
} else {
const link = document.createElement("a");
link.href = att.url;
link.target = "_blank";
link.rel = "noopener";
link.className = "non-image";
link.download = att.original_filename || "file";
link.textContent = att.original_filename || "file";
item.appendChild(link);
}
gallery.appendChild(item);
}
return gallery;
}
insertBubble(bubble) {
if (!this.thread) return;
if (this.typingEl && this.typingEl.parentElement === this.thread) {
this.thread.insertBefore(bubble, this.typingEl);
} else {
this.thread.appendChild(bubble);
}
this.scrollThreadToEnd();
}
markReceipts() {
this.thread.querySelectorAll(".message-bubble.mine .message-receipt").forEach((el) => {
el.hidden = false;
});
}
markRead() {
if (this._socketReady && this.otherUid) {
this.socket.send({ type: "read", with_uid: this.otherUid });
}
}
showTyping() {
if (!this.typingEl) return;
this.typingEl.hidden = false;
this.scrollThreadToEnd();
clearTimeout(this._typingHideTimer);
this._typingHideTimer = setTimeout(() => {
this.typingEl.hidden = true;
}, TYPING_HIDE_MS);
}
bumpConversation(frame, mine) {
const partnerUid = mine ? frame.receiver_uid : frame.sender_uid;
const item = document.querySelector(`.conversation-item[data-conv-uid="${partnerUid}"]`);
if (!item) return;
const preview = item.querySelector(".conversation-preview");
if (preview) preview.textContent = contentRenderer.preview(frame.content, 60);
const dot = item.querySelector(".conversation-unread-dot");
if (dot) {
dot.hidden = mine || partnerUid === this.otherUid;
}
const list = item.parentElement;
if (list && list.firstElementChild !== item) {
list.insertBefore(item, list.firstElementChild);
}
}
scrollThreadToEnd() {
if (!this.thread) return;
if (!this._userAtBottom) return;
this.thread.scrollTop = this.thread.scrollHeight;
}
}
@@ -13,7 +13,7 @@ export class AppContent extends Component {
return; return;
} }
if (!contentRenderer.emojiLoaded) { if (!contentRenderer.emojiLoaded) {
contentRenderer.ready.then(() => this.connectedCallback()).catch(console.error); contentRenderer.ready.then(() => this.connectedCallback());
return; return;
} }
this._rendered = true; this._rendered = true;
+20
View File
@@ -166,3 +166,23 @@ def test_send_attachment_only_empty_content_succeeds(seeded_db):
refresh_snapshot() refresh_snapshot()
row = get_table("messages").find_one(uid=msg["uid"]) row = get_table("messages").find_one(uid=msg["uid"])
assert row["content"] == "" assert row["content"] == ""
def test_duplicate_message_returns_same_uid(seeded_db):
s, _ = _member()
receiver = _db_user("bob_test")["uid"]
content = _unique("dupmsg")
first = s.post(
f"{BASE_URL}/messages/send",
headers=JSON_audit_log,
data={"content": content, "receiver_uid": receiver},
).json()["data"]
second = s.post(
f"{BASE_URL}/messages/send",
headers=JSON_audit_log,
data={"content": content, "receiver_uid": receiver},
).json()["data"]
assert first["uid"] == second["uid"], "duplicate messages should return the same uid"
-40
View File
@@ -263,43 +263,3 @@ def test_xss_legitimate_link_survives_audit():
out = str(render_content("see https://example.com/page ok")) out = str(render_content("see https://example.com/page ok"))
assert 'href="https://example.com/page"' in out assert 'href="https://example.com/page"' in out
assert_no_executable_html(out) assert_no_executable_html(out)
def test_url_inside_fenced_code_block_is_not_embedded():
out = str(render_content("```\nhttps://example.com/video.mp4\n```"))
assert "<pre" in out
assert "<video" not in out
assert "<iframe" not in out
assert "example.com/video.mp4" in out
def test_mention_does_not_match_email_address():
out = str(render_content("contact me at user@domain.com for info"))
assert "mention-link" not in out
assert "domain.com" in out
def test_bold_inside_inline_code_is_plain():
out = str(render_content("use `**not bold**` here"))
assert "<code>" in out
assert "<strong>" not in out
def test_consecutive_line_breaks_preserve_paragraphs():
out = str(render_content("line one\n\nline two\n\n\nline three"))
assert "<p>line one</p>" in out
assert "<p>line two</p>" in out
assert "<p>line three</p>" in out
def test_table_with_inline_links_renders_correctly():
out = str(render_content(
"| Name | Link |\n"
"|------|------|\n"
"| Dev | https://dev.place |\n"
"| Docs | https://docs.place |\n"
))
assert "<table>" in out
assert 'href="https://dev.place"' in out
assert 'href="https://docs.place"' in out
assert "<th" in out