Update channel message.

Reviewed-on: retoor/snek#67
Reviewed-by: retoor <retoor@noreply@molodetz.nl>

pyproject.toml

@@ -39,7 +39,8 @@ dependencies = [
     "Pillow",
     "pillow-heif",
     "IP2Location",
-    "bleach"
+    "bleach",
+    "sentry-sdk"
 ]
 
 [tool.setuptools.packages.find]

src/snek/__main__.py

@@ -9,6 +9,8 @@ from snek.shell import Shell
 from snek.app import Application
 
 
+
+
 @click.group()
 def cli():
     pass
@@ -122,6 +124,12 @@ def shell(db_path):
     Shell(db_path).run()
 
 def main():
+    try:
+        import sentry_sdk
+        sentry_sdk.init("https://ab6147c2f3354c819768c7e89455557b@gt.molodetz.nl/1")
+    except ImportError:
+        print("Could not import sentry_sdk")
+
     cli()
 
 

src/snek/app.py

@@ -6,6 +6,7 @@ import uuid
 import signal
 from datetime import datetime
 from contextlib import asynccontextmanager
+import aiohttp_debugtoolbar
 
 from snek import snode
 from snek.view.threads import ThreadsView
@@ -176,6 +177,8 @@ class Application(BaseApplication):
         self.on_startup.append(self.start_ssh_server)
         self.on_startup.append(self.prepare_database)
         
+
+
     async def prepare_stats(self, app):
         app['stats'] = create_stats_structure()
         print("Stats prepared", flush=True)
@@ -287,9 +290,9 @@ class Application(BaseApplication):
         self.router.add_view("/login.json", LoginView)
         self.router.add_view("/register.html", RegisterView)
         self.router.add_view("/register.json", RegisterView)
-        self.router.add_view("/drive/{rel_path:.*}", DriveView)
-        self.router.add_view("/drive.bin", UploadView)
-        self.router.add_view("/drive.bin/{uid}.{ext}", UploadView)
+       # self.router.add_view("/drive/{rel_path:.*}", DriveView)
+       ## self.router.add_view("/drive.bin", UploadView)
+       # self.router.add_view("/drive.bin/{uid}.{ext}", UploadView)
         self.router.add_view("/search-user.html", SearchUserView)
         self.router.add_view("/search-user.json", SearchUserView)
         self.router.add_view("/avatar/{uid}.svg", AvatarView)
@@ -297,25 +300,25 @@ class Application(BaseApplication):
         self.router.add_get("/http-photo", self.handle_http_photo)
         self.router.add_get("/rpc.ws", RPCView)
         self.router.add_get("/c/{channel:.*}", ChannelView)
-        self.router.add_view(
-            "/channel/{channel_uid}/attachment.bin", ChannelAttachmentView
-        )
-        self.router.add_view(
-            "/channel/{channel_uid}/drive.json", ChannelDriveApiView
-        )
+        #self.router.add_view(
+        #    "/channel/{channel_uid}/attachment.bin", ChannelAttachmentView
+        #)
+        #self.router.add_view(
+        #    "/channel/{channel_uid}/drive.json", ChannelDriveApiView
+        #)
         self.router.add_view(
             "/channel/{channel_uid}/attachment.sock", ChannelAttachmentUploadView
         )
         self.router.add_view(
             "/channel/attachment/{relative_url:.*}", ChannelAttachmentView
-        )
+        )#
         self.router.add_view("/channel/{channel}.html", WebView)
         self.router.add_view("/threads.html", ThreadsView)
         self.router.add_view("/terminal.ws", TerminalSocketView)
         self.router.add_view("/terminal.html", TerminalView)
-        self.router.add_view("/drive.json", DriveApiView)
-        self.router.add_view("/drive.html", DriveView)
-        self.router.add_view("/drive/{drive}.json", DriveView)
+        #self.router.add_view("/drive.json", DriveApiView)
+        #self.router.add_view("/drive.html", DriveView)
+        #self.router.add_view("/drive/{drive}.json", DriveView)
         self.router.add_get("/stats.html", stats_handler)
         self.router.add_view("/stats.json", StatsView)
         self.router.add_view("/user/{user}.html", UserView)
@@ -496,6 +499,7 @@ class Application(BaseApplication):
             raise raised_exception
 
 app = Application(db_path="sqlite:///snek.db")
+#aiohttp_debugtoolbar.setup(app)
 
 
 async def main():

src/snek/model/channel.py

@@ -23,7 +23,7 @@ class ChannelModel(BaseModel):
             history_start_filter = f" AND created_at > '{self['history_start']}' "
         try:
             async for model in self.app.services.channel_message.query(
-                "SELECT uid FROM channel_message WHERE channel_uid=:channel_uid" + history_start_filter + " ORDER BY created_at DESC LIMIT 1",
+                "SELECT uid FROM channel_message WHERE channel_uid=:channel_uid" + history_start_filter + " ORDER BY id DESC LIMIT 1",
                 {"channel_uid": self["uid"]},
             ):
 

src/snek/service/channel_message.py

@@ -1,6 +1,21 @@
 from snek.system.service import BaseService
-from snek.system.template import whitelist_attributes
+from snek.system.template import sanitize_html
 import time
+import asyncio
+from concurrent.futures import ProcessPoolExecutor
+import json
+
+from jinja2 import Environment, FileSystemLoader
+global jinja2_env
+import pathlib 
+template_path = pathlib.Path(__file__).parent.parent.joinpath("templates")
+
+
+def render(context):
+    template =jinja2_env.get_template("message.html")
+    return sanitize_html(template.render(**context))
+
+
 
 class ChannelMessageService(BaseService):
     mapper_name = "channel_message"
@@ -8,6 +23,19 @@ class ChannelMessageService(BaseService):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._configured_indexes = False
+        self._executor_pools = {}
+        global jinja2_env
+        jinja2_env = self.app.jinja2_env
+        self._max_workers = 1
+    def get_or_create_executor(self, uid):
+        if not uid in self._executor_pools:
+            self._executor_pools[uid] = ProcessPoolExecutor(max_workers=5)
+        return self._executor_pools[uid]
+    
+    def delete_executor(self, uid):
+        if uid in self._executor_pools:
+            self._executor_pools[uid].shutdown()
+            del self._executor_pools[uid]
 
     async def maintenance(self):
         args = {}
@@ -69,10 +97,14 @@ class ChannelMessageService(BaseService):
                 "color": user["color"],
             }
         )
-        context['message'] = whitelist_attributes(context['message'])
+        loop = asyncio.get_event_loop()
         try:
-            template = self.app.jinja2_env.get_template("message.html")
-            model["html"] = template.render(**context)
+
+            context = json.loads(json.dumps(context, default=str)) 
+
+
+            model["html"] = await loop.run_in_executor(self.get_or_create_executor(model["uid"]), render,context)
+            #model['html'] = await loop.run_in_executor(self.get_or_create_executor(user["uid"]), sanitize_html,model['html'])
         except Exception as ex:
             print(ex, flush=True)
 
@@ -91,6 +123,8 @@ class ChannelMessageService(BaseService):
                         ["deleted_at"], unique=False
                     )
             self._configured_indexes = True
+            if model['is_final']:
+                self.delete_executor(model['uid'])
             return model
         raise Exception(f"Failed to create channel message: {model.errors}.")
 
@@ -118,7 +152,6 @@ class ChannelMessageService(BaseService):
     async def save(self, model):
         context = {}
         context.update(model.record)
-        context['message'] = whitelist_attributes(context['message'])
         user = await self.app.services.user.get(model["user_uid"])
         context.update(
             {
@@ -128,9 +161,15 @@ class ChannelMessageService(BaseService):
                 "color": user["color"],
             }
         )
-        template = self.app.jinja2_env.get_template("message.html")
-        model["html"] = template.render(**context)
-        return await super().save(model)
+        context = json.loads(json.dumps(context, default=str)) 
+        loop = asyncio.get_event_loop()
+        model["html"] = await loop.run_in_executor(self.get_or_create_executor(model["uid"]), render, context)
+        #model['html'] = await loop.run_in_executor(self.get_or_create_executor(user["uid"]), sanitize_html,model['html'])
+
+        result = await super().save(model)
+        if model['is_final']:
+            self.delete_executor(model['uid'])
+        return result 
 
     async def offset(self, channel_uid, page=0, timestamp=None, page_size=30):
         channel = await self.services.channel.get(uid=channel_uid)

src/snek/service/socket.py

@@ -44,19 +44,29 @@ class SocketService(BaseService):
 
     async def user_availability_service(self):
         logger.info("User availability update service started.")
+        logger.debug("Entering the main loop.")
         while True:
             logger.info("Updating user availability...")
+            logger.debug("Initializing users_updated list.")
             users_updated = []
+            logger.debug("Iterating over sockets.")
             for s in self.sockets:
+                logger.debug(f"Checking connection status for socket: {s}.")
                 if not s.is_connected:
+                    logger.debug("Socket is not connected, continuing to next socket.")
                     continue
+                logger.debug(f"Checking if user {s.user} is already updated.")
                 if s.user not in users_updated:
+                    logger.debug(f"Updating last_ping for user: {s.user}.")
                     s.user["last_ping"] = now()
+                    logger.debug(f"Saving user {s.user} to the database.")
                     await self.app.services.user.save(s.user)
+                    logger.debug(f"Adding user {s.user} to users_updated list.")
                     users_updated.append(s.user)
             logger.info(
                 f"Updated user availability for {len(users_updated)} online users."
             )
+            logger.debug("Sleeping for 60 seconds before the next update.")
             await asyncio.sleep(60)
 
     async def add(self, ws, user_uid):
@@ -77,7 +87,7 @@ class SocketService(BaseService):
 
     async def send_to_user(self, user_uid, message):
         count = 0
-        for s in self.users.get(user_uid, []):
+        for s in list(self.users.get(user_uid, [])):
             if await s.send_json(message):
                 count += 1
         return count

src/snek/static/editor.js

@@ -1,226 +1,494 @@
-import { NjetComponent} from "/njet.js"
+import { NjetComponent } from "/njet.js"
 
-    class NjetEditor extends NjetComponent {
-      constructor() {
-        super();
-        this.attachShadow({ mode: 'open' });
+class NjetEditor extends NjetComponent {
+  constructor() {
+    super();
+    this.attachShadow({ mode: 'open' });
 
-        const style = document.createElement('style');
-        style.textContent = `
-          #editor {
-            padding: 1rem;
-            outline: none;
-            white-space: pre-wrap;
-            line-height: 1.5;
-            height: 100%;
-            overflow-y: auto;
-            background: #1e1e1e;
-            color: #d4d4d4;
-          }
-          #command-line {
-            position: absolute;
-            bottom: 0;
-            left: 0;
-            width: 100%;
-            padding: 0.2rem 1rem;
-            background: #333;
-            color: #0f0;
-            display: none;
-            font-family: monospace;
-          }
-        `;
+    const style = document.createElement('style');
+    style.textContent = `
+      :host {
+        display: block;
+        position: relative;
+        height: 100%;
+        font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
+      }
+      
+      #editor {
+        padding: 1rem;
+        outline: none;
+        white-space: pre-wrap;
+        line-height: 1.5;
+        height: calc(100% - 30px);
+        overflow-y: auto;
+        background: #1e1e1e;
+        color: #d4d4d4;
+        font-size: 14px;
+        caret-color: #fff;
+      }
+      
+      #editor.insert-mode {
+        caret-color: #4ec9b0;
+      }
+      
+      #editor.visual-mode {
+        caret-color: #c586c0;
+      }
+      
+      #editor::selection {
+        background: #264f78;
+      }
+      
+      #status-bar {
+        position: absolute;
+        bottom: 0;
+        left: 0;
+        right: 0;
+        height: 30px;
+        background: #007acc;
+        color: #fff;
+        display: flex;
+        align-items: center;
+        padding: 0 1rem;
+        font-size: 12px;
+        font-weight: 500;
+      }
+      
+      #mode-indicator {
+        text-transform: uppercase;
+        margin-right: 20px;
+        font-weight: bold;
+      }
+      
+      #command-line {
+        position: absolute;
+        bottom: 30px;
+        left: 0;
+        width: 100%;
+        padding: 0.3rem 1rem;
+        background: #2d2d2d;
+        color: #d4d4d4;
+        display: none;
+        font-family: inherit;
+        border-top: 1px solid #3e3e3e;
+        font-size: 14px;
+      }
+      
+      #command-input {
+        background: transparent;
+        border: none;
+        color: inherit;
+        outline: none;
+        font-family: inherit;
+        font-size: inherit;
+        width: calc(100% - 20px);
+      }
+      
+      .visual-selection {
+        background: #264f78 !important;
+      }
+    `;
 
-        this.editor = document.createElement('div');
-        this.editor.id = 'editor';
-        this.editor.contentEditable = true;
-        this.editor.innerText = `Welcome to VimEditor Component
+    this.editor = document.createElement('div');
+    this.editor.id = 'editor';
+    this.editor.contentEditable = true;
+    this.editor.spellcheck = false;
+    this.editor.innerText = `Welcome to VimEditor Component
 Line 2 here
 Another line
 Try i, Esc, v, :, yy, dd, 0, $, gg, G, and p`;
 
-        this.cmdLine = document.createElement('div');
-        this.cmdLine.id = 'command-line';
-        this.shadowRoot.append(style, this.editor, this.cmdLine);
+    this.cmdLine = document.createElement('div');
+    this.cmdLine.id = 'command-line';
+    
+    const cmdPrompt = document.createElement('span');
+    cmdPrompt.textContent = ':';
+    
+    this.cmdInput = document.createElement('input');
+    this.cmdInput.id = 'command-input';
+    this.cmdInput.type = 'text';
+    
+    this.cmdLine.append(cmdPrompt, this.cmdInput);
 
-        this.mode = 'normal'; // normal | insert | visual | command
-        this.keyBuffer = '';
-        this.lastDeletedLine = '';
-        this.yankedLine = '';
+    this.statusBar = document.createElement('div');
+    this.statusBar.id = 'status-bar';
+    
+    this.modeIndicator = document.createElement('span');
+    this.modeIndicator.id = 'mode-indicator';
+    this.modeIndicator.textContent = 'NORMAL';
+    
+    this.statusBar.appendChild(this.modeIndicator);
 
-        this.editor.addEventListener('keydown', this.handleKeydown.bind(this));
-      }
+    this.shadowRoot.append(style, this.editor, this.cmdLine, this.statusBar);
 
-      connectedCallback() {
+    this.mode = 'normal';
+    this.keyBuffer = '';
+    this.lastDeletedLine = '';
+    this.yankedLine = '';
+    this.visualStartOffset = null;
+    this.visualEndOffset = null;
+
+    // Bind event handlers
+    this.handleKeydown = this.handleKeydown.bind(this);
+    this.handleCmdKeydown = this.handleCmdKeydown.bind(this);
+    this.updateVisualSelection = this.updateVisualSelection.bind(this);
+    
+    this.editor.addEventListener('keydown', this.handleKeydown);
+    this.cmdInput.addEventListener('keydown', this.handleCmdKeydown);
+    this.editor.addEventListener('beforeinput', this.handleBeforeInput.bind(this));
+  }
+
+  connectedCallback() {
+    this.editor.focus();
+  }
+
+  setMode(mode) {
+    this.mode = mode;
+    this.modeIndicator.textContent = mode.toUpperCase();
+    
+    // Update editor classes
+    this.editor.classList.remove('insert-mode', 'visual-mode', 'normal-mode');
+    this.editor.classList.add(`${mode}-mode`);
+    
+    if (mode === 'visual') {
+      this.visualStartOffset = this.getCaretOffset();
+      this.editor.addEventListener('selectionchange', this.updateVisualSelection);
+    } else {
+      this.clearVisualSelection();
+      this.editor.removeEventListener('selectionchange', this.updateVisualSelection);
+    }
+    
+    if (mode === 'command') {
+      this.cmdLine.style.display = 'block';
+      this.cmdInput.value = '';
+      this.cmdInput.focus();
+    } else {
+      this.cmdLine.style.display = 'none';
+      if (mode !== 'insert') {
+        // Keep focus on editor for all non-insert modes
         this.editor.focus();
       }
+    }
+  }
 
-      getCaretOffset() {
-        let caretOffset = 0;
-        const sel = this.shadowRoot.getSelection();
-        if (!sel || sel.rangeCount === 0) return 0;
+  updateVisualSelection() {
+    if (this.mode !== 'visual') return;
+    this.visualEndOffset = this.getCaretOffset();
+  }
 
-        const range = sel.getRangeAt(0);
-        const preCaretRange = range.cloneRange();
-        preCaretRange.selectNodeContents(this.editor);
-        preCaretRange.setEnd(range.endContainer, range.endOffset);
-        caretOffset = preCaretRange.toString().length;
-        return caretOffset;
+  clearVisualSelection() {
+    const sel = this.shadowRoot.getSelection();
+    if (sel) sel.removeAllRanges();
+    this.visualStartOffset = null;
+    this.visualEndOffset = null;
+  }
+
+  getCaretOffset() {
+    const sel = this.shadowRoot.getSelection();
+    if (!sel || sel.rangeCount === 0) return 0;
+
+    const range = sel.getRangeAt(0);
+    const preCaretRange = range.cloneRange();
+    preCaretRange.selectNodeContents(this.editor);
+    preCaretRange.setEnd(range.endContainer, range.endOffset);
+    return preCaretRange.toString().length;
+  }
+
+  setCaretOffset(offset) {
+    const textContent = this.editor.innerText;
+    offset = Math.max(0, Math.min(offset, textContent.length));
+    
+    const range = document.createRange();
+    const sel = this.shadowRoot.getSelection();
+    const walker = document.createTreeWalker(
+      this.editor, 
+      NodeFilter.SHOW_TEXT, 
+      null, 
+      false
+    );
+
+    let currentOffset = 0;
+    let node;
+    
+    while ((node = walker.nextNode())) {
+      const nodeLength = node.textContent.length;
+      if (currentOffset + nodeLength >= offset) {
+        range.setStart(node, offset - currentOffset);
+        range.collapse(true);
+        sel.removeAllRanges();
+        sel.addRange(range);
+        return;
       }
+      currentOffset += nodeLength;
+    }
+    
+    // If we couldn't find the position, set to end
+    if (this.editor.lastChild) {
+      range.selectNodeContents(this.editor.lastChild);
+      range.collapse(false);
+      sel.removeAllRanges();
+      sel.addRange(range);
+    }
+  }
 
-      setCaretOffset(offset) {
-        const range = document.createRange();
-        const sel = this.shadowRoot.getSelection();
-        const walker = document.createTreeWalker(this.editor, NodeFilter.SHOW_TEXT, null, false);
+  handleBeforeInput(e) {
+    if (this.mode !== 'insert') {
+      e.preventDefault();
+    }
+  }
 
-        let currentOffset = 0;
-        let node;
-        while ((node = walker.nextNode())) {
-          if (currentOffset + node.length >= offset) {
-            range.setStart(node, offset - currentOffset);
-            range.collapse(true);
-            sel.removeAllRanges();
-            sel.addRange(range);
-            return;
-          }
-          currentOffset += node.length;
+  handleCmdKeydown(e) {
+    if (e.key === 'Enter') {
+      e.preventDefault();
+      this.executeCommand(this.cmdInput.value);
+      this.setMode('normal');
+    } else if (e.key === 'Escape') {
+      e.preventDefault();
+      this.setMode('normal');
+    }
+  }
+
+  executeCommand(cmd) {
+    const trimmedCmd = cmd.trim();
+    
+    // Handle basic vim commands
+    if (trimmedCmd === 'w' || trimmedCmd === 'write') {
+      console.log('Save command (not implemented)');
+    } else if (trimmedCmd === 'q' || trimmedCmd === 'quit') {
+      console.log('Quit command (not implemented)');
+    } else if (trimmedCmd === 'wq' || trimmedCmd === 'x') {
+      console.log('Save and quit command (not implemented)');
+    } else if (/^\d+$/.test(trimmedCmd)) {
+      // Go to line number
+      const lineNum = parseInt(trimmedCmd, 10) - 1;
+      this.goToLine(lineNum);
+    }
+  }
+
+  goToLine(lineNum) {
+    const lines = this.editor.innerText.split('\n');
+    if (lineNum < 0 || lineNum >= lines.length) return;
+    
+    let offset = 0;
+    for (let i = 0; i < lineNum; i++) {
+      offset += lines[i].length + 1;
+    }
+    this.setCaretOffset(offset);
+  }
+
+  getCurrentLineInfo() {
+    const text = this.editor.innerText;
+    const caretPos = this.getCaretOffset();
+    const lines = text.split('\n');
+    
+    let charCount = 0;
+    for (let i = 0; i < lines.length; i++) {
+      if (caretPos <= charCount + lines[i].length) {
+        return {
+          lineIndex: i,
+          lines: lines,
+          lineStartOffset: charCount,
+          positionInLine: caretPos - charCount
+        };
+      }
+      charCount += lines[i].length + 1;
+    }
+    
+    return {
+      lineIndex: lines.length - 1,
+      lines: lines,
+      lineStartOffset: charCount - lines[lines.length - 1].length - 1,
+      positionInLine: 0
+    };
+  }
+
+  handleKeydown(e) {
+    if (this.mode === 'insert') {
+      if (e.key === 'Escape') {
+        e.preventDefault();
+        this.setMode('normal');
+        // Move cursor one position left (vim behavior)
+        const offset = this.getCaretOffset();
+        if (offset > 0) {
+          this.setCaretOffset(offset - 1);
         }
       }
+      return;
+    }
 
-      handleKeydown(e) {
-        const key = e.key;
+    if (this.mode === 'command') {
+      return; // Command mode input is handled by cmdInput
+    }
 
-        if (this.mode === 'insert') {
-          if (key === 'Escape') {
-            e.preventDefault();
-            this.mode = 'normal';
-            this.editor.blur();
-            this.editor.focus();
-          }
-          return;
+    if (this.mode === 'visual') {
+      if (e.key === 'Escape') {
+        e.preventDefault();
+        this.setMode('normal');
+        return;
+      }
+      
+      // Allow movement in visual mode
+      if (['ArrowLeft', 'ArrowRight', 'ArrowUp', 'ArrowDown'].includes(e.key)) {
+        return; // Let default behavior handle selection
+      }
+      
+      if (e.key === 'y') {
+        e.preventDefault();
+        // Yank selected text
+        const sel = this.shadowRoot.getSelection();
+        if (sel && sel.rangeCount > 0) {
+          this.yankedLine = sel.toString();
         }
-
-        if (this.mode === 'command') {
-          if (key === 'Enter' || key === 'Escape') {
-            e.preventDefault();
-            this.cmdLine.style.display = 'none';
-            this.mode = 'normal';
-            this.keyBuffer = '';
-          }
-          return;
-        }
-
-        if (this.mode === 'visual') {
-          if (key === 'Escape') {
-            e.preventDefault();
-            this.mode = 'normal';
-          }
-          return;
-        }
-
-        // Handle normal mode
-        this.keyBuffer += key;
-
-        const text = this.editor.innerText;
-        const caretPos = this.getCaretOffset();
-        const lines = text.split('\n');
-
-        let charCount = 0, lineIdx = 0;
-        for (let i = 0; i < lines.length; i++) {
-          if (caretPos <= charCount + lines[i].length) {
-            lineIdx = i;
-            break;
-          }
-          charCount += lines[i].length + 1;
-        }
-
-        const offsetToLine = idx =>
-          text.split('\n').slice(0, idx).reduce((acc, l) => acc + l.length + 1, 0);
-
-        switch (this.keyBuffer) {
-          case 'i':
-            e.preventDefault();
-            this.mode = 'insert';
-            this.keyBuffer = '';
-            break;
-
-          case 'v':
-            e.preventDefault();
-            this.mode = 'visual';
-            this.keyBuffer = '';
-            break;
-
-          case ':':
-            e.preventDefault();
-            this.mode = 'command';
-            this.cmdLine.style.display = 'block';
-            this.cmdLine.textContent = ':';
-            this.keyBuffer = '';
-            break;
-
-          case 'yy':
-            e.preventDefault();
-            this.yankedLine = lines[lineIdx];
-            this.keyBuffer = '';
-            break;
-
-          case 'dd':
-            e.preventDefault();
-            this.lastDeletedLine = lines[lineIdx];
-            lines.splice(lineIdx, 1);
-            this.editor.innerText = lines.join('\n');
-            this.setCaretOffset(offsetToLine(lineIdx));
-            this.keyBuffer = '';
-            break;
-
-          case 'p':
-            e.preventDefault();
-            const lineToPaste = this.yankedLine || this.lastDeletedLine;
-            if (lineToPaste) {
-              lines.splice(lineIdx + 1, 0, lineToPaste);
-              this.editor.innerText = lines.join('\n');
-              this.setCaretOffset(offsetToLine(lineIdx + 1));
-            }
-            this.keyBuffer = '';
-            break;
-
-          case '0':
-            e.preventDefault();
-            this.setCaretOffset(offsetToLine(lineIdx));
-            this.keyBuffer = '';
-            break;
-
-          case '$':
-            e.preventDefault();
-            this.setCaretOffset(offsetToLine(lineIdx) + lines[lineIdx].length);
-            this.keyBuffer = '';
-            break;
-
-          case 'gg':
-            e.preventDefault();
-            this.setCaretOffset(0);
-            this.keyBuffer = '';
-            break;
-
-          case 'G':
-            e.preventDefault();
-            this.setCaretOffset(text.length);
-            this.keyBuffer = '';
-            break;
-
-          case 'Escape':
-            e.preventDefault();
-            this.mode = 'normal';
-            this.keyBuffer = '';
-            this.cmdLine.style.display = 'none';
-            break;
-
-          default:
-            // allow up to 2 chars for combos
-            if (this.keyBuffer.length > 2) this.keyBuffer = '';
-            break;
+        this.setMode('normal');
+        return;
+      }
+      
+      if (e.key === 'd' || e.key === 'x') {
+        e.preventDefault();
+        // Delete selected text
+        const sel = this.shadowRoot.getSelection();
+        if (sel && sel.rangeCount > 0) {
+          this.lastDeletedLine = sel.toString();
+          document.execCommand('delete');
         }
+        this.setMode('normal');
+        return;
       }
     }
 
-    customElements.define('njet-editor', NjetEditor);
-export {NjetEditor}
+    // Normal mode handling
+    e.preventDefault();
+    
+    // Special keys that should be handled immediately
+    if (e.key === 'Escape') {
+      this.keyBuffer = '';
+      this.setMode('normal');
+      return;
+    }
+    
+    // Build key buffer for commands
+    this.keyBuffer += e.key;
+    
+    const lineInfo = this.getCurrentLineInfo();
+    const { lineIndex, lines, lineStartOffset, positionInLine } = lineInfo;
+
+    // Process commands
+    switch (this.keyBuffer) {
+      case 'i':
+        this.keyBuffer = '';
+        this.setMode('insert');
+        break;
+
+      case 'a':
+        this.keyBuffer = '';
+        this.setCaretOffset(this.getCaretOffset() + 1);
+        this.setMode('insert');
+        break;
+
+      case 'v':
+        this.keyBuffer = '';
+        this.setMode('visual');
+        break;
+
+      case ':':
+        this.keyBuffer = '';
+        this.setMode('command');
+        break;
+
+      case 'yy':
+        this.keyBuffer = '';
+        this.yankedLine = lines[lineIndex];
+        break;
+
+      case 'dd':
+        this.keyBuffer = '';
+        this.lastDeletedLine = lines[lineIndex];
+        lines.splice(lineIndex, 1);
+        if (lines.length === 0) lines.push('');
+        this.editor.innerText = lines.join('\n');
+        this.setCaretOffset(lineStartOffset);
+        break;
+
+      case 'p':
+        this.keyBuffer = '';
+        const lineToPaste = this.yankedLine || this.lastDeletedLine;
+        if (lineToPaste) {
+          lines.splice(lineIndex + 1, 0, lineToPaste);
+          this.editor.innerText = lines.join('\n');
+          this.setCaretOffset(lineStartOffset + lines[lineIndex].length + 1);
+        }
+        break;
+
+      case '0':
+        this.keyBuffer = '';
+        this.setCaretOffset(lineStartOffset);
+        break;
+
+      case '$':
+        this.keyBuffer = '';
+        this.setCaretOffset(lineStartOffset + lines[lineIndex].length);
+        break;
+
+      case 'gg':
+        this.keyBuffer = '';
+        this.setCaretOffset(0);
+        break;
+
+      case 'G':
+        this.keyBuffer = '';
+        this.setCaretOffset(this.editor.innerText.length);
+        break;
+
+      case 'h':
+      case 'ArrowLeft':
+        this.keyBuffer = '';
+        const currentOffset = this.getCaretOffset();
+        if (currentOffset > 0) {
+          this.setCaretOffset(currentOffset - 1);
+        }
+        break;
+
+      case 'l':
+      case 'ArrowRight':
+        this.keyBuffer = '';
+        this.setCaretOffset(this.getCaretOffset() + 1);
+        break;
+
+      case 'j':
+      case 'ArrowDown':
+        this.keyBuffer = '';
+        if (lineIndex < lines.length - 1) {
+          const nextLineStart = lineStartOffset + lines[lineIndex].length + 1;
+          const nextLineLength = lines[lineIndex + 1].length;
+          const newPosition = Math.min(positionInLine, nextLineLength);
+          this.setCaretOffset(nextLineStart + newPosition);
+        }
+        break;
+
+      case 'k':
+      case 'ArrowUp':
+        this.keyBuffer = '';
+        if (lineIndex > 0) {
+          let prevLineStart = 0;
+          for (let i = 0; i < lineIndex - 1; i++) {
+            prevLineStart += lines[i].length + 1;
+          }
+          const prevLineLength = lines[lineIndex - 1].length;
+          const newPosition = Math.min(positionInLine, prevLineLength);
+          this.setCaretOffset(prevLineStart + newPosition);
+        }
+        break;
+
+      default:
+        // Clear buffer if it gets too long or contains invalid sequences
+        if (this.keyBuffer.length > 2 || 
+            (this.keyBuffer.length === 2 && !['dd', 'yy', 'gg'].includes(this.keyBuffer))) {
+          this.keyBuffer = '';
+        }
+        break;
+    }
+  }
+}
+
+customElements.define('njet-editor', NjetEditor);
+export { NjetEditor }

src/snek/static/message-list.js

@@ -5,75 +5,65 @@
 // The code seems to rely on some external dependencies like 'models.Message', 'app', and 'Schedule'. These should be imported or defined elsewhere in your application.
 
 // MIT License: This is free software. Permission is granted to use, copy, modify, and/or distribute this software for any purpose with or without fee. The software is provided "as is" without any warranty.
-import {app} from "./app.js";
+import { app } from "./app.js";
 
-const LONG_TIME = 1000 * 60 * 20
+const LONG_TIME = 1000 * 60 * 20;
 
 export class ReplyEvent extends Event {
-    constructor(messageTextTarget) {
-      super('reply', { bubbles: true, composed: true });
-      this.messageTextTarget = messageTextTarget;
+  constructor(messageTextTarget) {
+    super('reply', { bubbles: true, composed: true });
+    this.messageTextTarget = messageTextTarget;
 
-      const newMessage = messageTextTarget.cloneNode(true);
-      newMessage.style.maxHeight = "0"
-      messageTextTarget.parentElement.insertBefore(newMessage, messageTextTarget);
+    // Clone and sanitize message node to text-only reply
+    const newMessage = messageTextTarget.cloneNode(true);
+    newMessage.style.maxHeight = "0";
+    messageTextTarget.parentElement.insertBefore(newMessage, messageTextTarget);
 
-      newMessage.querySelectorAll('.embed-url-link').forEach(link => {
-        link.remove()
-      })
+    // Remove all .embed-url-link
+    newMessage.querySelectorAll('.embed-url-link').forEach(link => link.remove());
 
-      newMessage.querySelectorAll('picture').forEach(picture => {
-        const img = picture.querySelector('img');
-        if (img) {
-          picture.replaceWith(img);
-        }
-      })
+    // Replace <picture> with their <img>
+    newMessage.querySelectorAll('picture').forEach(picture => {
+      const img = picture.querySelector('img');
+      if (img) picture.replaceWith(img);
+    });
 
-      newMessage.querySelectorAll('img').forEach(img => {
-        const src = img.src || img.currentSrc;
-        img.replaceWith(document.createTextNode(src));
-      })
+    // Replace <img> with just their src
+    newMessage.querySelectorAll('img').forEach(img => {
+      const src = img.src || img.currentSrc;
+      img.replaceWith(document.createTextNode(src));
+    });
 
-      newMessage.querySelectorAll('iframe').forEach(iframe => {
-        const src = iframe.src || iframe.currentSrc;
-        iframe.replaceWith(document.createTextNode(src));
-      })
+    // Replace <iframe> with their src
+    newMessage.querySelectorAll('iframe').forEach(iframe => {
+      const src = iframe.src || iframe.currentSrc;
+      iframe.replaceWith(document.createTextNode(src));
+    });
 
-      newMessage.querySelectorAll('a').forEach(a => {
-        const href = a.getAttribute('href');
-        const text = a.innerText || a.textContent;
-        if (text === href || text === '') {
-          a.replaceWith(document.createTextNode(href));
-        } else {
-          a.replaceWith(document.createTextNode(`[${text}](${href})`));
-        }
-      })
+    // Replace <a> with href or markdown
+    newMessage.querySelectorAll('a').forEach(a => {
+      const href = a.getAttribute('href');
+      const text = a.innerText || a.textContent;
+      if (text === href || text === '') {
+        a.replaceWith(document.createTextNode(href));
+      } else {
+        a.replaceWith(document.createTextNode(`[${text}](${href})`));
+      }
+    });
 
-      this.replyText = newMessage.innerText.replaceAll("\n\n", "\n").trim();
-      newMessage.remove()
-    }
+    this.replyText = newMessage.innerText.replaceAll("\n\n", "\n").trim();
+    newMessage.remove();
+  }
 }
 
 class MessageElement extends HTMLElement {
-  // static observedAttributes = ['data-uid', 'data-color', 'data-channel_uid', 'data-user_nick', 'data-created_at', 'data-user_uid'];
-
-  isVisible() {
-    if (!this) return false;
-    const rect = this.getBoundingClientRect();
-    return (
-      rect.top >= 0 &&
-      rect.left >= 0 &&
-      rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
-      rect.right <= (window.innerWidth || document.documentElement.clientWidth)
-    );
-  }
-
   updateUI() {
     if (this._originalChildren === undefined) {
-      const { color,  user_nick, created_at, user_uid} = this.dataset;
+      const { color, user_nick, created_at, user_uid } = this.dataset;
       this.classList.add('message');
       this.style.maxWidth = '100%';
       this._originalChildren = Array.from(this.children);
+
       this.innerHTML = `
         <a class="avatar" style="background-color: ${color || ''}; color: black;" href="/user/${user_uid || ''}.html">
           <img class="avatar-img" width="40" height="40" src="/avatar/${user_uid || ''}.svg" alt="${user_nick || ''}" loading="lazy">
@@ -83,29 +73,30 @@ class MessageElement extends HTMLElement {
           <div class="text"></div>
           <div class="time no-select" data-created_at="${created_at || ''}">
             <span></span>
-            <a href="#reply">reply</a></div>
+            <a href="#reply">reply</a>
           </div>
-    `;
+        </div>
+      `;
 
       this.messageDiv = this.querySelector('.text');
-
       if (this._originalChildren && this._originalChildren.length > 0) {
-          this._originalChildren.forEach(child => {
-              this.messageDiv.appendChild(child);
-          });
+        this._originalChildren.forEach(child => {
+          this.messageDiv.appendChild(child);
+        });
       }
 
       this.timeDiv = this.querySelector('.time span');
       this.replyDiv = this.querySelector('.time a');
 
-        this.replyDiv.addEventListener('click', (e) => {
-          e.preventDefault();
-          this.dispatchEvent(new ReplyEvent(this.messageDiv));
-        })
+      this.replyDiv.addEventListener('click', (e) => {
+        e.preventDefault();
+        this.dispatchEvent(new ReplyEvent(this.messageDiv));
+      });
     }
 
-    if (!this.siblingGenerated && this.nextElementSibling) {
-      this.siblingGenerated = true;
+    // Sibling logic for user switches and long time gaps
+    if ((!this.siblingGenerated || this.siblingGenerated !== this.nextElementSibling) && this.nextElementSibling) {
+      this.siblingGenerated = this.nextElementSibling;
       if (this.nextElementSibling?.dataset?.user_uid !== this.dataset.user_uid) {
         this.classList.add('switch-user');
       } else {
@@ -113,7 +104,7 @@ class MessageElement extends HTMLElement {
         const siblingTime = new Date(this.nextElementSibling.dataset.created_at);
         const currentTime = new Date(this.dataset.created_at);
 
-        if (currentTime.getTime()  - siblingTime.getTime() > LONG_TIME) {
+        if (currentTime.getTime() - siblingTime.getTime() > LONG_TIME) {
           this.classList.add('long-time');
         } else {
           this.classList.remove('long-time');
@@ -126,7 +117,7 @@ class MessageElement extends HTMLElement {
 
   updateMessage(...messages) {
     if (this._originalChildren) {
-      this.messageDiv.replaceChildren(...messages)
+      this.messageDiv.replaceChildren(...messages);
       this._originalChildren = messages;
     }
   }
@@ -135,38 +126,26 @@ class MessageElement extends HTMLElement {
     this.updateUI();
   }
 
-  disconnectedCallback() {
-  }
-
-  connectedMoveCallback() {
-  }
+  disconnectedCallback() {}
+  connectedMoveCallback() {}
 
   attributeChangedCallback(name, oldValue, newValue) {
-    this.updateUI()
+    this.updateUI();
   }
 }
 
 class MessageList extends HTMLElement {
   constructor() {
     super();
-    app.ws.addEventListener("update_message_text", (data) => {
-      if (this.messageMap.has(data.uid)) {
-        this.upsertMessage(data);
-      }
-    });
-    app.ws.addEventListener("set_typing", (data) => {
-      this.triggerGlow(data.user_uid,data.color);
-    });
-
     this.messageMap = new Map();
     this.visibleSet = new Set();
+
     this._observer = new IntersectionObserver((entries) => {
       entries.forEach((entry) => {
         if (entry.isIntersecting) {
           this.visibleSet.add(entry.target);
-          const messageElement = entry.target;
-          if (messageElement instanceof MessageElement) {
-              messageElement.updateUI();
+          if (entry.target instanceof MessageElement) {
+            entry.target.updateUI();
           }
         } else {
           this.visibleSet.delete(entry.target);
@@ -175,28 +154,42 @@ class MessageList extends HTMLElement {
     }, {
       root: this,
       threshold: 0,
-    })
-
-    for(const c of this.children) {
-      this._observer.observe(c);
-      if (c instanceof MessageElement) {
-          this.messageMap.set(c.dataset.uid, c);
-      }
-    }
+    });
 
+    // End-of-messages marker
     this.endOfMessages = document.createElement('div');
     this.endOfMessages.classList.add('message-list-bottom');
     this.prepend(this.endOfMessages);
 
+    // Observe existing children and index by uid
+    for (const c of this.children) {
+      this._observer.observe(c);
+      if (c instanceof MessageElement) {
+        this.messageMap.set(c.dataset.uid, c);
+      }
+    }
+
+    // Wire up socket events
+    app.ws.addEventListener("update_message_text", (data) => {
+      if (this.messageMap.has(data.uid)) {
+        this.upsertMessage(data);
+      }
+    });
+    app.ws.addEventListener("set_typing", (data) => {
+      this.triggerGlow(data.user_uid, data.color);
+    });
+
     this.scrollToBottom(true);
   }
 
   connectedCallback() {
     this.addEventListener('click', (e) => {
-      if (e.target.tagName !== 'IMG' || e.target.classList.contains('avatar-img')) return;
+      if (
+        e.target.tagName !== 'IMG' ||
+        e.target.classList.contains('avatar-img')
+      ) return;
 
       const img = e.target;
-
       const overlay = document.createElement('div');
       overlay.style.cssText = 'position:fixed;top:0;left:0;width:100%;height:100%;background:rgba(0,0,0,0.9);display:flex;justify-content:center;align-items:center;z-index:9999;cursor:pointer;';
 
@@ -217,12 +210,11 @@ class MessageList extends HTMLElement {
 
       overlay.appendChild(fullImg);
       document.body.appendChild(overlay);
+
       overlay.addEventListener('click', () => {
-        if (overlay.parentNode) {
-          overlay.parentNode.removeChild(overlay);
-        }
+        if (overlay.parentNode) overlay.parentNode.removeChild(overlay);
       });
-      // Optional: ESC key closes overlay
+      // ESC to close
       const escListener = (evt) => {
         if (evt.key === 'Escape') {
           if (overlay.parentNode) overlay.parentNode.removeChild(overlay);
@@ -230,8 +222,9 @@ class MessageList extends HTMLElement {
         }
       };
       document.addEventListener('keydown', escListener);
-    })
+    });
   }
+
   isElementVisible(element) {
     if (!element) return false;
     const rect = element.getBoundingClientRect();
@@ -242,14 +235,16 @@ class MessageList extends HTMLElement {
       rect.right <= (window.innerWidth || document.documentElement.clientWidth)
     );
   }
+
   isScrolledToBottom() {
-    return this.isElementVisible(this.endOfMessages);
+    return this.visibleSet.has(this.endOfMessages);
   }
-  scrollToBottom(force = false, behavior= 'instant') {
+
+  scrollToBottom(force = false, behavior = 'instant') {
     if (force || !this.isScrolledToBottom()) {
-      this.firstElementChild.scrollIntoView({ behavior, block: 'end' });
+      this.endOfMessages.scrollIntoView({ behavior, block: 'end' });
       setTimeout(() => {
-        this.firstElementChild.scrollIntoView({ behavior, block: 'end' });
+        this.endOfMessages.scrollIntoView({ behavior, block: 'end' });
       }, 200);
     }
   }
@@ -261,7 +256,9 @@ class MessageList extends HTMLElement {
     this.querySelectorAll('.avatar').forEach((el) => {
       const anchor = el.closest('a');
       if (anchor && typeof anchor.href === 'string' && anchor.href.includes(uid)) {
-        lastElement = el;
+        if(!lastElement)
+          lastElement = el;
+
       }
     });
     if (lastElement) {
@@ -272,40 +269,44 @@ class MessageList extends HTMLElement {
     }
   }
 
-
   updateTimes() {
     this.visibleSet.forEach((messageElement) => {
-        if (messageElement instanceof MessageElement) {
-            messageElement.updateUI();
-        }
-    })
+      if (messageElement instanceof MessageElement) {
+        messageElement.updateUI();
+      }
+    });
   }
 
   upsertMessage(data) {
     let message = this.messageMap.get(data.uid);
-    if (message) {
-        message.parentElement?.removeChild(message);
+    if (message && (data.is_final || !data.message)) {
+      message.parentElement?.removeChild(message);
+      // TO force insert 
+      message = null;
+      
     }
-
-    if (!data.message) return
+    if (!data.message) return;
 
     const wrapper = document.createElement("div");
-
     wrapper.innerHTML = data.html;
 
     if (message) {
+      // If the old element is already custom, only update its message children
       message.updateMessage(...(wrapper.firstElementChild._originalChildren || wrapper.firstElementChild.children));
     } else {
+      // If not, insert the new one and observe
       message = wrapper.firstElementChild;
       this.messageMap.set(data.uid, message);
       this._observer.observe(message);
+      this.endOfMessages.after(message);
     }
 
     const scrolledToBottom = this.isScrolledToBottom();
-    this.prepend(message);
+
     if (scrolledToBottom) this.scrollToBottom(true);
   }
 }
 
 customElements.define("chat-message", MessageElement);
 customElements.define("message-list", MessageList);
+

src/snek/system/template.py

@@ -82,6 +82,32 @@ emoji.EMOJI_DATA[
 ALLOWED_TAGS = list(bleach.sanitizer.ALLOWED_TAGS) + ["picture"]
 
 def sanitize_html(value):
+
+    soup = BeautifulSoup(value, 'html.parser')
+
+    for script in soup.find_all('script'):
+        script.decompose()
+
+    #for iframe in soup.find_all('iframe'):
+        #iframe.decompose()
+
+    for tag in soup.find_all(['object', 'embed']):
+        tag.decompose()
+
+    for tag in soup.find_all():
+        event_attributes = ['onclick', 'onerror', 'onload', 'onmouseover', 'onfocus']
+        for attr in event_attributes:
+            if attr in tag.attrs:
+                del tag[attr]
+
+    for img in soup.find_all('img'):
+        if 'onerror' in img.attrs:
+            img.decompose()
+
+    return soup.prettify()
+
+
+def sanitize_html2(value):
     return bleach.clean(
         value,
         protocols=list(bleach.sanitizer.ALLOWED_PROTOCOLS) + ["data"],

src/snek/templates/sandbox.html

@@ -12,7 +12,7 @@ function showTerm(options){
 
 
 class StarField {
-  constructor({ count = 100, container = document.body } = {}) {
+  constructor({ count = 50, container = document.body } = {}) {
     this.container = container;
     this.starCount = count;
     this.stars = [];
@@ -567,7 +567,7 @@ const count = Array.from(messages).filter(el => el.textContent.trim() === text).
 
 
 
-const starField = new StarField({starCount: 100});
+const starField = new StarField({starCount: 50});
 app.starField = starField;
 
 class DemoSequence {

src/snek/view/channel.py

@@ -29,7 +29,7 @@ class ChannelDriveApiView(DriveApiView):
 
 class ChannelAttachmentView(BaseView):
 
-    login_required=True
+    login_required=False
 
     async def get(self):
         relative_path = self.request.match_info.get("relative_url")

src/snek/view/rpc.py

@@ -11,7 +11,7 @@ import asyncio
 import json
 import logging
 import traceback
-
+import random
 from aiohttp import web
 
 from snek.system.model import now
@@ -529,8 +529,8 @@ class RPCView(BaseView):
             try:
                 await self.ws.send_str(json.dumps(obj, default=str))
             except Exception as ex:
+                print("THIS IS THE DeAL>",str(ex), flush=True)
                 await self.services.socket.delete(self.ws)
-                await self.ws.close()
         
         async def get_online_users(self, channel_uid):
             self._require_login()
@@ -638,7 +638,7 @@ class RPCView(BaseView):
                 try:
                     await rpc(msg.json())
                 except Exception as ex:
-                    print("Deleting socket", ex, flush=True)
+                    print("XXXXXXXXXX Deleting socket", ex, flush=True)
                     logger.exception(ex)
                     await self.services.socket.delete(ws)
                     break