Merge pull request 'Added youtube music url to options for embedding' (#56) from BordedDev/snek:feat/embed-youtube-music into main

Reviewed-on: retoor/snek#56

src/snek/app.py

@@ -5,6 +5,7 @@ import ssl
 import uuid
 import signal
 from datetime import datetime
+from contextlib import asynccontextmanager
 
 from snek import snode
 from snek.view.threads import ThreadsView
@@ -231,6 +232,7 @@ class Application(BaseApplication):
                 print(ex)
             self.db.commit()
     
+
     async def prepare_database(self, app):
         self.db.query("PRAGMA journal_mode=WAL")
         self.db.query("PRAGMA syncnorm=off")
@@ -245,7 +247,7 @@ class Application(BaseApplication):
         except:
             pass
 
-        await app.services.drive.prepare_all()
+        await self.services.drive.prepare_all()
         self.loop.create_task(self.task_runner())
 
     def setup_router(self):
@@ -407,6 +409,11 @@ class Application(BaseApplication):
             request.session.get("uid")
         )
         
+        try:
+            context["nonce"] = request['csp_nonce']
+        except:
+            context['nonce'] = '?' 
+
         rendered = await super().render_template(template, request, context)
 
         self.jinja2_env.loader = self.original_loader
@@ -453,6 +460,27 @@ class Application(BaseApplication):
         template_paths.append(self.template_path)
         return FileSystemLoader(template_paths)
     
+    @asynccontextmanager
+    async def no_save(self):
+        stats = {
+            'count': 0
+        }
+        async def patched_save(*args, **kwargs):
+            await self.cache.set(args[0]["uid"], args[0])
+            stats['count'] = stats['count'] + 1
+            print(f"save is ignored {stats['count']} times")
+            return args[0]
+        save_original = self.services.channel_message.mapper.save
+        self.services.channel_message.mapper.save = patched_save
+        raised_exception = None
+        try:
+            yield 
+        except Exception as ex:
+            raised_exception = ex
+        finally:
+            self.services.channel_message.mapper.save = save_original
+        if raised_exception:
+            raise raised_exception
 
 app = Application(db_path="sqlite:///snek.db")
 

src/snek/model/channel.py

@@ -11,11 +11,15 @@ class ChannelModel(BaseModel):
     is_listed = ModelField(name="is_listed", required=True, kind=bool, value=True)
     index = ModelField(name="index", required=True, kind=int, value=1000)
     last_message_on = ModelField(name="last_message_on", required=False, kind=str)
+    history_start = ModelField(name="history_start", required=False, kind=str)
 
     async def get_last_message(self) -> ChannelMessageModel:
+        history_start_filter = ""
+        if self["history_start"]:
+            history_start_filter = f" AND created_at > '{self['history_start']}' "
         try:
             async for model in self.app.services.channel_message.query(
-                "SELECT uid FROM channel_message WHERE channel_uid=:channel_uid ORDER BY created_at DESC LIMIT 1",
+                "SELECT uid FROM channel_message WHERE channel_uid=:channel_uid" + history_start_filter + " ORDER BY created_at DESC LIMIT 1",
                 {"channel_uid": self["uid"]},
             ):
 

src/snek/service/channel.py

@@ -113,6 +113,12 @@ class ChannelService(BaseService):
             channel = await self.get(uid=channel_member["channel_uid"])
             yield channel
 
+    async def clear(self, channel_uid):
+        model = await self.get(uid=channel_uid)
+        model['history_from'] = datetime.now()
+        await self.save(model)
+
+
     async def ensure_public_channel(self, created_by_uid):
         model = await self.get(is_listed=True, tag="public")
         is_moderator = False

src/snek/service/channel_message.py

@@ -6,8 +6,10 @@ class ChannelMessageService(BaseService):
     mapper_name = "channel_message"
 
     async def maintenance(self):
+        args = {}
         async for message in self.find():
             updated_at = message["updated_at"]
+            message["is_final"] = True
             html = message["html"]
             await self.save(message)
 
@@ -21,6 +23,19 @@ class ChannelMessageService(BaseService):
             if html != message["html"]:
                 print("Reredefined message", message["uid"])
         
+        while True:
+            changed = 0
+            async for message in self.find(is_final=False):
+                message["is_final"] = True
+                await self.save(message)
+                changed += 1
+            async for message in self.find(is_final=None):
+                message["is_final"] = False
+                await self.save(message)
+                changed += 1
+            if not changed:
+                break
+
         
     async def create(self, channel_uid, user_uid, message, is_final=True):
         model = await self.new()
@@ -87,13 +102,20 @@ class ChannelMessageService(BaseService):
         model["html"] = whitelist_attributes(model["html"])
         return await super().save(model)
 
+
     async def offset(self, channel_uid, page=0, timestamp=None, page_size=30):
+        channel = await self.services.channel.get(uid=channel_uid)
+        if not channel:
+            return []
+        history_start_filter = ""
+        if channel["history_start"]:
+            history_start_filter = f" AND created_at > '{channel['history_start']}'"    
         results = []
         offset = page * page_size
         try:
             if timestamp:
                 async for model in self.query(
-                    "SELECT * FROM channel_message WHERE channel_uid=:channel_uid AND created_at < :timestamp ORDER BY created_at DESC LIMIT :page_size OFFSET :offset",
+                    f"SELECT * FROM channel_message WHERE channel_uid=:channel_uid AND created_at < :timestamp {history_start_filter} ORDER BY created_at DESC LIMIT :page_size OFFSET :offset",
                     {
                         "channel_uid": channel_uid,
                         "page_size": page_size,
@@ -104,7 +126,7 @@ class ChannelMessageService(BaseService):
                     results.append(model)
             elif page > 0:
                 async for model in self.query(
-                    "SELECT * FROM channel_message WHERE channel_uid=:channel_uid WHERE created_at < :timestamp ORDER BY created_at DESC LIMIT :page_size",
+                    f"SELECT * FROM channel_message WHERE channel_uid=:channel_uid WHERE created_at < :timestamp {history_start_filter} ORDER BY created_at DESC LIMIT :page_size",
                     {
                         "channel_uid": channel_uid,
                         "page_size": page_size,
@@ -115,7 +137,7 @@ class ChannelMessageService(BaseService):
                     results.append(model)
             else:
                 async for model in self.query(
-                    "SELECT * FROM channel_message WHERE channel_uid=:channel_uid ORDER BY created_at DESC LIMIT :page_size OFFSET :offset",
+                    f"SELECT * FROM channel_message WHERE channel_uid=:channel_uid {history_start_filter} ORDER BY created_at DESC LIMIT :page_size OFFSET :offset",
                     {
                         "channel_uid": channel_uid,
                         "page_size": page_size,
@@ -124,7 +146,7 @@ class ChannelMessageService(BaseService):
                 ):
                     results.append(model)
 
-        except:
+        except Exception as ex:
-            pass
+            print(ex)
         results.sort(key=lambda x: x["created_at"])
         return results

src/snek/service/chat.py

@@ -36,9 +36,21 @@ class ChatService(BaseService):
         channel = await self.services.channel.get(uid=channel_uid)
         if not channel:
             raise Exception("Channel not found.")
-        channel_message = await self.services.channel_message.create(
+        channel_message = await self.services.channel_message.get(
-            channel_uid, user_uid, message, is_final
+            channel_uid=channel_uid,user_uid=user_uid, is_final=False
         )
+        if channel_message:
+            channel_message["message"] = message
+            channel_message["is_final"] = is_final
+            if not channel_message["is_final"]:
+                async with self.app.no_save():
+                    await self.services.channel_message.save(channel_message)
+            else:
+                await self.services.channel_message.save(channel_message)
+        else:
+            channel_message = await self.services.channel_message.create(
+                channel_uid, user_uid, message, is_final
+            )
         channel_message_uid = channel_message["uid"]
 
         user = await self.services.user.get(uid=user_uid)

src/snek/service/user.py

@@ -12,6 +12,7 @@ class UserService(BaseService):
 
     async def search(self, query, **kwargs):
         query = query.strip().lower()
+        kwarggs["deleted_at"] = None
         if not query:
             return []
         results = []

src/snek/static/chat-input.js

@@ -1,22 +1,23 @@
 import { app } from "./app.js";
-import { NjetComponent } from "./njet.js";
+import { NjetComponent,eventBus } from "./njet.js";
 import { FileUploadGrid } from "./file-upload-grid.js";
+
 class ChatInputComponent extends NjetComponent {
   autoCompletions = {
-    "example 1": () => {
+    "example 1": () => {},
-    },
+    "example 2": () => {},
-    "example 2": () => {
+  };
-    },
+
-  }
   hiddenCompletions = {
     "/starsRender": () => {
       app.rpc.starsRender(this.channelUid, this.value.replace("/starsRender ", ""))
     }
-  }
+  };
-  users = []
+
-  textarea = null
+  users = [];
-  _value = ""
+  textarea = null;
-  lastUpdateEvent = null
+  _value = "";
+  lastUpdateEvent = null;
   expiryTimer = null;
   queuedMessage = null;
   lastMessagePromise = null;
@@ -38,7 +39,7 @@ class ChatInputComponent extends NjetComponent {
   }
 
   get allAutoCompletions() {
-    return Object.assign({}, this.autoCompletions, this.hiddenCompletions)
+    return Object.assign({}, this.autoCompletions, this.hiddenCompletions);
   }
 
   resolveAutoComplete(input) {
@@ -65,7 +66,7 @@ class ChatInputComponent extends NjetComponent {
   }
 
   getAuthors() {
-    return this.users.flatMap((user) => [user.username, user.nick])
+    return this.users.flatMap((user) => [user.username, user.nick]);
   }
 
   extractMentions(text) {
@@ -82,17 +83,14 @@ class ChatInputComponent extends NjetComponent {
         const lowerAuthor = author.toLowerCase();
         let distance = this.levenshteinDistance(lowerMention, lowerAuthor);
 
-
         if (!this.isSubsequence(lowerMention, lowerAuthor)) {
-          distance += 10
+          distance += 10;
         }
 
         if (distance < minDistance) {
           minDistance = distance;
           closestAuthor = author;
-
         }
-
       });
 
       return { mention, closestAuthor, distance: minDistance };
@@ -128,7 +126,6 @@ class ChatInputComponent extends NjetComponent {
     return matrix[b.length][a.length];
   }
 
-
   replaceMentionsWithAuthors(text) {
     const authors = this.getAuthors();
     const mentions = this.extractMentions(text);
@@ -143,21 +140,19 @@ class ChatInputComponent extends NjetComponent {
     return updatedText;
   }
 
-
   async connectedCallback() {
-    this.user = null
+    this.user = null;
     app.rpc.getUser(null).then((user) => {
-      this.user = user
+      this.user = user;
-    })
+    });
 
-    this.liveType = this.getAttribute("live-type") === "true";
+    this.liveType = this.getAttribute("live-type") !== "true";
-    this.liveTypeInterval =
+    this.liveTypeInterval = parseInt(this.getAttribute("live-type-interval")) || 6;
-      parseInt(this.getAttribute("live-type-interval")) || 6;
     this.channelUid = this.getAttribute("channel");
 
     app.rpc.getRecentUsers(this.channelUid).then(users => {
-      this.users = users
+      this.users = users;
-    })
+    });
     this.messageUid = null;
 
     this.classList.add("chat-input");
@@ -181,24 +176,36 @@ class ChatInputComponent extends NjetComponent {
       this.dispatchEvent(new CustomEvent("uploaded", e));
     });
     this.uploadButton.addEventListener("click", (e) => {
-    //    e.preventDefault();
+      e.preventDefault();
-      //  this.fileUploadGrid.openFileDialog()
+      this.fileUploadGrid.openFileDialog();
-
+    });
-    })
+    eventBus.subscribe("file-uploading", (e) => {
-    this.subscribe("file-uploading", (e) => {
+      this.fileUploadGrid.style.display = "block";
-        
+      this.uploadButton.style.display = "none";
-        this.fileUploadGrid.style.display = "block";   
+      this.textarea.style.display = "none";
-
-        this.uploadButton.style.display = "none";
-        this.textarea.style.display = "none";
     })
+    document.eventBus = eventBus;
     this.appendChild(this.uploadButton);
+
     this.textarea.addEventListener("blur", () => {
       this.updateFromInput("");
     });
+
+    eventBus.subscribe("file-uploads-done", (data)=>{
+        console.info("JEEJ", data)
+        this.textarea.style.display = "block";
+      this.uploadButton.style.display = "block";
+      this.fileUploadGrid.style.display = "none";
+      let message =data.reduce((file) => {
+        return `${message}[${file.filename}](/channel/attachment/${file.file})`;
+      }, '');
+      app.rpc.sendMessage(this.channelUid, message, true);
+    });
+
+    
+
     this.textarea.addEventListener("keyup", (e) => {
       if (e.key === "Enter" && !e.shiftKey) {
-
         const message = this.replaceMentionsWithAuthors(this.value);
         e.target.value = "";
 
@@ -210,12 +217,10 @@ class ChatInputComponent extends NjetComponent {
           autoCompletionHandler();
           this.value = "";
           e.target.value = "";
-
           return;
         }
 
-        this.finalizeMessage(this.messageUid)
+        this.finalizeMessage(this.messageUid);
-
         return;
       }
 
@@ -254,9 +259,10 @@ class ChatInputComponent extends NjetComponent {
       }, '');
       app.rpc.sendMessage(this.channelUid, message, true);
     });
+
     setTimeout(() => {
       this.focus();
-    }, 1000)
+    }, 1000);
   }
 
   trackSecondsBetweenEvents(event1Time, event2Time) {
@@ -278,33 +284,16 @@ class ChatInputComponent extends NjetComponent {
   flagTyping() {
     if (this.trackSecondsBetweenEvents(this.lastUpdateEvent, new Date()) >= 1) {
       this.lastUpdateEvent = new Date();
-      app.rpc.set_typing(this.channelUid, this.user.color).catch(() => {
+      app.rpc.set_typing(this.channelUid, this.user?.color).catch(() => {});
-      });
     }
   }
 
-  finalizeMessage(messageUid) {
+  async finalizeMessage(messageUid) {
-    if (!messageUid) {
+    await app.rpc.sendMessage(this.channelUid, this.replaceMentionsWithAuthors(this.value), true);
-      if (this.value.trim() === "") {
-        return;
-      }
-      this.sendMessage(this.channelUid, this.replaceMentionsWithAuthors(this.value), !this.liveType);
-    } else if (messageUid.startsWith("?")) {
-      const lastQueuedMessage = this.queuedMessage;
-
-      this.lastMessagePromise?.then((uid) => {
-        const updatePromise = lastQueuedMessage ? app.rpc.updateMessageText(uid, lastQueuedMessage) : Promise.resolve();
-        return updatePromise.finally(() => {
-          return app.rpc.finalizeMessage(uid);
-        })
-      })
-    } else {
-      app.rpc.finalizeMessage(messageUid)
-    }
     this.value = "";
     this.messageUid = null;
     this.queuedMessage = null;
-    this.lastMessagePromise = null
+    this.lastMessagePromise = null;
   }
 
   updateFromInput(value) {
@@ -315,39 +304,12 @@ class ChatInputComponent extends NjetComponent {
 
     this.value = value;
 
-    this.flagTyping()
+    this.flagTyping();
 
     if (this.liveType && value[0] !== "/") {
-      this.expiryTimer = setTimeout(() => {
-        this.finalizeMessage(this.messageUid)
-      }, this.liveTypeInterval * 1000);
-
-
       const messageText = this.replaceMentionsWithAuthors(value);
-      if (this.messageUid?.startsWith("?")) {
+      this.messageUid = this.sendMessage(this.channelUid, messageText, !this.liveType);
-        this.queuedMessage = messageText;
+      return this.messageUid;
-      } else if (this.messageUid) {
-        app.rpc.updateMessageText(this.messageUid, messageText).then((d) => {
-          if (!d.success) {
-            this.messageUid = null
-            this.updateFromInput(value)
-          }
-        })
-      } else {
-        const placeHolderId = "?" + crypto.randomUUID();
-        this.messageUid = placeHolderId;
-
-        this.lastMessagePromise = this.sendMessage(this.channelUid, messageText, !this.liveType).then(async (uid) => {
-          if (this.liveType && this.messageUid === placeHolderId) {
-            if (this.queuedMessage && this.queuedMessage !== messageText) {
-              await app.rpc.updateMessageText(uid, this.queuedMessage)
-            }
-            this.messageUid = uid;
-          }
-
-          return uid
-        });
-      }
     }
   }
 
@@ -360,3 +322,4 @@ class ChatInputComponent extends NjetComponent {
 }
 
 customElements.define("chat-input", ChatInputComponent);
+

src/snek/static/editor.js

@@ -0,0 +1,226 @@
+import { NjetComponent} from "/njet.js"
+
+    class NjetEditor extends NjetComponent {
+      constructor() {
+        super();
+        this.attachShadow({ mode: 'open' });
+
+        const style = document.createElement('style');
+        style.textContent = `
+          #editor {
+            padding: 1rem;
+            outline: none;
+            white-space: pre-wrap;
+            line-height: 1.5;
+            height: 100%;
+            overflow-y: auto;
+            background: #1e1e1e;
+            color: #d4d4d4;
+          }
+          #command-line {
+            position: absolute;
+            bottom: 0;
+            left: 0;
+            width: 100%;
+            padding: 0.2rem 1rem;
+            background: #333;
+            color: #0f0;
+            display: none;
+            font-family: monospace;
+          }
+        `;
+
+        this.editor = document.createElement('div');
+        this.editor.id = 'editor';
+        this.editor.contentEditable = true;
+        this.editor.innerText = `Welcome to VimEditor Component
+Line 2 here
+Another line
+Try i, Esc, v, :, yy, dd, 0, $, gg, G, and p`;
+
+        this.cmdLine = document.createElement('div');
+        this.cmdLine.id = 'command-line';
+        this.shadowRoot.append(style, this.editor, this.cmdLine);
+
+        this.mode = 'normal'; // normal | insert | visual | command
+        this.keyBuffer = '';
+        this.lastDeletedLine = '';
+        this.yankedLine = '';
+
+        this.editor.addEventListener('keydown', this.handleKeydown.bind(this));
+      }
+
+      connectedCallback() {
+        this.editor.focus();
+      }
+
+      getCaretOffset() {
+        let caretOffset = 0;
+        const sel = this.shadowRoot.getSelection();
+        if (!sel || sel.rangeCount === 0) return 0;
+
+        const range = sel.getRangeAt(0);
+        const preCaretRange = range.cloneRange();
+        preCaretRange.selectNodeContents(this.editor);
+        preCaretRange.setEnd(range.endContainer, range.endOffset);
+        caretOffset = preCaretRange.toString().length;
+        return caretOffset;
+      }
+
+      setCaretOffset(offset) {
+        const range = document.createRange();
+        const sel = this.shadowRoot.getSelection();
+        const walker = document.createTreeWalker(this.editor, NodeFilter.SHOW_TEXT, null, false);
+
+        let currentOffset = 0;
+        let node;
+        while ((node = walker.nextNode())) {
+          if (currentOffset + node.length >= offset) {
+            range.setStart(node, offset - currentOffset);
+            range.collapse(true);
+            sel.removeAllRanges();
+            sel.addRange(range);
+            return;
+          }
+          currentOffset += node.length;
+        }
+      }
+
+      handleKeydown(e) {
+        const key = e.key;
+
+        if (this.mode === 'insert') {
+          if (key === 'Escape') {
+            e.preventDefault();
+            this.mode = 'normal';
+            this.editor.blur();
+            this.editor.focus();
+          }
+          return;
+        }
+
+        if (this.mode === 'command') {
+          if (key === 'Enter' || key === 'Escape') {
+            e.preventDefault();
+            this.cmdLine.style.display = 'none';
+            this.mode = 'normal';
+            this.keyBuffer = '';
+          }
+          return;
+        }
+
+        if (this.mode === 'visual') {
+          if (key === 'Escape') {
+            e.preventDefault();
+            this.mode = 'normal';
+          }
+          return;
+        }
+
+        // Handle normal mode
+        this.keyBuffer += key;
+
+        const text = this.editor.innerText;
+        const caretPos = this.getCaretOffset();
+        const lines = text.split('\n');
+
+        let charCount = 0, lineIdx = 0;
+        for (let i = 0; i < lines.length; i++) {
+          if (caretPos <= charCount + lines[i].length) {
+            lineIdx = i;
+            break;
+          }
+          charCount += lines[i].length + 1;
+        }
+
+        const offsetToLine = idx =>
+          text.split('\n').slice(0, idx).reduce((acc, l) => acc + l.length + 1, 0);
+
+        switch (this.keyBuffer) {
+          case 'i':
+            e.preventDefault();
+            this.mode = 'insert';
+            this.keyBuffer = '';
+            break;
+
+          case 'v':
+            e.preventDefault();
+            this.mode = 'visual';
+            this.keyBuffer = '';
+            break;
+
+          case ':':
+            e.preventDefault();
+            this.mode = 'command';
+            this.cmdLine.style.display = 'block';
+            this.cmdLine.textContent = ':';
+            this.keyBuffer = '';
+            break;
+
+          case 'yy':
+            e.preventDefault();
+            this.yankedLine = lines[lineIdx];
+            this.keyBuffer = '';
+            break;
+
+          case 'dd':
+            e.preventDefault();
+            this.lastDeletedLine = lines[lineIdx];
+            lines.splice(lineIdx, 1);
+            this.editor.innerText = lines.join('\n');
+            this.setCaretOffset(offsetToLine(lineIdx));
+            this.keyBuffer = '';
+            break;
+
+          case 'p':
+            e.preventDefault();
+            const lineToPaste = this.yankedLine || this.lastDeletedLine;
+            if (lineToPaste) {
+              lines.splice(lineIdx + 1, 0, lineToPaste);
+              this.editor.innerText = lines.join('\n');
+              this.setCaretOffset(offsetToLine(lineIdx + 1));
+            }
+            this.keyBuffer = '';
+            break;
+
+          case '0':
+            e.preventDefault();
+            this.setCaretOffset(offsetToLine(lineIdx));
+            this.keyBuffer = '';
+            break;
+
+          case '$':
+            e.preventDefault();
+            this.setCaretOffset(offsetToLine(lineIdx) + lines[lineIdx].length);
+            this.keyBuffer = '';
+            break;
+
+          case 'gg':
+            e.preventDefault();
+            this.setCaretOffset(0);
+            this.keyBuffer = '';
+            break;
+
+          case 'G':
+            e.preventDefault();
+            this.setCaretOffset(text.length);
+            this.keyBuffer = '';
+            break;
+
+          case 'Escape':
+            e.preventDefault();
+            this.mode = 'normal';
+            this.keyBuffer = '';
+            this.cmdLine.style.display = 'none';
+            break;
+
+          default:
+            // allow up to 2 chars for combos
+            if (this.keyBuffer.length > 2) this.keyBuffer = '';
+            break;
+        }
+      }
+    }
+
+    customElements.define('njet-editor', NjetEditor);
+export {NjetEditor}

src/snek/static/file-upload-grid.js

@@ -57,6 +57,7 @@ class FileUploadGrid extends NjetComponent {
     }
 
     reset(){
+        this.uploadResponses = [];
         this.uploadsDone = 0;
         this.uploadsStarted = 0;
         this._grid.innerHTML = '';
@@ -70,6 +71,7 @@ class FileUploadGrid extends NjetComponent {
         this.reset()
         this.uploadsDone = 0;
         this.uploadsStarted = files.length;
+        
         [...files].forEach(file => this.createTile(file));
     }
     connectedCallback() {
@@ -130,6 +132,7 @@ class FileUploadGrid extends NjetComponent {
     startUpload(file, tile, progress) {
 
         this.publish('file-uploading', {file: file, tile: tile, progress: progress});
+        console.info("File uploading",file)
 
         const protocol = location.protocol === "https:" ? "wss://" : "ws://";
         const ws = new WebSocket(`${protocol}${location.host}/channel/${this.channelUid}/attachment.sock`);
@@ -148,7 +151,7 @@ class FileUploadGrid extends NjetComponent {
         };
 
         ws.onmessage = (event) => {
-
+            console.info(event.data)
 
             const data = JSON.parse(event.data);
             if (data.type === 'progress') {
@@ -156,14 +159,20 @@ class FileUploadGrid extends NjetComponent {
                 progress.style.width = pct + '%';
                 this.publish('file-uploading', {file: file, tile: tile, progress: progress});
             } else if (data.type === 'done') {
-                
+                console.info("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAH")
+                console.info("Done")
+                console.info(this.uploadResponses)
                 this.uploadsDone += 1;
                 this.publish('file-uploaded', {file: file, tile: tile, progress: progress});
                 progress.style.width = '100%';
                 tile.classList.add('fug-done');
-
+                console.info("Closed")
                 ws.close();
-
+                this.uploadResponses.push({file:file, remoteFile:data.file})
+                console.info(this.uploadsDone, this.uploadsStarted)
+                if(this.uploadsDone == this.uploadsStarted){
+                    this.publish('file-uploads-done', this.uploadResponses);
+                }
                 this.reset()
             }
         };

src/snek/static/njet.js

@@ -210,19 +210,18 @@ class Njet extends HTMLElement {
         customElements.define(name, component);
     }
 
-    constructor(config = {}) {
+    constructor() {
         super();
         if (!Njet._root) {
             Njet._root = this
-            Njet._rest = new RestClient({ baseURL: config.baseURL || null })
+            Njet._rest = new RestClient({ baseURL: '/' || null })
         }
         this.root._elements.push(this)
         this.classList.add('njet');
-        this.config = config;
         this.render.call(this);
-        this.initProps(config);
+        //this.initProps(config);
-        if (typeof this.construct === 'function')
+        //if (typeof this.config.construct === 'function')
-            this.construct.call(this)
+        //    this.config.construct.call(this)
     }
 
     initProps(config) {
@@ -284,9 +283,12 @@ class Njet extends HTMLElement {
 
     render() {}
 }
+Njet.registerComponent('njet-root', Njet);
 
 class Component extends Njet {}
 
+Njet.registerComponent('njet-component', Component);
+
 class NjetPanel extends Component {
     render() {
         this.innerHTML = '';
@@ -491,18 +493,67 @@ document.body.appendChild(dialog);
 */
 
 class NjetComponent extends Component {}
-    const njet = Njet;
+    const njet = Njet
 njet.showDialog = function(args){
     const dialog = new NjetDialog(args) 
     dialog.show()
     return dialog 
 }
+
+class EventBus extends EventTarget {
+    constructor() {
+        super();
+        this.eventMap = new Map();
+    }
+
+    subscribe(eventName, callback) {
+        this.addEventListener(eventName, callback);
+        
+        if (!this.eventMap.has(eventName)) {
+            this.eventMap.set(eventName, []);
+        }
+        this.eventMap.get(eventName).push(callback);
+    }
+
+    publish(eventName, detail = {}) {
+        const event = new CustomEvent(eventName, { 
+            detail, 
+            bubbles: true, 
+            cancelable: true 
+        });
+        
+        document.dispatchEvent(event);
+    }
+
+    unsubscribe(eventName, callback) {
+        this.removeEventListener(eventName, callback);
+        
+        const subscribers = this.eventMap.get(eventName);
+        if (subscribers) {
+            const index = subscribers.indexOf(callback);
+            if (index > -1) subscribers.splice(index, 1);
+        }
+    }
+}
+
+const eventBus = new EventBus()
+
+
 njet.showWindow = function(args) {
     const w = new NjetWindow(args)
     w.show()
     return w
 }
+njet.publish = function(event, data) {
+    if (this.root._subscriptions[event]) {
+        this.root._subscriptions[event].forEach(callback => callback(data))
+    }
+}
+njet.subscribe = function(event, callback) {
+    if (!this.root._subscriptions[event]) {
+        this.root._subscriptions[event] = []
+    }
+    this.root._subscriptions[event].push(callback)
+}
 
-window.njet = njet 
+export { Njet, NjetButton, NjetPanel, NjetDialog, NjetGrid, NjetComponent, njet, NjetWindow,eventBus };
-
-export { Njet, NjetButton, NjetPanel, NjetDialog, NjetGrid, NjetComponent, njet, NjetWindow };

src/snek/static/upload-button.js

@@ -112,9 +112,9 @@ class UploadButtonElement extends HTMLElement {
     this.channelUid = this.getAttribute("channel");
     this.uploadButton = this.container.querySelector(".upload-button");
     this.fileInput = this.container.querySelector(".hidden-input");
-    this.uploadButton.addEventListener("click", () => {
+    /*this.uploadButton.addEventListener("click", () => {
       this.fileInput.click();
-    });
+    });*/
     this.fileInput.addEventListener("change", () => {
       this.uploadFiles();
     });

src/snek/system/cache.py

@@ -14,7 +14,7 @@ class Cache:
         self.cache = {}
         self.max_items = max_items
         self.stats = {}
-        self.enabled = False
+        self.enabled = True
         self.lru = []
         self.version = ((42 + 420 + 1984 + 1990 + 10 + 6 + 71 + 3004 + 7245) ^ 1337) + 4
 

src/snek/system/mapper.py

@@ -25,9 +25,20 @@ class BaseMapper:
         return asyncio.get_event_loop()
 
     async def run_in_executor(self, func, *args, **kwargs):
-        async with self.semaphore:
+        use_semaphore = kwargs.pop("use_semaphore", False)
-            return func(*args, **kwargs)
+        if use_semaphore:
-            # return await self.loop.run_in_executor(None, lambda: func(*args, **kwargs))
+            async with self.semaphore:
+                database_exception = None 
+                for x in range(20):
+                    try:
+                        result = func(*args, **kwargs)
+                        self.db.commit()
+                        return result
+                    except Exception as ex:
+                        await asyncio.sleep(0)
+                        database_exception = ex
+                raise database_exception
+        return await self.loop.run_in_executor(None, lambda: func(*args, **kwargs))
 
     async def new(self):
         return self.model_class(mapper=self, app=self.app)
@@ -39,7 +50,8 @@ class BaseMapper:
     async def get(self, uid: str = None, **kwargs) -> BaseModel:
         if uid:
             kwargs["uid"] = uid
-
+        if not kwargs.get("deleted_at"):
+            kwargs["deleted_at"] = None
         record = await self.run_in_executor(self.table.find_one, **kwargs)
         if not record:
             return None
@@ -48,7 +60,6 @@ class BaseMapper:
         for key, value in record.items():
             model[key] = value
         return model
-        return await self.model_class.from_record(mapper=self, record=record)
 
     async def exists(self, **kwargs):
         return await self.run_in_executor(self.table.exists, **kwargs)
@@ -60,26 +71,39 @@ class BaseMapper:
         if not model.record.get("uid"):
             raise Exception(f"Attempt to save without uid: {model.record}.")
         model.updated_at.update()
-        return await self.run_in_executor(self.table.upsert, model.record, ["uid"])
+        return await self.run_in_executor(self.table.upsert, model.record, ["uid"],use_semaphore=True)
 
     async def find(self, **kwargs) -> typing.AsyncGenerator:
         if not kwargs.get("_limit"):
             kwargs["_limit"] = self.default_limit
+        if not kwargs.get("deleted_at"):
+            kwargs["deleted_at"] = None
         for record in await self.run_in_executor(self.table.find, **kwargs):
             model = await self.new()
             for key, value in record.items():
                 model[key] = value
             yield model
 
+    async def _use_semaphore(self, sql):
+        sql = sql.lower().strip()
+        return "insert" in sql or "update" in sql or "delete" in sql
+
     async def query(self, sql, *args):
-        for record in await self.run_in_executor(self.db.query, sql, *args):
+        for record in await self.run_in_executor(self.db.query, sql, *args, use_semaphore=await self._use_semaphore(sql)):
             yield dict(record)
 
     async def update(self, model):
+        if not model["deleted_at"] is None:
+            raise Exception("Can't update deleted record.")
         model.updated_at.update()
-        return await self.run_in_executor(self.table.update, model.record, ["uid"])
+        return await self.run_in_executor(self.table.update, model.record, ["uid"],use_semaphore=True)
+
+    async def upsert(self, model):
+        model.updated_at.update()
+        return await self.run_in_executor(self.table.upsert, model.record, ["uid"],use_semaphore=True)
 
     async def delete(self, **kwargs) -> int:
         if not kwargs or not isinstance(kwargs, dict):
             raise Exception("Can't execute delete with no filter.")
+        kwargs["use_semaphore"] = True
         return await self.run_in_executor(self.table.delete, **kwargs)

src/snek/system/middleware.py

@@ -1,35 +1,33 @@
 # Written by retoor@molodetz.nl
 
-# This code provides middleware functions for an aiohttp server to manage and modify CORS (Cross-Origin Resource Sharing) headers.
+# This code provides middleware functions for an aiohttp server to manage and modify CSP, CORS, and authentication headers.
-
-# Imports from 'aiohttp' library are used to create middleware; they are not part of Python's standard library.
-
-# MIT License: This code is distributed under the MIT License.
 
 import secrets
-
 from aiohttp import web
 
-csp_policy = (
-    "default-src 'self'; "
-    "script-src 'self' https://*.cloudflare.com https://molodetz.nl 'nonce-{nonce}'; "
-    "style-src 'self' https://*.cloudflare.com https://molodetz.nl; "
-    "img-src 'self' https://*.cloudflare.com https://molodetz.nl data:; "
-    "connect-src 'self' https://*.cloudflare.com https://molodetz.nl;"
-)
-
-
-def generate_nonce():
-    return secrets.token_hex(16)
-
 
 @web.middleware
 async def csp_middleware(request, handler):
-
+    nonce = secrets.token_hex(16)
+    origin = request.headers.get('Origin')
+    csp_policy = (
+        "default-src 'self'; "
+        f"script-src 'self' {origin} 'nonce-{nonce}'; "
+        f"style-src 'self' 'unsafe-inline' {origin} 'nonce-{nonce}'; "
+        "img-src *; "
+        "connect-src 'self' https://umami.molodetz.nl; "
+        "font-src *; "
+        "object-src 'none'; "
+        "base-uri 'self'; "
+        "form-action 'self'; "
+        "frame-src 'self'; "
+        "worker-src *; "
+        "media-src *; "
+        "manifest-src 'self';"
+    )
+    request['csp_nonce'] = nonce
     response = await handler(request)
-    return response
+    #response.headers['Content-Security-Policy'] = csp_policy
-    nonce = generate_nonce()
-    response.headers["Content-Security-Policy"] = csp_policy.format(nonce=nonce)
     return response
 
 
@@ -39,7 +37,6 @@ async def no_cors_middleware(request, handler):
     response.headers.pop("Access-Control-Allow-Origin", None)
     return response
 
-
 @web.middleware
 async def cors_allow_middleware(request, handler):
     response = await handler(request)
@@ -51,7 +48,6 @@ async def cors_allow_middleware(request, handler):
     response.headers["Access-Control-Allow-Credentials"] = "true"
     return response
 
-
 @web.middleware
 async def auth_middleware(request, handler):
     request["user"] = None
@@ -61,7 +57,6 @@ async def auth_middleware(request, handler):
         )
     return await handler(request)
 
-
 @web.middleware
 async def cors_middleware(request, handler):
     if request.headers.get("Allow"):

src/snek/system/service.py

@@ -40,11 +40,11 @@ class BaseService:
             yield record
 
     async def get(self, uid=None, **kwargs):
+        kwargs["deleted_at"] = None 
         if uid:
-            if not kwargs:
+            result = await self.cache.get(uid)
-                result = await self.cache.get(uid)
+            if result and result.__class__ == self.mapper.model_class:
-                if False and result and result.__class__ == self.mapper.model_class:
+                return result
-                    return result
             kwargs["uid"] = uid
 
         result = await self.mapper.get(**kwargs)
@@ -52,7 +52,7 @@ class BaseService:
             await self.cache.set(result["uid"], result)
         return result
 
-    async def save(self, model: UserModel):
+    async def save(self, model):
         # if model.is_valid: You Know why not
         if await self.mapper.save(model):
             await self.cache.set(model["uid"], model)

src/snek/system/template.py

@@ -151,6 +151,14 @@ SAFE_ATTRIBUTES = {
     "aria-hidden",
     "aria-label",
     "srcset",
+    "target",
+    "rel",
+    "referrerpolicy",
+    "controls",
+    "frameborder",
+    "allow",
+    "allowfullscreen",
+    "referrerpolicy",
 }
 
 
@@ -180,6 +188,7 @@ def embed_youtube(text):
             or url.hostname
             in [
                 "www.youtube.com",
+                "music.youtube.com",
                 "youtube.com",
                 "www.youtube-nocookie.com",
                 "youtube-nocookie.com",
@@ -330,7 +339,7 @@ def embed_url(text):
     attachments = {}
 
     for element in soup.find_all("a"):
-        if "href" in element.attrs and element.attrs["href"].startswith("http"):
+        if "href" in element.attrs and element.attrs["href"].startswith("http") and ("data-noembed" not in element.attrs):
             page_url = urlparse(element.attrs["href"])
             page = get_url_content(element.attrs["href"])
             if page:
@@ -447,7 +456,7 @@ def embed_url(text):
 
                     description_element.append(
                         BeautifulSoup(
-                            f"<p class='page-description'>{page_description or "No description available."}</p>",
+                            f"<p class='page-description'>{page_description or 'No description available.'}</p>",
                             "html.parser",
                         )
                     )
@@ -556,3 +565,4 @@ class PythonExtension(Extension):
             return "".join(to_write)
 
         return str(fn(caller()))
+

src/snek/templates/app.html

@@ -18,6 +18,7 @@
   <script src="/generic-form.js" type="module"></script>
   <script src="/html-frame.js" type="module"></script>
   <script src="/app.js" type="module"></script>
+  <script src="/editor.js" type="module"></script> 
   <script src="/file-manager.js" type="module"></script>
   <script src="/user-list.js"></script>
   <script src="/message-list.js" type="module"></script>
@@ -30,7 +31,7 @@
   <link rel="stylesheet" href="/base.css">
   <link rel="icon" type="image/png" href="/image/snek_logo_32x32.png" sizes="32x32">
   <link rel="icon" type="image/png" href="/image/snek_logo_64x64.png" sizes="64x64">
-<script defer src="https://umami.molodetz.nl/script.js" data-website-id="d127c3e4-dc70-4041-a1c8-bcc32c2492ea"></script>
+  <script nonce="{{nonce}}" defer src="https://umami.molodetz.nl/script.js" data-website-id="d127c3e4-dc70-4041-a1c8-bcc32c2492ea"></script>
 </head>
 <body>
   <header>

src/snek/templates/sandbox.html

@@ -1,6 +1,6 @@
 <div id="star-tooltip" class="star-tooltip"></div>
 <div id="star-popup" class="star-popup"></div>
-<script type="module">
+<script type="module" nonce="{{nonce}}">
 import { app } from "/app.js";
 import {WebTerminal} from "/dumb-term.js";
 

src/snek/view/channel.py

@@ -192,16 +192,22 @@ class ChannelAttachmentUploadView(BaseView):
             channel_uid=channel_uid, name=filename, user_uid=user_uid
         )
         pathlib.Path(attachment["path"]).parent.mkdir(parents=True, exist_ok=True)
-        async with aiofiles.open(attachment["path"], "wb") as f:
+        
+        async with aiofiles.open(attachment["path"], "wb") as file:
+            print("File openend.", filename)
             async for msg in ws:
                 if msg.type == web.WSMsgType.BINARY:
+                    print("Binary",filename)
                     if file is not None:
                         await file.write(msg.data)
-                        await ws.send_json({"type": "progress", "filename": filename, "bytes": file.tell()})
+                        await ws.send_json({"type": "progress", "filename": filename, "bytes": await file.tell()})
                 elif msg.type == web.WSMsgType.TEXT:
+                    print("TExt",filename)
+                    print(msg.json())
                     data = msg.json()                   
                     if data.get('type') == 'end':
-                        await ws.send_json({"type": "done", "filename": filename})
+                        relative_url = urllib.parse.quote(attachment_record["relative_url"])
+                        await ws.send_json({"type": "done", "file": relative_url, "filename": filename})
                 elif msg.type == web.WSMsgType.ERROR:
                     break
         return ws

src/snek/view/rpc.py

@@ -30,6 +30,7 @@ class RPCView(BaseView):
             self.ws = ws
             self.user_session = {}
             self._scheduled = []
+            self._finalize_task = None
 
         async def _session_ensure(self):
             uid = await self.view.session_get("uid")
@@ -203,6 +204,13 @@ class RPCView(BaseView):
                 )
             return channels
         
+        async def clear_channel(self, channel_uid):
+            self._require_login()
+            user = await self.services.user.get(uid=self.user_uid)
+            if not user["is_admin"]:
+                raise Exception("Not allowed")
+            return await self.services.channel_message.clear(channel_uid)
+
         async def write_container(self, channel_uid, content,timeout=3):
             self._require_login()
             channel_member = await self.services.channel_member.get(
@@ -259,6 +267,21 @@ class RPCView(BaseView):
                 }
             return result
 
+        async def send_message(self, channel_uid, message, is_final=True):
+            self._require_login()
+            
+            #if not is_final:
+            #    check_message = await self.services.channel_message.get(channel_uid=channel_uid, user_uid=self.user_uid,is_final=False)
+            #    if check_message:
+            #        return await self.update_message_text(check_message["uid"], message)
+            is_final = True            
+            message = await self.services.chat.send(
+                self.user_uid, channel_uid, message, is_final
+            )
+
+            return message["uid"]
+
+
         async def start_container(self, channel_uid):
             self._require_login()
             channel_member = await self.services.channel_member.get(
@@ -286,6 +309,8 @@ class RPCView(BaseView):
                 raise Exception("Not allowed")
             return await self.services.container.get_status(channel_uid)
         
+        
+
         async def finalize_message(self, message_uid):
             self._require_login()
             message = await self.services.channel_message.get(message_uid)
@@ -301,48 +326,56 @@ class RPCView(BaseView):
             return True
 
         async def update_message_text(self, message_uid, text):
+            async with self.app.no_save():
+                self._require_login()
+                message = await self.services.channel_message.get(message_uid)
+                if message["user_uid"] != self.user_uid:
+                    raise Exception("Not allowed")
+
+                if message.get_seconds_since_last_update() > 5:
+                    return {
+                        "error": "Message too old",
+                        "seconds_since_last_update": message.get_seconds_since_last_update(),
+                        "success": False,
+                    }
+
+                message["message"] = text
+                if not text:
+                    message["deleted_at"] = now()
+                else:
+                    message["deleted_at"] = None
+
+                await self.services.channel_message.save(message)
+                data = message.record
+                data["text"] = message["message"]
+                data["message_uid"] = message_uid
+
+                await self.services.socket.broadcast(
+                    message["channel_uid"],
+                    {
+                        "channel_uid": message["channel_uid"],
+                        "event": "update_message_text",
+                        "data": message.record,
+                    },
+                )
+
+                return {"success": True}
+
+
+
+
+        async def clear_channel(self, channel_uid):
             self._require_login()
-            message = await self.services.channel_message.get(message_uid)
+            user = await self.services.user.get(uid=self.user_uid)
-            if message["user_uid"] != self.user_uid:
+            if not user["is_admin"]:
                 raise Exception("Not allowed")
+            channel = await self.services.channel.get(uid=channel_uid)
+            if not channel:
+                raise Exception("Channel not found")
+            channel['history_start'] = datetime.now()
+            await self.services.channel.save(channel)
+            return await self.services.channel_message.clear(channel_uid)
 
-            if message.get_seconds_since_last_update() > 3:
-                await self.finalize_message(message["uid"])
-                return {
-                    "error": "Message too old",
-                    "seconds_since_last_update": message.get_seconds_since_last_update(),
-                    "success": False,
-                }
-
-            message["message"] = text
-            if not text:
-                message["deleted_at"] = now()
-            else:
-                message["deleted_at"] = None
-
-            await self.services.channel_message.save(message)
-            data = message.record
-            data["text"] = message["message"]
-            data["message_uid"] = message_uid
-
-            await self.services.socket.broadcast(
-                message["channel_uid"],
-                {
-                    "channel_uid": message["channel_uid"],
-                    "event": "update_message_text",
-                    "data": message.record,
-                },
-            )
-
-            return {"success": True}
-
-        async def send_message(self, channel_uid, message, is_final=True):
-            self._require_login()
-            message = await self.services.chat.send(
-                self.user_uid, channel_uid, message, is_final
-            )
-
-            return message["uid"]
 
         async def echo(self, *args):
             self._require_login()

src/snek/view/web.py

@@ -71,6 +71,7 @@ class WebView(BaseView):
         await self.app.services.channel_member.save(channel_member)
 
         user = await self.services.user.get(uid=self.session.get("uid"))
+
         messages = [
             await self.app.services.channel_message.to_extended_dict(message)
             for message in await self.app.services.channel_message.offset(
@@ -81,7 +82,7 @@ class WebView(BaseView):
             await self.app.services.notification.mark_as_read(
                 self.session.get("uid"), message["uid"]
             )
-
+        print(messages)
         name = await channel_member.get_name()
         return await self.render_template(
             "web.html",