Initial commit

.gitignore

@@ -0,0 +1,7 @@
+.vscode
+.history
+.venv
+__pycache__
+dist
+mololog.db*
+.trigger-2024-12-02 13:37:42

Makefile

@@ -0,0 +1,35 @@
+BIN = ./.venv/bin/
+PYTHON = ./.venv/bin/python
+PIP = ./.venv/bin/pip
+
+APP_NAME=app
+
+all: install build test
+
+ensure_repo:
+	-@git init 
+
+ensure_env: ensure_repo
+	-@python3 -m venv .venv 
+
+install: ensure_env
+	$(PIP) install -e . 
+
+format: ensure_env
+	$(PIP) install shed 
+	. $(BIN)/activate && shed
+
+build: ensure_env
+	$(MAKE) format
+	$(PIP) install build
+	$(PYTHON) -m build 
+
+serve: ensure_env
+	$(BIN)mololog.serve --host=0.0.0.0 --port=3016 --db=mololog.db
+
+test: ensure_env 
+	$(BIN)mololog.test --url=http://localhost:3016
+
+bench: ensure_env
+	$(BIN)mololog.bench --url=http://localhost:3016
+

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

setup.cfg

@@ -0,0 +1,29 @@
+[metadata]
+name = Mololog
+version = 1.3.37
+description = HTTP Logging service
+author = Retoor
+author_email = retoor@molodetz.nl
+license = MIT
+long_description = file: README.md
+long_description_content_type = text/markdown
+
+[options]
+packages = find:
+package_dir =
+    = src
+python_requires = >=3.7
+install_requires =
+    requests==2.32.3
+    aiohttp
+    app @ git+https://retoor.molodetz.nl/retoor/app@main
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+console_scripts =
+    mololog.serve = mololog.server:serve
+    mololog.test = mololog.client:test
+    mololog.bench = mololog.client:bench
+

src/Mololog.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: Mololog
+Version: 1.3.37
+Summary: HTTP Logging service
+Author: Retoor
+Author-email: retoor@molodetz.nl
+License: MIT
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: requests==2.32.3
+Requires-Dist: aiohttp
+Requires-Dist: app@ git+https://retoor.molodetz.nl/retoor/app@main

src/Mololog.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+pyproject.toml
+setup.cfg
+src/Mololog.egg-info/PKG-INFO
+src/Mololog.egg-info/SOURCES.txt
+src/Mololog.egg-info/dependency_links.txt
+src/Mololog.egg-info/entry_points.txt
+src/Mololog.egg-info/requires.txt
+src/Mololog.egg-info/top_level.txt
+src/mololog/__init__.py
+src/mololog/__main__.py
+src/mololog/client.py
+src/mololog/server.py

src/Mololog.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

src/Mololog.egg-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+mololog.bench = mololog.client:bench
+mololog.serve = mololog.server:serve
+mololog.test = mololog.client:test

src/Mololog.egg-info/requires.txt

@@ -0,0 +1,3 @@
+requests==2.32.3
+aiohttp
+app@ git+https://retoor.molodetz.nl/retoor/app@main

src/Mololog.egg-info/top_level.txt

@@ -0,0 +1 @@
+mololog

src/mololog/client.py

@@ -0,0 +1,104 @@
+import logging 
+import json 
+import code
+import requests
+import argparse 
+import time 
+from concurrent.futures import ThreadPoolExecutor as Executor 
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Mololog client.")
+
+    parser.add_argument(
+        "--url",
+        type=str,
+        required=True
+    )
+    return parser.parse_args()
+
+
+
+from requests.adapters import HTTPAdapter
+
+
+log = logging.getLogger("mololog")
+
+class HTTPLogger(logging.Handler):
+    def __init__(self, url):
+
+        self.url = url.rstrip("/") + "/emit"
+        self.session = requests.Session()
+        self.session.max_redirects =100
+        self.adapter = HTTPAdapter(max_retries=10)
+        self.session.mount(self.url[:self.url.find("://")+2], self.adapter)
+        self.total_sent = 0
+        self.total_send = 0
+        self.total_queued = 0
+        self.enabled = True
+        self.executor = Executor()
+        super().__init__()
+
+    def emit(self, record):
+        if not self.enabled:
+            return False 
+        self.total_send += 1
+        self.total_queued += 1
+        def send(me, record):
+            me.session.post(me.url,data=record)
+            me.total_sent += 1
+            me.total_queued -= 1
+            print("DONE",me.total_queued,flush=True)
+        try:
+            self.executor.submit(send,self,json.dumps(record.__dict__,default=str))
+        except Exception as ex:
+            print(ex)
+    
+            print("Disabling mololog logger.")
+            self.enabled = False 
+            log.exception(ex)
+    
+    def shutdown(self):
+        self.executor.shutdown(wait=True)
+
+    def __del__(self):
+        self.shutdown()
+        
+def get_logger_names():
+    root_logger = logging.getLogger()
+    for child in root_logger.getChildren():
+        yield child.name
+
+http_logger = None
+
+def patch(url,level=logging.INFO):
+    global http_logger
+    http_logger = HTTPLogger(url)
+    for name in get_logger_names():
+        if "http" in name or "request" in name:
+            continue
+        logger = logging.getLogger(name)
+        logger.setLevel(level)
+        logger.addHandler(http_logger)
+
+def test():
+    args = parse_args()
+    test_logger = logging.getLogger("mololog.test")
+    patch(args.url)
+    time.sleep(2)
+    test_logger.warn("Test 1")
+    test_logger.warn("Test 2")
+    test_logger.warn("Test 3")
+
+def bench():
+    args = parse_args()
+    bench_logger = logging.getLogger("mololog.bench")
+    patch(args.url)
+    time.sleep(2)
+    time_start = time.time()
+    count = 0
+    while time.time() - time_start <= 10:
+        bench_logger.info("New line on {}".format(time.time()))
+        count += 1
+        print(http_logger.total_queued)
+    print("Requests per second: {}".format(count / 10))
+    http_logger.shutdown()

src/mololog/server.py

@@ -0,0 +1,54 @@
+from app.app import Application as BaseApplication
+import asyncio
+from aiohttp import web
+import json
+import uuid
+
+class Application(BaseApplication):
+
+
+    def __init__(self,*args,**kwargs):
+        
+        super().__init__(*args, **kwargs)
+        
+        self.router.add_post("/emit",self.handle_emit)
+
+    async def handle_emit(self, request):
+        data = await request.json()
+        for key,value in data.items():
+            if type(value) == list or type(value) == dict or type(value) == tuple:
+                data[key] = json.dumps(value,default=str)
+        data['uid'] = str(uuid.uuid4())
+        result = await self.insert("log", data)
+        return web.json_response(data['uid'],content_type="application/json")
+
+import argparse 
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Mololog server.")
+
+    parser.add_argument(
+        "--host",
+        type=str,
+        required=True
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        required=True
+    )
+    parser.add_argument(
+        "--db",
+        type=str,
+        required=True
+    )
+    return parser.parse_args()
+
+
+    
+def serve():
+    args = parse_args()
+    app = Application(db_path="sqlite:///{}".format(args.db))
+    app.run(host=args.host, port=args.port)
+
+