commit d3b97c8e83eb44c30ddb19ddefc81a8fda4141c4 Author: retoor Date: Thu Dec 5 07:35:28 2024 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7132562 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.vscode +.history +.venv +__pycache__ +dist +mololog.db* +.trigger-2024-12-02 13:37:42 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1375d65 --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +BIN = ./.venv/bin/ +PYTHON = ./.venv/bin/python +PIP = ./.venv/bin/pip + +APP_NAME=app + +all: install build test + +ensure_repo: + -@git init + +ensure_env: ensure_repo + -@python3 -m venv .venv + +install: ensure_env + $(PIP) install -e . + +format: ensure_env + $(PIP) install shed + . $(BIN)/activate && shed + +build: ensure_env + $(MAKE) format + $(PIP) install build + $(PYTHON) -m build + +serve: ensure_env + $(BIN)mololog.serve --host=0.0.0.0 --port=3016 --db=mololog.db + +test: ensure_env + $(BIN)mololog.test --url=http://localhost:3016 + +bench: ensure_env + $(BIN)mololog.bench --url=http://localhost:3016 + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..07de284 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..95c8027 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,29 @@ +[metadata] +name = Mololog +version = 1.3.37 +description = HTTP Logging service +author = Retoor +author_email = retoor@molodetz.nl +license = MIT +long_description = file: README.md +long_description_content_type = text/markdown + +[options] +packages = find: +package_dir = + = src +python_requires = >=3.7 +install_requires = + requests==2.32.3 + aiohttp + app @ git+https://retoor.molodetz.nl/retoor/app@main + +[options.packages.find] +where = src + +[options.entry_points] +console_scripts = + mololog.serve = mololog.server:serve + mololog.test = mololog.client:test + mololog.bench = mololog.client:bench + diff --git a/src/Mololog.egg-info/PKG-INFO b/src/Mololog.egg-info/PKG-INFO new file mode 100644 index 0000000..264313c --- /dev/null +++ b/src/Mololog.egg-info/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 2.1 +Name: Mololog +Version: 1.3.37 +Summary: HTTP Logging service +Author: Retoor +Author-email: retoor@molodetz.nl +License: MIT +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +Requires-Dist: requests==2.32.3 +Requires-Dist: aiohttp +Requires-Dist: app@ git+https://retoor.molodetz.nl/retoor/app@main diff --git a/src/Mololog.egg-info/SOURCES.txt b/src/Mololog.egg-info/SOURCES.txt new file mode 100644 index 0000000..e395a6f --- /dev/null +++ b/src/Mololog.egg-info/SOURCES.txt @@ -0,0 +1,12 @@ +pyproject.toml +setup.cfg +src/Mololog.egg-info/PKG-INFO +src/Mololog.egg-info/SOURCES.txt +src/Mololog.egg-info/dependency_links.txt +src/Mololog.egg-info/entry_points.txt +src/Mololog.egg-info/requires.txt +src/Mololog.egg-info/top_level.txt +src/mololog/__init__.py +src/mololog/__main__.py +src/mololog/client.py +src/mololog/server.py \ No newline at end of file diff --git a/src/Mololog.egg-info/dependency_links.txt b/src/Mololog.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/Mololog.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/Mololog.egg-info/entry_points.txt b/src/Mololog.egg-info/entry_points.txt new file mode 100644 index 0000000..c9af3f7 --- /dev/null +++ b/src/Mololog.egg-info/entry_points.txt @@ -0,0 +1,4 @@ +[console_scripts] +mololog.bench = mololog.client:bench +mololog.serve = mololog.server:serve +mololog.test = mololog.client:test diff --git a/src/Mololog.egg-info/requires.txt b/src/Mololog.egg-info/requires.txt new file mode 100644 index 0000000..64006c6 --- /dev/null +++ b/src/Mololog.egg-info/requires.txt @@ -0,0 +1,3 @@ +requests==2.32.3 +aiohttp +app@ git+https://retoor.molodetz.nl/retoor/app@main diff --git a/src/Mololog.egg-info/top_level.txt b/src/Mololog.egg-info/top_level.txt new file mode 100644 index 0000000..0977044 --- /dev/null +++ b/src/Mololog.egg-info/top_level.txt @@ -0,0 +1 @@ +mololog diff --git a/src/mololog/__init__.py b/src/mololog/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mololog/__main__.py b/src/mololog/__main__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mololog/client.py b/src/mololog/client.py new file mode 100644 index 0000000..7c7bd15 --- /dev/null +++ b/src/mololog/client.py @@ -0,0 +1,104 @@ +import logging +import json +import code +import requests +import argparse +import time +from concurrent.futures import ThreadPoolExecutor as Executor + +def parse_args(): + parser = argparse.ArgumentParser(description="Mololog client.") + + parser.add_argument( + "--url", + type=str, + required=True + ) + return parser.parse_args() + + + +from requests.adapters import HTTPAdapter + + +log = logging.getLogger("mololog") + +class HTTPLogger(logging.Handler): + def __init__(self, url): + + self.url = url.rstrip("/") + "/emit" + self.session = requests.Session() + self.session.max_redirects =100 + self.adapter = HTTPAdapter(max_retries=10) + self.session.mount(self.url[:self.url.find("://")+2], self.adapter) + self.total_sent = 0 + self.total_send = 0 + self.total_queued = 0 + self.enabled = True + self.executor = Executor() + super().__init__() + + def emit(self, record): + if not self.enabled: + return False + self.total_send += 1 + self.total_queued += 1 + def send(me, record): + me.session.post(me.url,data=record) + me.total_sent += 1 + me.total_queued -= 1 + print("DONE",me.total_queued,flush=True) + try: + self.executor.submit(send,self,json.dumps(record.__dict__,default=str)) + except Exception as ex: + print(ex) + + print("Disabling mololog logger.") + self.enabled = False + log.exception(ex) + + def shutdown(self): + self.executor.shutdown(wait=True) + + def __del__(self): + self.shutdown() + +def get_logger_names(): + root_logger = logging.getLogger() + for child in root_logger.getChildren(): + yield child.name + +http_logger = None + +def patch(url,level=logging.INFO): + global http_logger + http_logger = HTTPLogger(url) + for name in get_logger_names(): + if "http" in name or "request" in name: + continue + logger = logging.getLogger(name) + logger.setLevel(level) + logger.addHandler(http_logger) + +def test(): + args = parse_args() + test_logger = logging.getLogger("mololog.test") + patch(args.url) + time.sleep(2) + test_logger.warn("Test 1") + test_logger.warn("Test 2") + test_logger.warn("Test 3") + +def bench(): + args = parse_args() + bench_logger = logging.getLogger("mololog.bench") + patch(args.url) + time.sleep(2) + time_start = time.time() + count = 0 + while time.time() - time_start <= 10: + bench_logger.info("New line on {}".format(time.time())) + count += 1 + print(http_logger.total_queued) + print("Requests per second: {}".format(count / 10)) + http_logger.shutdown() diff --git a/src/mololog/server.py b/src/mololog/server.py new file mode 100644 index 0000000..b250d4c --- /dev/null +++ b/src/mololog/server.py @@ -0,0 +1,54 @@ +from app.app import Application as BaseApplication +import asyncio +from aiohttp import web +import json +import uuid + +class Application(BaseApplication): + + + def __init__(self,*args,**kwargs): + + super().__init__(*args, **kwargs) + + self.router.add_post("/emit",self.handle_emit) + + async def handle_emit(self, request): + data = await request.json() + for key,value in data.items(): + if type(value) == list or type(value) == dict or type(value) == tuple: + data[key] = json.dumps(value,default=str) + data['uid'] = str(uuid.uuid4()) + result = await self.insert("log", data) + return web.json_response(data['uid'],content_type="application/json") + +import argparse + +def parse_args(): + parser = argparse.ArgumentParser(description="Mololog server.") + + parser.add_argument( + "--host", + type=str, + required=True + ) + parser.add_argument( + "--port", + type=int, + required=True + ) + parser.add_argument( + "--db", + type=str, + required=True + ) + return parser.parse_args() + + + +def serve(): + args = parse_args() + app = Application(db_path="sqlite:///{}".format(args.db)) + app.run(host=args.host, port=args.port) + +