diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml new file mode 100644 index 0000000..1b06ed2 --- /dev/null +++ b/.gitea/workflows/build.yaml @@ -0,0 +1,26 @@ +name: Build +run-name: Build and test +on: [push] + +jobs: + Build: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Update repo + run: git pull + - name: List files in the repository + run: | + ls ${{ gitea.workspace }} + - name: Update apt + run: apt update + - name: Installing system dependencies + run: apt install python3 python3-pip python3-venv make -y + - name: Run Make + run: make + - run: git add . + - run: git config --global user.email "bot@molodetz.com" + - run: git config --global user.name "bot" + - run: git commit -a -m "Automated update of package." + - run: git push diff --git a/Makefile b/Makefile index 1375d65..da08547 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ serve: ensure_env $(BIN)mololog.serve --host=0.0.0.0 --port=3016 --db=mololog.db test: ensure_env - $(BIN)mololog.test --url=http://localhost:3016 + $(PYTHON) test.py bench: ensure_env $(BIN)mololog.bench --url=http://localhost:3016 diff --git a/setup.cfg b/setup.cfg index 95c8027..73bff88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = Mololog +name = mololog version = 1.3.37 description = HTTP Logging service author = Retoor @@ -16,7 +16,8 @@ python_requires = >=3.7 install_requires = requests==2.32.3 aiohttp - app @ git+https://retoor.molodetz.nl/retoor/app@main + app @ git+https://retoor.molodetz.nl/retoor/app.git +# zhurnal @ git+https://retoor.molodetz.nl/retoor/zhurnal.git [options.packages.find] where = src diff --git a/src/Mololog.egg-info/PKG-INFO b/src/Mololog.egg-info/PKG-INFO index 264313c..7d571a6 100644 --- a/src/Mololog.egg-info/PKG-INFO +++ b/src/Mololog.egg-info/PKG-INFO @@ -10,3 +10,4 @@ Description-Content-Type: text/markdown Requires-Dist: requests==2.32.3 Requires-Dist: aiohttp Requires-Dist: app@ git+https://retoor.molodetz.nl/retoor/app@main +Requires-Dist: zhurnal@ git+https://retoor.molodetz.nl/retoor/zhurnal@main diff --git a/src/Mololog.egg-info/requires.txt b/src/Mololog.egg-info/requires.txt index 64006c6..b7a2745 100644 --- a/src/Mololog.egg-info/requires.txt +++ b/src/Mololog.egg-info/requires.txt @@ -1,3 +1,4 @@ requests==2.32.3 aiohttp app@ git+https://retoor.molodetz.nl/retoor/app@main +zhurnal@ git+https://retoor.molodetz.nl/retoor/zhurnal@main diff --git a/src/mololog.egg-info/PKG-INFO b/src/mololog.egg-info/PKG-INFO new file mode 100644 index 0000000..b2da6ef --- /dev/null +++ b/src/mololog.egg-info/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 2.1 +Name: mololog +Version: 1.3.37 +Summary: HTTP Logging service +Author: Retoor +Author-email: retoor@molodetz.nl +License: MIT +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +Requires-Dist: requests==2.32.3 +Requires-Dist: aiohttp +Requires-Dist: app@ git+https://retoor.molodetz.nl/retoor/app.git diff --git a/src/mololog.egg-info/SOURCES.txt b/src/mololog.egg-info/SOURCES.txt new file mode 100644 index 0000000..0fbb675 --- /dev/null +++ b/src/mololog.egg-info/SOURCES.txt @@ -0,0 +1,12 @@ +pyproject.toml +setup.cfg +src/mololog/__init__.py +src/mololog/__main__.py +src/mololog/client.py +src/mololog/server.py +src/mololog.egg-info/PKG-INFO +src/mololog.egg-info/SOURCES.txt +src/mololog.egg-info/dependency_links.txt +src/mololog.egg-info/entry_points.txt +src/mololog.egg-info/requires.txt +src/mololog.egg-info/top_level.txt \ No newline at end of file diff --git a/src/mololog.egg-info/dependency_links.txt b/src/mololog.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/mololog.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/mololog.egg-info/entry_points.txt b/src/mololog.egg-info/entry_points.txt new file mode 100644 index 0000000..c9af3f7 --- /dev/null +++ b/src/mololog.egg-info/entry_points.txt @@ -0,0 +1,4 @@ +[console_scripts] +mololog.bench = mololog.client:bench +mololog.serve = mololog.server:serve +mololog.test = mololog.client:test diff --git a/src/mololog.egg-info/requires.txt b/src/mololog.egg-info/requires.txt new file mode 100644 index 0000000..201f32b --- /dev/null +++ b/src/mololog.egg-info/requires.txt @@ -0,0 +1,3 @@ +requests==2.32.3 +aiohttp +app@ git+https://retoor.molodetz.nl/retoor/app.git diff --git a/src/mololog.egg-info/top_level.txt b/src/mololog.egg-info/top_level.txt new file mode 100644 index 0000000..0977044 --- /dev/null +++ b/src/mololog.egg-info/top_level.txt @@ -0,0 +1 @@ +mololog diff --git a/src/mololog/client.py b/src/mololog/client.py index 7c7bd15..1f6f8ba 100644 --- a/src/mololog/client.py +++ b/src/mololog/client.py @@ -1,36 +1,32 @@ -import logging -import json -import code +import argparse +import json +import logging +import time +from concurrent.futures import ThreadPoolExecutor as Executor + import requests -import argparse -import time -from concurrent.futures import ThreadPoolExecutor as Executor + def parse_args(): parser = argparse.ArgumentParser(description="Mololog client.") - parser.add_argument( - "--url", - type=str, - required=True - ) + parser.add_argument("--url", type=str, required=True) return parser.parse_args() - from requests.adapters import HTTPAdapter - log = logging.getLogger("mololog") + class HTTPLogger(logging.Handler): def __init__(self, url): self.url = url.rstrip("/") + "/emit" self.session = requests.Session() - self.session.max_redirects =100 + self.session.max_redirects = 100 self.adapter = HTTPAdapter(max_retries=10) - self.session.mount(self.url[:self.url.find("://")+2], self.adapter) + self.session.mount(self.url[: self.url.find("://") + 2], self.adapter) self.total_sent = 0 self.total_send = 0 self.total_queued = 0 @@ -40,37 +36,41 @@ class HTTPLogger(logging.Handler): def emit(self, record): if not self.enabled: - return False + return False self.total_send += 1 self.total_queued += 1 + def send(me, record): - me.session.post(me.url,data=record) + me.session.post(me.url, data=record) me.total_sent += 1 me.total_queued -= 1 - print("DONE",me.total_queued,flush=True) + try: - self.executor.submit(send,self,json.dumps(record.__dict__,default=str)) + self.executor.submit(send, self, json.dumps(record.__dict__, default=str)) except Exception as ex: print(ex) - + print("Disabling mololog logger.") - self.enabled = False + self.enabled = False log.exception(ex) - + def shutdown(self): self.executor.shutdown(wait=True) def __del__(self): self.shutdown() - + + def get_logger_names(): root_logger = logging.getLogger() for child in root_logger.getChildren(): yield child.name + http_logger = None -def patch(url,level=logging.INFO): + +def patch(url, level=logging.INFO): global http_logger http_logger = HTTPLogger(url) for name in get_logger_names(): @@ -80,6 +80,7 @@ def patch(url,level=logging.INFO): logger.setLevel(level) logger.addHandler(http_logger) + def test(): args = parse_args() test_logger = logging.getLogger("mololog.test") @@ -89,6 +90,7 @@ def test(): test_logger.warn("Test 2") test_logger.warn("Test 3") + def bench(): args = parse_args() bench_logger = logging.getLogger("mololog.bench") @@ -97,8 +99,8 @@ def bench(): time_start = time.time() count = 0 while time.time() - time_start <= 10: - bench_logger.info("New line on {}".format(time.time())) + bench_logger.info(f"New line on {time.time()}") count += 1 print(http_logger.total_queued) - print("Requests per second: {}".format(count / 10)) + print(f"Requests per second: {count / 10}") http_logger.shutdown() diff --git a/src/mololog/server.py b/src/mololog/server.py index b250d4c..7205aa3 100644 --- a/src/mololog/server.py +++ b/src/mololog/server.py @@ -1,54 +1,64 @@ -from app.app import Application as BaseApplication -import asyncio -from aiohttp import web import json import uuid +import asyncio +from aiohttp import web +from app.app import Application as BaseApplication + class Application(BaseApplication): - - def __init__(self,*args,**kwargs): + def __init__(self, test=0, *args, **kwargs): + + self.test = test + self.total_received = 0 + super().__init__(*args, **kwargs) - self.router.add_post("/emit",self.handle_emit) + self.middlewares.append(self.test_middleware) + + self.router.add_post("/emit", self.handle_emit) + + async def exit_app(self): + await asyncio.sleep(1) + exit(0) + + @web.middleware + async def test_middleware(self,request,handler): + result = await handler(request) + + + if self.total_received == self.test: + await asyncio.create_task(self.exit_app()) + + return result + async def handle_emit(self, request): data = await request.json() - for key,value in data.items(): + for key, value in data.items(): if type(value) == list or type(value) == dict or type(value) == tuple: - data[key] = json.dumps(value,default=str) - data['uid'] = str(uuid.uuid4()) - result = await self.insert("log", data) - return web.json_response(data['uid'],content_type="application/json") + data[key] = json.dumps(value, default=str) + data["uid"] = str(uuid.uuid4()) + await self.insert("log", data) + self.total_received += 1 + return web.json_response(data["uid"], content_type="application/json") + + +import argparse -import argparse def parse_args(): parser = argparse.ArgumentParser(description="Mololog server.") - parser.add_argument( - "--host", - type=str, - required=True - ) - parser.add_argument( - "--port", - type=int, - required=True - ) - parser.add_argument( - "--db", - type=str, - required=True - ) + parser.add_argument("--host", type=str, required=True) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--db", type=str, required=True) + parser.add_argument("--test", type=int, required=False,default=0) return parser.parse_args() - def serve(): args = parse_args() - app = Application(db_path="sqlite:///{}".format(args.db)) + app = Application(db_path=f"sqlite:///{args.db}",test=args.test) app.run(host=args.host, port=args.port) - - diff --git a/test.py b/test.py new file mode 100644 index 0000000..5ca3b50 --- /dev/null +++ b/test.py @@ -0,0 +1,21 @@ +import subprocess +import shlex +import time + +processes = [ + subprocess.Popen(shlex.split("./.venv/bin/mololog.serve --host=127.0.0.1 --port=3999 --db=':memory:' --test=3")), + subprocess.Popen(shlex.split("./.venv/bin/mololog.test --url=http://localhost:3999")) +] + +for process in processes: + while True: + process.poll() + if not process.returncode is None: + if process.returncode != 0: + exit(process.returncode) + break + time.sleep(1) + +print("Test sucess!") + +