Update.

Browse Source
This commit is contained in:
retoor 2025-01-03 00:09:54 +01:00
parent 5434a4e9bc
commit bbe1907063
3 changed files with 139 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
src/metriki/app.py
View File

@ -5,6 +5,25 @@ import json
import uuid import uuid
from datetime import datetime from datetime import datetime
import pathlib import pathlib
import time
import socket
hostname = socket.gethostname()
dns_resolved_addr = socket.gethostbyname(hostname)
class TimeIt:
def __init__(self, name):
self.name = name
self.start = time.time()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.end = time.time()
self.duration = self.end - self.start
print(f"{self.name} took {self.duration} seconds")
class BaseView(web.View): class BaseView(web.View):
@ -27,14 +46,14 @@ class BaseView(web.View):
ip=self.ip, ip=self.ip,
key=key, key=key,
value=json.dumps(value,default=str), value=json.dumps(value,default=str),
created=str(datetime.now()) created=datetime.now()
) )
record.update(kwargs) record.update(kwargs)
self.db['session'].upsert(dict( self.db['session'].upsert(dict(
key=key, key=key,
value=json.dumps(value,default=str), value=json.dumps(value,default=str),
ip=self.ip, ip=self.ip,
created=str(datetime.now()) created=datetime.now()
),['id','key']) ),['id','key'])
return record return record
@ -46,9 +65,9 @@ class BaseView(web.View):
def insert(self, table, data): def insert(self, table, data):
data['ip'] = self.ip data['ip'] = self.ip
data['created'] = str(datetime.now()) data['created'] = datetime.now()
for key,value in data.items(): for key,value in data.items():
data[key] = str(value) data[key] = value
#data = json.loads(json.dumps(data,default=str)) #data = json.loads(json.dumps(data,default=str))
return self.db[table].insert(data) return self.db[table].insert(data)
@ -58,6 +77,24 @@ class BaseView(web.View):
data['ip'] = self.ip data['ip'] = self.ip
return [dict(d) for d in self.db[table].find(**data)] return [dict(d) for d in self.db[table].find(**data)]
class Visit:
def __init__(self, db, visit):
self.__dict__.update(dict(visit))
self._db = db
self.html_bytes = len(self.html)
self.count = db['event'].count(visit_id=self.visit_id)
def __str__(self):
return json.dumps(self.__dict__,default=str)
@classmethod
def find(cls, db, where=None):
if not where:
where = {}
return [cls(db, visit) for visit in db['visit'].find(**where)]
class EventView(BaseView): class EventView(BaseView):
document_fields = ['title','html','domain','href'] document_fields = ['title','html','domain','href']
@ -71,38 +108,19 @@ class EventView(BaseView):
return data.get('event') == 'click' return data.get('event') == 'click'
def get_record(self, data): def get_record(self, data):
record = {}
try: for field in self.document_fields:
record = {} if field in data:
for field in self.document_fields: record[field] = data[field]
if field in data: for field in self.event_fields:
record[field] = data[field] if field in data:
if len(record.keys()) == len(self.document_fields): record[field] = data[field]
return record for field in self.click_fields:
except KeyError: if field in data:
pass record[field] = data[field]
if not record.get('event') and not record.get('html'):
try: return None
record = {} return record
for field in self.event_fields:
if field in data:
record[field] = data[field]
if len(record.keys()) == len(self.event_fields):
return record
except:
pass
try:
record = {}
for field in self.click_fields:
if field in data:
record[field] = data[field]
if len(record.keys()) == len(self.click_fields):
return record
except:
pass
return None
async def get(self): async def get(self):
ws = web.WebSocketResponse() ws = web.WebSocketResponse()
@ -113,47 +131,47 @@ class EventView(BaseView):
print(f"Socket connected from {self.ip}.") print(f"Socket connected from {self.ip}.")
async for msg in ws: async for msg in ws:
if msg.type == web.WSMsgType.TEXT: with TimeIt("process_event"):
# Echo the received message back to the client if msg.type == web.WSMsgType.TEXT:
try: # Echo the received message back to the client
data = msg.json() try:
except Exception as ex: data = msg.json()
print(ex) except Exception as ex:
continue print(ex)
continue
original_data = data original_data = data
data = self.get_record(data) data = self.get_record(data)
if not data: if not data:
print("Not a valid record:", original_data) print("Not a valid record:", original_data)
exit(1) exit(1)
continue continue
if self.is_document_record(data): if self.is_document_record(data):
html = data.get('html') html = data.get('html')
record = dict( record = dict(
html=html, html=html,
visit_id=visit_id, visit_id=visit_id,
domain=data.get('domain'), domain=data.get('domain'),
href=data.get('href'), href=data.get('href'),
title=data.get('title') title=data.get('title')
) )
self.insert('visit', record) self.insert('visit', record)
self.app.view_count += 1 self.app.view_count += 1
elif self.is_click_record(data): elif self.is_click_record(data):
data['visit_id'] = visit_id data['visit_id'] = visit_id
record = data record = data
self.insert('click', record) self.insert('click', record)
else: else:
data['visit_id'] = visit_id data['visit_id'] = visit_id
record = data record = data
self.insert('event', record) self.insert('event', record)
self.app.event_count += 1 self.app.event_count += 1
print(record) self.app.sset('view_count',self.app.view_count)
print(self.app.view_count, self.app.event_count) self.app.sset('event_count',self.app.event_count)
elif msg.type == web.WSMsgType.ERROR:
elif msg.type == web.WSMsgType.ERROR: print(f'Socket closed: {ws.exception()}')
print(f'Socket closed: {ws.exception()}')
print("Socket cracefully closed.") print("Socket cracefully closed.")
return ws return ws
@ -168,10 +186,37 @@ class Application(BaseApplication):
super().__init__(template_path=self.template_path,db_path=db_path, *args, **kwargs) super().__init__(template_path=self.template_path,db_path=db_path, *args, **kwargs)
self.router.add_get("/", self.index_handler) self.router.add_get("/", self.index_handler)
self.router.add_view("/event", EventView) self.router.add_view("/event", EventView)
self.router.add_view("/dashboard", self.dashboard_handler)
self.router.add_view("/visit", self.visit_handler)
self.router.add_static("/static", pathlib.Path(__file__).parent.joinpath("static")) self.router.add_static("/static", pathlib.Path(__file__).parent.joinpath("static"))
async def visit_handler(self,handler):
totals = {}
with TimeIt("visit_handler"):
views = [dict(view) for view in self.db['visit'].find()]
date_start = None
for view in views:
if not date_start:
date_start = view['created']
view['count'] = self.db['event'].count(visit_id=view['visit_id'])
date_end = view['created']
if not view['href'] in totals:
totals[view['href']] = {"title": view['title'], "count": view['count']}
totals[view['href']]['count'] += view['count']
views = json.loads(json.dumps(views,default=str))
return web.json_response(dict(date_start=str(date_start),date_end=str(date_end),view_count=self.sget("view_count"),event_count=self.sget("event_count"),totals=totals))
def ip(self, request):
return request.headers.get("X-Forwarded-For",request.remote)
async def index_handler(self, request): async def index_handler(self, request):
return await self.render_template("index.html",request, dict(view_count=self.view_count,event_count=self.event_count)) return await self.render_template("index.html",request, dict(view_count=self.view_count,event_count=self.event_count,ip=self.ip(request),hostname=hostname,dns_resolved_addr=dns_resolved_addr))
async def dashboard_handler(self, request):
visits = Visit.find(self.db)
db_mbs = pathlib.Path(self.db_path.lstrip("sqlite:///")).stat().st_size * 0.000001
return await self.render_template("dashboard.html", request, dict(view_count=self.view_count,event_count=self.event_count,ip=self.ip(request),hostname=hostname,dns_resolved_addr=dns_resolved_addr,visits=visits,visits_count=len(visits),db_mbs=db_mbs))

23
src/metriki/static/metriki.js
View File

@ -79,19 +79,26 @@ class Metriki {
document.addEventListener('click', (e) => { document.addEventListener('click', (e) => {
me.emit({ me.emit({
"event":"click", "event":"click",
"target":e.target.id || e.target.className || e.target.tagName "target":e.target.id || e.target.className || e.target.tagName,
"scroll_height":document.documentElement.scrollHeight,
"scroll_left":document.documentElement.scrollLeft,
"scroll_top":document.documentElement.scrollTop,
"client_width":document.documentElement.clientWidth,
"client_height":document.documentElement.clientHeight,
"page_x":e.pageX,
"page_y":e.pageY,
"screen_x":e.screenX,
"screen_y":e.screenY,
"client_x":e.clientX,
"client_y":e.clientY,
"target_x":e.target.offsetLeft,
"target_y":e.target.offsetTop
}) })
}) })
} }
constructor() { constructor() {
this.url = window.location.href.replace("http://", "ws://").replace("https://", "wss://").replace("www.", "") this.url = window.location.protocol == 'http:' ? 'ws://localhost:4000/event' : 'wss://metriki.molodetz.nl/event'
if(this.url.endsWith("/")){
this.url += "event"
}else{
this.url += "/event"
}
this.url = "wss://metriki.molodetz.nl/event"
const me = this const me = this
this.ws = new WebSocket(this.url) this.ws = new WebSocket(this.url)
this.ws.onopen = () => { this.ws.onopen = () => {

3
src/metriki/templates/index.html
View File

@ -7,8 +7,7 @@
<body> <body>
<h1>Metriki</h1> <h1>Metriki</h1>
{{ dns_resolved_addr }}
</body> </body>
</html> </html>