diff --git a/src/metriki/app.py b/src/metriki/app.py index 81a82f4..8551335 100644 --- a/src/metriki/app.py +++ b/src/metriki/app.py @@ -5,6 +5,25 @@ import json import uuid from datetime import datetime import pathlib +import time +import socket +hostname = socket.gethostname() +dns_resolved_addr = socket.gethostbyname(hostname) + + +class TimeIt: + + def __init__(self, name): + self.name = name + self.start = time.time() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.end = time.time() + self.duration = self.end - self.start + print(f"{self.name} took {self.duration} seconds") class BaseView(web.View): @@ -27,14 +46,14 @@ class BaseView(web.View): ip=self.ip, key=key, value=json.dumps(value,default=str), - created=str(datetime.now()) + created=datetime.now() ) record.update(kwargs) self.db['session'].upsert(dict( key=key, value=json.dumps(value,default=str), ip=self.ip, - created=str(datetime.now()) + created=datetime.now() ),['id','key']) return record @@ -46,9 +65,9 @@ class BaseView(web.View): def insert(self, table, data): data['ip'] = self.ip - data['created'] = str(datetime.now()) + data['created'] = datetime.now() for key,value in data.items(): - data[key] = str(value) + data[key] = value #data = json.loads(json.dumps(data,default=str)) return self.db[table].insert(data) @@ -58,6 +77,24 @@ class BaseView(web.View): data['ip'] = self.ip return [dict(d) for d in self.db[table].find(**data)] +class Visit: + + def __init__(self, db, visit): + self.__dict__.update(dict(visit)) + self._db = db + self.html_bytes = len(self.html) + + self.count = db['event'].count(visit_id=self.visit_id) + + def __str__(self): + return json.dumps(self.__dict__,default=str) + + @classmethod + def find(cls, db, where=None): + if not where: + where = {} + return [cls(db, visit) for visit in db['visit'].find(**where)] + class EventView(BaseView): document_fields = ['title','html','domain','href'] @@ -71,38 +108,19 @@ class EventView(BaseView): return data.get('event') == 'click' def get_record(self, data): - - try: - record = {} - for field in self.document_fields: - if field in data: - record[field] = data[field] - if len(record.keys()) == len(self.document_fields): - return record - except KeyError: - pass - - try: - record = {} - for field in self.event_fields: - if field in data: - record[field] = data[field] - if len(record.keys()) == len(self.event_fields): - return record - except: - pass - - try: - record = {} - for field in self.click_fields: - if field in data: - record[field] = data[field] - if len(record.keys()) == len(self.click_fields): - return record - except: - pass - - return None + record = {} + for field in self.document_fields: + if field in data: + record[field] = data[field] + for field in self.event_fields: + if field in data: + record[field] = data[field] + for field in self.click_fields: + if field in data: + record[field] = data[field] + if not record.get('event') and not record.get('html'): + return None + return record async def get(self): ws = web.WebSocketResponse() @@ -113,47 +131,47 @@ class EventView(BaseView): print(f"Socket connected from {self.ip}.") async for msg in ws: - if msg.type == web.WSMsgType.TEXT: - # Echo the received message back to the client - try: - data = msg.json() - except Exception as ex: - print(ex) - continue - - original_data = data - data = self.get_record(data) - if not data: - print("Not a valid record:", original_data) - exit(1) - continue - - if self.is_document_record(data): - html = data.get('html') - record = dict( - html=html, - visit_id=visit_id, - domain=data.get('domain'), - href=data.get('href'), - title=data.get('title') - ) - self.insert('visit', record) - self.app.view_count += 1 - elif self.is_click_record(data): - data['visit_id'] = visit_id - record = data - self.insert('click', record) - else: - data['visit_id'] = visit_id - record = data - self.insert('event', record) - self.app.event_count += 1 - - print(record) - print(self.app.view_count, self.app.event_count) - - elif msg.type == web.WSMsgType.ERROR: - print(f'Socket closed: {ws.exception()}') + with TimeIt("process_event"): + if msg.type == web.WSMsgType.TEXT: + # Echo the received message back to the client + try: + data = msg.json() + except Exception as ex: + print(ex) + continue + + original_data = data + data = self.get_record(data) + if not data: + print("Not a valid record:", original_data) + exit(1) + continue + + if self.is_document_record(data): + html = data.get('html') + record = dict( + html=html, + visit_id=visit_id, + domain=data.get('domain'), + href=data.get('href'), + title=data.get('title') + ) + self.insert('visit', record) + self.app.view_count += 1 + elif self.is_click_record(data): + data['visit_id'] = visit_id + record = data + self.insert('click', record) + else: + data['visit_id'] = visit_id + record = data + self.insert('event', record) + self.app.event_count += 1 + + self.app.sset('view_count',self.app.view_count) + self.app.sset('event_count',self.app.event_count) + elif msg.type == web.WSMsgType.ERROR: + print(f'Socket closed: {ws.exception()}') print("Socket cracefully closed.") return ws @@ -168,10 +186,37 @@ class Application(BaseApplication): super().__init__(template_path=self.template_path,db_path=db_path, *args, **kwargs) self.router.add_get("/", self.index_handler) self.router.add_view("/event", EventView) + self.router.add_view("/dashboard", self.dashboard_handler) + self.router.add_view("/visit", self.visit_handler) self.router.add_static("/static", pathlib.Path(__file__).parent.joinpath("static")) + + + async def visit_handler(self,handler): + totals = {} + + with TimeIt("visit_handler"): + views = [dict(view) for view in self.db['visit'].find()] + date_start = None + for view in views: + if not date_start: + date_start = view['created'] + view['count'] = self.db['event'].count(visit_id=view['visit_id']) + date_end = view['created'] + if not view['href'] in totals: + totals[view['href']] = {"title": view['title'], "count": view['count']} + totals[view['href']]['count'] += view['count'] + views = json.loads(json.dumps(views,default=str)) + return web.json_response(dict(date_start=str(date_start),date_end=str(date_end),view_count=self.sget("view_count"),event_count=self.sget("event_count"),totals=totals)) + + def ip(self, request): + return request.headers.get("X-Forwarded-For",request.remote) async def index_handler(self, request): - return await self.render_template("index.html",request, dict(view_count=self.view_count,event_count=self.event_count)) + return await self.render_template("index.html",request, dict(view_count=self.view_count,event_count=self.event_count,ip=self.ip(request),hostname=hostname,dns_resolved_addr=dns_resolved_addr)) + async def dashboard_handler(self, request): + visits = Visit.find(self.db) + db_mbs = pathlib.Path(self.db_path.lstrip("sqlite:///")).stat().st_size * 0.000001 + return await self.render_template("dashboard.html", request, dict(view_count=self.view_count,event_count=self.event_count,ip=self.ip(request),hostname=hostname,dns_resolved_addr=dns_resolved_addr,visits=visits,visits_count=len(visits),db_mbs=db_mbs)) diff --git a/src/metriki/static/metriki.js b/src/metriki/static/metriki.js index c687f71..f4ccd20 100644 --- a/src/metriki/static/metriki.js +++ b/src/metriki/static/metriki.js @@ -79,19 +79,26 @@ class Metriki { document.addEventListener('click', (e) => { me.emit({ "event":"click", - "target":e.target.id || e.target.className || e.target.tagName + "target":e.target.id || e.target.className || e.target.tagName, + "scroll_height":document.documentElement.scrollHeight, + "scroll_left":document.documentElement.scrollLeft, + "scroll_top":document.documentElement.scrollTop, + "client_width":document.documentElement.clientWidth, + "client_height":document.documentElement.clientHeight, + "page_x":e.pageX, + "page_y":e.pageY, + "screen_x":e.screenX, + "screen_y":e.screenY, + "client_x":e.clientX, + "client_y":e.clientY, + "target_x":e.target.offsetLeft, + "target_y":e.target.offsetTop }) }) } constructor() { - this.url = window.location.href.replace("http://", "ws://").replace("https://", "wss://").replace("www.", "") - if(this.url.endsWith("/")){ - this.url += "event" - }else{ - this.url += "/event" - } - this.url = "wss://metriki.molodetz.nl/event" + this.url = window.location.protocol == 'http:' ? 'ws://localhost:4000/event' : 'wss://metriki.molodetz.nl/event' const me = this this.ws = new WebSocket(this.url) this.ws.onopen = () => { diff --git a/src/metriki/templates/index.html b/src/metriki/templates/index.html index e7b941b..7f30f04 100644 --- a/src/metriki/templates/index.html +++ b/src/metriki/templates/index.html @@ -7,8 +7,7 @@