diff --git a/README.md b/README.md index e1021b8..5a7f94c 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,7 @@ Simple project to determine health of the devrant platform. +## Credits +Thanks to Rohan Burke (coolq). The creator of the dr api wrapper this project uses. Since it isn't made like a package, i had to copy his source files to my source folder. His library: https://github.com/coolq1000/devrant-python-api/ + diff --git a/drstats.db b/drstats.db new file mode 100644 index 0000000..bf0308d Binary files /dev/null and b/drstats.db differ diff --git a/setup.cfg b/setup.cfg index f467468..e58b50e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,12 +16,14 @@ python_requires = >=3.7 install_requires = aiohttp>=3.10.10 dataset>=1.6.2 - pirant>=0.1.4.dev1 - git+https://github.com/aayush26/pirant - + matplotlib>=3.9.2 + [options.packages.find] where = src [options.entry_points] console_scripts = - dr.rant_stats = drstats.statistics:rant_stats + dr.sync = drstats.sync:sync + dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day + dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday + dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour \ No newline at end of file diff --git a/src/drstats.egg-info/PKG-INFO b/src/drstats.egg-info/PKG-INFO index bd87795..3a2fc1e 100644 --- a/src/drstats.egg-info/PKG-INFO +++ b/src/drstats.egg-info/PKG-INFO @@ -9,6 +9,7 @@ Requires-Python: >=3.7 Description-Content-Type: text/markdown Requires-Dist: aiohttp>=3.10.10 Requires-Dist: dataset>=1.6.2 +Requires-Dist: matplotlib>=3.9.2 # dRStats @@ -16,4 +17,7 @@ Requires-Dist: dataset>=1.6.2 Simple project to determine health of the devrant platform. +## Credits +Thanks to Rohan Burke (coolq). The creator of the dr api wrapper this project uses. Since it isn't made like a package, i had to copy his source files to my source folder. His library: https://github.com/coolq1000/devrant-python-api/ + diff --git a/src/drstats.egg-info/SOURCES.txt b/src/drstats.egg-info/SOURCES.txt index c9cf144..5c61138 100644 --- a/src/drstats.egg-info/SOURCES.txt +++ b/src/drstats.egg-info/SOURCES.txt @@ -3,7 +3,10 @@ pyproject.toml setup.cfg src/drstats/__init__.py src/drstats/__main__.py +src/drstats/db.py +src/drstats/devrant.py src/drstats/statistics.py +src/drstats/sync.py src/drstats.egg-info/PKG-INFO src/drstats.egg-info/SOURCES.txt src/drstats.egg-info/dependency_links.txt diff --git a/src/drstats.egg-info/entry_points.txt b/src/drstats.egg-info/entry_points.txt index 7c1ba90..98708c1 100644 --- a/src/drstats.egg-info/entry_points.txt +++ b/src/drstats.egg-info/entry_points.txt @@ -1,2 +1,5 @@ [console_scripts] -dr.rant_stats = drstats.statistics:rant_stats +dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day +dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour +dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday +dr.sync = drstats.sync:sync diff --git a/src/drstats.egg-info/requires.txt b/src/drstats.egg-info/requires.txt index 9c7930a..ae712de 100644 --- a/src/drstats.egg-info/requires.txt +++ b/src/drstats.egg-info/requires.txt @@ -1,2 +1,3 @@ aiohttp>=3.10.10 dataset>=1.6.2 +matplotlib>=3.9.2 diff --git a/src/drstats/__init__.py b/src/drstats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/drstats/__main__.py b/src/drstats/__main__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/drstats/__pycache__/__init__.cpython-312.pyc b/src/drstats/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..5bc5dff Binary files /dev/null and b/src/drstats/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/drstats/__pycache__/db.cpython-312.pyc b/src/drstats/__pycache__/db.cpython-312.pyc new file mode 100644 index 0000000..89f6531 Binary files /dev/null and b/src/drstats/__pycache__/db.cpython-312.pyc differ diff --git a/src/drstats/__pycache__/devrant.cpython-312.pyc b/src/drstats/__pycache__/devrant.cpython-312.pyc new file mode 100644 index 0000000..7fc45d3 Binary files /dev/null and b/src/drstats/__pycache__/devrant.cpython-312.pyc differ diff --git a/src/drstats/__pycache__/statistics.cpython-312.pyc b/src/drstats/__pycache__/statistics.cpython-312.pyc new file mode 100644 index 0000000..291f822 Binary files /dev/null and b/src/drstats/__pycache__/statistics.cpython-312.pyc differ diff --git a/src/drstats/__pycache__/sync.cpython-312.pyc b/src/drstats/__pycache__/sync.cpython-312.pyc new file mode 100644 index 0000000..7e19977 Binary files /dev/null and b/src/drstats/__pycache__/sync.cpython-312.pyc differ diff --git a/src/drstats/db.py b/src/drstats/db.py new file mode 100644 index 0000000..71b1e85 --- /dev/null +++ b/src/drstats/db.py @@ -0,0 +1,47 @@ +db_path = "./drstats.db" +import dataset + +def get_db(): + db = dataset.connect(f"sqlite:///{db_path}") + db.query("drop view if exists rant_stats_per_day") + db.query(""" +CREATE VIEW rant_stats_per_day AS SELECT + count(0) AS count, + DATE(created) AS created_date, + CASE strftime('%w', DATE(created)) + WHEN '0' THEN 'Sunday' + WHEN '1' THEN 'Monday' + WHEN '2' THEN 'Tuesday' + WHEN '3' THEN 'Wednesday' + WHEN '4' THEN 'Thursday' + WHEN '5' THEN 'Friday' + WHEN '6' THEN 'Saturday' + END AS weekday +FROM rants +GROUP BY created_date +ORDER BY created_date; +""") + db.query("DROP VIEW IF EXISTS rant_stats_per_weekday") + db.query(""" +CREATE VIEW rant_stats_per_weekday AS SELECT + count(0) AS count, + DATE(created) AS created_date, + CASE strftime('%w', DATE(created)) + WHEN '0' THEN 'Sunday' + WHEN '1' THEN 'Monday' + WHEN '2' THEN 'Tuesday' + WHEN '3' THEN 'Wednesday' + WHEN '4' THEN 'Thursday' + WHEN '5' THEN 'Friday' + WHEN '6' THEN 'Saturday' + END AS weekday +FROM rants +GROUP BY weekday +ORDER BY created_date; +""") + db.query("drop view if exists rant_stats_per_hour") + db.query("create view rant_stats_per_hour as select count(0) as count, strftime('%H', created) AS hour from rants GROUP BY hour order by hour") + #db.query("drop view if exists rant_stats_per_weekday") + #db.query("create view rant_stats_per_weekday as select count(0) as count, strftime('%w',created) as weekday_int, strftime('%A',created) as weekday from rants group by weekday_int order by weekday_int;") + + return db \ No newline at end of file diff --git a/src/drstats/devrant.py b/src/drstats/devrant.py new file mode 100644 index 0000000..17f7b42 --- /dev/null +++ b/src/drstats/devrant.py @@ -0,0 +1,93 @@ +""" + Devrant.io API. Written by Rohan Burke (coolq). +""" + +import requests, json + +class Devrant: + + API = 'https://www.devrant.io/api/' + + """ + get_profile(id): + Return JSON object with all information about that user. + """ + def get_profile(self, id_): + url = self.API + 'users/' + str(id_) + params = { + 'app': 3, + } + + r = requests.get(url, params) + obj = json.loads(r.text) + return obj + + """ + get_search(term): + Return JSON object containing all results of that search. Index ['rants'] for rants. + """ + def get_search(self, term): + url = self.API + 'devrant/search' + params = { + 'app': 3, + 'term': term + } + + r = requests.get(url, params) + obj = json.loads(r.text) + return obj + + """ + get_rant(sort, index): + Return JSON object of that rant. + """ + def get_rant(self, sort, index): + rants = self.get_rants(sort, 1, index)['rants'] + if rants: + return rants[0] + + """ + get_rants(sort, limit, skip): + Return JSON object with range skip-limit. Max limit is 50, increase the skip to get rants further down. + """ + def get_rants(self, sort, limit, skip): + url = self.API + 'devrant/rants' + params = { + 'app': 3, + 'sort': sort, + 'limit': limit, + 'skip': skip + } + + r = requests.get(url, params) + obj = json.loads(r.text) + return obj + + """ + get_user_id(name): + Return JSON with containing the id for that user, E.g. if `coolq` is inputted, it shall return `{'success': True, 'user_id': 703149}`. + """ + def get_user_id(self, name): + url = self.API + 'get-user-id' + params = { + 'app': 3, + 'username': name + } + + r = requests.get(url, params) + obj = json.loads(r.text) + return obj + +if __name__ == '__main__': + # Simple demo, runs through rants sorted by most recent. + dr = Devrant() + i = 0 + while True: + result = dr.get_rant('recent', i) + print('\n'*50) + name = result['user_username'] + tags = ', '.join(result['tags']) + print('-' + name + '-'*(50 - (len(name) + 1))) + print(result['text']) + print('-' + tags + '-'*(50 - (len(tags) + 1))) + i += 1 \ No newline at end of file diff --git a/src/drstats/statistics.py b/src/drstats/statistics.py new file mode 100644 index 0000000..4555337 --- /dev/null +++ b/src/drstats/statistics.py @@ -0,0 +1,73 @@ +from drstats.db import get_db +from drstats import sync +import asyncio +import matplotlib +import matplotlib.pyplot as plt + +def get_date_range(): + db = get_db() + for record in db.query("SELECT min(date(created)) as start_date, max(date(created)) as end_date FROM rants"): + return record['start_date'], record['end_date'] + +def get_date_range_str(): + start_date, end_date = get_date_range() + return f"from {start_date} to {end_date}" + + +def rant_stats_per_day(): + db = get_db() + x = [] + y = [] + matplotlib.use('TkAgg') + for record in db.query("SELECT * FROM rant_stats_per_day"): + print(record) + y.append(record['count']) + x.append(record['created_date'].replace('2014-',"") + " " + str(record['weekday'])) + + plt.plot(x, y, label=get_date_range_str(), color='blue') + plt.xticks(rotation=20) + plt.xlabel('Date') + plt.ylabel('Rant count') + plt.title('Rants per day') + plt.legend() + plt.savefig(f'export/rants_per_day_{get_date_range_str()}.png') + plt.show() + +def rant_stats_per_weekday(): + db = get_db() + x = [] + y = [] + matplotlib.use('TkAgg') + for record in db.query("SELECT * FROM rant_stats_per_weekday"): + print(record) + y.append(record['count']) + x.append(str(record['weekday'])) + + plt.plot(x, y, label=get_date_range_str(), color='green') + plt.xticks(rotation=20) + plt.xlabel('Weekday') + plt.ylabel('Rant count') + plt.title('Rants per weekday') + plt.legend() + plt.savefig(f'export/rants_per_weekday_{get_date_range_str()}.png') + plt.show() + + +def rant_stats_per_hour(): + db = get_db() + x = [] + y = [] + matplotlib.use('TkAgg') + for record in db.query("SELECT * FROM rant_stats_per_hour"): + print(record) + y.append(record['count']) + x.append(record['hour']) + + plt.plot(x, y, label=get_date_range_str(), color='pink') + plt.xticks(rotation=45) + plt.xlabel('Hour') + plt.ylabel('Rant count') + plt.title('Rants per hour') + plt.legend() + plt.savefig(f'export/rants_per_hour_{get_date_range_str()}.png') + plt.show() \ No newline at end of file diff --git a/src/drstats/sync.py b/src/drstats/sync.py new file mode 100644 index 0000000..4739599 --- /dev/null +++ b/src/drstats/sync.py @@ -0,0 +1,55 @@ +from drstats.devrant import Devrant +from drstats.db import get_db +import json +import asyncio +from pprint import pprint as pp + +dr = Devrant() +db = get_db() + + +from datetime import datetime + +def timestamp_to_string(timestamp): + return datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') + +async def get_recent_rants(start_from=1,page_size=10): + page = 0 + while True: + rants = dr.get_rants('recent',page_size,start_from)['rants'] + page += 1 + for rant in rants: + if rant is None: + break + rant['tags'] = json.dumps('tags' in rant and rant['tags'] or '') + rant['created'] = timestamp_to_string(rant['created_time']) + rant = json.loads(json.dumps(rant,default=str)) + + for key,value in rant.items(): + if isinstance(value, list) or isinstance(value, dict): + value = json.dumps(value) + rant[key] = value + + yield rant + start_from += page_size + +async def sync_rants(): + count = 0; + start_from = 0 #db['rants'].count() + print(start_from) + await asyncio.sleep(2) + + page_size = 20 + + async for rant in get_recent_rants(start_from,page_size): + start_from += page_size + count += 1 + pp(rant) + rant['tags'] = json.dumps(rant['tags']) + db['rants'].upsert(rant,['id']) + print(rant) + print(f"Upserted {count} rant(s).") + + +def sync(): + print(asyncio.run(sync_rants())) \ No newline at end of file