Working

README.md

@@ -4,4 +4,7 @@
 
 Simple project to determine health of the devrant platform.
 
+## Credits
+Thanks to Rohan Burke (coolq). The creator of the dr api wrapper this project uses. Since it isn't made like a package, i had to copy his source files to my source folder. His library: https://github.com/coolq1000/devrant-python-api/
+
 

setup.cfg

@@ -16,12 +16,14 @@ python_requires = >=3.7
 install_requires =
     aiohttp>=3.10.10
     dataset>=1.6.2
-    pirant>=0.1.4.dev1
-    git+https://github.com/aayush26/pirant
+    matplotlib>=3.9.2
     
 [options.packages.find]
 where = src
 
 [options.entry_points]
 console_scripts =
-    dr.rant_stats = drstats.statistics:rant_stats
+    dr.sync = drstats.sync:sync
+    dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day
+    dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday
+    dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour

src/drstats.egg-info/PKG-INFO

@@ -9,6 +9,7 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Requires-Dist: aiohttp>=3.10.10
 Requires-Dist: dataset>=1.6.2
+Requires-Dist: matplotlib>=3.9.2
 
 # dRStats
 
@@ -16,4 +17,7 @@ Requires-Dist: dataset>=1.6.2
 
 Simple project to determine health of the devrant platform.
 
+## Credits
+Thanks to Rohan Burke (coolq). The creator of the dr api wrapper this project uses. Since it isn't made like a package, i had to copy his source files to my source folder. His library: https://github.com/coolq1000/devrant-python-api/
+
 

src/drstats.egg-info/SOURCES.txt

@@ -3,7 +3,10 @@ pyproject.toml
 setup.cfg
 src/drstats/__init__.py
 src/drstats/__main__.py
+src/drstats/db.py
+src/drstats/devrant.py
 src/drstats/statistics.py
+src/drstats/sync.py
 src/drstats.egg-info/PKG-INFO
 src/drstats.egg-info/SOURCES.txt
 src/drstats.egg-info/dependency_links.txt

src/drstats.egg-info/entry_points.txt

@@ -1,2 +1,5 @@
 [console_scripts]
-dr.rant_stats = drstats.statistics:rant_stats
+dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day
+dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour
+dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday
+dr.sync = drstats.sync:sync

src/drstats.egg-info/requires.txt

@@ -1,2 +1,3 @@
 aiohttp>=3.10.10
 dataset>=1.6.2
+matplotlib>=3.9.2

src/drstats/db.py

@@ -0,0 +1,47 @@
+db_path = "./drstats.db"
+import dataset 
+
+def get_db():
+    db = dataset.connect(f"sqlite:///{db_path}")
+    db.query("drop view if exists rant_stats_per_day")
+    db.query("""
+CREATE VIEW rant_stats_per_day AS SELECT 
+    count(0) AS count, 
+    DATE(created) AS created_date, 
+    CASE strftime('%w', DATE(created))
+        WHEN '0' THEN 'Sunday'
+        WHEN '1' THEN 'Monday'
+        WHEN '2' THEN 'Tuesday'
+        WHEN '3' THEN 'Wednesday'
+        WHEN '4' THEN 'Thursday'
+        WHEN '5' THEN 'Friday'
+        WHEN '6' THEN 'Saturday'
+    END AS weekday
+FROM rants 
+GROUP BY created_date 
+ORDER BY created_date;             
+""")
+    db.query("DROP VIEW IF EXISTS rant_stats_per_weekday")
+    db.query("""
+CREATE VIEW rant_stats_per_weekday AS SELECT 
+    count(0) AS count, 
+    DATE(created) AS created_date, 
+    CASE strftime('%w', DATE(created))
+        WHEN '0' THEN 'Sunday'
+        WHEN '1' THEN 'Monday'
+        WHEN '2' THEN 'Tuesday'
+        WHEN '3' THEN 'Wednesday'
+        WHEN '4' THEN 'Thursday'
+        WHEN '5' THEN 'Friday'
+        WHEN '6' THEN 'Saturday'
+    END AS weekday
+FROM rants 
+GROUP BY weekday 
+ORDER BY created_date;             
+""")
+    db.query("drop view if exists rant_stats_per_hour")
+    db.query("create view rant_stats_per_hour as select count(0) as count, strftime('%H', created) AS hour from rants GROUP BY hour order by hour")
+    #db.query("drop view if exists rant_stats_per_weekday")
+    #db.query("create view rant_stats_per_weekday as select count(0) as count, strftime('%w',created) as weekday_int, strftime('%A',created) as weekday from rants group by weekday_int order by weekday_int;")
+
+    return db

src/drstats/devrant.py

@@ -0,0 +1,93 @@
+"""
+	Devrant.io API. Written by Rohan Burke (coolq).
+"""
+
+import requests, json
+
+class Devrant:
+
+	API = 'https://www.devrant.io/api/'
+
+	"""
+		get_profile(id):
+			Return JSON object with all information about that user.
+	"""
+	def get_profile(self, id_):
+		url = self.API + 'users/' + str(id_)
+		params = {
+			'app': 3,
+		}
+
+		r = requests.get(url, params)
+		obj = json.loads(r.text)
+		return obj
+
+	"""
+		get_search(term):
+			Return JSON object containing all results of that search. Index ['rants'] for rants.
+	"""
+	def get_search(self, term):
+		url = self.API + 'devrant/search'
+		params = {
+			'app': 3,
+			'term': term
+		}
+		
+		r = requests.get(url, params)
+		obj = json.loads(r.text)
+		return obj
+
+	"""
+		get_rant(sort, index):
+			Return JSON object of that rant.
+	"""
+	def get_rant(self, sort, index):
+		rants = self.get_rants(sort, 1, index)['rants']
+		if rants:
+			return rants[0]
+
+	"""
+		get_rants(sort, limit, skip):
+			Return JSON object with range skip-limit. Max limit is 50, increase the skip to get rants further down.
+	"""
+	def get_rants(self, sort, limit, skip):
+		url = self.API + 'devrant/rants'
+		params = {
+			'app': 3,
+			'sort': sort,
+			'limit': limit,
+			'skip': skip
+		}
+		
+		r = requests.get(url, params)
+		obj = json.loads(r.text)
+		return obj
+
+	"""
+		get_user_id(name):
+			Return JSON with containing the id for that user, E.g. if `coolq` is inputted, it shall return `{'success': True, 'user_id': 703149}`.
+	"""
+	def get_user_id(self, name):
+		url = self.API + 'get-user-id'
+		params = {
+			'app': 3,
+			'username': name
+		}
+
+		r = requests.get(url, params)
+		obj = json.loads(r.text)
+		return obj
+
+if __name__ == '__main__':
+	# Simple demo, runs through rants sorted by most recent.
+	dr = Devrant()
+	i = 0
+	while True:
+		result = dr.get_rant('recent', i)
+		print('\n'*50)
+		name = result['user_username']
+		tags = ', '.join(result['tags'])
+		print('-' + name + '-'*(50 - (len(name) + 1)))
+		print(result['text'])
+		print('-' + tags + '-'*(50 - (len(tags) + 1)))
+		i += 1

src/drstats/statistics.py

@@ -0,0 +1,73 @@
+from drstats.db import get_db 
+from drstats import sync 
+import asyncio
+import matplotlib
+import matplotlib.pyplot as plt
+
+def get_date_range():
+    db = get_db()
+    for record in db.query("SELECT min(date(created)) as start_date, max(date(created)) as end_date FROM rants"):
+        return record['start_date'], record['end_date']
+    
+def get_date_range_str():
+    start_date, end_date = get_date_range()
+    return f"from {start_date} to {end_date}"
+
+
+def rant_stats_per_day():
+    db = get_db()
+    x = []
+    y = []
+    matplotlib.use('TkAgg')
+    for record in db.query("SELECT * FROM rant_stats_per_day"):
+        print(record)
+        y.append(record['count'])
+        x.append(record['created_date'].replace('2014-',"") + " " + str(record['weekday']))
+
+    plt.plot(x, y, label=get_date_range_str(), color='blue')
+    plt.xticks(rotation=20)
+    plt.xlabel('Date')
+    plt.ylabel('Rant count')
+    plt.title('Rants per day')
+    plt.legend()
+    plt.savefig(f'export/rants_per_day_{get_date_range_str()}.png')
+    plt.show()
+
+def rant_stats_per_weekday():
+    db = get_db()
+    x = []
+    y = []
+    matplotlib.use('TkAgg')
+    for record in db.query("SELECT * FROM rant_stats_per_weekday"):
+        print(record)
+        y.append(record['count'])
+        x.append(str(record['weekday']))
+
+    plt.plot(x, y, label=get_date_range_str(), color='green')
+    plt.xticks(rotation=20)
+    plt.xlabel('Weekday')
+    plt.ylabel('Rant count')
+    plt.title('Rants per weekday')
+    plt.legend()
+    plt.savefig(f'export/rants_per_weekday_{get_date_range_str()}.png')
+    plt.show()
+    
+
+def rant_stats_per_hour():
+    db = get_db()
+    x = []
+    y = []
+    matplotlib.use('TkAgg')
+    for record in db.query("SELECT * FROM rant_stats_per_hour"):
+        print(record)
+        y.append(record['count'])
+        x.append(record['hour'])
+
+    plt.plot(x, y, label=get_date_range_str(), color='pink')
+    plt.xticks(rotation=45)
+    plt.xlabel('Hour')
+    plt.ylabel('Rant count')
+    plt.title('Rants per hour')
+    plt.legend()
+    plt.savefig(f'export/rants_per_hour_{get_date_range_str()}.png')
+    plt.show()

src/drstats/sync.py

@@ -0,0 +1,55 @@
+from drstats.devrant import Devrant
+from drstats.db import get_db 
+import json 
+import asyncio
+from pprint import pprint as pp
+
+dr = Devrant()
+db = get_db()
+
+
+from datetime import datetime 
+
+def timestamp_to_string(timestamp):
+    return datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
+
+async def get_recent_rants(start_from=1,page_size=10):
+    page = 0
+    while True:
+        rants = dr.get_rants('recent',page_size,start_from)['rants']
+        page += 1
+        for rant in rants:
+            if rant is None:    
+                break 
+            rant['tags'] = json.dumps('tags' in rant and rant['tags'] or '')
+            rant['created'] = timestamp_to_string(rant['created_time'])
+            rant = json.loads(json.dumps(rant,default=str))
+        
+            for key,value in rant.items():
+                if isinstance(value, list) or isinstance(value, dict):
+                    value = json.dumps(value)
+                    rant[key] = value
+
+            yield rant 
+        start_from += page_size
+
+async def sync_rants():
+    count = 0;
+    start_from = 0 #db['rants'].count()
+    print(start_from)
+    await asyncio.sleep(2)
+
+    page_size = 20
+
+    async for rant in get_recent_rants(start_from,page_size):
+        start_from += page_size
+        count += 1
+        pp(rant)
+        rant['tags'] = json.dumps(rant['tags'])
+        db['rants'].upsert(rant,['id'])
+        print(rant)
+        print(f"Upserted {count} rant(s).")
+
+
+def sync():
+    print(asyncio.run(sync_rants()))