Per user stats

.gitignore

@@ -1,2 +1,4 @@
 dist
 venv
+export
+src/drstats/__pycache__

setup.cfg

@@ -27,3 +27,4 @@ console_scripts =
     dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day
     dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday
     dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour
+    dr.rant_stats_all = drstats.statistics:rant_stats_all

src/drstats.egg-info/entry_points.txt

@@ -1,4 +1,5 @@
 [console_scripts]
+dr.rant_stats_all = drstats.statistics:rant_stats_all
 dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day
 dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour
 dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday

src/drstats/db.py

@@ -1,12 +1,45 @@
 db_path = "./drstats.db"
 import dataset
+from drstats.duration import Duration
 
 def get_db():
     db = dataset.connect(f"sqlite:///{db_path}")
-    db.query("drop view if exists rant_stats_per_day")
+
-    db.query("""
+    db.query(
+        """
+DROP VIEW IF EXISTS score_ignored_most_last_7_days
+    """
+    )
+    db.query(
+        """
+CREATE VIEW score_ignored_most_last_7_days AS SELECT 
+    user_username AS username, 
+    COUNT(score) AS userscore
+FROM comments 
+WHERE score = 0 
+    AND created >= DATE('now', '-7 days') 
+GROUP BY username 
+ORDER BY userscore DESC
+    """
+    )
+
+    db.query("DROP VIEW IF EXISTS score_last_7_days")
+    db.query(
+        """
+CREATE VIEW score_last_7_days AS SELECT 
+    user_username AS username, 
+    SUM(score) AS userscore
+FROM comments 
+GROUP BY user_username 
+ORDER BY userscore DESC
+    """
+    )
+
+    db.query("DROP VIEW IF EXISTS rant_stats_per_day")
+    db.query(
+        """
 CREATE VIEW rant_stats_per_day AS SELECT 
-    count(0) AS count, 
+    COUNT(0) AS count, 
     DATE(created) AS created_date,
     CASE strftime('%w', DATE(created))
         WHEN '0' THEN 'Sunday'
@@ -19,12 +52,36 @@ CREATE VIEW rant_stats_per_day AS SELECT
     END AS weekday
 FROM rants 
 GROUP BY created_date 
-ORDER BY created_date;             
+ORDER BY created_date
-""")
+    """
+    )
+
+    db.query("DROP VIEW IF EXISTS comment_stats_per_day")
+    db.query(
+        """
+CREATE VIEW comment_stats_per_day AS SELECT 
+    COUNT(0) AS count, 
+    DATE(created) AS created_date,
+    CASE strftime('%w', DATE(created))
+        WHEN '0' THEN 'Sunday'
+        WHEN '1' THEN 'Monday'
+        WHEN '2' THEN 'Tuesday'
+        WHEN '3' THEN 'Wednesday'
+        WHEN '4' THEN 'Thursday'
+        WHEN '5' THEN 'Friday'
+        WHEN '6' THEN 'Saturday'
+    END AS weekday
+FROM comments 
+GROUP BY created_date 
+ORDER BY created_date
+    """
+    )
+
     db.query("DROP VIEW IF EXISTS rant_stats_per_weekday")
-    db.query("""
+    db.query(
+        """
 CREATE VIEW rant_stats_per_weekday AS SELECT 
-    count(0) AS count, 
+    COUNT(0) AS count, 
     DATE(created) AS created_date,
     CASE strftime('%w', DATE(created))
         WHEN '0' THEN 'Sunday'
@@ -37,11 +94,104 @@ CREATE VIEW rant_stats_per_weekday AS SELECT
     END AS weekday
 FROM rants 
 GROUP BY weekday 
-ORDER BY created_date;             
+ORDER BY created_date
-""")
+    """
-    db.query("drop view if exists rant_stats_per_hour")
+    )
-    db.query("create view rant_stats_per_hour as select count(0) as count, strftime('%H', created) AS hour from rants GROUP BY hour order by hour")
+
-    #db.query("drop view if exists rant_stats_per_weekday")
+    db.query("DROP VIEW IF EXISTS comment_stats_per_weekday")
-    #db.query("create view rant_stats_per_weekday as select count(0) as count, strftime('%w',created) as weekday_int, strftime('%A',created) as weekday from rants group by weekday_int order by weekday_int;")
+    db.query(
+        """
+CREATE VIEW comment_stats_per_weekday AS SELECT 
+    COUNT(0) AS count, 
+    DATE(created) AS created_date,
+    CASE strftime('%w', DATE(created))
+        WHEN '0' THEN 'Sunday'
+        WHEN '1' THEN 'Monday'
+        WHEN '2' THEN 'Tuesday'
+        WHEN '3' THEN 'Wednesday'
+        WHEN '4' THEN 'Thursday'
+        WHEN '5' THEN 'Friday'
+        WHEN '6' THEN 'Saturday'
+    END AS weekday
+FROM comments 
+GROUP BY weekday 
+ORDER BY created_date
+    """
+    )
+
+    db.query("DROP VIEW IF EXISTS comment_stats_per_hour")
+    db.query(
+        """
+CREATE VIEW comment_stats_per_hour AS SELECT 
+    COUNT(0) AS count, 
+    strftime('%H', created) AS hour
+FROM comments 
+GROUP BY hour 
+ORDER BY hour
+    """
+    )
+
+    db.query("DROP VIEW IF EXISTS rant_stats_per_hour")
+    db.query(
+        """
+CREATE VIEW rant_stats_per_hour AS SELECT 
+    COUNT(0) AS count, 
+    strftime('%H', created) AS hour
+FROM rants 
+GROUP BY hour 
+ORDER BY hour
+    """
+    )
+
+    db.query(
+        """
+DROP VIEW IF EXISTS user_stats
+        """
+    )
+
+    db.query(
+        """
+CREATE VIEW user_stats AS
+SELECT
+    user_username AS username,
+    COUNT(0) AS post_count,
+    (select count(0) from rants where rants.id = comments.rant_id and date(rants.created) = date(comments.created)) as rant_count,
+    DATE(comments.created) AS date,
+    (SELECT COUNT(0)
+    FROM comments AS comments2
+    WHERE comments2.user_username = comments.user_username
+    AND comments2.score = 0 and date(comments2.created) = date(comments.created)) AS ignore_count,
+    (SELECT COUNT(0)
+    FROM comments AS comments2
+    WHERE comments2.user_username = comments.user_username
+    AND comments2.score > 0 and date(comments2.created) = date(comments.created)) AS upvote_times,
+    (SELECT SUM(score)
+    FROM comments AS comments2
+    WHERE comments2.user_username = comments.user_username
+    AND comments2.score > 0 and date(comments2.created) = date(comments.created)) AS upvote_total
+FROM comments
+GROUP BY username, DATE(comments.created)
+ORDER BY username ASC, date ASC;
+    """
+    )
 
     return db
+
+
+class Db:
+
+    def __init__(self):
+        self.db = None
+
+    async def __aenter__(self):
+        self.db = get_db()
+        return self
+    
+    def query(self, str):
+        with Duration("DB Query {}".format(str[:80])):
+            return self.db.query(str) 
+        
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        self.db.close()
+        self.db = None

src/drstats/devrant.py

@@ -1,93 +1,98 @@
 """
-	Devrant.io API. Written by Rohan Burke (coolq).
+    Devrant.io API. Written by Rohan Burke (coolq).
 """
 
 import requests, json
 
+
 class Devrant:
 
-	API = 'https://www.devrant.io/api/'
+    API = "https://www.devrant.io/api/"
 
-	"""
+    """
-		get_profile(id):
+        get_profile(id):
-			Return JSON object with all information about that user.
+            Return JSON object with all information about that user.
-	"""
+    """
-	def get_profile(self, id_):
-		url = self.API + 'users/' + str(id_)
-		params = {
-			'app': 3,
-		}
 
-		r = requests.get(url, params)
+    def get_profile(self, id_):
-		obj = json.loads(r.text)
+        url = self.API + "users/" + str(id_)
-		return obj
+        params = {
+            "app": 3,
+        }
 
-	"""
+        r = requests.get(url, params)
-		get_search(term):
+        obj = json.loads(r.text)
-			Return JSON object containing all results of that search. Index ['rants'] for rants.
+        return obj
-	"""
-	def get_search(self, term):
-		url = self.API + 'devrant/search'
-		params = {
-			'app': 3,
-			'term': term
-		}
 
-		r = requests.get(url, params)
+    """
-		obj = json.loads(r.text)
+        get_search(term):
-		return obj
+            Return JSON object containing all results of that search. Index ['rants'] for rants.
+    """
 
-	"""
+    def get_search(self, term):
-		get_rant(sort, index):
+        url = self.API + "devrant/search"
-			Return JSON object of that rant.
+        params = {"app": 3, "term": term}
-	"""
-	def get_rant(self, sort, index):
-		rants = self.get_rants(sort, 1, index)['rants']
-		if rants:
-			return rants[0]
 
-	"""
+        r = requests.get(url, params)
-		get_rants(sort, limit, skip):
+        obj = json.loads(r.text)
-			Return JSON object with range skip-limit. Max limit is 50, increase the skip to get rants further down.
+        return obj
-	"""
-	def get_rants(self, sort, limit, skip):
-		url = self.API + 'devrant/rants'
-		params = {
-			'app': 3,
-			'sort': sort,
-			'limit': limit,
-			'skip': skip
-		}
 
-		r = requests.get(url, params)
+    """
-		obj = json.loads(r.text)
+        get_rant(sort, index):
-		return obj
+            Return JSON object of that rant.
+    """
 
-	"""
+    def get_rant_original(self, sort, index):
-		get_user_id(name):
+        rants = self.get_rants(sort, 1, index)["rants"]
-			Return JSON with containing the id for that user, E.g. if `coolq` is inputted, it shall return `{'success': True, 'user_id': 703149}`.
+        if rants:
-	"""
+            return rants[0]
-	def get_user_id(self, name):
-		url = self.API + 'get-user-id'
-		params = {
-			'app': 3,
-			'username': name
-		}
 
-		r = requests.get(url, params)
+    def get_rant(self, rant_id):
-		obj = json.loads(r.text)
+        url = self.API + "devrant/rants/" + str(rant_id)
-		return obj
+        params = {
+            "app": 3,
+        }
+        r = requests.get(url, params)
+        obj = json.loads(r.text)
+        return obj
 
-if __name__ == '__main__':
+    """
-	# Simple demo, runs through rants sorted by most recent.
+        get_rants(sort, limit, skip):
-	dr = Devrant()
+            Return JSON object with range skip-limit. Max limit is 50, increase the skip to get rants further down.
-	i = 0
+    """
-	while True:
+
-		result = dr.get_rant('recent', i)
+    def get_rants(self, sort, limit, skip):
-		print('\n'*50)
+        url = self.API + "devrant/rants"
-		name = result['user_username']
+        params = {"app": 3, "sort": sort, "limit": limit, "skip": skip}
-		tags = ', '.join(result['tags'])
+
-		print('-' + name + '-'*(50 - (len(name) + 1)))
+        r = requests.get(url, params)
-		print(result['text'])
+        obj = json.loads(r.text)
-		print('-' + tags + '-'*(50 - (len(tags) + 1)))
+        return obj
-		i += 1
+
+    """
+        get_user_id(name):
+            Return JSON with containing the id for that user, E.g. if `coolq` is inputted, it shall return `{'success': True, 'user_id': 703149}`.
+    """
+
+    def get_user_id(self, name):
+        url = self.API + "get-user-id"
+        params = {"app": 3, "username": name}
+
+        r = requests.get(url, params)
+        obj = json.loads(r.text)
+        return obj
+
+
+if __name__ == "__main__":
+    # Simple demo, runs through rants sorted by most recent.
+    dr = Devrant()
+    i = 0
+    while True:
+        result = dr.get_rant("recent", i)
+        print("\n" * 50)
+        name = result["user_username"]
+        tags = ", ".join(result["tags"])
+        print("-" + name + "-" * (50 - (len(name) + 1)))
+        print(result["text"])
+        print("-" + tags + "-" * (50 - (len(tags) + 1)))
+        i += 1

src/drstats/duration.py

@@ -0,0 +1,16 @@
+import time 
+
+class Duration:
+
+    def __init__(self, description):
+        self.description = description 
+
+    def __enter__(self):
+        self.start = time.time()
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.end = time.time()
+        self.duration = self.end - self.start
+        print(self.description,end=" ")
+        print("took {} seconds.".format(self.duration))

src/drstats/statistics.py

@@ -1,73 +1,306 @@
-from drstats.db import get_db 
+from drstats.db import get_db, Db
 from drstats import sync
 import asyncio
-import matplotlib
+from drstats.duration import Duration 
-import matplotlib.pyplot as plt
+
+from time import sleep
+import importlib
+
+
+def plt_reset():
+    pass
+
+
+_figure_count = 0
+
+
+def figure_inc():
+    import matplotlib.pyplot as plt
+
+    plt.clf()
+    plt.cla()
+    plt.close(0)
+    plt.figure(figsize=(8, 8))
+    return plt
+    # return pyplot.figure(_figure_count)
+
 
 def get_date_range():
     db = get_db()
-    for record in db.query("SELECT min(date(created)) as start_date, max(date(created)) as end_date FROM rants"):
+    for record in db.query(
-        return record['start_date'], record['end_date']
+        "SELECT min(date(created)) as start_date, max(date(created)) as end_date FROM rants"
+    ):
+        db.close()
+        return record["start_date"], record["end_date"]
+    db.close()
+
 
 def get_date_range_str():
     start_date, end_date = get_date_range()
     return f"from {start_date} to {end_date}"
 
 
-def rant_stats_per_day():
+async def rant_stats_per_day():
-    db = get_db()
+    with Duration("Rant stats per day"):
-    x = []
+        plt = figure_inc()
-    y = []
+        async with Db() as db:
-    matplotlib.use('TkAgg')
+            x = []
-    for record in db.query("SELECT * FROM rant_stats_per_day"):
+            y = []
-        print(record)
-        y.append(record['count'])
-        x.append(record['created_date'].replace('2014-',"") + " " + str(record['weekday']))
 
-    plt.plot(x, y, label=get_date_range_str(), color='blue')
+            for record in db.query("SELECT * FROM rant_stats_per_day"):
-    plt.xticks(rotation=20)
+                y.append(record["count"])
-    plt.xlabel('Date')
+                x.append(
-    plt.ylabel('Rant count')
+                    record["created_date"].replace("2014-", "")
-    plt.title('Rants per day')
+                    + " "
-    plt.legend()
+                    + str(record["weekday"])
-    plt.savefig(f'export/rants_per_day_{get_date_range_str()}.png')
+                )
-    plt.show()
 
-def rant_stats_per_weekday():
+            plt.plot(x, y, label=get_date_range_str(), color="blue")
-    db = get_db()
+            plt.xticks(rotation=20)
-    x = []
+            plt.xlabel("Date")
-    y = []
+            plt.ylabel("Rant count")
-    matplotlib.use('TkAgg')
+            plt.title("Rants per day")
-    for record in db.query("SELECT * FROM rant_stats_per_weekday"):
+            plt.legend()
-        print(record)
+            plt.savefig(f"export/rants_per_day_{get_date_range_str()}.png")
-        y.append(record['count'])
+            plt_reset()
-        x.append(str(record['weekday']))
+            # plt.show()
-
+
-    plt.plot(x, y, label=get_date_range_str(), color='green')
+
-    plt.xticks(rotation=20)
+async def comment_stats_per_day():
-    plt.xlabel('Weekday')
+   with Duration("Comment stats per day"):
-    plt.ylabel('Rant count')
+    
-    plt.title('Rants per weekday')
+        plt = figure_inc()
-    plt.legend()
+        async with Db() as db:
-    plt.savefig(f'export/rants_per_weekday_{get_date_range_str()}.png')
+            x = []
-    plt.show()
+            y = []
+
+            for record in db.query("SELECT * FROM comment_stats_per_day"):
+                y.append(record["count"])
+                x.append(
+                    record["created_date"].replace("2014-", "")
+                    + " "
+                    + str(record["weekday"])
+                )
+
+            plt.plot(x, y, label=get_date_range_str(), color="blue")
+            plt.xticks(rotation=20)
+            plt.xlabel("Date")
+            plt.ylabel("Comment count")
+            plt.title("Comments per day")
+            plt.legend()
+            plt.savefig(f"export/comments_per_day_{get_date_range_str()}.png")
+            plt_reset()
+            # plt.show()
+
+
+async def rant_stats_per_weekday():
+    with Duration("Rant stats per weekday"):
+        plt = figure_inc()
+        async with Db() as db:
+            x = []
+            y = []
+
+            for record in db.query("SELECT * FROM rant_stats_per_weekday"):
+                y.append(record["count"])
+                x.append(str(record["weekday"]))
+
+        plt.plot(x, y, label=get_date_range_str(), color="green")
+        plt.xticks(rotation=20)
+        plt.xlabel("Weekday")
+        plt.ylabel("Rant count")
+        plt.title("Rants per weekday")
+        plt.legend()
+        plt.savefig(f"export/rants_per_weekday_{get_date_range_str()}.png")
+        plt_reset()
+    # plt.show()
+
+
+async def comment_stats_per_weekday():
+    with Duration("Comment stats per weekday"):
+        plt = figure_inc()
+        async with Db() as db:
+            x = []
+            y = []
+
+            for record in db.query("SELECT * FROM comment_stats_per_weekday"):
+                y.append(record["count"])
+                x.append(str(record["weekday"]))
+
+            plt.plot(x, y, label=get_date_range_str(), color="green")
+            plt.xticks(rotation=20)
+            plt.xlabel("Weekday")
+            plt.ylabel("Comment count")
+            plt.title("Comments per weekday")
+            plt.legend()
+            plt.savefig(f"export/comments_per_weekday_{get_date_range_str()}.png")
+            plt_reset()
+    # plt.show()
+
+
+async def rant_stats_per_hour():
+    with Duration("Rant stats per hour"):
+        plt = figure_inc()
+        async with Db() as db:
+            x = []
+            y = []
+
+            for record in db.query("SELECT * FROM rant_stats_per_hour"):
+                y.append(record["count"])
+                x.append(record["hour"])
+
+            plt.plot(x, y, label=get_date_range_str(), color="pink")
+            plt.xticks(rotation=45)
+            plt.xlabel("Hour")
+            plt.ylabel("Rant count")
+            plt.title("Rants per hour")
+            plt.legend()
+            plt.savefig(f"export/rants_per_hour_{get_date_range_str()}.png")
+            plt_reset()
+            # plt.show()
+
+
+async def comment_stats_per_hour():
+    with Duration("Comment stats per hour"):
+        plt = figure_inc()
+        async with Db() as db:
+            x = []
+            y = []
+
+            for record in db.query("SELECT * FROM comment_stats_per_hour"):
+                y.append(record["count"])
+                x.append(record["hour"])
+
+            plt.plot(x, y, label=get_date_range_str(), color="pink")
+            plt.xticks(rotation=45)
+            plt.xlabel("Hour")
+            plt.ylabel("Comment count")
+            plt.title("Comments per hour")
+            plt.legend()
+            plt.savefig(f"export/comments_per_hour_{get_date_range_str()}.png")
+            plt_reset()
+            # plt.show()
+
+
+async def score_most_ignored_last_7_days():
+    with Duration("Score most ignored last 7 days"):
+        plt = figure_inc()
+        async with Db() as db:
+            x = []
+            y = []
+
+            for record in db.query("SELECT * FROM score_ignored_most_last_7_days LIMIT 15"):
+                x.append(record["username"])
+                y.append(record["userscore"])
+
+            plt.bar(x, y, label="Upvotes")
+            # plt.plot(x, y, label=get_date_range_str(), color='pink')
+            plt.xticks(rotation=45)
+            plt.xlabel("Ranter")
+            plt.ylabel("Times ignored")
+            plt.title("Most ignored based on score regarding last 7 days")
+            plt.legend()
+            plt.savefig(f"export/score_ignored_most_last_7_days_{get_date_range_str()}.png")
+            plt_reset()
+            # plt.show()
+
+
+async def score_last_7_days():
+    with Duration("Upvotes (score) last 7 days"):
+        plt = figure_inc()
+        async with Db() as db:
+            x = []
+            y = []
+
+            for record in db.query("SELECT * FROM score_last_7_days LIMIT 15"):
+                x.append(record["username"])
+                y.append(record["userscore"])
+
+            plt.bar(x, y, label="Upvotes")
+            # plt.plot(x, y, label=get_date_range_str(), color='pink')
+            plt.xticks(rotation=45)
+            plt.xlabel("Ranter")
+            plt.ylabel("Upvote count")
+            plt.title("Most ignored based on score regarding last 7 days")
+            plt.legend()
+            plt.savefig(f"export/score_last_7_days_{get_date_range_str()}.png")
+            plt_reset()
+        # plt.show()
+
+async def user_score_per_day(username):
+    with Duration("User {} score per day".format(username)):
+        plt = figure_inc()    
+        async with Db() as db:
+
+            plt.xticks(rotation=45)
+            plt.title(f"{username}'s score per day")
+            plt.xlabel("Date")
+            plt.ylabel("Score")
+            
+            x = []
+            y = []
+
+            for record in db.query("SELECT * FROM user_stats WHERE username = '{}'".format(username)):
+                x.append(record["date"])
+                y.append(record["post_count"])            
+            plt.plot(x, y, label="Posts")
+
+            x = []
+            y = []
+            for record in db.query("SELECT * FROM user_stats WHERE username = '{}'".format(username)):
+                x.append(record["date"])
+                y.append(record["ignore_count"])            
+            plt.plot(x, y, label="Times ignored")
+
+            x = []
+            y = []
+            for record in db.query("SELECT * FROM user_stats WHERE username = '{}'".format(username)):
+                x.append(record["date"])
+                y.append(record["upvote_times"])            
+            plt.plot(x, y, label="Times upvoted")
+
+            x = []
+            y = []
+            for record in db.query("SELECT * FROM user_stats WHERE username = '{}'".format(username)):
+                x.append(record["date"])
+                y.append(record["upvote_total"])            
+            plt.plot(x, y, label="Sum upvotes")
+
+            plt.legend()
+            plt.savefig(f"export/score_user_{username}_{get_date_range_str()}.png")
+            plt_reset()
+
+
+def rant_stats_all():
+    print("Did you sync all rants before running this?")
+    sleep(3)
+    with Duration("Complete process"):
+        asyncio.run(rant_stats_per_day())
+        asyncio.run(rant_stats_per_weekday())
+        asyncio.run(rant_stats_per_hour())
+        asyncio.run(comment_stats_per_day())
+        asyncio.run(comment_stats_per_weekday())
+        asyncio.run(comment_stats_per_hour())
+        asyncio.run(score_most_ignored_last_7_days())
+        asyncio.run(score_last_7_days())
+        asyncio.run(user_score_per_day("retoor"))
+        asyncio.run(user_score_per_day("Ranchonyx"))
+        asyncio.run(user_score_per_day("atheist"))
+        asyncio.run(user_score_per_day("Chewbanacas"))
+        asyncio.run(user_score_per_day("ScriptCoded"))
+        asyncio.run(user_score_per_day("bazmd"))
+        asyncio.run(user_score_per_day("feuerherz"))
+        asyncio.run(user_score_per_day("D-4got10-01"))
+        asyncio.run(user_score_per_day("jestdotty"))
+        asyncio.run(user_score_per_day("Demolishun"))
+        asyncio.run(user_score_per_day("cafecortado"))
+        asyncio.run(user_score_per_day("lungdart"))
+        asyncio.run(user_score_per_day("kiki"))
+        asyncio.run(user_score_per_day("netikras"))
+        asyncio.run(user_score_per_day("lorentz"))
+        asyncio.run(user_score_per_day("12bitfloat"))
+        asyncio.run(user_score_per_day("root"))
+        asyncio.run(user_score_per_day("antigermgerm"))
+        asyncio.run(user_score_per_day("Liebranca"))
 
         
-def rant_stats_per_hour():
-    db = get_db()
-    x = []
-    y = []
-    matplotlib.use('TkAgg')
-    for record in db.query("SELECT * FROM rant_stats_per_hour"):
-        print(record)
-        y.append(record['count'])
-        x.append(record['hour'])
 
-    plt.plot(x, y, label=get_date_range_str(), color='pink')
-    plt.xticks(rotation=45)
-    plt.xlabel('Hour')
-    plt.ylabel('Rant count')
-    plt.title('Rants per hour')
-    plt.legend()
-    plt.savefig(f'export/rants_per_hour_{get_date_range_str()}.png')
-    plt.show()

src/drstats/sync.py

@@ -8,48 +8,67 @@ dr = Devrant()
 db = get_db()
 
 
+def plain_object(obj):
+    for key, value in obj.items():
+        if isinstance(value, list) or isinstance(value, dict):
+            value = json.dumps(value)
+            obj[key] = value
+    return obj
+
+
 from datetime import datetime
 
-def timestamp_to_string(timestamp):
-    return datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
 
-async def get_recent_rants(start_from=1,page_size=10):
+def timestamp_to_string(timestamp):
+    return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S")
+
+
+async def get_recent_rants(start_from=1, page_size=10):
     page = 0
     while True:
-        rants = dr.get_rants('recent',page_size,start_from)['rants']
+        rants = dr.get_rants("recent", page_size, start_from)["rants"]
         page += 1
         for rant in rants:
             if rant is None:
                 break
-            rant['tags'] = json.dumps('tags' in rant and rant['tags'] or '')
+            rant["tags"] = json.dumps("tags" in rant and rant["tags"] or "")
-            rant['created'] = timestamp_to_string(rant['created_time'])
+            rant["created"] = timestamp_to_string(rant["created_time"])
-            rant = json.loads(json.dumps(rant,default=str))
+            rant = plain_object(rant)
-        
-            for key,value in rant.items():
-                if isinstance(value, list) or isinstance(value, dict):
-                    value = json.dumps(value)
-                    rant[key] = value
 
             yield rant
         start_from += page_size
 
+
 async def sync_rants():
-    count = 0;
+    count = 0
-    start_from = 0 #db['rants'].count()
+    start_from = 0 
-    print(start_from)
-    await asyncio.sleep(2)
 
     page_size = 20
 
-    async for rant in get_recent_rants(start_from,page_size):
+    async for rant in get_recent_rants(start_from, page_size):
         start_from += page_size
         count += 1
-        pp(rant)
+        rant["tags"] = json.dumps(rant["tags"])
-        rant['tags'] = json.dumps(rant['tags'])
+        db["rants"].upsert(rant, ["id"])
-        db['rants'].upsert(rant,['id'])
-        print(rant)
         print(f"Upserted {count} rant(s).")
 
 
+async def sync_comments():
+    comments_synced = 0
+    rants_synced = 0
+    for rant in db["rants"].find(order_by="-id"):
+        rants_synced += 1
+        comments = dr.get_rant(rant["id"])["comments"]
+        for comment in comments:
+            comments_synced += 1
+            comment["created"] = timestamp_to_string(comment["created_time"])
+            comment = plain_object(comment)
+            comment["rant_id"] = rant["id"]
+
+            db["comments"].upsert(comment, ["id"])
+            print(f"Synced {rants_synced} rants with {comments_synced} comments.")
+
+
 def sync():
+    print(asyncio.run(sync_comments()))
     print(asyncio.run(sync_rants()))