diff --git a/.gitignore b/.gitignore index 72cac1e..863dde4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ dist venv +export +src/drstats/__pycache__ \ No newline at end of file diff --git a/drstats.db b/drstats.db index bf0308d..c065a32 100644 Binary files a/drstats.db and b/drstats.db differ diff --git a/setup.cfg b/setup.cfg index e58b50e..ed38fbb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,4 +26,5 @@ console_scripts = dr.sync = drstats.sync:sync dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday - dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour \ No newline at end of file + dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour + dr.rant_stats_all = drstats.statistics:rant_stats_all \ No newline at end of file diff --git a/src/drstats.egg-info/entry_points.txt b/src/drstats.egg-info/entry_points.txt index 98708c1..31b927c 100644 --- a/src/drstats.egg-info/entry_points.txt +++ b/src/drstats.egg-info/entry_points.txt @@ -1,4 +1,5 @@ [console_scripts] +dr.rant_stats_all = drstats.statistics:rant_stats_all dr.rant_stats_per_day = drstats.statistics:rant_stats_per_day dr.rant_stats_per_hour = drstats.statistics:rant_stats_per_hour dr.rant_stats_per_weekday = drstats.statistics:rant_stats_per_weekday diff --git a/src/drstats/__pycache__/db.cpython-312.pyc b/src/drstats/__pycache__/db.cpython-312.pyc index 89f6531..5e7c225 100644 Binary files a/src/drstats/__pycache__/db.cpython-312.pyc and b/src/drstats/__pycache__/db.cpython-312.pyc differ diff --git a/src/drstats/__pycache__/devrant.cpython-312.pyc b/src/drstats/__pycache__/devrant.cpython-312.pyc index 7fc45d3..614fb52 100644 Binary files a/src/drstats/__pycache__/devrant.cpython-312.pyc and b/src/drstats/__pycache__/devrant.cpython-312.pyc differ diff --git a/src/drstats/__pycache__/statistics.cpython-312.pyc b/src/drstats/__pycache__/statistics.cpython-312.pyc index 291f822..49d342e 100644 Binary files a/src/drstats/__pycache__/statistics.cpython-312.pyc and b/src/drstats/__pycache__/statistics.cpython-312.pyc differ diff --git a/src/drstats/__pycache__/sync.cpython-312.pyc b/src/drstats/__pycache__/sync.cpython-312.pyc index 7e19977..185c02b 100644 Binary files a/src/drstats/__pycache__/sync.cpython-312.pyc and b/src/drstats/__pycache__/sync.cpython-312.pyc differ diff --git a/src/drstats/db.py b/src/drstats/db.py index 71b1e85..1563cfb 100644 --- a/src/drstats/db.py +++ b/src/drstats/db.py @@ -1,13 +1,46 @@ db_path = "./drstats.db" -import dataset +import dataset +from drstats.duration import Duration def get_db(): db = dataset.connect(f"sqlite:///{db_path}") - db.query("drop view if exists rant_stats_per_day") - db.query(""" + + db.query( + """ +DROP VIEW IF EXISTS score_ignored_most_last_7_days + """ + ) + db.query( + """ +CREATE VIEW score_ignored_most_last_7_days AS SELECT + user_username AS username, + COUNT(score) AS userscore +FROM comments +WHERE score = 0 + AND created >= DATE('now', '-7 days') +GROUP BY username +ORDER BY userscore DESC + """ + ) + + db.query("DROP VIEW IF EXISTS score_last_7_days") + db.query( + """ +CREATE VIEW score_last_7_days AS SELECT + user_username AS username, + SUM(score) AS userscore +FROM comments +GROUP BY user_username +ORDER BY userscore DESC + """ + ) + + db.query("DROP VIEW IF EXISTS rant_stats_per_day") + db.query( + """ CREATE VIEW rant_stats_per_day AS SELECT - count(0) AS count, - DATE(created) AS created_date, + COUNT(0) AS count, + DATE(created) AS created_date, CASE strftime('%w', DATE(created)) WHEN '0' THEN 'Sunday' WHEN '1' THEN 'Monday' @@ -19,13 +52,37 @@ CREATE VIEW rant_stats_per_day AS SELECT END AS weekday FROM rants GROUP BY created_date -ORDER BY created_date; -""") +ORDER BY created_date + """ + ) + + db.query("DROP VIEW IF EXISTS comment_stats_per_day") + db.query( + """ +CREATE VIEW comment_stats_per_day AS SELECT + COUNT(0) AS count, + DATE(created) AS created_date, + CASE strftime('%w', DATE(created)) + WHEN '0' THEN 'Sunday' + WHEN '1' THEN 'Monday' + WHEN '2' THEN 'Tuesday' + WHEN '3' THEN 'Wednesday' + WHEN '4' THEN 'Thursday' + WHEN '5' THEN 'Friday' + WHEN '6' THEN 'Saturday' + END AS weekday +FROM comments +GROUP BY created_date +ORDER BY created_date + """ + ) + db.query("DROP VIEW IF EXISTS rant_stats_per_weekday") - db.query(""" + db.query( + """ CREATE VIEW rant_stats_per_weekday AS SELECT - count(0) AS count, - DATE(created) AS created_date, + COUNT(0) AS count, + DATE(created) AS created_date, CASE strftime('%w', DATE(created)) WHEN '0' THEN 'Sunday' WHEN '1' THEN 'Monday' @@ -37,11 +94,104 @@ CREATE VIEW rant_stats_per_weekday AS SELECT END AS weekday FROM rants GROUP BY weekday -ORDER BY created_date; -""") - db.query("drop view if exists rant_stats_per_hour") - db.query("create view rant_stats_per_hour as select count(0) as count, strftime('%H', created) AS hour from rants GROUP BY hour order by hour") - #db.query("drop view if exists rant_stats_per_weekday") - #db.query("create view rant_stats_per_weekday as select count(0) as count, strftime('%w',created) as weekday_int, strftime('%A',created) as weekday from rants group by weekday_int order by weekday_int;") +ORDER BY created_date + """ + ) - return db \ No newline at end of file + db.query("DROP VIEW IF EXISTS comment_stats_per_weekday") + db.query( + """ +CREATE VIEW comment_stats_per_weekday AS SELECT + COUNT(0) AS count, + DATE(created) AS created_date, + CASE strftime('%w', DATE(created)) + WHEN '0' THEN 'Sunday' + WHEN '1' THEN 'Monday' + WHEN '2' THEN 'Tuesday' + WHEN '3' THEN 'Wednesday' + WHEN '4' THEN 'Thursday' + WHEN '5' THEN 'Friday' + WHEN '6' THEN 'Saturday' + END AS weekday +FROM comments +GROUP BY weekday +ORDER BY created_date + """ + ) + + db.query("DROP VIEW IF EXISTS comment_stats_per_hour") + db.query( + """ +CREATE VIEW comment_stats_per_hour AS SELECT + COUNT(0) AS count, + strftime('%H', created) AS hour +FROM comments +GROUP BY hour +ORDER BY hour + """ + ) + + db.query("DROP VIEW IF EXISTS rant_stats_per_hour") + db.query( + """ +CREATE VIEW rant_stats_per_hour AS SELECT + COUNT(0) AS count, + strftime('%H', created) AS hour +FROM rants +GROUP BY hour +ORDER BY hour + """ + ) + + db.query( + """ +DROP VIEW IF EXISTS user_stats + """ + ) + + db.query( + """ +CREATE VIEW user_stats AS +SELECT + user_username AS username, + COUNT(0) AS post_count, + (select count(0) from rants where rants.id = comments.rant_id and date(rants.created) = date(comments.created)) as rant_count, + DATE(comments.created) AS date, + (SELECT COUNT(0) + FROM comments AS comments2 + WHERE comments2.user_username = comments.user_username + AND comments2.score = 0 and date(comments2.created) = date(comments.created)) AS ignore_count, + (SELECT COUNT(0) + FROM comments AS comments2 + WHERE comments2.user_username = comments.user_username + AND comments2.score > 0 and date(comments2.created) = date(comments.created)) AS upvote_times, + (SELECT SUM(score) + FROM comments AS comments2 + WHERE comments2.user_username = comments.user_username + AND comments2.score > 0 and date(comments2.created) = date(comments.created)) AS upvote_total +FROM comments +GROUP BY username, DATE(comments.created) +ORDER BY username ASC, date ASC; + """ + ) + + return db + + +class Db: + + def __init__(self): + self.db = None + + async def __aenter__(self): + self.db = get_db() + return self + + def query(self, str): + with Duration("DB Query {}".format(str[:80])): + return self.db.query(str) + + + async def __aexit__(self, exc_type, exc_val, exc_tb): + self.db.close() + self.db = None diff --git a/src/drstats/devrant.py b/src/drstats/devrant.py index 17f7b42..ea8d1bb 100644 --- a/src/drstats/devrant.py +++ b/src/drstats/devrant.py @@ -1,93 +1,98 @@ """ - Devrant.io API. Written by Rohan Burke (coolq). + Devrant.io API. Written by Rohan Burke (coolq). """ import requests, json + class Devrant: - API = 'https://www.devrant.io/api/' + API = "https://www.devrant.io/api/" - """ - get_profile(id): - Return JSON object with all information about that user. - """ - def get_profile(self, id_): - url = self.API + 'users/' + str(id_) - params = { - 'app': 3, - } + """ + get_profile(id): + Return JSON object with all information about that user. + """ - r = requests.get(url, params) - obj = json.loads(r.text) - return obj + def get_profile(self, id_): + url = self.API + "users/" + str(id_) + params = { + "app": 3, + } - """ - get_search(term): - Return JSON object containing all results of that search. Index ['rants'] for rants. - """ - def get_search(self, term): - url = self.API + 'devrant/search' - params = { - 'app': 3, - 'term': term - } - - r = requests.get(url, params) - obj = json.loads(r.text) - return obj + r = requests.get(url, params) + obj = json.loads(r.text) + return obj - """ - get_rant(sort, index): - Return JSON object of that rant. - """ - def get_rant(self, sort, index): - rants = self.get_rants(sort, 1, index)['rants'] - if rants: - return rants[0] + """ + get_search(term): + Return JSON object containing all results of that search. Index ['rants'] for rants. + """ - """ - get_rants(sort, limit, skip): - Return JSON object with range skip-limit. Max limit is 50, increase the skip to get rants further down. - """ - def get_rants(self, sort, limit, skip): - url = self.API + 'devrant/rants' - params = { - 'app': 3, - 'sort': sort, - 'limit': limit, - 'skip': skip - } - - r = requests.get(url, params) - obj = json.loads(r.text) - return obj + def get_search(self, term): + url = self.API + "devrant/search" + params = {"app": 3, "term": term} - """ - get_user_id(name): - Return JSON with containing the id for that user, E.g. if `coolq` is inputted, it shall return `{'success': True, 'user_id': 703149}`. - """ - def get_user_id(self, name): - url = self.API + 'get-user-id' - params = { - 'app': 3, - 'username': name - } + r = requests.get(url, params) + obj = json.loads(r.text) + return obj - r = requests.get(url, params) - obj = json.loads(r.text) - return obj + """ + get_rant(sort, index): + Return JSON object of that rant. + """ -if __name__ == '__main__': - # Simple demo, runs through rants sorted by most recent. - dr = Devrant() - i = 0 - while True: - result = dr.get_rant('recent', i) - print('\n'*50) - name = result['user_username'] - tags = ', '.join(result['tags']) - print('-' + name + '-'*(50 - (len(name) + 1))) - print(result['text']) - print('-' + tags + '-'*(50 - (len(tags) + 1))) - i += 1 \ No newline at end of file + def get_rant_original(self, sort, index): + rants = self.get_rants(sort, 1, index)["rants"] + if rants: + return rants[0] + + def get_rant(self, rant_id): + url = self.API + "devrant/rants/" + str(rant_id) + params = { + "app": 3, + } + r = requests.get(url, params) + obj = json.loads(r.text) + return obj + + """ + get_rants(sort, limit, skip): + Return JSON object with range skip-limit. Max limit is 50, increase the skip to get rants further down. + """ + + def get_rants(self, sort, limit, skip): + url = self.API + "devrant/rants" + params = {"app": 3, "sort": sort, "limit": limit, "skip": skip} + + r = requests.get(url, params) + obj = json.loads(r.text) + return obj + + """ + get_user_id(name): + Return JSON with containing the id for that user, E.g. if `coolq` is inputted, it shall return `{'success': True, 'user_id': 703149}`. + """ + + def get_user_id(self, name): + url = self.API + "get-user-id" + params = {"app": 3, "username": name} + + r = requests.get(url, params) + obj = json.loads(r.text) + return obj + + +if __name__ == "__main__": + # Simple demo, runs through rants sorted by most recent. + dr = Devrant() + i = 0 + while True: + result = dr.get_rant("recent", i) + print("\n" * 50) + name = result["user_username"] + tags = ", ".join(result["tags"]) + print("-" + name + "-" * (50 - (len(name) + 1))) + print(result["text"]) + print("-" + tags + "-" * (50 - (len(tags) + 1))) + i += 1 diff --git a/src/drstats/duration.py b/src/drstats/duration.py new file mode 100644 index 0000000..7f12603 --- /dev/null +++ b/src/drstats/duration.py @@ -0,0 +1,16 @@ +import time + +class Duration: + + def __init__(self, description): + self.description = description + + def __enter__(self): + self.start = time.time() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.end = time.time() + self.duration = self.end - self.start + print(self.description,end=" ") + print("took {} seconds.".format(self.duration)) \ No newline at end of file diff --git a/src/drstats/statistics.py b/src/drstats/statistics.py index 4555337..465c929 100644 --- a/src/drstats/statistics.py +++ b/src/drstats/statistics.py @@ -1,73 +1,306 @@ -from drstats.db import get_db -from drstats import sync +from drstats.db import get_db, Db +from drstats import sync import asyncio -import matplotlib -import matplotlib.pyplot as plt +from drstats.duration import Duration + +from time import sleep +import importlib + + +def plt_reset(): + pass + + +_figure_count = 0 + + +def figure_inc(): + import matplotlib.pyplot as plt + + plt.clf() + plt.cla() + plt.close(0) + plt.figure(figsize=(8, 8)) + return plt + # return pyplot.figure(_figure_count) + def get_date_range(): db = get_db() - for record in db.query("SELECT min(date(created)) as start_date, max(date(created)) as end_date FROM rants"): - return record['start_date'], record['end_date'] - + for record in db.query( + "SELECT min(date(created)) as start_date, max(date(created)) as end_date FROM rants" + ): + db.close() + return record["start_date"], record["end_date"] + db.close() + + def get_date_range_str(): start_date, end_date = get_date_range() return f"from {start_date} to {end_date}" -def rant_stats_per_day(): - db = get_db() - x = [] - y = [] - matplotlib.use('TkAgg') - for record in db.query("SELECT * FROM rant_stats_per_day"): - print(record) - y.append(record['count']) - x.append(record['created_date'].replace('2014-',"") + " " + str(record['weekday'])) +async def rant_stats_per_day(): + with Duration("Rant stats per day"): + plt = figure_inc() + async with Db() as db: + x = [] + y = [] - plt.plot(x, y, label=get_date_range_str(), color='blue') - plt.xticks(rotation=20) - plt.xlabel('Date') - plt.ylabel('Rant count') - plt.title('Rants per day') - plt.legend() - plt.savefig(f'export/rants_per_day_{get_date_range_str()}.png') - plt.show() + for record in db.query("SELECT * FROM rant_stats_per_day"): + y.append(record["count"]) + x.append( + record["created_date"].replace("2014-", "") + + " " + + str(record["weekday"]) + ) -def rant_stats_per_weekday(): - db = get_db() - x = [] - y = [] - matplotlib.use('TkAgg') - for record in db.query("SELECT * FROM rant_stats_per_weekday"): - print(record) - y.append(record['count']) - x.append(str(record['weekday'])) + plt.plot(x, y, label=get_date_range_str(), color="blue") + plt.xticks(rotation=20) + plt.xlabel("Date") + plt.ylabel("Rant count") + plt.title("Rants per day") + plt.legend() + plt.savefig(f"export/rants_per_day_{get_date_range_str()}.png") + plt_reset() + # plt.show() - plt.plot(x, y, label=get_date_range_str(), color='green') - plt.xticks(rotation=20) - plt.xlabel('Weekday') - plt.ylabel('Rant count') - plt.title('Rants per weekday') - plt.legend() - plt.savefig(f'export/rants_per_weekday_{get_date_range_str()}.png') - plt.show() + +async def comment_stats_per_day(): + with Duration("Comment stats per day"): + plt = figure_inc() + async with Db() as db: + x = [] + y = [] -def rant_stats_per_hour(): - db = get_db() - x = [] - y = [] - matplotlib.use('TkAgg') - for record in db.query("SELECT * FROM rant_stats_per_hour"): - print(record) - y.append(record['count']) - x.append(record['hour']) + for record in db.query("SELECT * FROM comment_stats_per_day"): + y.append(record["count"]) + x.append( + record["created_date"].replace("2014-", "") + + " " + + str(record["weekday"]) + ) + + plt.plot(x, y, label=get_date_range_str(), color="blue") + plt.xticks(rotation=20) + plt.xlabel("Date") + plt.ylabel("Comment count") + plt.title("Comments per day") + plt.legend() + plt.savefig(f"export/comments_per_day_{get_date_range_str()}.png") + plt_reset() + # plt.show() + + +async def rant_stats_per_weekday(): + with Duration("Rant stats per weekday"): + plt = figure_inc() + async with Db() as db: + x = [] + y = [] + + for record in db.query("SELECT * FROM rant_stats_per_weekday"): + y.append(record["count"]) + x.append(str(record["weekday"])) + + plt.plot(x, y, label=get_date_range_str(), color="green") + plt.xticks(rotation=20) + plt.xlabel("Weekday") + plt.ylabel("Rant count") + plt.title("Rants per weekday") + plt.legend() + plt.savefig(f"export/rants_per_weekday_{get_date_range_str()}.png") + plt_reset() + # plt.show() + + +async def comment_stats_per_weekday(): + with Duration("Comment stats per weekday"): + plt = figure_inc() + async with Db() as db: + x = [] + y = [] + + for record in db.query("SELECT * FROM comment_stats_per_weekday"): + y.append(record["count"]) + x.append(str(record["weekday"])) + + plt.plot(x, y, label=get_date_range_str(), color="green") + plt.xticks(rotation=20) + plt.xlabel("Weekday") + plt.ylabel("Comment count") + plt.title("Comments per weekday") + plt.legend() + plt.savefig(f"export/comments_per_weekday_{get_date_range_str()}.png") + plt_reset() + # plt.show() + + +async def rant_stats_per_hour(): + with Duration("Rant stats per hour"): + plt = figure_inc() + async with Db() as db: + x = [] + y = [] + + for record in db.query("SELECT * FROM rant_stats_per_hour"): + y.append(record["count"]) + x.append(record["hour"]) + + plt.plot(x, y, label=get_date_range_str(), color="pink") + plt.xticks(rotation=45) + plt.xlabel("Hour") + plt.ylabel("Rant count") + plt.title("Rants per hour") + plt.legend() + plt.savefig(f"export/rants_per_hour_{get_date_range_str()}.png") + plt_reset() + # plt.show() + + +async def comment_stats_per_hour(): + with Duration("Comment stats per hour"): + plt = figure_inc() + async with Db() as db: + x = [] + y = [] + + for record in db.query("SELECT * FROM comment_stats_per_hour"): + y.append(record["count"]) + x.append(record["hour"]) + + plt.plot(x, y, label=get_date_range_str(), color="pink") + plt.xticks(rotation=45) + plt.xlabel("Hour") + plt.ylabel("Comment count") + plt.title("Comments per hour") + plt.legend() + plt.savefig(f"export/comments_per_hour_{get_date_range_str()}.png") + plt_reset() + # plt.show() + + +async def score_most_ignored_last_7_days(): + with Duration("Score most ignored last 7 days"): + plt = figure_inc() + async with Db() as db: + x = [] + y = [] + + for record in db.query("SELECT * FROM score_ignored_most_last_7_days LIMIT 15"): + x.append(record["username"]) + y.append(record["userscore"]) + + plt.bar(x, y, label="Upvotes") + # plt.plot(x, y, label=get_date_range_str(), color='pink') + plt.xticks(rotation=45) + plt.xlabel("Ranter") + plt.ylabel("Times ignored") + plt.title("Most ignored based on score regarding last 7 days") + plt.legend() + plt.savefig(f"export/score_ignored_most_last_7_days_{get_date_range_str()}.png") + plt_reset() + # plt.show() + + +async def score_last_7_days(): + with Duration("Upvotes (score) last 7 days"): + plt = figure_inc() + async with Db() as db: + x = [] + y = [] + + for record in db.query("SELECT * FROM score_last_7_days LIMIT 15"): + x.append(record["username"]) + y.append(record["userscore"]) + + plt.bar(x, y, label="Upvotes") + # plt.plot(x, y, label=get_date_range_str(), color='pink') + plt.xticks(rotation=45) + plt.xlabel("Ranter") + plt.ylabel("Upvote count") + plt.title("Most ignored based on score regarding last 7 days") + plt.legend() + plt.savefig(f"export/score_last_7_days_{get_date_range_str()}.png") + plt_reset() + # plt.show() + +async def user_score_per_day(username): + with Duration("User {} score per day".format(username)): + plt = figure_inc() + async with Db() as db: + + plt.xticks(rotation=45) + plt.title(f"{username}'s score per day") + plt.xlabel("Date") + plt.ylabel("Score") + + x = [] + y = [] + + for record in db.query("SELECT * FROM user_stats WHERE username = '{}'".format(username)): + x.append(record["date"]) + y.append(record["post_count"]) + plt.plot(x, y, label="Posts") + + x = [] + y = [] + for record in db.query("SELECT * FROM user_stats WHERE username = '{}'".format(username)): + x.append(record["date"]) + y.append(record["ignore_count"]) + plt.plot(x, y, label="Times ignored") + + x = [] + y = [] + for record in db.query("SELECT * FROM user_stats WHERE username = '{}'".format(username)): + x.append(record["date"]) + y.append(record["upvote_times"]) + plt.plot(x, y, label="Times upvoted") + + x = [] + y = [] + for record in db.query("SELECT * FROM user_stats WHERE username = '{}'".format(username)): + x.append(record["date"]) + y.append(record["upvote_total"]) + plt.plot(x, y, label="Sum upvotes") + + plt.legend() + plt.savefig(f"export/score_user_{username}_{get_date_range_str()}.png") + plt_reset() + + +def rant_stats_all(): + print("Did you sync all rants before running this?") + sleep(3) + with Duration("Complete process"): + asyncio.run(rant_stats_per_day()) + asyncio.run(rant_stats_per_weekday()) + asyncio.run(rant_stats_per_hour()) + asyncio.run(comment_stats_per_day()) + asyncio.run(comment_stats_per_weekday()) + asyncio.run(comment_stats_per_hour()) + asyncio.run(score_most_ignored_last_7_days()) + asyncio.run(score_last_7_days()) + asyncio.run(user_score_per_day("retoor")) + asyncio.run(user_score_per_day("Ranchonyx")) + asyncio.run(user_score_per_day("atheist")) + asyncio.run(user_score_per_day("Chewbanacas")) + asyncio.run(user_score_per_day("ScriptCoded")) + asyncio.run(user_score_per_day("bazmd")) + asyncio.run(user_score_per_day("feuerherz")) + asyncio.run(user_score_per_day("D-4got10-01")) + asyncio.run(user_score_per_day("jestdotty")) + asyncio.run(user_score_per_day("Demolishun")) + asyncio.run(user_score_per_day("cafecortado")) + asyncio.run(user_score_per_day("lungdart")) + asyncio.run(user_score_per_day("kiki")) + asyncio.run(user_score_per_day("netikras")) + asyncio.run(user_score_per_day("lorentz")) + asyncio.run(user_score_per_day("12bitfloat")) + asyncio.run(user_score_per_day("root")) + asyncio.run(user_score_per_day("antigermgerm")) + asyncio.run(user_score_per_day("Liebranca")) + + - plt.plot(x, y, label=get_date_range_str(), color='pink') - plt.xticks(rotation=45) - plt.xlabel('Hour') - plt.ylabel('Rant count') - plt.title('Rants per hour') - plt.legend() - plt.savefig(f'export/rants_per_hour_{get_date_range_str()}.png') - plt.show() \ No newline at end of file diff --git a/src/drstats/sync.py b/src/drstats/sync.py index 4739599..3ad9691 100644 --- a/src/drstats/sync.py +++ b/src/drstats/sync.py @@ -1,6 +1,6 @@ from drstats.devrant import Devrant -from drstats.db import get_db -import json +from drstats.db import get_db +import json import asyncio from pprint import pprint as pp @@ -8,48 +8,67 @@ dr = Devrant() db = get_db() -from datetime import datetime +def plain_object(obj): + for key, value in obj.items(): + if isinstance(value, list) or isinstance(value, dict): + value = json.dumps(value) + obj[key] = value + return obj + + +from datetime import datetime + def timestamp_to_string(timestamp): - return datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') + return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S") -async def get_recent_rants(start_from=1,page_size=10): + +async def get_recent_rants(start_from=1, page_size=10): page = 0 while True: - rants = dr.get_rants('recent',page_size,start_from)['rants'] + rants = dr.get_rants("recent", page_size, start_from)["rants"] page += 1 for rant in rants: - if rant is None: - break - rant['tags'] = json.dumps('tags' in rant and rant['tags'] or '') - rant['created'] = timestamp_to_string(rant['created_time']) - rant = json.loads(json.dumps(rant,default=str)) - - for key,value in rant.items(): - if isinstance(value, list) or isinstance(value, dict): - value = json.dumps(value) - rant[key] = value + if rant is None: + break + rant["tags"] = json.dumps("tags" in rant and rant["tags"] or "") + rant["created"] = timestamp_to_string(rant["created_time"]) + rant = plain_object(rant) - yield rant + yield rant start_from += page_size + async def sync_rants(): - count = 0; - start_from = 0 #db['rants'].count() - print(start_from) - await asyncio.sleep(2) + count = 0 + start_from = 0 page_size = 20 - async for rant in get_recent_rants(start_from,page_size): + async for rant in get_recent_rants(start_from, page_size): start_from += page_size count += 1 - pp(rant) - rant['tags'] = json.dumps(rant['tags']) - db['rants'].upsert(rant,['id']) - print(rant) + rant["tags"] = json.dumps(rant["tags"]) + db["rants"].upsert(rant, ["id"]) print(f"Upserted {count} rant(s).") +async def sync_comments(): + comments_synced = 0 + rants_synced = 0 + for rant in db["rants"].find(order_by="-id"): + rants_synced += 1 + comments = dr.get_rant(rant["id"])["comments"] + for comment in comments: + comments_synced += 1 + comment["created"] = timestamp_to_string(comment["created_time"]) + comment = plain_object(comment) + comment["rant_id"] = rant["id"] + + db["comments"].upsert(comment, ["id"]) + print(f"Synced {rants_synced} rants with {comments_synced} comments.") + + def sync(): - print(asyncio.run(sync_rants())) \ No newline at end of file + print(asyncio.run(sync_comments())) + print(asyncio.run(sync_rants()))