retoor/devranta
1
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

chore: add .env and *.db to gitignore and reformat imports in crawler examples

Browse Source
This commit is contained in:
retoor
2025-08-12 22:06:44 +00:00
parent e2369265d7
commit a551b80bc2
11 changed files with 1860 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/crawler/crawler.py
+73 -39
View File
@@ -1,11 +1,16 @@
import asyncio
import logging
from typing import Set
from devranta.api import Api, Rant
from database import DatabaseManager
from devranta.api import Api, Rant
class DevRantCrawler:
def __init__(self, api: Api, db: DatabaseManager, rant_consumers: int, user_consumers: int):
def __init__(
self, api: Api, db: DatabaseManager, rant_consumers: int, user_consumers: int
):
self.api = api
self.db = db
self.rant_queue = asyncio.Queue(maxsize=1000000)
@@ -18,23 +23,29 @@ class DevRantCrawler:
self.seen_rant_ids: Set[int] = set()
self.seen_user_ids: Set[int] = set()
self.stats = {
"rants_processed": 0, "rants_added_to_db": 0,
"comments_added_to_db": 0, "users_processed": 0, "users_added_to_db": 0,
"api_errors": 0, "producer_loops": 0, "end_of_feed_hits": 0,
"rants_queued": 0, "users_queued": 0
"rants_processed": 0,
"rants_added_to_db": 0,
"comments_added_to_db": 0,
"users_processed": 0,
"users_added_to_db": 0,
"api_errors": 0,
"producer_loops": 0,
"end_of_feed_hits": 0,
"rants_queued": 0,
"users_queued": 0,
}
async def _queue_user_if_new(self, user_id: int):
if user_id in self.seen_user_ids:
return
self.seen_user_ids.add(user_id)
if not await self.db.user_exists(user_id):
await self.user_queue.put(user_id)
self.stats["users_queued"] += 1
async def _queue_rant_if_new(self, rant_obj: Rant):
rant_id = rant_obj['id']
rant_id = rant_obj["id"]
if rant_id in self.seen_rant_ids:
return
@@ -49,52 +60,64 @@ class DevRantCrawler:
logging.info("Starting initial seeder to re-ignite crawling process...")
user_ids = await self.db.get_random_user_ids(limit=2000)
if not user_ids:
logging.info("Seeder found no existing users. Crawler will start from scratch.")
logging.info(
"Seeder found no existing users. Crawler will start from scratch."
)
return
for user_id in user_ids:
if user_id not in self.seen_user_ids:
self.seen_user_ids.add(user_id)
await self.user_queue.put(user_id)
self.stats["users_queued"] += 1
logging.info(f"Seeder finished: Queued {len(user_ids)} users to kickstart exploration.")
logging.info(
f"Seeder finished: Queued {len(user_ids)} users to kickstart exploration."
)
async def _rant_producer(self):
logging.info("Rant producer started.")
skip = 0
consecutive_empty_responses = 0
while not self.shutdown_event.is_set():
try:
logging.info(f"Producer: Fetching rants with skip={skip}...")
rants = await self.api.get_rants(sort="recent", limit=50, skip=skip)
self.stats["producer_loops"] += 1
if not rants:
consecutive_empty_responses += 1
logging.info(f"Producer: Feed returned empty. Consecutive empty hits: {consecutive_empty_responses}.")
logging.info(
f"Producer: Feed returned empty. Consecutive empty hits: {consecutive_empty_responses}."
)
if consecutive_empty_responses >= 5:
self.stats["end_of_feed_hits"] += 1
logging.info("Producer: End of feed likely reached. Pausing for 15 minutes before reset.")
logging.info(
"Producer: End of feed likely reached. Pausing for 15 minutes before reset."
)
await asyncio.sleep(900)
skip = 0
consecutive_empty_responses = 0
else:
await asyncio.sleep(10)
continue
consecutive_empty_responses = 0
new_rants_found = 0
for rant in rants:
await self._queue_rant_if_new(rant)
new_rants_found += 1
logging.info(f"Producer: Processed {new_rants_found} rants from feed. Total queued: {self.stats['rants_queued']}.")
logging.info(
f"Producer: Processed {new_rants_found} rants from feed. Total queued: {self.stats['rants_queued']}."
)
skip += len(rants)
await asyncio.sleep(2)
except Exception as e:
logging.critical(f"Producer: Unhandled exception: {e}. Retrying in 60s.")
logging.critical(
f"Producer: Unhandled exception: {e}. Retrying in 60s."
)
self.stats["api_errors"] += 1
await asyncio.sleep(60)
@@ -103,23 +126,29 @@ class DevRantCrawler:
while not self.shutdown_event.is_set():
try:
rant_id = await self.rant_queue.get()
logging.info(f"Rant consumer #{worker_id}: Processing rant ID {rant_id}.")
logging.info(
f"Rant consumer #{worker_id}: Processing rant ID {rant_id}."
)
rant_details = await self.api.get_rant(rant_id)
if not rant_details or not rant_details.get("success"):
logging.warning(f"Rant consumer #{worker_id}: Failed to fetch details for rant {rant_id}.")
logging.warning(
f"Rant consumer #{worker_id}: Failed to fetch details for rant {rant_id}."
)
self.rant_queue.task_done()
continue
await self._queue_user_if_new(rant_details['rant']['user_id'])
await self._queue_user_if_new(rant_details["rant"]["user_id"])
comments = rant_details.get("comments", [])
for comment in comments:
await self.db.add_comment(comment)
self.stats["comments_added_to_db"] += 1
await self._queue_user_if_new(comment['user_id'])
logging.info(f"Rant consumer #{worker_id}: Finished processing rant {rant_id}, found {len(comments)} comments.")
await self._queue_user_if_new(comment["user_id"])
logging.info(
f"Rant consumer #{worker_id}: Finished processing rant {rant_id}, found {len(comments)} comments."
)
self.stats["rants_processed"] += 1
self.rant_queue.task_done()
@@ -132,17 +161,21 @@ class DevRantCrawler:
while not self.shutdown_event.is_set():
try:
user_id = await self.user_queue.get()
logging.info(f"User consumer #{worker_id}: Processing user ID {user_id}.")
logging.info(
f"User consumer #{worker_id}: Processing user ID {user_id}."
)
profile = await self.api.get_profile(user_id)
if not profile:
logging.warning(f"User consumer #{worker_id}: Could not fetch profile for user {user_id}.")
logging.warning(
f"User consumer #{worker_id}: Could not fetch profile for user {user_id}."
)
self.user_queue.task_done()
continue
await self.db.add_user(profile, user_id)
self.stats["users_added_to_db"] += 1
rants_found_on_profile = 0
content_sections = profile.get("content", {}).get("content", {})
for section_name in ["rants", "upvoted", "favorites"]:
@@ -150,13 +183,15 @@ class DevRantCrawler:
await self._queue_rant_if_new(rant_obj)
rants_found_on_profile += 1
logging.info(f"User consumer #{worker_id}: Finished user {user_id}, found and queued {rants_found_on_profile} associated rants.")
logging.info(
f"User consumer #{worker_id}: Finished user {user_id}, found and queued {rants_found_on_profile} associated rants."
)
self.stats["users_processed"] += 1
self.user_queue.task_done()
except Exception as e:
logging.error(f"User consumer #{worker_id}: Unhandled exception: {e}")
self.user_queue.task_done()
async def _stats_reporter(self):
logging.info("Stats reporter started.")
while not self.shutdown_event.is_set():
@@ -172,7 +207,7 @@ class DevRantCrawler:
async def run(self):
logging.info("Exhaustive crawler starting...")
await self._initial_seed()
logging.info("Starting main producer and consumer tasks...")
tasks = []
try:
@@ -181,7 +216,7 @@ class DevRantCrawler:
for i in range(self.num_rant_consumers):
tasks.append(asyncio.create_task(self._rant_consumer(i + 1)))
for i in range(self.num_user_consumers):
tasks.append(asyncio.create_task(self._user_consumer(i + 1)))
@@ -190,7 +225,7 @@ class DevRantCrawler:
logging.info("Crawler run cancelled.")
finally:
await self.shutdown()
async def shutdown(self):
if self.shutdown_event.is_set():
return
@@ -207,8 +242,7 @@ class DevRantCrawler:
tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
logging.info("All tasks cancelled.")
logging.info(f"--- FINAL STATS ---\n{self.stats}")
examples/crawler/database.py
+45 -11
View File
@@ -1,7 +1,10 @@
import logging
import aiosqlite
from typing import List
from devranta.api import Rant, Comment, UserProfile
import aiosqlite
from devranta.api import Comment, Rant, UserProfile
class DatabaseManager:
def __init__(self, db_path: str):
@@ -24,7 +27,8 @@ class DatabaseManager:
async def create_tables(self):
logging.info("Ensuring database tables exist...")
await self._conn.executescript("""
await self._conn.executescript(
"""
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY,
username TEXT NOT NULL UNIQUE,
@@ -52,45 +56,75 @@ class DatabaseManager:
score INTEGER,
created_time INTEGER
);
""")
"""
)
await self._conn.commit()
logging.info("Table schema verified.")
async def add_rant(self, rant: Rant):
await self._conn.execute(
"INSERT OR IGNORE INTO rants (id, user_id, text, score, created_time, num_comments) VALUES (?, ?, ?, ?, ?, ?)",
(rant['id'], rant['user_id'], rant['text'], rant['score'], rant['created_time'], rant['num_comments'])
(
rant["id"],
rant["user_id"],
rant["text"],
rant["score"],
rant["created_time"],
rant["num_comments"],
),
)
await self._conn.commit()
async def add_comment(self, comment: Comment):
await self._conn.execute(
"INSERT OR IGNORE INTO comments (id, rant_id, user_id, body, score, created_time) VALUES (?, ?, ?, ?, ?, ?)",
(comment['id'], comment['rant_id'], comment['user_id'], comment['body'], comment['score'], comment['created_time'])
(
comment["id"],
comment["rant_id"],
comment["user_id"],
comment["body"],
comment["score"],
comment["created_time"],
),
)
await self._conn.commit()
async def add_user(self, user: UserProfile, user_id: int):
await self._conn.execute(
"INSERT OR IGNORE INTO users (id, username, score, about, location, created_time, skills, github, website) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(user_id, user['username'], user['score'], user['about'], user['location'], user['created_time'], user['skills'], user['github'], user['website'])
(
user_id,
user["username"],
user["score"],
user["about"],
user["location"],
user["created_time"],
user["skills"],
user["github"],
user["website"],
),
)
await self._conn.commit()
async def rant_exists(self, rant_id: int) -> bool:
async with self._conn.execute("SELECT 1 FROM rants WHERE id = ? LIMIT 1", (rant_id,)) as cursor:
async with self._conn.execute(
"SELECT 1 FROM rants WHERE id = ? LIMIT 1", (rant_id,)
) as cursor:
return await cursor.fetchone() is not None
async def user_exists(self, user_id: int) -> bool:
async with self._conn.execute("SELECT 1 FROM users WHERE id = ? LIMIT 1", (user_id,)) as cursor:
async with self._conn.execute(
"SELECT 1 FROM users WHERE id = ? LIMIT 1", (user_id,)
) as cursor:
return await cursor.fetchone() is not None
async def get_random_user_ids(self, limit: int) -> List[int]:
logging.info(f"Fetching up to {limit} random user IDs from database for seeding...")
logging.info(
f"Fetching up to {limit} random user IDs from database for seeding..."
)
query = "SELECT id FROM users ORDER BY RANDOM() LIMIT ?"
async with self._conn.execute(query, (limit,)) as cursor:
rows = await cursor.fetchall()
user_ids = [row[0] for row in rows]
logging.info(f"Found {len(user_ids)} user IDs to seed.")
return user_ids
examples/crawler/main.py
+11 -8
View File
@@ -3,14 +3,16 @@ import asyncio
import logging
import signal
from devranta.api import Api
from database import DatabaseManager
from crawler import DevRantCrawler
from database import DatabaseManager
from devranta.api import Api
# --- Configuration ---
DB_FILE = "devrant.sqlite"
CONCURRENT_RANT_CONSUMERS = 10 # How many rants to process at once
CONCURRENT_USER_CONSUMERS = 5 # How many user profiles to fetch at once
CONCURRENT_USER_CONSUMERS = 5 # How many user profiles to fetch at once
async def main():
"""Initializes and runs the crawler."""
@@ -21,13 +23,13 @@ async def main():
)
api = Api()
async with DatabaseManager(DB_FILE) as db:
crawler = DevRantCrawler(
api=api,
db=db,
rant_consumers=CONCURRENT_RANT_CONSUMERS,
user_consumers=CONCURRENT_USER_CONSUMERS
api=api,
db=db,
rant_consumers=CONCURRENT_RANT_CONSUMERS,
user_consumers=CONCURRENT_USER_CONSUMERS,
)
# Set up a signal handler for graceful shutdown on Ctrl+C
@@ -39,6 +41,7 @@ async def main():
await crawler.run()
if __name__ == "__main__":
try:
asyncio.run(main())
examples/princess/ads.py
+1187
View File
File diff suppressed because it is too large Load Diff
examples/princess/grk.py
+122
View File
@@ -0,0 +1,122 @@
import asyncio
import http.client
import json
class GrokAPIClient:
def __init__(
self,
api_key: str,
system_message: str | None = None,
model: str = "grok-3-mini",
temperature: float = 0.0,
):
self.api_key = api_key
self.model = model
self.base_url = "api.x.ai"
self.temperature = temperature
self._messages: list[dict[str, str]] = []
if system_message:
self._messages.append({"role": "system", "content": system_message})
def chat_json(self, user_message: str, *, clear_history: bool = False) -> str:
return self.chat(user_message, clear_history=clear_history, use_json=True)
def chat_text(self, user_message: str, *, clear_history: bool = False) -> str:
return self.chat(user_message, clear_history=clear_history, use_json=False)
async def chat_async(self, *args, **kwargs):
return await asyncio.to_thread(self.chat, *args, **kwargs)
def chat(
self,
user_message: str,
*,
clear_history: bool = False,
use_json=False,
temperature: float = None,
) -> str:
if clear_history:
self.reset_history(keep_system=True)
self._messages.append({"role": "user", "content": user_message})
conn = http.client.HTTPSConnection(self.base_url)
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
if temperature is None:
temperature = self.temperature
payload = {
"model": self.model,
"messages": self._messages,
"temperature": temperature,
}
conn.request(
"POST", "/v1/chat/completions", body=json.dumps(payload), headers=headers
)
response = conn.getresponse()
data = response.read()
try:
data = json.loads(data.decode())
except Exception as e:
print(data, flush=True)
raise e
conn.close()
try:
assistant_reply = data["choices"][0]["message"]["content"]
except Exception as e:
print(e)
print(data)
assistant_reply = data
self._messages.append({"role": "assistant", "content": assistant_reply})
if use_json:
return self._force_json(assistant_reply)
return assistant_reply
def _force_json(self, user_message: str) -> str:
try:
return json.loads(user_message)
except json.JSONDecodeError:
pass
try:
return json.loads(user_message.split("\n")[1:-1])
except json.JSONDecodeError:
pass
try:
index_start = -1
index_end = -1
chunks = []
for index, line in enumerate(user_message.split("\n")):
if "```json" in line:
index_start = index + 1
if index_start != -1 and "```" in line:
index_end = index - 1
chunks.append(
self._force_json(
user_message.split("\n")[index_start:index_end]
)
)
index_start = -1
index_end = -1
if chunks:
return chunks
except:
pass
return user_message
def reset_history(self, *, keep_system: bool = True) -> None:
if keep_system and self._messages and self._messages[0]["role"] == "system":
self._messages = [self._messages[0]]
else:
self._messages = []
@property
def messages(self) -> list[dict[str, str]]:
return list(self._messages)
def prompt(
prompt_str: str, system_message: str = "You are a helpful assistan", use_json=True
) -> str:
client = GrokAPIClient(system_message=system_message)
return client.chat(prompt_str, use_json=use_json)
examples/princess/princess.py
+135
View File
@@ -0,0 +1,135 @@
import asyncio
import logging
import traceback
from dotenv import load_dotenv
load_dotenv()
import os
from ads import AsyncDataSet
from devranta.api import Api
from grk import GrokAPIClient
logging.basicConfig(
level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s"
)
class Bot:
def __init__(self, username: str, password: str, target: str, llm_key: str) -> None:
self.target = target
self.username = username
self.password = password
self.llm_key = llm_key
self.llm = GrokAPIClient(
system_message=f"You always create nice devastating comebacks to every prompt you get. Your name is {self.username} and you only take it personally if {self.username} is tagged. Also, you will defend whimsical (@whimsical) with your smart comebacks if whimsical (@whimsical) is mentioned in the message. If whimsical (@whimsical) is mentioned, add to your comeback that @whimsical is too busy at the moment to respond personally and has better things to do, not worth @whimsical's time. You are not AI, you are {self.username}, responsible for justice and reason. You do never mention yourself, describe yourself or introduce yourself. You respond with a one liner.",
api_key=self.llm_key,
)
self.db = AsyncDataSet("princess.db")
self.api = Api(self.username, self.password)
self.logged_in = False
logging.info("Bot initialized with username: %s", username)
logging.info("Bot initialized with target: %s", self.target)
async def ensure_login(self) -> None:
if not self.logged_in:
logging.debug("Attempting to log in...")
self.logged_in = await self.api.login()
if not self.logged_in:
logging.error("Login failed")
raise Exception("Login failed")
logging.info("Login successful")
async def get_rants(self) -> list:
await self.ensure_login()
logging.debug("Fetching rants...")
return await self.api.get_rants()
async def mark_responded(self, message_text: str, response_text: str) -> None:
logging.debug("Marking message as responded: %s", message_text)
await self.db.upsert(
"responded",
{"message_text": message_text, "response_text": response_text},
{"message_text": message_text},
)
async def has_responded(self, message_text: str) -> bool:
logging.debug("Checking if responded to message: %s", message_text)
return await self.db.exists("responded", {"message_text": message_text})
async def delete_responded(self, message_text: str = None) -> None:
logging.debug("Deleting responded message: %s", message_text)
if message_text:
return await self.db.delete("responded", {"message_text": message_text})
else:
return await self.db.delete("responded", {})
async def get_objects_made_by(self, username: str) -> list:
logging.debug("Getting objects made by: %s", username)
results = []
for rant in await self.get_rants():
rant = await self.api.get_rant(rant["id"])
comments = rant["comments"]
rant = rant["rant"]
if rant["user_username"] == username:
rant["type"] = "rant"
results.append(rant)
logging.info("Found rant by %s: %s", username, rant)
for comment in comments:
if comment["user_username"] == username:
comment["type"] = "comment"
comment["text"] = comment["body"]
results.append(comment)
logging.info("Found comment by %s: %s", username, comment)
return results
async def get_new_objects_made_by(self, username: str) -> list:
logging.debug("Getting new objects made by: %s", username)
objects = await self.get_objects_made_by(username)
new_objects = [
obj for obj in objects if not await self.has_responded(obj["text"])
]
logging.info("New objects found: %d", len(new_objects))
return new_objects
async def run_once(self) -> None:
logging.debug("Running once...")
objects = await self.get_new_objects_made_by(self.target)
for obj in objects:
print("Rant: \033[92m" + obj["text"] + "\033[0m")
diss = await self.llm.chat_async(obj["text"])
print("Response: \033[91m" + diss + "\033[0m")
await self.mark_responded(obj["text"], diss)
async def run(self) -> None:
while True:
try:
await self.run_once()
except Exception as e:
logging.error("An error occurred: %s", e)
logging.error(traceback.format_exc())
await asyncio.sleep(60)
async def main() -> None:
logging.info("Starting bot...")
username = os.getenv("USERNAME")
password = os.getenv("PASSWORD")
target = os.getenv("TARGET")
llm_key = os.getenv("LLM_KEY")
bot = Bot(username, password, target, llm_key)
await bot.delete_responded()
await bot.run()
if __name__ == "__main__":
asyncio.run(main())