devplacepy/devplacepy/database.py

import dataset
import logging
from collections import defaultdict
from datetime import datetime, timezone
from devplacepy.cache import TTLCache
from devplacepy.config import DATABASE_URL

logger = logging.getLogger(__name__)

db = dataset.connect(
    DATABASE_URL,
    engine_kwargs={
        "connect_args": {
            "timeout": 30,
            "check_same_thread": False,
        },
    },
    on_connect_statements=[
        "PRAGMA journal_mode=WAL",
        "PRAGMA synchronous=NORMAL",
        "PRAGMA busy_timeout=30000",
        "PRAGMA cache_size=-8000",
        "PRAGMA temp_store=MEMORY",
        "PRAGMA mmap_size=268435456",
    ],
)


def _index(db, table, name, columns):
    try:
        if table in db.tables:
            cols = ", ".join(columns)
            db.query(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({cols})")
    except Exception as e:
        logger.warning(f"Could not create index {name} on {table}: {e}")


def init_db():
    tables = db.tables
    _index(db, "users", "idx_users_username", ["username"])
    _index(db, "users", "idx_users_email", ["email"])
    _index(db, "posts", "idx_posts_user_uid", ["user_uid"])
    _index(db, "posts", "idx_posts_created_at", ["created_at"])
    _index(db, "posts", "idx_posts_topic", ["topic"])
    _index(db, "comments", "idx_comments_post_uid", ["post_uid"])
    _index(db, "comments", "idx_comments_target", ["target_type", "target_uid"])
    _index(db, "comments", "idx_comments_user_uid", ["user_uid"])
    _index(db, "comments", "idx_comments_created_at", ["created_at"])
    _index(db, "votes", "idx_votes_target", ["target_uid", "target_type"])
    _index(db, "messages", "idx_messages_sender", ["sender_uid"])
    _index(db, "messages", "idx_messages_receiver", ["receiver_uid"])
    _index(db, "notifications", "idx_notifications_user", ["user_uid"])
    _index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
    _index(db, "push_registration", "idx_push_registration_user", ["user_uid"])
    _index(db, "sessions", "idx_sessions_token", ["session_token"])
    _index(db, "projects", "idx_projects_user", ["user_uid"])
    _index(db, "badges", "idx_badges_user", ["user_uid"])
    _index(db, "follows", "idx_follows_follower", ["follower_uid"])
    _index(db, "follows", "idx_follows_following", ["following_uid"])
    _index(db, "password_resets", "idx_password_resets_token", ["token"])
    _index(db, "gists", "idx_gists_user_uid", ["user_uid"])
    _index(db, "gists", "idx_gists_language", ["language"])
    _index(db, "attachments", "idx_attachments_resource", ["resource_type", "resource_uid"])
    _index(db, "attachments", "idx_attachments_target", ["target_type", "target_uid"])

    if "site_settings" in tables:
        defaults = {"site_name": "DevPlace", "site_description": "The Developer Social Network", "site_tagline": "Track industry shifts. Discover bold releases. Share what you are building in an open, uncensored environment."}
        for key, value in defaults.items():
            existing = db["site_settings"].find_one(key=key)
            if not existing:
                db["site_settings"].insert({"uid": f"default_{key}", "key": key, "value": value})

    _index(db, "news", "idx_news_external_id", ["external_id"])
    _index(db, "news", "idx_news_synced_at", ["synced_at"])
    _index(db, "news", "idx_news_status", ["status"])
    _index(db, "news_images", "idx_news_images_news_uid", ["news_uid"])
    _index(db, "news_sync", "idx_news_sync_external_id", ["external_id"])

    if "news" in tables:
        for article in db["news"].find(status=None):
            was_featured = article.get("featured", 0)
            db["news"].update({
                "uid": article["uid"],
                "status": "published" if was_featured else "draft",
            }, ["uid"])
        for article in db["news"].find(show_on_landing=None):
            db["news"].update({
                "uid": article["uid"],
                "show_on_landing": 0,
            }, ["uid"])
        for article in db["news"].find(slug=None):
            from devplacepy.utils import make_combined_slug
            slug = make_combined_slug(article.get("title", "") or "news", article["uid"])
            db["news"].update({
                "uid": article["uid"],
                "slug": slug,
            }, ["uid"])

    if "news_sync" in tables:
        for entry in db["news_sync"].find():
            current = entry.get("status", "")
            if current in ("below_threshold", ""):
                db["news_sync"].update({
                    "id": entry["id"],
                    "status": "graded",
                }, ["id"])

    if "site_settings" in tables:
        news_defaults = {
            "news_grade_threshold": "7",
            "news_api_url": "https://news.app.molodetz.nl/api",
            "news_ai_url": "https://openai.app.molodetz.nl/v1/chat/completions",
            "news_ai_model": "molodetz",
        }
        for key, value in news_defaults.items():
            existing = db["site_settings"].find_one(key=key)
            if not existing:
                db["site_settings"].insert({"uid": f"default_{key}", "key": key, "value": value})

        upload_defaults = {
            "max_upload_size_mb": "10",
            "allowed_file_types": "",
            "max_attachments_per_resource": "10",
        }
        for key, value in upload_defaults.items():
            existing = db["site_settings"].find_one(key=key)
            if not existing:
                db["site_settings"].insert({"uid": f"default_{key}", "key": key, "value": value})

    _backfill_gamification()

    logger.info("Database initialized")


def _backfill_gamification():
    if "users" not in db.tables:
        return
    from devplacepy.utils import (
        level_for_xp, check_milestone_badges,
        XP_POST, XP_COMMENT, XP_PROJECT, XP_GIST, XP_UPVOTE, XP_FOLLOW,
    )

    pending = list(db["users"].find(xp=0))
    if not pending:
        return

    xp_by_user = defaultdict(int)

    def add_counts(table, column, points):
        if table not in db.tables:
            return
        for row in db.query(f"SELECT {column} AS uid, COUNT(*) AS c FROM {table} GROUP BY {column}"):
            if row["uid"]:
                xp_by_user[row["uid"]] += row["c"] * points

    add_counts("posts", "user_uid", XP_POST)
    add_counts("comments", "user_uid", XP_COMMENT)
    add_counts("projects", "user_uid", XP_PROJECT)
    add_counts("gists", "user_uid", XP_GIST)
    add_counts("follows", "following_uid", XP_FOLLOW)

    if "votes" in db.tables:
        for content_table, target_type in (("posts", "post"), ("projects", "project"), ("gists", "gist"), ("comments", "comment")):
            if content_table not in db.tables:
                continue
            rows = db.query(
                f"SELECT c.user_uid AS uid, COUNT(*) AS c "
                f"FROM votes v JOIN {content_table} c ON v.target_uid = c.uid "
                f"WHERE v.target_type = :t AND v.value = 1 GROUP BY c.user_uid",
                t=target_type,
            )
            for row in rows:
                if row["uid"]:
                    xp_by_user[row["uid"]] += row["c"] * XP_UPVOTE

    for user in pending:
        xp = xp_by_user.get(user["uid"], 0)
        if xp <= 0:
            continue
        db["users"].update({"uid": user["uid"], "xp": xp, "level": level_for_xp(xp)}, ["uid"])

    _authors_cache.clear()
    for user in pending:
        check_milestone_badges(user["uid"])
    logger.info(f"Gamification backfill processed {len(pending)} users")


def get_table(name):
    return db[name]


def _in_clause(uids, prefix="p"):
    placeholders = ", ".join(f":{prefix}{i}" for i in range(len(uids)))
    params = {f"{prefix}{i}": uid for i, uid in enumerate(uids)}
    return placeholders, params


def get_users_by_uids(uids):
    if not uids:
        return {}
    seen = set()
    unique = [u for u in uids if u not in seen and not seen.add(u)]
    return {u["uid"]: u for u in db["users"].find(db["users"].table.columns.uid.in_(unique))}


def get_comment_counts_by_post_uids(post_uids):
    if not post_uids or "comments" not in db.tables:
        return {}
    placeholders, params = _in_clause(post_uids)
    rows = db.query(f"SELECT target_uid, COUNT(*) as c FROM comments WHERE target_type='post' AND target_uid IN ({placeholders}) GROUP BY target_uid", **params)
    return {r["target_uid"]: r["c"] for r in rows}


def get_post_counts_by_user_uids(user_uids):
    if not user_uids or "posts" not in db.tables:
        return {}
    placeholders, params = _in_clause(user_uids)
    rows = db.query(f"SELECT user_uid, COUNT(*) as c FROM posts WHERE user_uid IN ({placeholders}) GROUP BY user_uid", **params)
    return {r["user_uid"]: r["c"] for r in rows}


def get_vote_counts(target_uids):
    if not target_uids or "votes" not in db.tables:
        return {}, {}
    placeholders, params = _in_clause(target_uids)
    rows = db.query(f"SELECT target_uid, value, COUNT(*) as c FROM votes WHERE target_uid IN ({placeholders}) GROUP BY target_uid, value", **params)
    ups = {}
    downs = {}
    for r in rows:
        if r["value"] == 1:
            ups[r["target_uid"]] = r["c"]
        else:
            downs[r["target_uid"]] = r["c"]
    return ups, downs


def get_user_votes(user_uid, target_uids):
    if not user_uid or not target_uids or "votes" not in db.tables:
        return {}
    placeholders, params = _in_clause(target_uids)
    params["uid"] = user_uid
    rows = db.query(f"SELECT target_uid, value FROM votes WHERE user_uid = :uid AND target_uid IN ({placeholders})", **params)
    return {r["target_uid"]: r["value"] for r in rows}


def load_comments(target_type, target_uid, user=None):
    if "comments" not in db.tables:
        return []
    comments_table = db["comments"]
    raw = list(comments_table.find(target_type=target_type, target_uid=target_uid, order_by=["created_at"]))
    if not raw and target_type == "post":
        raw = list(comments_table.find(post_uid=target_uid, order_by=["created_at"]))
    if not raw:
        return []
    uids = [c["user_uid"] for c in raw]
    cids = [c["uid"] for c in raw]
    users = get_users_by_uids(uids)
    ups, downs = get_vote_counts(cids)
    my_votes = get_user_votes(user["uid"], cids) if user else {}
    from devplacepy.utils import time_ago
    from devplacepy.attachments import get_attachments_batch as _gab
    atts_map = _gab("comment", cids) if "attachments" in db.tables else {}
    cmap = {}
    for c in raw:
        cmap[c["uid"]] = {
            "comment": c,
            "author": users.get(c["user_uid"]),
            "time_ago": time_ago(c["created_at"]),
            "votes": {"up": ups.get(c["uid"], 0), "down": downs.get(c["uid"], 0)},
            "my_vote": my_votes.get(c["uid"], 0),
            "children": [],
            "attachments": atts_map.get(c["uid"], []),
        }
    top = []
    for item in cmap.values():
        parent = item["comment"].get("parent_uid")
        if parent and parent in cmap:
            cmap[parent]["children"].append(item)
        else:
            top.append(item)
    return top


def get_attachments(resource_type: str, resource_uid: str) -> list:
    if "attachments" not in db.tables:
        return []
    return list(db["attachments"].find(resource_type=resource_type, resource_uid=resource_uid, order_by=["created_at"]))


def get_attachments_by_type(resource_type: str, resource_uids: list) -> dict:
    if not resource_uids or "attachments" not in db.tables:
        return {}
    rows = list(db["attachments"].find(db["attachments"].table.columns.resource_uid.in_(resource_uids), resource_type=resource_type))
    result = {}
    for a in rows:
        key = a["resource_uid"]
        if key not in result:
            result[key] = []
        result[key].append(a)
    return result


def get_news_images_by_uids(news_uids: list) -> dict:
    if not news_uids or "news_images" not in db.tables:
        return {}
    images_table = db["news_images"]
    rows = images_table.find(images_table.table.columns.news_uid.in_(news_uids), order_by=["uid"])
    result = {}
    for r in rows:
        result.setdefault(r["news_uid"], r["url"])
    return result


def delete_attachment_record(uid: str) -> None:
    if "attachments" not in db.tables:
        return
    att = db["attachments"].find_one(uid=uid)
    if att:
        _delete_attachment_file(att.get("storage_path", ""))
        db["attachments"].delete(id=att["id"])


def delete_attachments(resource_type: str, resource_uid: str) -> None:
    if "attachments" not in db.tables:
        return
    for a in db["attachments"].find(resource_type=resource_type, resource_uid=resource_uid):
        _delete_attachment_file(a.get("storage_path", ""))
    db["attachments"].delete(resource_type=resource_type, resource_uid=resource_uid)


def _delete_attachment_file(storage_path: str) -> None:
    if not storage_path:
        return
    from devplacepy.config import STATIC_DIR
    file_path = STATIC_DIR / "uploads" / storage_path
    try:
        file_path.unlink(missing_ok=True)
        parent = file_path.parent
        if parent.exists() and not any(parent.iterdir()):
            parent.rmdir()
        grandparent = parent.parent
        if grandparent.exists() and not any(grandparent.iterdir()):
            grandparent.rmdir()
    except Exception as e:
        logger.warning(f"Failed to delete attachment file {storage_path}: {e}")


_settings_cache = TTLCache(ttl=60)


def get_setting(key: str, default: str = "") -> str:
    cached = _settings_cache.get(key)
    if cached is not None:
        return cached
    if "site_settings" not in db.tables:
        return default
    entry = db["site_settings"].find_one(key=key)
    if entry is None:
        return default
    _settings_cache.set(key, entry["value"])
    return entry["value"]


def get_int_setting(key: str, default: int) -> int:
    raw = get_setting(key, str(default))
    try:
        return int(raw)
    except (TypeError, ValueError):
        return default


def clear_settings_cache() -> None:
    _settings_cache.clear()


_stats_cache = TTLCache(ttl=30)


def get_site_stats() -> dict:
    cached = _stats_cache.get("site")
    if cached is not None:
        return cached
    today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0).isoformat()
    stats = {
        "total_members": db["users"].count() if "users" in db.tables else 0,
        "posts_today": db["posts"].count(created_at={">=": today_start}) if "posts" in db.tables else 0,
        "total_projects": db["projects"].count() if "projects" in db.tables else 0,
        "total_gists": db["gists"].count() if "gists" in db.tables else 0,
    }
    _stats_cache.set("site", stats)
    return stats


_gist_languages_cache = TTLCache(ttl=60)


def get_gist_languages() -> set[str]:
    cached = _gist_languages_cache.get("codes")
    if cached is not None:
        return cached
    codes: set[str] = set()
    if "gists" in db.tables:
        for row in db.query("SELECT DISTINCT language FROM gists"):
            language = row.get("language")
            if language:
                codes.add(language)
    _gist_languages_cache.set("codes", codes)
    return codes


_STARRED_CONTENT_TABLES = ("posts", "projects", "gists")

_authors_cache = TTLCache(ttl=60)


def _ranked_authors() -> list:
    cached = _authors_cache.get("ranked")
    if cached is not None:
        return cached
    sources = [table for table in _STARRED_CONTENT_TABLES if table in db.tables]
    if not sources:
        _authors_cache.set("ranked", [])
        return []
    union = " UNION ALL ".join(f"SELECT user_uid, stars FROM {table}" for table in sources)
    rows = db.query(
        f"SELECT user_uid, SUM(stars) AS total FROM ({union}) "
        f"GROUP BY user_uid HAVING total > 0 ORDER BY total DESC"
    )
    ranked = [(row["user_uid"], row["total"]) for row in rows]
    users_map = get_users_by_uids([uid for uid, _ in ranked])
    authors = []
    for uid, total in ranked:
        user = users_map.get(uid)
        if user:
            author = dict(user)
            author["stars"] = total
            authors.append(author)
    _authors_cache.set("ranked", authors)
    return authors


def get_top_authors(limit: int = 5) -> list:
    return _ranked_authors()[:limit]


def get_leaderboard(limit: int = 50, offset: int = 0) -> list:
    sliced = _ranked_authors()[offset:offset + limit]
    leaderboard = []
    for position, author in enumerate(sliced, start=offset + 1):
        entry = dict(author)
        entry["rank"] = position
        leaderboard.append(entry)
    return leaderboard


def get_user_rank(user_uid: str):
    for position, author in enumerate(_ranked_authors(), start=1):
        if author["uid"] == user_uid:
            return position
    return None


def get_user_stars(user_uid: str) -> int:
    total = 0
    for table in _STARRED_CONTENT_TABLES:
        if table in db.tables:
            for row in db.query(f"SELECT COALESCE(SUM(stars), 0) AS s FROM {table} WHERE user_uid = :u", u=user_uid):
                total += row["s"] or 0
    return total


def resolve_by_slug(table, slug):
    entry = table.find_one(slug=slug)
    if not entry:
        entry = table.find_one(uid=slug)
    return entry


def resolve_object_url(target_type: str, target_uid: str) -> str:
    if target_type == "post":
        post = resolve_by_slug(get_table("posts"), target_uid)
        return f"/posts/{post['slug'] or post['uid']}" if post else "/feed"
    if target_type == "project":
        project = resolve_by_slug(get_table("projects"), target_uid)
        return f"/projects/{project['slug'] or project['uid']}" if project else "/projects"
    if target_type == "news":
        article = resolve_by_slug(get_table("news"), target_uid)
        if article:
            return f"/news/{article.get('slug', '') or article['uid']}"
        return "/news"
    if target_type == "bug":
        return f"/bugs?highlight={target_uid}"
    if target_type == "gist":
        gist = resolve_by_slug(get_table("gists"), target_uid)
        return f"/gists/{gist['slug'] or gist['uid']}" if gist else "/gists"
    if target_type == "comment":
        comment = get_table("comments").find_one(uid=target_uid)
        if not comment:
            return "/feed"
        parent_url = resolve_object_url(comment.get("target_type", "post"), comment.get("target_uid") or comment.get("post_uid", ""))
        return f"{parent_url}#comment-{target_uid}"
    return "/feed"


VOTABLE_TARGETS: dict[str, str] = {
    "post": "posts",
    "project": "projects",
    "gist": "gists",
    "comment": "comments",
}

STAR_TARGETS: set[str] = {"post", "project", "gist"}


def update_target_stars(target_type: str, target_uid: str, net_stars: int) -> None:
    table_name = VOTABLE_TARGETS.get(target_type)
    if not table_name or target_type not in STAR_TARGETS:
        return
    get_table(table_name).update({"uid": target_uid, "stars": net_stars}, ["uid"])
    _authors_cache.clear()


def get_target_owner_uid(target_type: str, target_uid: str) -> str | None:
    table_name = VOTABLE_TARGETS.get(target_type)
    if not table_name:
        return None
    row = get_table(table_name).find_one(uid=target_uid)
    return row["user_uid"] if row else None


def build_pagination(page, total, per_page=25):
    total_pages = max(1, __import__("math").ceil(total / per_page))
    page = max(1, min(page, total_pages))
    return {
        "page": page,
        "per_page": per_page,
        "total": total,
        "total_pages": total_pages,
        "has_prev": page > 1,
        "has_next": page < total_pages,
        "prev_page": page - 1,
        "next_page": page + 1,
    }


def get_daily_topic():
    if "news" in db.tables:
        article = db["news"].find_one(status="published", order_by=["-synced_at"])
        if article:
            desc = (article.get("description") or "")[:200] or (article.get("content") or "")[:200]
            return {
                "title": article.get("title", ""),
                "summary": desc,
                "slug": article.get("slug", ""),
                "url": article.get("url", ""),
            }
    return {"title": "Welcome to DevPlace", "summary": "Stay tuned for the latest dev news."}


def get_featured_news(limit=5):
    if "news" not in db.tables:
        return []
    from devplacepy.utils import time_ago
    rows = list(db["news"].find(show_on_landing=1, order_by=["-synced_at"], _limit=limit))
    articles = []
    for article in rows:
        summary = (article.get("description") or "")[:120] or (article.get("content") or "")[:120]
        articles.append({
            "title": article.get("title", ""),
            "summary": summary,
            "slug": article.get("slug", ""),
            "url": article.get("url", ""),
            "source_name": article.get("source_name", ""),
            "time_ago": time_ago(article["synced_at"]) if article.get("synced_at") else "",
        })
    return articles