AI gateway: - Add a generic, admin-selectable `client_profile` field on gateway_providers (e.g. "opencode") so a provider needing special request headers (OpenCode Zen's client-identity spoofing) is configured like any other provider, not hardcoded by name. - Track per-(provider, model) reliability/speed/latency health in memory, seeded from the existing gateway_usage_ledger at startup - purely observational, never influences routing. - New Stats tab on /admin/gateway: request volume, latency, per-model breakdowns, and reliability weight, charted with a vendored Chart.js and devplace's own theme tokens. - Record which model a failed request actually fell back to (fallback_used_route), surfaced in the Recent Failures table. - Stop excluding context_length errors from fallback, and skip a primary attempt outright when its known context window is already too small for the estimated request size, going straight to the fallback. - gateway_usage_ledger's provider/fallback_used_route columns and indexes are ensured centrally in database/schema.py's init_db(), the single point of truth for this table's schema. - Non-OpenAI upstream routing and client-model passthrough; trust only the upstream's own X-Gateway-Model header for served-model attribution. Devii agent: - Fix a real lockup: plan/verify tools could be individually disabled via the admin tool toggles while still being required by the protocol gate, permanently bricking any task that needed tools. They can no longer be disabled, and the gate now also checks the tool is actually offered. - Fix compaction being silently calibrated for a 1M-token model while running a much smaller one: context budget is now percentage-based and the summarizer's own request is sized to fit the real model. - Give a specific, actionable retry message when plan()'s own arguments get cut off by the output limit, and tighten its schema to discourage overlong plans. Other: - Backup service: offload completed backups to a remote Hetzner Storage Box. - Container manager: fix orphan blob leaks from sync races, add a two-phase plan/execute `system prune` CLI command. - Admin gateway UI: replace the JS-rendered model/provider tables with server-rendered forms and pages. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DhmEkvutuwtzFVcLbTrhdo
2407 lines
80 KiB
Python
2407 lines
80 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from .core import AQUALITY_NEWS_GRADING_MODEL, AQUALITY_NEWS_GRADING_URL, DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger
|
|
from .settings import get_setting, set_setting
|
|
from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns
|
|
from .ranking import _authors_cache
|
|
|
|
|
|
BUG_TABLE_RENAMES = (
|
|
("bug_tickets", "issue_tickets"),
|
|
("bug_comment_authors", "issue_comment_authors"),
|
|
)
|
|
|
|
|
|
def migrate_bug_tables_to_issue_tables() -> None:
|
|
for source_name, destination_name in BUG_TABLE_RENAMES:
|
|
if source_name not in db.tables:
|
|
continue
|
|
source = db[source_name]
|
|
destination = get_table(destination_name)
|
|
copied = 0
|
|
for row in source.all():
|
|
payload = {key: value for key, value in row.items() if key != "id"}
|
|
destination.insert_ignore(payload, ["uid"])
|
|
copied += 1
|
|
logger.info(
|
|
"Migrated %s rows from %s into %s", copied, source_name, destination_name
|
|
)
|
|
source.drop()
|
|
logger.info("Dropped table %s after migration", source_name)
|
|
|
|
|
|
_TUNNEL_STATUS_RANK = {
|
|
"active": 4,
|
|
"provisioning": 3,
|
|
"pending": 2,
|
|
"failed": 1,
|
|
"suspended": 0,
|
|
}
|
|
|
|
|
|
def _dedupe_tunnel_hostnames() -> None:
|
|
if "tunnels" not in db.tables:
|
|
return
|
|
table = get_table("tunnels")
|
|
if not table.has_column("hostname") or not table.has_column("uid"):
|
|
return
|
|
groups = list(
|
|
db.query(
|
|
"SELECT hostname FROM tunnels "
|
|
"WHERE hostname IS NOT NULL AND hostname != '' "
|
|
"GROUP BY hostname HAVING COUNT(*) > 1"
|
|
)
|
|
)
|
|
for group in groups:
|
|
hostname = group["hostname"]
|
|
dupes = list(table.find(hostname=hostname))
|
|
dupes.sort(
|
|
key=lambda r: (
|
|
_TUNNEL_STATUS_RANK.get(r.get("status") or "", -1),
|
|
r.get("deleted_at") is None,
|
|
r.get("created_at") or "",
|
|
r.get("id") or 0,
|
|
),
|
|
reverse=True,
|
|
)
|
|
losers = [r["uid"] for r in dupes[1:]]
|
|
if not losers:
|
|
continue
|
|
with db:
|
|
for uid in losers:
|
|
db.query("DELETE FROM tunnels WHERE uid = :uid", uid=uid)
|
|
logger.warning(
|
|
"Removed %d duplicate tunnel row(s) for hostname %s, kept %s",
|
|
len(losers),
|
|
hostname,
|
|
dupes[0]["uid"],
|
|
)
|
|
|
|
|
|
def init_db():
|
|
tables = db.tables
|
|
_index(db, "users", "idx_users_username", ["username"])
|
|
_index(db, "users", "idx_users_email", ["email"])
|
|
_index(db, "users", "idx_users_api_key", ["api_key"])
|
|
_index(db, "users", "idx_users_role", ["role", "created_at"])
|
|
_index(db, "users", "idx_users_last_seen", ["last_seen"])
|
|
_index(db, "users", "idx_users_created_at", ["created_at"])
|
|
_ensure_users_uid()
|
|
_index(db, "posts", "idx_posts_user_uid", ["user_uid"])
|
|
_index(db, "posts", "idx_posts_created_at", ["created_at"])
|
|
_index(db, "posts", "idx_posts_topic", ["topic"])
|
|
_index(db, "posts", "idx_posts_slug", ["slug"])
|
|
_index(db, "posts", "idx_posts_project_uid", ["project_uid"])
|
|
if "posts" in tables:
|
|
posts_table = get_table("posts")
|
|
if not posts_table.has_column("tags"):
|
|
posts_table.create_column_by_example("tags", "")
|
|
if "devrant_tokens" in tables:
|
|
devrant_tokens = get_table("devrant_tokens")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("key", ""),
|
|
("user_uid", ""),
|
|
("user_id", 0),
|
|
("expire_time", 0),
|
|
("created_at", ""),
|
|
):
|
|
if not devrant_tokens.has_column(column):
|
|
devrant_tokens.create_column_by_example(column, example)
|
|
_index(db, "devrant_tokens", "idx_devrant_tokens_key", ["key"])
|
|
_index(db, "devrant_tokens", "idx_devrant_tokens_user", ["user_uid"])
|
|
if "access_tokens" in tables:
|
|
access_tokens = get_table("access_tokens")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("token", ""),
|
|
("user_uid", ""),
|
|
("label", ""),
|
|
("expires_at", ""),
|
|
("created_at", ""),
|
|
):
|
|
if not access_tokens.has_column(column):
|
|
access_tokens.create_column_by_example(column, example)
|
|
_index(db, "access_tokens", "idx_access_tokens_token", ["token"])
|
|
_index(db, "access_tokens", "idx_access_tokens_user", ["user_uid"])
|
|
if "telegram_pairings" in tables:
|
|
telegram_pairings = get_table("telegram_pairings")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("code_hash", ""),
|
|
("expires_at", ""),
|
|
("used", 0),
|
|
("created_at", ""),
|
|
):
|
|
if not telegram_pairings.has_column(column):
|
|
telegram_pairings.create_column_by_example(column, example)
|
|
_index(db, "telegram_pairings", "idx_telegram_pairings_code", ["code_hash"])
|
|
_index(db, "telegram_pairings", "idx_telegram_pairings_user", ["user_uid"])
|
|
if "telegram_links" in tables:
|
|
telegram_links = get_table("telegram_links")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("chat_id", 0),
|
|
("from_id", 0),
|
|
("created_at", ""),
|
|
):
|
|
if not telegram_links.has_column(column):
|
|
telegram_links.create_column_by_example(column, example)
|
|
_index(db, "telegram_links", "idx_telegram_links_chat", ["chat_id"])
|
|
_index(db, "telegram_links", "idx_telegram_links_user", ["user_uid"])
|
|
telegram_outbox = get_table("telegram_outbox")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("chat_id", 0),
|
|
("text", ""),
|
|
("status", "pending"),
|
|
("attempts", 0),
|
|
("created_at", ""),
|
|
("sent_at", ""),
|
|
):
|
|
if not telegram_outbox.has_column(column):
|
|
telegram_outbox.create_column_by_example(column, example)
|
|
_index(db, "telegram_outbox", "idx_telegram_outbox_status", ["status", "id"])
|
|
_index(db, "comments", "idx_comments_post_uid", ["post_uid"])
|
|
_index(db, "comments", "idx_comments_target", ["target_type", "target_uid"])
|
|
_index(db, "comments", "idx_comments_user_uid", ["user_uid"])
|
|
_index(db, "comments", "idx_comments_created_at", ["created_at"])
|
|
_index(db, "votes", "idx_votes_target", ["target_uid", "target_type"])
|
|
messages = get_table("messages")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("sender_uid", ""),
|
|
("receiver_uid", ""),
|
|
("content", ""),
|
|
("read", False),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not messages.has_column(column):
|
|
messages.create_column_by_example(column, example)
|
|
_index(db, "messages", "idx_messages_sender", ["sender_uid"])
|
|
_index(db, "messages", "idx_messages_receiver", ["receiver_uid"])
|
|
_index(
|
|
db, "messages", "idx_messages_conversation", ["sender_uid", "receiver_uid"]
|
|
)
|
|
_index(
|
|
db,
|
|
"messages",
|
|
"idx_messages_conversation_rev",
|
|
["receiver_uid", "sender_uid"],
|
|
)
|
|
_index(db, "messages", "idx_messages_updated_at", ["updated_at"])
|
|
_index(db, "notifications", "idx_notifications_user", ["user_uid"])
|
|
_index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
|
|
push_registration = get_table("push_registration")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("provider", "webpush"),
|
|
("endpoint", ""),
|
|
("key_auth", ""),
|
|
("key_p256dh", ""),
|
|
("token", ""),
|
|
("client_id", ""),
|
|
("environment", ""),
|
|
("created_at", ""),
|
|
("registered_at", ""),
|
|
("deleted_at", ""),
|
|
):
|
|
if not push_registration.has_column(column):
|
|
push_registration.create_column_by_example(column, example)
|
|
_index(db, "push_registration", "idx_push_registration_user", ["user_uid"])
|
|
_index(db, "push_registration", "idx_push_registration_provider", ["provider"])
|
|
_index(
|
|
db,
|
|
"push_registration",
|
|
"idx_push_registration_client",
|
|
["user_uid", "provider", "client_id"],
|
|
)
|
|
_index(
|
|
db,
|
|
"push_registration",
|
|
"idx_push_registration_token",
|
|
["user_uid", "provider", "token"],
|
|
)
|
|
if "push_registration" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"UPDATE push_registration SET provider = 'webpush' "
|
|
"WHERE provider IS NULL OR provider = ''"
|
|
)
|
|
db.query(
|
|
"UPDATE push_registration SET registered_at = created_at "
|
|
"WHERE registered_at IS NULL OR registered_at = ''"
|
|
)
|
|
_index(db, "sessions", "idx_sessions_token", ["session_token"])
|
|
projects = get_table("projects")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("slug", ""),
|
|
("created_at", ""),
|
|
("stars", 0),
|
|
("project_type", ""),
|
|
("is_private", 0),
|
|
("read_only", 0),
|
|
("updated_at", ""),
|
|
("title", ""),
|
|
("description", ""),
|
|
("status", ""),
|
|
("platforms", ""),
|
|
("release_date", ""),
|
|
("demo_date", ""),
|
|
("website_url", ""),
|
|
("repo_url", ""),
|
|
("cover_attachment_uid", ""),
|
|
("logo_attachment_uid", ""),
|
|
):
|
|
if not projects.has_column(column):
|
|
projects.create_column_by_example(column, example)
|
|
|
|
_index(db, "projects", "idx_projects_user", ["user_uid"])
|
|
_index(db, "projects", "idx_projects_slug", ["slug"])
|
|
_index(db, "projects", "idx_projects_private", ["is_private"])
|
|
_index(db, "projects", "idx_projects_stars", ["stars"])
|
|
_index(db, "projects", "idx_projects_type", ["project_type"])
|
|
project_files = get_table("project_files")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("project_uid", ""),
|
|
("user_uid", ""),
|
|
("path", ""),
|
|
("name", ""),
|
|
("parent_path", ""),
|
|
("type", ""),
|
|
("content", ""),
|
|
("is_binary", 0),
|
|
("stored_name", ""),
|
|
("directory", ""),
|
|
("mime_type", ""),
|
|
("size", 0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not project_files.has_column(column):
|
|
project_files.create_column_by_example(column, example)
|
|
_index(db, "project_files", "idx_project_files_path", ["project_uid", "path"])
|
|
_index(
|
|
db, "project_files", "idx_project_files_parent", ["project_uid", "parent_path"]
|
|
)
|
|
project_file_sync_state = get_table("project_file_sync_state")
|
|
for column, example in (
|
|
("project_uid", ""),
|
|
("path", ""),
|
|
("db_epoch", 0.0),
|
|
("fs_epoch", 0.0),
|
|
):
|
|
if not project_file_sync_state.has_column(column):
|
|
project_file_sync_state.create_column_by_example(column, example)
|
|
_index(
|
|
db,
|
|
"project_file_sync_state",
|
|
"idx_project_file_sync_state_path",
|
|
["project_uid", "path"],
|
|
unique=True,
|
|
)
|
|
_index(db, "badges", "idx_badges_user", ["user_uid"])
|
|
_index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"])
|
|
_drop_index(db, "idx_follows_follower")
|
|
_drop_index(db, "idx_follows_following")
|
|
_index(db, "follows", "idx_follows_follower_created", ["follower_uid", "created_at"])
|
|
_index(db, "follows", "idx_follows_following_created", ["following_uid", "created_at"])
|
|
user_relations = get_table("user_relations")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("target_uid", ""),
|
|
("kind", ""),
|
|
("created_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not user_relations.has_column(column):
|
|
user_relations.create_column_by_example(column, example)
|
|
_index(db, "user_relations", "idx_user_relations_owner_kind", ["user_uid", "kind"])
|
|
_index(db, "user_relations", "idx_user_relations_target_kind", ["target_uid", "kind"])
|
|
_index(db, "password_resets", "idx_password_resets_token", ["token"])
|
|
_index(db, "gists", "idx_gists_user_uid", ["user_uid"])
|
|
_index(db, "gists", "idx_gists_language", ["language"])
|
|
_index(db, "gists", "idx_gists_slug", ["slug"])
|
|
attachments = get_table("attachments")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("target_type", ""),
|
|
("target_uid", ""),
|
|
("resource_type", ""),
|
|
("resource_uid", ""),
|
|
("user_uid", ""),
|
|
("original_filename", ""),
|
|
("stored_name", ""),
|
|
("directory", ""),
|
|
("file_size", 0),
|
|
("mime_type", ""),
|
|
("image_width", 0),
|
|
("image_height", 0),
|
|
("has_thumbnail", 0),
|
|
("thumbnail_name", ""),
|
|
("gitea_asset_id", 0),
|
|
("created_at", ""),
|
|
("deleted_at", ""),
|
|
):
|
|
if not attachments.has_column(column):
|
|
attachments.create_column_by_example(column, example)
|
|
|
|
_index(
|
|
db, "attachments", "idx_attachments_resource", ["resource_type", "resource_uid"]
|
|
)
|
|
_index(db, "attachments", "idx_attachments_target", ["target_type", "target_uid"])
|
|
_index(db, "attachments", "idx_attachments_user_created", ["user_uid", "created_at"])
|
|
_index(db, "reactions", "idx_reactions_target", ["target_type", "target_uid"])
|
|
_index(
|
|
db,
|
|
"reactions",
|
|
"idx_reactions_user_target",
|
|
["user_uid", "target_type", "target_uid"],
|
|
)
|
|
_index(db, "bookmarks", "idx_bookmarks_user", ["user_uid"])
|
|
_index(db, "bookmarks", "idx_bookmarks_target", ["target_type", "target_uid"])
|
|
_index(db, "polls", "idx_polls_post", ["post_uid"])
|
|
_index(db, "poll_options", "idx_poll_options_poll", ["poll_uid"])
|
|
_index(db, "poll_votes", "idx_poll_votes_poll", ["poll_uid"])
|
|
_index(db, "poll_votes", "idx_poll_votes_user", ["poll_uid", "user_uid"])
|
|
|
|
if "site_settings" in tables:
|
|
defaults = {
|
|
"site_name": "DevPlace",
|
|
"site_description": "The Developer Social Network",
|
|
"site_tagline": "Track industry shifts. Discover bold releases. Share what you are building in an open environment built by developers, for developers.",
|
|
}
|
|
for key, value in defaults.items():
|
|
existing = db["site_settings"].find_one(key=key)
|
|
if not existing:
|
|
db["site_settings"].insert(
|
|
{"uid": f"default_{key}", "key": key, "value": value}
|
|
)
|
|
_index(db, "site_settings", "idx_site_settings_key", ["key"])
|
|
|
|
news = get_table("news")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("slug", ""),
|
|
("title", ""),
|
|
("external_id", ""),
|
|
("status", ""),
|
|
("source_name", ""),
|
|
("url", ""),
|
|
("description", ""),
|
|
("content", ""),
|
|
("synced_at", ""),
|
|
("grade", 0),
|
|
("ai_grade", 0),
|
|
("show_on_landing", 0),
|
|
("featured", 0),
|
|
("author", ""),
|
|
("article_published", ""),
|
|
("image_url", ""),
|
|
("has_unique_image", 0),
|
|
("featured_locked", 0),
|
|
("landing_locked", 0),
|
|
):
|
|
if not news.has_column(column):
|
|
news.create_column_by_example(column, example)
|
|
|
|
_index(db, "news", "idx_news_external_id", ["external_id"])
|
|
_index(db, "news", "idx_news_slug", ["slug"])
|
|
_index(db, "news", "idx_news_synced_at", ["synced_at"])
|
|
_index(db, "news", "idx_news_status", ["status"])
|
|
_index(db, "news", "idx_news_featured", ["featured"])
|
|
_index(db, "news", "idx_news_landing", ["show_on_landing"])
|
|
|
|
news_images = get_table("news_images")
|
|
for column, example in (
|
|
("news_uid", ""),
|
|
("url", ""),
|
|
("alt_text", ""),
|
|
("phash", ""),
|
|
("width", 0),
|
|
("height", 0),
|
|
("is_placeholder", 0),
|
|
):
|
|
if not news_images.has_column(column):
|
|
news_images.create_column_by_example(column, example)
|
|
|
|
news_sync = get_table("news_sync")
|
|
if not news_sync.has_column("external_id"):
|
|
news_sync.create_column_by_example("external_id", "")
|
|
|
|
_index(db, "news_images", "idx_news_images_news_uid", ["news_uid"])
|
|
_index(db, "news_sync", "idx_news_sync_external_id", ["external_id"])
|
|
|
|
issue_tickets = get_table("issue_tickets")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("gitea_number", 0),
|
|
("author_uid", ""),
|
|
("original_title", ""),
|
|
("original_description", ""),
|
|
("enhanced_title", ""),
|
|
("html_url", ""),
|
|
("last_status", ""),
|
|
("last_comment_count", 0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not issue_tickets.has_column(column):
|
|
issue_tickets.create_column_by_example(column, example)
|
|
|
|
issue_comment_authors = get_table("issue_comment_authors")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("gitea_comment_id", 0),
|
|
("gitea_number", 0),
|
|
("author_uid", ""),
|
|
("created_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not issue_comment_authors.has_column(column):
|
|
issue_comment_authors.create_column_by_example(column, example)
|
|
|
|
_index(db, "issue_tickets", "idx_issue_tickets_number", ["gitea_number"])
|
|
_index(db, "issue_tickets", "idx_issue_tickets_author", ["author_uid"])
|
|
_index(
|
|
db,
|
|
"issue_comment_authors",
|
|
"idx_issue_comment_authors_comment",
|
|
["gitea_comment_id"],
|
|
)
|
|
_index(
|
|
db, "issue_comment_authors", "idx_issue_comment_authors_number", ["gitea_number"]
|
|
)
|
|
|
|
ws_tickets = get_table("ws_tickets")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("token", ""),
|
|
("user_uid", ""),
|
|
("created_at", ""),
|
|
("expires_at", ""),
|
|
("used_at", ""),
|
|
):
|
|
if not ws_tickets.has_column(column):
|
|
ws_tickets.create_column_by_example(column, example)
|
|
_index(db, "ws_tickets", "idx_ws_tickets_token", ["token"], unique=True)
|
|
_index(db, "ws_tickets", "idx_ws_tickets_expires", ["expires_at"])
|
|
|
|
migrate_bug_tables_to_issue_tables()
|
|
_index(db, "service_state", "idx_service_state_name", ["name"])
|
|
if "devii_conversations" in db.tables:
|
|
conversations = get_table("devii_conversations")
|
|
if not conversations.has_column("channel"):
|
|
conversations.create_column_by_example("channel", "main")
|
|
try:
|
|
with db:
|
|
db.query(
|
|
"UPDATE devii_conversations SET channel='main' WHERE channel IS NULL"
|
|
)
|
|
except Exception as e: # noqa: BLE001
|
|
logger.warning(f"Could not backfill devii_conversations.channel: {e}")
|
|
_index(
|
|
db,
|
|
"devii_conversations",
|
|
"idx_devii_conv_owner",
|
|
["owner_kind", "owner_id", "channel"],
|
|
)
|
|
_index(
|
|
db,
|
|
"devii_usage_ledger",
|
|
"idx_devii_ledger_owner_time",
|
|
["owner_kind", "owner_id", "created_at"],
|
|
)
|
|
_index(
|
|
db,
|
|
"devii_turns",
|
|
"idx_devii_turns_owner_time",
|
|
["owner_kind", "owner_id", "started_at"],
|
|
)
|
|
if "devii_tasks" in db.tables:
|
|
tasks = get_table("devii_tasks")
|
|
for column, example in (
|
|
("expires_at", ""),
|
|
("failure_count", 0),
|
|
("notify", 0),
|
|
("tz", ""),
|
|
):
|
|
if not tasks.has_column(column):
|
|
tasks.create_column_by_example(column, example)
|
|
try:
|
|
with db:
|
|
db.query(
|
|
"UPDATE devii_tasks SET failure_count = 0 WHERE failure_count IS NULL"
|
|
)
|
|
except Exception as e: # noqa: BLE001
|
|
logger.warning(f"Could not backfill devii_tasks.failure_count: {e}")
|
|
task_runs = get_table("devii_task_runs")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("task_uid", ""),
|
|
("created_at", ""),
|
|
):
|
|
if not task_runs.has_column(column):
|
|
task_runs.create_column_by_example(column, example)
|
|
_index(
|
|
db,
|
|
"devii_task_runs",
|
|
"idx_devii_task_runs_owner_time",
|
|
["owner_kind", "owner_id", "created_at"],
|
|
)
|
|
_index(db, "devii_task_runs", "idx_devii_task_runs_time", ["created_at"])
|
|
_index(
|
|
db,
|
|
"devii_tasks",
|
|
"idx_devii_tasks_owner_created",
|
|
["owner_kind", "owner_id", "created_at"],
|
|
)
|
|
_index(db, "devii_tasks", "idx_devii_tasks_owner", ["owner_kind", "owner_id"])
|
|
_index(
|
|
db, "devii_tasks", "idx_devii_tasks_due", ["enabled", "status", "next_run_at"]
|
|
)
|
|
_index(db, "devii_lessons", "idx_devii_lessons_owner", ["owner_kind", "owner_id"])
|
|
_index(db, "devii_lessons", "idx_devii_lessons_owner_created", ["owner_kind", "owner_id", "created_at"])
|
|
_index(
|
|
db, "devii_virtual_tools", "idx_devii_vtools_owner", ["owner_kind", "owner_id"]
|
|
)
|
|
_index(
|
|
db,
|
|
"devii_virtual_tools",
|
|
"idx_devii_vtools_name",
|
|
["owner_kind", "owner_id", "name"],
|
|
)
|
|
_index(db, "devii_behavior", "idx_devii_behavior_owner", ["owner_kind", "owner_id"])
|
|
_index(db, "user_customizations", "idx_user_cust_owner", ["owner_kind", "owner_id"])
|
|
_index(
|
|
db,
|
|
"user_customizations",
|
|
"idx_user_cust_scope",
|
|
["owner_kind", "owner_id", "scope", "lang"],
|
|
)
|
|
gateway_usage_ledger = get_table("gateway_usage_ledger")
|
|
for column, example in (
|
|
("created_at", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("backend", ""),
|
|
("endpoint", ""),
|
|
("requested_model", ""),
|
|
("model", ""),
|
|
("status_code", 0),
|
|
("success", 0),
|
|
("error_category", ""),
|
|
("upstream_latency_ms", 0.0),
|
|
("gateway_overhead_ms", 0.0),
|
|
("queue_wait_ms", 0.0),
|
|
("connect_ms", 0.0),
|
|
("total_latency_ms", 0.0),
|
|
("prompt_tokens", 0),
|
|
("completion_tokens", 0),
|
|
("cache_hit_tokens", 0),
|
|
("cache_miss_tokens", 0),
|
|
("reasoning_tokens", 0),
|
|
("total_tokens", 0),
|
|
("tokens_per_second", 0.0),
|
|
("context_window", 0),
|
|
("context_utilization", 0.0),
|
|
("cost_usd", 0.0),
|
|
("input_cost_usd", 0.0),
|
|
("output_cost_usd", 0.0),
|
|
("native_cost", 0),
|
|
("stream_requested", 0),
|
|
("temperature", 0.0),
|
|
("top_p", 0.0),
|
|
("max_tokens", 0),
|
|
("has_tools", 0),
|
|
("retries_attempted", 0),
|
|
("retry_succeeded", 0),
|
|
("circuit_open", 0),
|
|
("user_agent", ""),
|
|
("app_reference", ""),
|
|
("ttft_ms", 0.0),
|
|
("inter_token_ms", 0.0),
|
|
("provider", ""),
|
|
("fallback_used_route", ""),
|
|
):
|
|
if not gateway_usage_ledger.has_column(column):
|
|
gateway_usage_ledger.create_column_by_example(column, example)
|
|
_index(db, "gateway_usage_ledger", "idx_gw_usage_time", ["created_at"])
|
|
_index(
|
|
db,
|
|
"gateway_usage_ledger",
|
|
"idx_gw_usage_provider_time",
|
|
["provider", "created_at"],
|
|
)
|
|
_index(
|
|
db,
|
|
"gateway_usage_ledger",
|
|
"idx_gw_usage_backend_time",
|
|
["backend", "created_at"],
|
|
)
|
|
_index(
|
|
db, "gateway_usage_ledger", "idx_gw_usage_model_time", ["model", "created_at"]
|
|
)
|
|
_index(
|
|
db,
|
|
"gateway_usage_ledger",
|
|
"idx_gw_usage_owner_time",
|
|
["owner_kind", "owner_id", "created_at"],
|
|
)
|
|
_index(
|
|
db,
|
|
"gateway_usage_ledger",
|
|
"idx_gw_usage_endpoint_time",
|
|
["endpoint", "created_at"],
|
|
)
|
|
_index(
|
|
db,
|
|
"gateway_usage_ledger",
|
|
"idx_gw_usage_appref_time",
|
|
["app_reference", "created_at"],
|
|
)
|
|
_index(db, "gateway_concurrency_samples", "idx_gw_conc_time", ["created_at"])
|
|
jobs_table = get_table("jobs")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("kind", ""),
|
|
("status", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("preferred_name", ""),
|
|
("payload", ""),
|
|
("result", ""),
|
|
("error", ""),
|
|
("retry_count", 0),
|
|
("created_at", ""),
|
|
("started_at", ""),
|
|
("completed_at", ""),
|
|
("updated_at", ""),
|
|
("duration_ms", 0),
|
|
("last_accessed_at", ""),
|
|
("expires_at", ""),
|
|
("bytes_in", 0),
|
|
("bytes_out", 0),
|
|
("item_count", 0),
|
|
):
|
|
if not jobs_table.has_column(column):
|
|
jobs_table.create_column_by_example(column, example)
|
|
_index(db, "jobs", "idx_jobs_kind_status", ["kind", "status"])
|
|
_index(db, "jobs", "idx_jobs_owner", ["owner_kind", "owner_id"])
|
|
_index(db, "jobs", "idx_jobs_expires", ["expires_at"])
|
|
_index(db, "project_forks", "idx_project_forks_source", ["source_project_uid"])
|
|
_index(db, "project_forks", "idx_project_forks_forked", ["forked_project_uid"])
|
|
instances = get_table("instances")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("project_uid", ""),
|
|
("slug", ""),
|
|
("name", ""),
|
|
("status", ""),
|
|
("created_at", ""),
|
|
("owner_uid", ""),
|
|
("created_by", ""),
|
|
("desired_state", ""),
|
|
("container_id", ""),
|
|
("ingress_slug", ""),
|
|
("ingress_port", 0),
|
|
("ports_json", ""),
|
|
("container_gateway", ""),
|
|
("run_as_uid", ""),
|
|
("boot_language", "none"),
|
|
("boot_script", ""),
|
|
("start_on_boot", 0),
|
|
("is_workspace", 0),
|
|
("workspace_owner_uid", ""),
|
|
("editor_port", 0),
|
|
("editor_host_port", 0),
|
|
("editor_password", ""),
|
|
("tunnel_name", ""),
|
|
("last_active_at", ""),
|
|
("idle_warned_at", ""),
|
|
("delete_warned_at", ""),
|
|
("disk_bytes", 0),
|
|
("disk_sampled_at", ""),
|
|
("egress_bytes", 0),
|
|
("request_count", 0),
|
|
("flagged_at", ""),
|
|
("flag_reason", ""),
|
|
("suspended_at", ""),
|
|
("suspended_by", ""),
|
|
("boot_marker", ""),
|
|
("workspace_cpu_millicores", 0),
|
|
("workspace_memory_mb", 0),
|
|
("workspace_disk_quota_mb", 0),
|
|
):
|
|
if not instances.has_column(column):
|
|
instances.create_column_by_example(column, example)
|
|
|
|
tunnels = get_table("tunnels")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("instance_uid", ""),
|
|
("project_uid", ""),
|
|
("user_uid", ""),
|
|
("hostname", ""),
|
|
("label", ""),
|
|
("container_port", 0),
|
|
("desired_state", "present"),
|
|
("status", "pending"),
|
|
("cert_status", ""),
|
|
("cert_checked_at", ""),
|
|
("request_count", 0),
|
|
("bytes_out", 0),
|
|
("last_request_at", ""),
|
|
("last_error", ""),
|
|
("last_synced_at", ""),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not tunnels.has_column(column):
|
|
tunnels.create_column_by_example(column, example)
|
|
|
|
quota_rules = get_table("workspace_quota_rules")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("label", ""),
|
|
("max_workspaces", 0),
|
|
("max_tunnels", 0),
|
|
("disk_quota_mb", 0),
|
|
("egress_quota_mb", 0),
|
|
("idle_stop_minutes", 0),
|
|
("retention_days", 0),
|
|
("cpu_millicores", 0),
|
|
("memory_mb", 0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not quota_rules.has_column(column):
|
|
quota_rules.create_column_by_example(column, example)
|
|
|
|
editor_prefs = get_table("workspace_editor_prefs")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("font_size", 0),
|
|
("terminal_font_size", 0),
|
|
("zoom_level", -99),
|
|
("theme", ""),
|
|
("layout", ""),
|
|
("panel_preset", ""),
|
|
("window_mode", ""),
|
|
("window_width", 0),
|
|
("window_height", 0),
|
|
("boot_agent", ""),
|
|
("boot_shell", -1),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not editor_prefs.has_column(column):
|
|
editor_prefs.create_column_by_example(column, example)
|
|
|
|
flags = get_table("workspace_flags")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("instance_uid", ""),
|
|
("user_uid", ""),
|
|
("kind", ""),
|
|
("severity", "warn"),
|
|
("detail", ""),
|
|
("metric_value", 0.0),
|
|
("threshold", 0.0),
|
|
("status", "open"),
|
|
("resolved_by", ""),
|
|
("resolved_at", ""),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not flags.has_column(column):
|
|
flags.create_column_by_example(column, example)
|
|
|
|
_index(db, "instances", "idx_instances_workspace", ["is_workspace", "status"])
|
|
_index(db, "instances", "idx_instances_workspace_owner", ["workspace_owner_uid"])
|
|
_index(db, "instances", "idx_instances_tunnel_name", ["tunnel_name"])
|
|
_dedupe_tunnel_hostnames()
|
|
_drop_index(db, "idx_tunnels_hostname")
|
|
_index(db, "tunnels", "idx_tunnels_hostname", ["hostname"], unique=True)
|
|
_index(db, "tunnels", "idx_tunnels_instance", ["instance_uid"])
|
|
_index(db, "tunnels", "idx_tunnels_user", ["user_uid"])
|
|
_index(db, "tunnels", "idx_tunnels_state", ["desired_state", "status"])
|
|
_index(
|
|
db,
|
|
"workspace_quota_rules",
|
|
"idx_workspace_quota_owner",
|
|
["owner_kind", "owner_id"],
|
|
)
|
|
_index(
|
|
db,
|
|
"workspace_editor_prefs",
|
|
"idx_workspace_editor_prefs_owner",
|
|
["owner_kind", "owner_id"],
|
|
)
|
|
_index(db, "workspace_flags", "idx_workspace_flags_open", ["status", "created_at"])
|
|
_index(
|
|
db,
|
|
"workspace_flags",
|
|
"idx_workspace_flags_instance",
|
|
["instance_uid", "kind"],
|
|
)
|
|
_index(db, "workspace_flags", "idx_workspace_flags_user", ["user_uid"])
|
|
|
|
_index(db, "instances", "idx_instances_project", ["project_uid"])
|
|
_index(db, "instances", "idx_instances_slug", ["slug"])
|
|
_index(db, "instances", "idx_instances_name", ["name"])
|
|
_index(db, "instances", "idx_instances_state", ["desired_state", "status"])
|
|
_index(db, "instances", "idx_instances_container", ["container_id"])
|
|
_index(db, "instances", "idx_instances_ingress", ["ingress_slug"])
|
|
_index(
|
|
db,
|
|
"instance_events",
|
|
"idx_instance_events_instance",
|
|
["instance_uid", "created_at"],
|
|
)
|
|
_index(
|
|
db,
|
|
"instance_metrics",
|
|
"idx_instance_metrics_instance_ts",
|
|
["instance_uid", "ts"],
|
|
)
|
|
_index(
|
|
db,
|
|
"instance_schedules",
|
|
"idx_instance_schedules_due",
|
|
["enabled", "next_run_at"],
|
|
)
|
|
|
|
from devplacepy.services.audit import store as audit_store
|
|
|
|
audit_store.ensure_tables()
|
|
|
|
from devplacepy.services.backup import store as backup_store
|
|
|
|
backup_store.ensure_tables()
|
|
|
|
from devplacepy.services.openai_gateway import routing as gateway_routing
|
|
|
|
gateway_routing.ensure_tables()
|
|
|
|
from devplacepy.services.openai_gateway import quota as gateway_quota
|
|
|
|
gateway_quota.ensure_tables()
|
|
_index(db, "audit_log", "idx_audit_created_at", ["created_at"])
|
|
_index(db, "audit_log", "idx_audit_event_key", ["event_key"])
|
|
_index(db, "audit_log", "idx_audit_category", ["category"])
|
|
_index(db, "audit_log", "idx_audit_actor_uid", ["actor_uid"])
|
|
_index(db, "audit_log", "idx_audit_actor_role", ["actor_role"])
|
|
_index(db, "audit_log", "idx_audit_origin", ["origin"])
|
|
_index(db, "audit_log", "idx_audit_target", ["target_type", "target_uid"])
|
|
_index(db, "audit_log", "idx_audit_result", ["result"])
|
|
_index(db, "audit_log_links", "idx_audit_links_audit", ["audit_uid"])
|
|
_index(db, "audit_log_links", "idx_audit_links_object", ["object_type", "object_uid"])
|
|
_index(db, "audit_log_links", "idx_audit_links_relation", ["relation"])
|
|
|
|
notification_preferences = get_table("notification_preferences")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("notification_type", ""),
|
|
("in_app_enabled", 1),
|
|
("push_enabled", 1),
|
|
("telegram_enabled", 1),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not notification_preferences.has_column(column):
|
|
notification_preferences.create_column_by_example(column, example)
|
|
_index(db, "notification_preferences", "idx_notif_prefs_user", ["user_uid"])
|
|
_index(
|
|
db,
|
|
"notification_preferences",
|
|
"idx_notif_prefs_user_type",
|
|
["user_uid", "notification_type"],
|
|
)
|
|
|
|
correction_usage = get_table("correction_usage")
|
|
for column, example in (
|
|
("user_uid", ""),
|
|
("calls", 0),
|
|
("prompt_tokens", 0),
|
|
("completion_tokens", 0),
|
|
("total_tokens", 0),
|
|
("cost_usd", 0.0),
|
|
("upstream_latency_ms", 0.0),
|
|
("total_latency_ms", 0.0),
|
|
("updated_at", ""),
|
|
):
|
|
if not correction_usage.has_column(column):
|
|
correction_usage.create_column_by_example(column, example)
|
|
try:
|
|
if "correction_usage" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user "
|
|
"ON correction_usage (user_uid)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create unique index on correction_usage: {e}")
|
|
|
|
modifier_usage = get_table("modifier_usage")
|
|
for column, example in (
|
|
("user_uid", ""),
|
|
("calls", 0),
|
|
("prompt_tokens", 0),
|
|
("completion_tokens", 0),
|
|
("total_tokens", 0),
|
|
("cost_usd", 0.0),
|
|
("upstream_latency_ms", 0.0),
|
|
("total_latency_ms", 0.0),
|
|
("updated_at", ""),
|
|
):
|
|
if not modifier_usage.has_column(column):
|
|
modifier_usage.create_column_by_example(column, example)
|
|
try:
|
|
if "modifier_usage" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user "
|
|
"ON modifier_usage (user_uid)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create unique index on modifier_usage: {e}")
|
|
|
|
news_usage = get_table("news_usage")
|
|
for column, example in (
|
|
("user_uid", ""),
|
|
("calls", 0),
|
|
("prompt_tokens", 0),
|
|
("completion_tokens", 0),
|
|
("total_tokens", 0),
|
|
("cost_usd", 0.0),
|
|
("upstream_latency_ms", 0.0),
|
|
("total_latency_ms", 0.0),
|
|
("updated_at", ""),
|
|
):
|
|
if not news_usage.has_column(column):
|
|
news_usage.create_column_by_example(column, example)
|
|
try:
|
|
if "news_usage" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user "
|
|
"ON news_usage (user_uid)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create unique index on news_usage: {e}")
|
|
|
|
issue_usage = get_table("issue_usage")
|
|
for column, example in (
|
|
("user_uid", ""),
|
|
("calls", 0),
|
|
("prompt_tokens", 0),
|
|
("completion_tokens", 0),
|
|
("total_tokens", 0),
|
|
("cost_usd", 0.0),
|
|
("upstream_latency_ms", 0.0),
|
|
("total_latency_ms", 0.0),
|
|
("updated_at", ""),
|
|
):
|
|
if not issue_usage.has_column(column):
|
|
issue_usage.create_column_by_example(column, example)
|
|
try:
|
|
if "issue_usage" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user "
|
|
"ON issue_usage (user_uid)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create unique index on issue_usage: {e}")
|
|
|
|
seo_usage = get_table("seo_usage")
|
|
for column, example in (
|
|
("user_uid", ""),
|
|
("calls", 0),
|
|
("prompt_tokens", 0),
|
|
("completion_tokens", 0),
|
|
("total_tokens", 0),
|
|
("cost_usd", 0.0),
|
|
("upstream_latency_ms", 0.0),
|
|
("total_latency_ms", 0.0),
|
|
("updated_at", ""),
|
|
):
|
|
if not seo_usage.has_column(column):
|
|
seo_usage.create_column_by_example(column, example)
|
|
try:
|
|
if "seo_usage" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user "
|
|
"ON seo_usage (user_uid)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create unique index on seo_usage: {e}")
|
|
|
|
award_usage = get_table("award_usage")
|
|
for column, example in (
|
|
("user_uid", ""),
|
|
("calls", 0),
|
|
("prompt_tokens", 0),
|
|
("completion_tokens", 0),
|
|
("total_tokens", 0),
|
|
("cost_usd", 0.0),
|
|
("upstream_latency_ms", 0.0),
|
|
("total_latency_ms", 0.0),
|
|
("updated_at", ""),
|
|
):
|
|
if not award_usage.has_column(column):
|
|
award_usage.create_column_by_example(column, example)
|
|
try:
|
|
if "award_usage" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_award_usage_user "
|
|
"ON award_usage (user_uid)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create unique index on award_usage: {e}")
|
|
|
|
awards = get_table("awards")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("slug", ""),
|
|
("description", ""),
|
|
("giver_uid", ""),
|
|
("receiver_uid", ""),
|
|
("attachment_uid_512", ""),
|
|
("attachment_uid_256", ""),
|
|
("attachment_uid_64", ""),
|
|
("generated_at", ""),
|
|
("created_at", ""),
|
|
("job_uid", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not awards.has_column(column):
|
|
awards.create_column_by_example(column, example)
|
|
try:
|
|
if "awards" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_awards_slug ON awards (slug)"
|
|
)
|
|
db.query(
|
|
"CREATE INDEX IF NOT EXISTS idx_awards_receiver_created "
|
|
"ON awards (receiver_uid, created_at)"
|
|
)
|
|
db.query(
|
|
"CREATE INDEX IF NOT EXISTS idx_awards_giver_created "
|
|
"ON awards (giver_uid, created_at)"
|
|
)
|
|
db.query(
|
|
"CREATE INDEX IF NOT EXISTS idx_awards_generated "
|
|
"ON awards (receiver_uid, generated_at)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create awards indexes: {e}")
|
|
|
|
seo_metadata = get_table("seo_metadata")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("target_type", ""),
|
|
("target_uid", ""),
|
|
("seo_title", ""),
|
|
("seo_description", ""),
|
|
("seo_keywords", ""),
|
|
("status", ""),
|
|
("source", ""),
|
|
("generated_at", ""),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not seo_metadata.has_column(column):
|
|
seo_metadata.create_column_by_example(column, example)
|
|
_index(
|
|
db,
|
|
"seo_metadata",
|
|
"idx_seo_metadata_target",
|
|
["target_type", "target_uid"],
|
|
unique=True,
|
|
)
|
|
|
|
email_accounts = get_table("email_accounts")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("label", ""),
|
|
("imap_host", ""),
|
|
("imap_port", 993),
|
|
("imap_ssl", 1),
|
|
("imap_starttls", 0),
|
|
("smtp_host", ""),
|
|
("smtp_port", 587),
|
|
("smtp_ssl", 0),
|
|
("smtp_starttls", 1),
|
|
("username", ""),
|
|
("password", ""),
|
|
("from_address", ""),
|
|
("from_name", ""),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not email_accounts.has_column(column):
|
|
email_accounts.create_column_by_example(column, example)
|
|
_index(db, "email_accounts", "idx_email_accounts_owner", ["owner_kind", "owner_id"])
|
|
_index(
|
|
db,
|
|
"email_accounts",
|
|
"idx_email_accounts_label",
|
|
["owner_kind", "owner_id", "label"],
|
|
)
|
|
|
|
user_activity = get_table("user_activity")
|
|
for column, example in (
|
|
("user_uid", ""),
|
|
("action", ""),
|
|
("count", 0),
|
|
("first_at", ""),
|
|
("last_at", ""),
|
|
):
|
|
if not user_activity.has_column(column):
|
|
user_activity.create_column_by_example(column, example)
|
|
try:
|
|
if "user_activity" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action "
|
|
"ON user_activity (user_uid, action)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create unique index on user_activity: {e}")
|
|
|
|
user_activity_seen = get_table("user_activity_seen")
|
|
for column, example in (
|
|
("user_uid", ""),
|
|
("action", ""),
|
|
("target", ""),
|
|
("created_at", ""),
|
|
):
|
|
if not user_activity_seen.has_column(column):
|
|
user_activity_seen.create_column_by_example(column, example)
|
|
try:
|
|
if "user_activity_seen" in db.tables:
|
|
with db:
|
|
db.query(
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique "
|
|
"ON user_activity_seen (user_uid, action, target)"
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Could not create unique index on user_activity_seen: {e}")
|
|
|
|
deepsearch_sessions = get_table("deepsearch_sessions")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("query", ""),
|
|
("status", ""),
|
|
("depth", 0),
|
|
("max_pages", 0),
|
|
("score", 0),
|
|
("confidence", 0.0),
|
|
("source_diversity", 0.0),
|
|
("page_count", 0),
|
|
("chunk_count", 0),
|
|
("collection", ""),
|
|
("summary", ""),
|
|
("created_at", ""),
|
|
("completed_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not deepsearch_sessions.has_column(column):
|
|
deepsearch_sessions.create_column_by_example(column, example)
|
|
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_owner", ["owner_kind", "owner_id"])
|
|
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_status", ["status"])
|
|
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_created", ["created_at"])
|
|
|
|
deepsearch_messages = get_table("deepsearch_messages")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("session_uid", ""),
|
|
("role", ""),
|
|
("content", ""),
|
|
("citations", ""),
|
|
("created_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not deepsearch_messages.has_column(column):
|
|
deepsearch_messages.create_column_by_example(column, example)
|
|
_index(db, "deepsearch_messages", "idx_deepsearch_messages_session", ["session_uid"])
|
|
|
|
deepsearch_url_cache = get_table("deepsearch_url_cache")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("url_hash", ""),
|
|
("url", ""),
|
|
("title", ""),
|
|
("content_hash", ""),
|
|
("status", 0),
|
|
("byte_size", 0),
|
|
("fetched_at", ""),
|
|
):
|
|
if not deepsearch_url_cache.has_column(column):
|
|
deepsearch_url_cache.create_column_by_example(column, example)
|
|
_index(db, "deepsearch_url_cache", "idx_deepsearch_url_cache_hash", ["url_hash"])
|
|
|
|
isslop_analyses = get_table("isslop_analyses")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("source_url", ""),
|
|
("source_kind", ""),
|
|
("status", ""),
|
|
("created_at", ""),
|
|
("finished_at", ""),
|
|
("content_hash", ""),
|
|
("grade", ""),
|
|
("slop_score", 0.0),
|
|
("origin_score", 0.0),
|
|
("quality_deficit_score", 0.0),
|
|
("human_percent", 0.0),
|
|
("ai_percent", 0.0),
|
|
("category", ""),
|
|
("confidence", ""),
|
|
("files_total", 0),
|
|
("files_analyzed", 0),
|
|
("error_message_text", ""),
|
|
("detected_builder", ""),
|
|
("dom_slop_score", 0.0),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not isslop_analyses.has_column(column):
|
|
isslop_analyses.create_column_by_example(column, example)
|
|
_index(db, "isslop_analyses", "idx_isslop_analyses_owner", ["owner_kind", "owner_id", "created_at"])
|
|
_index(db, "isslop_analyses", "idx_isslop_analyses_status", ["status"])
|
|
_index(db, "isslop_analyses", "idx_isslop_analyses_hash", ["content_hash"])
|
|
|
|
isslop_events = get_table("isslop_events")
|
|
for column, example in (
|
|
("analysis_uid", ""),
|
|
("seq", 0),
|
|
("kind", ""),
|
|
("message", ""),
|
|
("payload", ""),
|
|
("created_at", ""),
|
|
):
|
|
if not isslop_events.has_column(column):
|
|
isslop_events.create_column_by_example(column, example)
|
|
_index(db, "isslop_events", "idx_isslop_events_analysis", ["analysis_uid", "seq"])
|
|
|
|
isslop_file_results = get_table("isslop_file_results")
|
|
for column, example in (
|
|
("analysis_uid", ""),
|
|
("path", ""),
|
|
("language", ""),
|
|
("lines", 0),
|
|
("origin_score", 0.0),
|
|
("quality_deficit_score", 0.0),
|
|
("category", ""),
|
|
("signals", ""),
|
|
("source", ""),
|
|
):
|
|
if not isslop_file_results.has_column(column):
|
|
isslop_file_results.create_column_by_example(column, example)
|
|
_index(db, "isslop_file_results", "idx_isslop_file_results_analysis", ["analysis_uid"])
|
|
|
|
isslop_image_results = get_table("isslop_image_results")
|
|
for column, example in (
|
|
("analysis_uid", ""),
|
|
("path", ""),
|
|
("ai_probability", 0.0),
|
|
("grade", ""),
|
|
("verdict", ""),
|
|
("image_kind", ""),
|
|
("tells", ""),
|
|
("description", ""),
|
|
("thumb", ""),
|
|
):
|
|
if not isslop_image_results.has_column(column):
|
|
isslop_image_results.create_column_by_example(column, example)
|
|
_index(db, "isslop_image_results", "idx_isslop_image_results_analysis", ["analysis_uid"])
|
|
|
|
isslop_dom_results = get_table("isslop_dom_results")
|
|
for column, example in (
|
|
("analysis_uid", ""),
|
|
("url", ""),
|
|
("detected_builder", ""),
|
|
("signal_count", 0),
|
|
("screenshot", ""),
|
|
("signals", ""),
|
|
):
|
|
if not isslop_dom_results.has_column(column):
|
|
isslop_dom_results.create_column_by_example(column, example)
|
|
_index(db, "isslop_dom_results", "idx_isslop_dom_results_analysis", ["analysis_uid"])
|
|
|
|
isslop_reports = get_table("isslop_reports")
|
|
for column, example in (
|
|
("analysis_uid", ""),
|
|
("markdown", ""),
|
|
("model_used", ""),
|
|
("generated_at", ""),
|
|
):
|
|
if not isslop_reports.has_column(column):
|
|
isslop_reports.create_column_by_example(column, example)
|
|
_index(db, "isslop_reports", "idx_isslop_reports_analysis", ["analysis_uid"], unique=True)
|
|
|
|
game_farms = get_table("game_farms")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("coins", 0),
|
|
("xp", 0),
|
|
("level", 1),
|
|
("ci_tier", 1),
|
|
("plot_count", 0),
|
|
("total_harvests", 0),
|
|
("prestige", 0),
|
|
("streak", 0),
|
|
("last_daily_at", ""),
|
|
("perk_yield", 0),
|
|
("perk_growth", 0),
|
|
("perk_discount", 0),
|
|
("perk_xp", 0),
|
|
("stars", 0),
|
|
("legacy_autoharvest", 0),
|
|
("legacy_multiplier", 0),
|
|
("legacy_speed", 0),
|
|
("legacy_plots", 0),
|
|
("legacy_defense", 0),
|
|
("legacy_carryover", 0),
|
|
("last_grant_week", ""),
|
|
("prestiged_at", ""),
|
|
("mastery_points", 0),
|
|
("mastery_points_earned_total", 0),
|
|
("mastery_autoreplant", 0),
|
|
("mastery_analytics", 0),
|
|
("mastery_contracts", 0),
|
|
("lifetime_coins_earned", 0),
|
|
("lifetime_harvests", 0),
|
|
("infra_registry", 0),
|
|
("infra_canary", 0),
|
|
("infra_observability", 0),
|
|
("defense_level", 0),
|
|
("defense_last_upkeep_at", ""),
|
|
("upkeep_amnesty", 0),
|
|
("active_title", ""),
|
|
("underdog_boost_until", ""),
|
|
("contract_boost_until", ""),
|
|
("harvests_week", 0),
|
|
("harvests_week_start", ""),
|
|
("last_kernel_harvest_prestige", 0),
|
|
("time_to_kernel_seconds", 0),
|
|
("era_coins", 0),
|
|
("era_harvests", 0),
|
|
("era_joined_at", ""),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not game_farms.has_column(column):
|
|
game_farms.create_column_by_example(column, example)
|
|
_index(db, "game_farms", "idx_game_farms_user", ["user_uid"], unique=True)
|
|
_index(db, "game_farms", "idx_game_farms_rank", ["level", "xp"])
|
|
|
|
game_steals = get_table("game_steals")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("thief_uid", ""),
|
|
("owner_uid", ""),
|
|
("slot_index", 0),
|
|
("crop_key", ""),
|
|
("coins", 0),
|
|
("stolen_at", ""),
|
|
("created_at", ""),
|
|
):
|
|
if not game_steals.has_column(column):
|
|
game_steals.create_column_by_example(column, example)
|
|
_index(
|
|
db, "game_steals", "idx_game_steals_pair", ["thief_uid", "owner_uid", "stolen_at"]
|
|
)
|
|
_index(db, "game_steals", "idx_game_steals_owner_time", ["owner_uid", "stolen_at"])
|
|
|
|
game_quests = get_table("game_quests")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("farm_uid", ""),
|
|
("user_uid", ""),
|
|
("day", ""),
|
|
("scope", "daily"),
|
|
("slot_index", 0),
|
|
("kind", ""),
|
|
("label", ""),
|
|
("goal", 0),
|
|
("progress", 0),
|
|
("reward_coins", 0),
|
|
("reward_xp", 0),
|
|
("reward_stars", 0),
|
|
("claimed", 0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not game_quests.has_column(column):
|
|
game_quests.create_column_by_example(column, example)
|
|
with db:
|
|
db.query("UPDATE game_quests SET scope = 'daily' WHERE scope IS NULL OR scope = ''")
|
|
_index(db, "game_quests", "idx_game_quests_farm_day", ["farm_uid", "day"])
|
|
_index(db, "game_quests", "idx_game_quests_farm_day_scope", ["farm_uid", "day", "scope"])
|
|
|
|
game_plots = get_table("game_plots")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("farm_uid", ""),
|
|
("user_uid", ""),
|
|
("slot_index", 0),
|
|
("crop_key", ""),
|
|
("planted_at", ""),
|
|
("ready_at", ""),
|
|
("watered_by", "[]"),
|
|
("raided_fraction", 0.0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
):
|
|
if not game_plots.has_column(column):
|
|
game_plots.create_column_by_example(column, example)
|
|
_index(db, "game_plots", "idx_game_plots_farm", ["farm_uid", "slot_index"])
|
|
|
|
game_market_ticks = get_table("game_market_ticks")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("crop_key", ""),
|
|
("hour_bucket", ""),
|
|
("harvests", 0),
|
|
("updated_at", ""),
|
|
):
|
|
if not game_market_ticks.has_column(column):
|
|
game_market_ticks.create_column_by_example(column, example)
|
|
_index(
|
|
db,
|
|
"game_market_ticks",
|
|
"idx_game_market_ticks_bucket",
|
|
["crop_key", "hour_bucket"],
|
|
unique=True,
|
|
)
|
|
|
|
game_cosmetics = get_table("game_cosmetics")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("cosmetic_key", ""),
|
|
("purchased_at", ""),
|
|
("created_at", ""),
|
|
):
|
|
if not game_cosmetics.has_column(column):
|
|
game_cosmetics.create_column_by_example(column, example)
|
|
_index(
|
|
db,
|
|
"game_cosmetics",
|
|
"idx_game_cosmetics_owner",
|
|
["user_uid", "cosmetic_key"],
|
|
unique=True,
|
|
)
|
|
|
|
game_treasury = get_table("game_treasury")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("balance", 0),
|
|
("collected_total", 0),
|
|
("granted_total", 0),
|
|
("updated_at", ""),
|
|
):
|
|
if not game_treasury.has_column(column):
|
|
game_treasury.create_column_by_example(column, example)
|
|
|
|
game_eras = get_table("game_eras")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("era_number", 0),
|
|
("name", ""),
|
|
("started_at", ""),
|
|
("ends_at", ""),
|
|
("active", 0),
|
|
("created_at", ""),
|
|
):
|
|
if not game_eras.has_column(column):
|
|
game_eras.create_column_by_example(column, example)
|
|
_index(db, "game_eras", "idx_game_eras_active", ["active"])
|
|
|
|
game_era_results = get_table("game_era_results")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("era_number", 0),
|
|
("user_uid", ""),
|
|
("rank", 0),
|
|
("era_score", 0),
|
|
("era_coins_final", 0),
|
|
("joined_at", ""),
|
|
("reward_stars", 0),
|
|
("reward_cosmetic_key", ""),
|
|
("created_at", ""),
|
|
):
|
|
if not game_era_results.has_column(column):
|
|
game_era_results.create_column_by_example(column, example)
|
|
_index(db, "game_era_results", "idx_game_era_results_era", ["era_number", "rank"])
|
|
|
|
quizzes = get_table("quizzes")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("user_uid", ""),
|
|
("slug", ""),
|
|
("title", ""),
|
|
("description", ""),
|
|
("status", "draft"),
|
|
("published_at", ""),
|
|
("shuffle_questions", 0),
|
|
("shuffle_options", 0),
|
|
("reveal_answers", 0),
|
|
("allow_review", 0),
|
|
("time_limit_seconds", 0),
|
|
("pass_percent", 0),
|
|
("question_count", 0),
|
|
("total_points", 0),
|
|
("attempt_count", 0),
|
|
("stars", 0),
|
|
("content_version", 0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not quizzes.has_column(column):
|
|
quizzes.create_column_by_example(column, example)
|
|
_index(db, "quizzes", "idx_quizzes_slug", ["slug"], unique=True)
|
|
_index(db, "quizzes", "idx_quizzes_user_created", ["user_uid", "created_at"])
|
|
_index(db, "quizzes", "idx_quizzes_status_created", ["status", "created_at"])
|
|
_index(
|
|
db,
|
|
"quizzes",
|
|
"idx_quizzes_live_created",
|
|
["created_at"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
|
|
quiz_questions = get_table("quiz_questions")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("quiz_uid", ""),
|
|
("position", 0),
|
|
("kind", ""),
|
|
("prompt", ""),
|
|
("explanation", ""),
|
|
("points", 1),
|
|
("media_attachment_uid", ""),
|
|
("correct_boolean", 0),
|
|
("expected_answer", ""),
|
|
("grading_criteria", ""),
|
|
("numeric_value", 0.0),
|
|
("numeric_tolerance", 0.0),
|
|
("case_sensitive", 0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not quiz_questions.has_column(column):
|
|
quiz_questions.create_column_by_example(column, example)
|
|
_index(
|
|
db, "quiz_questions", "idx_quiz_questions_quiz_position", ["quiz_uid", "position"]
|
|
)
|
|
|
|
quiz_options = get_table("quiz_options")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("question_uid", ""),
|
|
("quiz_uid", ""),
|
|
("position", 0),
|
|
("label", ""),
|
|
("match_value", ""),
|
|
("is_correct", 0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not quiz_options.has_column(column):
|
|
quiz_options.create_column_by_example(column, example)
|
|
_index(
|
|
db,
|
|
"quiz_options",
|
|
"idx_quiz_options_question_position",
|
|
["question_uid", "position"],
|
|
)
|
|
_index(db, "quiz_options", "idx_quiz_options_quiz", ["quiz_uid"])
|
|
|
|
quiz_attempts = get_table("quiz_attempts")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("quiz_uid", ""),
|
|
("user_uid", ""),
|
|
("status", "in_progress"),
|
|
("question_order", "[]"),
|
|
("started_at", ""),
|
|
("expires_at", ""),
|
|
("completed_at", ""),
|
|
("answered_count", 0),
|
|
("score_points", 0.0),
|
|
("max_points", 0),
|
|
("score_percent", 0.0),
|
|
("passed", 0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not quiz_attempts.has_column(column):
|
|
quiz_attempts.create_column_by_example(column, example)
|
|
_index(db, "quiz_attempts", "idx_quiz_attempts_user_created", ["user_uid", "created_at"])
|
|
_index(db, "quiz_attempts", "idx_quiz_attempts_quiz_status", ["quiz_uid", "status"])
|
|
_index(db, "quiz_attempts", "idx_quiz_attempts_user_quiz", ["user_uid", "quiz_uid"])
|
|
_index(db, "quiz_attempts", "idx_quiz_attempts_status_user", ["status", "user_uid"])
|
|
|
|
quiz_answers = get_table("quiz_answers")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("attempt_uid", ""),
|
|
("question_uid", ""),
|
|
("quiz_uid", ""),
|
|
("position", 0),
|
|
("answer_text", ""),
|
|
("option_uids", "[]"),
|
|
("answered_at", ""),
|
|
("is_correct", 0),
|
|
("awarded_points", 0.0),
|
|
("feedback", ""),
|
|
("graded_by", ""),
|
|
("confidence", 0.0),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not quiz_answers.has_column(column):
|
|
quiz_answers.create_column_by_example(column, example)
|
|
_index(
|
|
db, "quiz_answers", "idx_quiz_answers_attempt_position", ["attempt_uid", "position"]
|
|
)
|
|
_index(db, "quiz_answers", "idx_quiz_answers_quiz", ["quiz_uid"])
|
|
|
|
opinion_wars = get_table("opinion_wars")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("post_uid", ""),
|
|
("user_uid", ""),
|
|
("faction_a", ""),
|
|
("faction_b", ""),
|
|
("hp_a", 0),
|
|
("hp_b", 0),
|
|
("leader", ""),
|
|
("status", "active"),
|
|
("winner", ""),
|
|
("created_at", ""),
|
|
("ends_at", ""),
|
|
("resolved_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not opinion_wars.has_column(column):
|
|
opinion_wars.create_column_by_example(column, example)
|
|
_index(db, "opinion_wars", "idx_opinion_wars_post", ["post_uid"], unique=True)
|
|
_index(db, "opinion_wars", "idx_opinion_wars_status_ends", ["status", "ends_at"])
|
|
_index(
|
|
db,
|
|
"opinion_wars",
|
|
"idx_opinion_wars_live_created",
|
|
["created_at"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
|
|
opinion_war_fighters = get_table("opinion_war_fighters")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("war_uid", ""),
|
|
("user_uid", ""),
|
|
("faction", ""),
|
|
("hp_a", 0),
|
|
("hp_b", 0),
|
|
("fight_count", 0),
|
|
("last_fight_at", ""),
|
|
("cooldown_notified_at", ""),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not opinion_war_fighters.has_column(column):
|
|
opinion_war_fighters.create_column_by_example(column, example)
|
|
_index(
|
|
db,
|
|
"opinion_war_fighters",
|
|
"idx_opinion_war_fighters_war_user",
|
|
["war_uid", "user_uid"],
|
|
unique=True,
|
|
)
|
|
_index(
|
|
db,
|
|
"opinion_war_fighters",
|
|
"idx_opinion_war_fighters_war_faction",
|
|
["war_uid", "faction"],
|
|
)
|
|
_index(
|
|
db,
|
|
"opinion_war_fighters",
|
|
"idx_opinion_war_fighters_last_fight",
|
|
["last_fight_at"],
|
|
)
|
|
|
|
opinion_war_events = get_table("opinion_war_events")
|
|
for column, example in (
|
|
("uid", ""),
|
|
("war_uid", ""),
|
|
("seq", 0),
|
|
("kind", ""),
|
|
("message", ""),
|
|
("payload", ""),
|
|
("actor_uid", ""),
|
|
("created_at", ""),
|
|
("deleted_at", ""),
|
|
("deleted_by", ""),
|
|
):
|
|
if not opinion_war_events.has_column(column):
|
|
opinion_war_events.create_column_by_example(column, example)
|
|
_index(
|
|
db,
|
|
"opinion_war_events",
|
|
"idx_opinion_war_events_war_seq",
|
|
["war_uid", "seq"],
|
|
unique=True,
|
|
)
|
|
|
|
_index(db, "posts", "idx_posts_user_created", ["user_uid", "created_at"])
|
|
_index(
|
|
db,
|
|
"posts",
|
|
"idx_posts_live_created",
|
|
["created_at"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
_index(
|
|
db,
|
|
"comments",
|
|
"idx_comments_target_created",
|
|
["target_type", "target_uid", "created_at"],
|
|
)
|
|
_index(db, "comments", "idx_comments_user_created", ["user_uid", "created_at"])
|
|
_index(db, "votes", "idx_votes_user_target", ["user_uid", "target_uid"])
|
|
_index(db, "gists", "idx_gists_user_created", ["user_uid", "created_at"])
|
|
_index(db, "projects", "idx_projects_user_created", ["user_uid", "created_at"])
|
|
_index(
|
|
db,
|
|
"notifications",
|
|
"idx_notifications_user_created",
|
|
["user_uid", "created_at"],
|
|
)
|
|
|
|
_ensure_moderation_tables()
|
|
|
|
for table in db.tables:
|
|
_uid_index(db, table)
|
|
|
|
for table in SOFT_DELETE_TABLES:
|
|
ensure_soft_delete_columns(table)
|
|
|
|
_index(
|
|
db,
|
|
"comments",
|
|
"idx_comments_target_live",
|
|
["target_type", "target_uid"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
_index(
|
|
db,
|
|
"votes",
|
|
"idx_votes_target_live",
|
|
["target_uid", "value"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
_index(
|
|
db,
|
|
"reactions",
|
|
"idx_reactions_target_live",
|
|
["target_type", "target_uid"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
_index(
|
|
db,
|
|
"gists",
|
|
"idx_gists_live_created",
|
|
["created_at"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
_index(
|
|
db,
|
|
"projects",
|
|
"idx_projects_live_created",
|
|
["created_at"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
_index(
|
|
db,
|
|
"attachments",
|
|
"idx_attachments_user_created_live",
|
|
["user_uid", "created_at"],
|
|
where="deleted_at IS NULL",
|
|
)
|
|
|
|
_index(db, "seo_metadata", "idx_seo_metadata_status", ["status", "deleted_at"])
|
|
|
|
if "news" in tables:
|
|
for article in db["news"].find(status=None):
|
|
was_featured = article.get("featured", 0)
|
|
db["news"].update(
|
|
{
|
|
"uid": article["uid"],
|
|
"status": "published" if was_featured else "draft",
|
|
},
|
|
["uid"],
|
|
)
|
|
for article in db["news"].find(show_on_landing=None):
|
|
db["news"].update(
|
|
{
|
|
"uid": article["uid"],
|
|
"show_on_landing": 0,
|
|
},
|
|
["uid"],
|
|
)
|
|
for article in db["news"].find(slug=None):
|
|
from devplacepy.utils import make_combined_slug
|
|
|
|
slug = make_combined_slug(
|
|
article.get("title", "") or "news", article["uid"]
|
|
)
|
|
db["news"].update(
|
|
{
|
|
"uid": article["uid"],
|
|
"slug": slug,
|
|
},
|
|
["uid"],
|
|
)
|
|
|
|
if "news_sync" in tables:
|
|
for entry in db["news_sync"].find():
|
|
current = entry.get("status", "")
|
|
if current in ("below_threshold", ""):
|
|
db["news_sync"].update(
|
|
{
|
|
"id": entry["id"],
|
|
"status": "graded",
|
|
},
|
|
["id"],
|
|
)
|
|
|
|
if "site_settings" in tables:
|
|
news_defaults = {
|
|
"news_grade_threshold": "7",
|
|
"news_api_url": "https://news.app.molodetz.nl/api",
|
|
"news_ai_url": AQUALITY_NEWS_GRADING_URL,
|
|
"news_ai_model": AQUALITY_NEWS_GRADING_MODEL,
|
|
}
|
|
for key, value in news_defaults.items():
|
|
existing = db["site_settings"].find_one(key=key)
|
|
if not existing:
|
|
db["site_settings"].insert(
|
|
{"uid": f"default_{key}", "key": key, "value": value}
|
|
)
|
|
|
|
upload_defaults = {
|
|
"max_upload_size_mb": "10",
|
|
"allowed_file_types": "",
|
|
"max_attachments_per_resource": "10",
|
|
}
|
|
for key, value in upload_defaults.items():
|
|
existing = db["site_settings"].find_one(key=key)
|
|
if not existing:
|
|
db["site_settings"].insert(
|
|
{"uid": f"default_{key}", "key": key, "value": value}
|
|
)
|
|
|
|
operational_defaults = {
|
|
"rate_limit_per_minute": "60",
|
|
"rate_limit_window_seconds": "60",
|
|
"news_service_interval": "3600",
|
|
"session_max_age_days": "7",
|
|
"session_remember_days": "30",
|
|
"registration_open": "1",
|
|
"maintenance_mode": "0",
|
|
"maintenance_message": "DevPlace is undergoing scheduled maintenance. Please check back shortly.",
|
|
"customization_enabled": "1",
|
|
"customization_js_enabled": "1",
|
|
"happy_404_enabled": "1",
|
|
"audit_log_retention_days": "90",
|
|
"statistics_tracking_enabled": "1",
|
|
"docs_search_mode": "agent",
|
|
"outbound_proxy_url": "",
|
|
"ios_app_url": "https://apps.apple.com/app/devplace/id6797215143",
|
|
"devii_lessons_max_per_owner": "500",
|
|
"devii_lessons_max_age_days": "90",
|
|
"moderation_sla_hours": "24",
|
|
"moderation_filter_mode": "review",
|
|
"moderation_filter_review_score": "2",
|
|
"moderation_minimum_age": "16",
|
|
"moderation_mature_default_hidden": "1",
|
|
"contact_email": "",
|
|
"contact_phone": "",
|
|
"contact_address": "",
|
|
"terms_version": "1",
|
|
"privacy_version": "1",
|
|
"guidelines_version": "1",
|
|
"ai_third_party_provider": "",
|
|
"account_deletion_grace_hours": "24",
|
|
}
|
|
for key, value in operational_defaults.items():
|
|
existing = db["site_settings"].find_one(key=key)
|
|
if not existing:
|
|
db["site_settings"].insert(
|
|
{"uid": f"default_{key}", "key": key, "value": value}
|
|
)
|
|
|
|
with db:
|
|
db.query(
|
|
"CREATE TABLE IF NOT EXISTS visit_stats_hourly ("
|
|
"bucket_start TEXT NOT NULL, "
|
|
"page_group TEXT NOT NULL, "
|
|
"referrer_group TEXT NOT NULL, "
|
|
"views INTEGER NOT NULL DEFAULT 0, "
|
|
"member_views INTEGER NOT NULL DEFAULT 0, "
|
|
"guest_views INTEGER NOT NULL DEFAULT 0)"
|
|
)
|
|
db.query(
|
|
"CREATE TABLE IF NOT EXISTS visit_unique_slots ("
|
|
"bucket_start TEXT NOT NULL, "
|
|
"visitor_hash TEXT NOT NULL, "
|
|
"page_group TEXT NOT NULL, "
|
|
"user_uid TEXT)"
|
|
)
|
|
_index(db, "visit_stats_hourly", "idx_visit_hourly_bucket", ["bucket_start"])
|
|
_index(
|
|
db,
|
|
"visit_stats_hourly",
|
|
"idx_visit_hourly_page_time",
|
|
["page_group", "bucket_start"],
|
|
)
|
|
_index(
|
|
db,
|
|
"visit_stats_hourly",
|
|
"idx_visit_hourly_ref_time",
|
|
["referrer_group", "bucket_start"],
|
|
)
|
|
_index(
|
|
db,
|
|
"visit_stats_hourly",
|
|
"idx_visit_hourly_unique_row",
|
|
["bucket_start", "page_group", "referrer_group"],
|
|
unique=True,
|
|
)
|
|
_index(db, "visit_unique_slots", "idx_visit_unique_bucket", ["bucket_start"])
|
|
_index(
|
|
db,
|
|
"visit_unique_slots",
|
|
"idx_visit_unique_hash",
|
|
["bucket_start", "visitor_hash"],
|
|
)
|
|
_index(
|
|
db,
|
|
"visit_unique_slots",
|
|
"idx_visit_unique_row",
|
|
["bucket_start", "visitor_hash", "page_group"],
|
|
unique=True,
|
|
)
|
|
_index(db, "issue_tickets", "idx_issue_tickets_created", ["created_at"])
|
|
_index(db, "devii_turns", "idx_devii_turns_started", ["started_at"])
|
|
_index(db, "instance_events", "idx_instance_events_created", ["created_at"])
|
|
_index(db, "game_steals", "idx_game_steals_stolen_at", ["stolen_at"])
|
|
_index(db, "messages", "idx_messages_created_at", ["created_at"])
|
|
_index(db, "notifications", "idx_notifications_created", ["created_at"])
|
|
_index(db, "follows", "idx_follows_created_at", ["created_at"])
|
|
_index(db, "reactions", "idx_reactions_created_at", ["created_at"])
|
|
_index(db, "votes", "idx_votes_created_at", ["created_at"])
|
|
_index(db, "bookmarks", "idx_bookmarks_created_at", ["created_at"])
|
|
_index(db, "badges", "idx_badges_created_at", ["created_at"])
|
|
_index(db, "audit_log", "idx_audit_result_created", ["result", "created_at"])
|
|
_index(db, "jobs", "idx_jobs_created_at", ["created_at"])
|
|
_index(db, "attachments", "idx_attachments_created_at", ["created_at"])
|
|
|
|
_backfill_gamification()
|
|
backfill_api_keys()
|
|
migrate_ai_gateway_settings()
|
|
_refresh_query_planner_stats()
|
|
|
|
|
|
MODERATION_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
|
|
"content_reports": (
|
|
("uid", ""),
|
|
("reporter_uid", ""),
|
|
("target_type", ""),
|
|
("target_uid", ""),
|
|
("owner_uid", ""),
|
|
("reason", ""),
|
|
("detail", ""),
|
|
("severity", "warn"),
|
|
("status", "open"),
|
|
("origin", "member"),
|
|
("categories", ""),
|
|
("resolved_by", ""),
|
|
("resolved_at", ""),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
),
|
|
"moderation_actions": (
|
|
("uid", ""),
|
|
("report_uid", ""),
|
|
("actor_uid", ""),
|
|
("action", ""),
|
|
("target_type", ""),
|
|
("target_uid", ""),
|
|
("subject_uid", ""),
|
|
("reason", ""),
|
|
("notes", ""),
|
|
("expires_at", ""),
|
|
("created_at", ""),
|
|
),
|
|
"content_maturity": (
|
|
("uid", ""),
|
|
("target_type", ""),
|
|
("target_uid", ""),
|
|
("level", "general"),
|
|
("source", ""),
|
|
("set_by", ""),
|
|
("created_at", ""),
|
|
("updated_at", ""),
|
|
),
|
|
"user_consents": (
|
|
("uid", ""),
|
|
("owner_kind", ""),
|
|
("owner_id", ""),
|
|
("kind", ""),
|
|
("version", "1"),
|
|
("state", ""),
|
|
("granted_at", ""),
|
|
("withdrawn_at", ""),
|
|
("created_at", ""),
|
|
),
|
|
}
|
|
|
|
MODERATION_INDEXES: tuple[tuple[str, str, list[str]], ...] = (
|
|
("content_reports", "idx_content_reports_queue", ["status", "created_at"]),
|
|
("content_reports", "idx_content_reports_target", ["target_type", "target_uid"]),
|
|
("content_reports", "idx_content_reports_reporter", ["reporter_uid"]),
|
|
("content_reports", "idx_content_reports_owner", ["owner_uid"]),
|
|
("moderation_actions", "idx_moderation_actions_report", ["report_uid"]),
|
|
("moderation_actions", "idx_moderation_actions_subject", ["subject_uid"]),
|
|
("moderation_actions", "idx_moderation_actions_created", ["created_at"]),
|
|
("content_maturity", "idx_content_maturity_target", ["target_type", "target_uid"]),
|
|
("user_consents", "idx_user_consents_owner", ["owner_kind", "owner_id", "kind"]),
|
|
)
|
|
|
|
|
|
def _ensure_moderation_tables() -> None:
|
|
for table_name, columns in MODERATION_COLUMNS.items():
|
|
table = get_table(table_name)
|
|
for column, example in columns:
|
|
if not table.has_column(column):
|
|
table.create_column_by_example(column, example)
|
|
for table_name, index_name, columns in MODERATION_INDEXES:
|
|
_index(db, table_name, index_name, columns)
|
|
|
|
|
|
def _refresh_query_planner_stats() -> None:
|
|
try:
|
|
has_stats = bool(
|
|
list(db.query("SELECT name FROM sqlite_master WHERE name='sqlite_stat1'"))
|
|
)
|
|
with db:
|
|
if not has_stats:
|
|
db.query("ANALYZE")
|
|
db.query("PRAGMA optimize")
|
|
except Exception as e:
|
|
logger.warning(f"Could not refresh query planner stats: {e}")
|
|
|
|
logger.info("Database initialized")
|
|
|
|
|
|
OLD_GATEWAY_URL = "https://openai.app.molodetz.nl/v1/chat/completions"
|
|
|
|
|
|
def migrate_ai_gateway_settings() -> None:
|
|
import os
|
|
import uuid
|
|
|
|
if not get_setting("gateway_internal_key", ""):
|
|
set_setting("gateway_internal_key", str(uuid.uuid4()))
|
|
logger.info("Generated internal gateway key")
|
|
deepseek = os.environ.get("DEEPSEEK_API_KEY", "")
|
|
openrouter = os.environ.get("OPENROUTER_API_KEY", "")
|
|
if not get_setting("gateway_api_key", "") and (deepseek or openrouter):
|
|
set_setting("gateway_api_key", deepseek or openrouter)
|
|
logger.info("Migrated gateway upstream key from environment")
|
|
if not get_setting("gateway_vision_key", "") and openrouter:
|
|
set_setting("gateway_vision_key", openrouter)
|
|
logger.info("Migrated gateway vision key from environment")
|
|
for key in ("news_ai_url", "bot_api_url", "devii_ai_url"):
|
|
if get_setting(key, "") == OLD_GATEWAY_URL:
|
|
set_setting(key, INTERNAL_GATEWAY_URL)
|
|
logger.info(f"Migrated {key} to the internal gateway")
|
|
if get_setting("news_ai_url", "") == INTERNAL_GATEWAY_URL:
|
|
set_setting("news_ai_url", AQUALITY_NEWS_GRADING_URL)
|
|
logger.info("Migrated news_ai_url to the free aquality grading model")
|
|
if get_setting("news_ai_model", "") in ("molodetz", ""):
|
|
set_setting("news_ai_model", AQUALITY_NEWS_GRADING_MODEL)
|
|
logger.info("Migrated news_ai_model to aquality")
|
|
if get_setting("bot_model", "") == "deepseek-chat":
|
|
set_setting("bot_model", "molodetz")
|
|
logger.info("Migrated bot_model to molodetz")
|
|
from devplacepy.services.openai_gateway.routing import (
|
|
migrate_retired_image_gateway,
|
|
seed_default_deepseek_routes,
|
|
seed_default_image_routes,
|
|
)
|
|
|
|
seed_default_deepseek_routes()
|
|
seed_default_image_routes()
|
|
migrate_retired_image_gateway()
|
|
|
|
|
|
def _ensure_users_uid() -> int:
|
|
if "users" not in db.tables:
|
|
return 0
|
|
users = get_table("users")
|
|
if not users.has_column("uid"):
|
|
users.create_column_by_example("uid", "")
|
|
import uuid_utils
|
|
|
|
updated = 0
|
|
for user in users.find():
|
|
if not user.get("uid"):
|
|
users.update(
|
|
{"id": user["id"], "uid": str(uuid_utils.uuid7())}, ["id"]
|
|
)
|
|
updated += 1
|
|
if updated:
|
|
logger.info("Backfilled uid for %s user(s)", updated)
|
|
return updated
|
|
|
|
|
|
def backfill_api_keys() -> int:
|
|
users = get_table("users")
|
|
if not users.has_column("api_key"):
|
|
users.create_column_by_example("api_key", "")
|
|
if not users.has_column("created_at"):
|
|
users.create_column_by_example("created_at", "")
|
|
if not users.has_column("cust_disable_global"):
|
|
users.create_column_by_example("cust_disable_global", 0)
|
|
if not users.has_column("cust_disable_pagetype"):
|
|
users.create_column_by_example("cust_disable_pagetype", 0)
|
|
if not users.has_column("ai_correction_enabled"):
|
|
users.create_column_by_example("ai_correction_enabled", 0)
|
|
if not users.has_column("ai_correction_prompt"):
|
|
users.create_column_by_example("ai_correction_prompt", DEFAULT_CORRECTION_PROMPT)
|
|
if not users.has_column("ai_correction_sync"):
|
|
users.create_column_by_example("ai_correction_sync", 0)
|
|
if not users.has_column("ai_modifier_enabled"):
|
|
users.create_column_by_example("ai_modifier_enabled", 1)
|
|
if not users.has_column("ai_modifier_sync"):
|
|
users.create_column_by_example("ai_modifier_sync", 1)
|
|
if not users.has_column("ai_modifier_prompt"):
|
|
users.create_column_by_example("ai_modifier_prompt", DEFAULT_MODIFIER_PROMPT)
|
|
if not users.has_column("interactions_enabled"):
|
|
users.create_column_by_example("interactions_enabled", -1)
|
|
if not users.has_column("timezone"):
|
|
users.create_column_by_example("timezone", "")
|
|
if not users.has_column("avatar_seed"):
|
|
users.create_column_by_example("avatar_seed", "")
|
|
if not users.has_column("last_seen"):
|
|
users.create_column_by_example("last_seen", "")
|
|
if not users.has_column("award_count"):
|
|
users.create_column_by_example("award_count", 0)
|
|
if not users.has_column("last_award_at"):
|
|
users.create_column_by_example("last_award_at", "")
|
|
if not users.has_column("last_award_slug"):
|
|
users.create_column_by_example("last_award_slug", "")
|
|
if not users.has_column("last_award_uid"):
|
|
users.create_column_by_example("last_award_uid", "")
|
|
if not users.has_column("terms_version"):
|
|
users.create_column_by_example("terms_version", "")
|
|
if not users.has_column("terms_accepted_at"):
|
|
users.create_column_by_example("terms_accepted_at", "")
|
|
if not users.has_column("age_band"):
|
|
users.create_column_by_example("age_band", "")
|
|
if not users.has_column("age_declared_at"):
|
|
users.create_column_by_example("age_declared_at", "")
|
|
if not users.has_column("mature_opt_in"):
|
|
users.create_column_by_example("mature_opt_in", 0)
|
|
if not users.has_column("suspended_until"):
|
|
users.create_column_by_example("suspended_until", "")
|
|
if not users.has_column("suspension_reason"):
|
|
users.create_column_by_example("suspension_reason", "")
|
|
if not users.has_column("deletion_requested_at"):
|
|
users.create_column_by_example("deletion_requested_at", "")
|
|
with db:
|
|
db.query(
|
|
"UPDATE users SET ai_modifier_enabled = 1 WHERE ai_modifier_enabled IS NULL"
|
|
)
|
|
db.query("UPDATE users SET ai_modifier_sync = 1 WHERE ai_modifier_sync IS NULL")
|
|
db.query(
|
|
"UPDATE users SET ai_modifier_prompt = :prompt "
|
|
"WHERE ai_modifier_prompt IS NULL OR ai_modifier_prompt = ''",
|
|
prompt=DEFAULT_MODIFIER_PROMPT,
|
|
)
|
|
db.query(
|
|
"UPDATE users SET interactions_enabled = -1 "
|
|
"WHERE interactions_enabled IS NULL"
|
|
)
|
|
db.query("UPDATE users SET mature_opt_in = 0 WHERE mature_opt_in IS NULL")
|
|
import uuid_utils
|
|
|
|
updated = 0
|
|
for user in users.find():
|
|
if not user.get("api_key"):
|
|
users.update(
|
|
{"uid": user["uid"], "api_key": str(uuid_utils.uuid7())}, ["uid"]
|
|
)
|
|
updated += 1
|
|
return updated
|
|
|
|
|
|
MILESTONE_SOURCES = (
|
|
("posts", "user_uid"),
|
|
("comments", "user_uid"),
|
|
("projects", "user_uid"),
|
|
("gists", "user_uid"),
|
|
("follows", "follower_uid"),
|
|
("follows", "following_uid"),
|
|
("user_activity", "user_uid"),
|
|
)
|
|
|
|
|
|
def _milestone_candidates() -> set:
|
|
tables = db.tables
|
|
candidates = set()
|
|
for table, column in MILESTONE_SOURCES:
|
|
if table not in tables:
|
|
continue
|
|
for row in db.query(f"SELECT DISTINCT {column} AS uid FROM {table}"):
|
|
if row["uid"]:
|
|
candidates.add(row["uid"])
|
|
return candidates
|
|
|
|
|
|
def _backfill_gamification():
|
|
if "users" not in db.tables:
|
|
return
|
|
from devplacepy.utils import (
|
|
level_for_xp,
|
|
check_milestone_badges,
|
|
XP_POST,
|
|
XP_COMMENT,
|
|
XP_PROJECT,
|
|
XP_GIST,
|
|
XP_UPVOTE,
|
|
XP_FOLLOW,
|
|
)
|
|
|
|
pending = list(db["users"].find(xp=0))
|
|
if not pending:
|
|
return
|
|
|
|
xp_by_user = defaultdict(int)
|
|
|
|
def add_counts(table, column, points):
|
|
if table not in db.tables:
|
|
return
|
|
for row in db.query(
|
|
f"SELECT {column} AS uid, COUNT(*) AS c FROM {table} GROUP BY {column}"
|
|
):
|
|
if row["uid"]:
|
|
xp_by_user[row["uid"]] += row["c"] * points
|
|
|
|
add_counts("posts", "user_uid", XP_POST)
|
|
add_counts("comments", "user_uid", XP_COMMENT)
|
|
add_counts("projects", "user_uid", XP_PROJECT)
|
|
add_counts("gists", "user_uid", XP_GIST)
|
|
add_counts("follows", "following_uid", XP_FOLLOW)
|
|
|
|
if "votes" in db.tables:
|
|
for content_table, target_type in (
|
|
("posts", "post"),
|
|
("projects", "project"),
|
|
("gists", "gist"),
|
|
("comments", "comment"),
|
|
):
|
|
if content_table not in db.tables:
|
|
continue
|
|
rows = db.query(
|
|
f"SELECT c.user_uid AS uid, COUNT(*) AS c "
|
|
f"FROM votes v JOIN {content_table} c ON v.target_uid = c.uid "
|
|
f"WHERE v.target_type = :t AND v.value = 1 GROUP BY c.user_uid",
|
|
t=target_type,
|
|
)
|
|
for row in rows:
|
|
if row["uid"]:
|
|
xp_by_user[row["uid"]] += row["c"] * XP_UPVOTE
|
|
|
|
for user in pending:
|
|
xp = xp_by_user.get(user["uid"], 0)
|
|
if xp <= 0:
|
|
continue
|
|
db["users"].update(
|
|
{"uid": user["uid"], "xp": xp, "level": level_for_xp(xp)}, ["uid"]
|
|
)
|
|
|
|
_authors_cache.clear()
|
|
candidates = _milestone_candidates()
|
|
checked = [user for user in pending if user["uid"] in candidates]
|
|
for user in checked:
|
|
check_milestone_badges(user["uid"])
|
|
logger.info(
|
|
f"Gamification backfill processed {len(pending)} users, "
|
|
f"{len(checked)} with milestone-eligible activity"
|
|
)
|
|
|
|
|