# retoor <retoor@molodetz.nl>
from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _index, _uid_index, db, defaultdict, get_table, logger
from .settings import get_setting, set_setting
from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns
from .ranking import _authors_cache
BUG_TABLE_RENAMES = (
("bug_tickets", "issue_tickets"),
("bug_comment_authors", "issue_comment_authors"),
)
def migrate_bug_tables_to_issue_tables() -> None:
for source_name, destination_name in BUG_TABLE_RENAMES:
if source_name not in db.tables:
continue
source = db[source_name]
destination = get_table(destination_name)
copied = 0
for row in source.all():
payload = {key: value for key, value in row.items() if key != "id"}
destination.insert_ignore(payload, ["uid"])
copied += 1
logger.info(
"Migrated %s rows from %s into %s", copied, source_name, destination_name
)
source.drop()
logger.info("Dropped table %s after migration", source_name)
def init_db():
tables = db.tables
_index(db, "users", "idx_users_username", ["username"])
_index(db, "users", "idx_users_email", ["email"])
_index(db, "users", "idx_users_api_key", ["api_key"])
_index(db, "users", "idx_users_role", ["role", "created_at"])
_index(db, "users", "idx_users_last_seen", ["last_seen"])
_index(db, "users", "idx_users_created_at", ["created_at"])
_index(db, "posts", "idx_posts_user_uid", ["user_uid"])
_index(db, "posts", "idx_posts_created_at", ["created_at"])
_index(db, "posts", "idx_posts_topic", ["topic"])
_index(db, "posts", "idx_posts_slug", ["slug"])
if "posts" in tables:
posts_table = get_table("posts")
if not posts_table.has_column("tags"):
posts_table.create_column_by_example("tags", "")
if "devrant_tokens" in tables:
devrant_tokens = get_table("devrant_tokens")
for column, example in (
("uid", ""),
("key", ""),
("user_uid", ""),
("user_id", 0),
("expire_time", 0),
("created_at", ""),
):
if not devrant_tokens.has_column(column):
devrant_tokens.create_column_by_example(column, example)
_index(db, "devrant_tokens", "idx_devrant_tokens_key", ["key"])
_index(db, "devrant_tokens", "idx_devrant_tokens_user", ["user_uid"])
if "access_tokens" in tables:
access_tokens = get_table("access_tokens")
for column, example in (
("uid", ""),
("token", ""),
("user_uid", ""),
("label", ""),
("expires_at", ""),
("created_at", ""),
):
if not access_tokens.has_column(column):
access_tokens.create_column_by_example(column, example)
_index(db, "access_tokens", "idx_access_tokens_token", ["token"])
_index(db, "access_tokens", "idx_access_tokens_user", ["user_uid"])
if "telegram_pairings" in tables:
telegram_pairings = get_table("telegram_pairings")
for column, example in (
("uid", ""),
("user_uid", ""),
("code_hash", ""),
("expires_at", ""),
("used", 0),
("created_at", ""),
):
if not telegram_pairings.has_column(column):
telegram_pairings.create_column_by_example(column, example)
_index(db, "telegram_pairings", "idx_telegram_pairings_code", ["code_hash"])
_index(db, "telegram_pairings", "idx_telegram_pairings_user", ["user_uid"])
if "telegram_links" in tables:
telegram_links = get_table("telegram_links")
for column, example in (
("uid", ""),
("user_uid", ""),
("chat_id", 0),
("from_id", 0),
("created_at", ""),
):
if not telegram_links.has_column(column):
telegram_links.create_column_by_example(column, example)
_index(db, "telegram_links", "idx_telegram_links_chat", ["chat_id"])
_index(db, "telegram_links", "idx_telegram_links_user", ["user_uid"])
telegram_outbox = get_table("telegram_outbox")
for column, example in (
("uid", ""),
("user_uid", ""),
("chat_id", 0),
("text", ""),
("status", "pending"),
("attempts", 0),
("created_at", ""),
("sent_at", ""),
):
if not telegram_outbox.has_column(column):
telegram_outbox.create_column_by_example(column, example)
_index(db, "telegram_outbox", "idx_telegram_outbox_status", ["status", "id"])
_index(db, "comments", "idx_comments_post_uid", ["post_uid"])
_index(db, "comments", "idx_comments_target", ["target_type", "target_uid"])
_index(db, "comments", "idx_comments_user_uid", ["user_uid"])
_index(db, "comments", "idx_comments_created_at", ["created_at"])
_index(db, "votes", "idx_votes_target", ["target_uid", "target_type"])
_index(db, "messages", "idx_messages_sender", ["sender_uid"])
_index(db, "messages", "idx_messages_receiver", ["receiver_uid"])
_index(
db, "messages", "idx_messages_conversation", ["sender_uid", "receiver_uid"]
)
_index(
db,
"messages",
"idx_messages_conversation_rev",
["receiver_uid", "sender_uid"],
)
_index(db, "notifications", "idx_notifications_user", ["user_uid"])
_index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
_index(db, "push_registration", "idx_push_registration_user", ["user_uid"])
_index(db, "sessions", "idx_sessions_token", ["session_token"])
projects = get_table("projects")
for column, example in (
("uid", ""),
("user_uid", ""),
("slug", ""),
("created_at", ""),
("stars", 0),
("project_type", ""),
("is_private", 0),
("read_only", 0),
("updated_at", ""),
):
if not projects.has_column(column):
projects.create_column_by_example(column, example)
_index(db, "projects", "idx_projects_user", ["user_uid"])
_index(db, "projects", "idx_projects_slug", ["slug"])
_index(db, "projects", "idx_projects_private", ["is_private"])
_index(db, "projects", "idx_projects_stars", ["stars"])
_index(db, "projects", "idx_projects_type", ["project_type"])
project_files = get_table("project_files")
for column, example in (
("uid", ""),
("project_uid", ""),
("user_uid", ""),
("path", ""),
("name", ""),
("parent_path", ""),
("type", ""),
("content", ""),
("is_binary", 0),
("stored_name", ""),
("directory", ""),
("mime_type", ""),
("size", 0),
("created_at", ""),
("updated_at", ""),
("deleted_at", ""),
("deleted_by", ""),
):
if not project_files.has_column(column):
project_files.create_column_by_example(column, example)
_index(db, "project_files", "idx_project_files_path", ["project_uid", "path"])
_index(
db, "project_files", "idx_project_files_parent", ["project_uid", "parent_path"]
)
_index(db, "badges", "idx_badges_user", ["user_uid"])
_index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"])
_index(db, "follows", "idx_follows_follower", ["follower_uid"])
_index(db, "follows", "idx_follows_following", ["following_uid"])
user_relations = get_table("user_relations")
for column, example in (
("uid", ""),
("user_uid", ""),
("target_uid", ""),
("kind", ""),
("created_at", ""),
("deleted_at", ""),
("deleted_by", ""),
):
if not user_relations.has_column(column):
user_relations.create_column_by_example(column, example)
_index(db, "user_relations", "idx_user_relations_owner_kind", ["user_uid", "kind"])
_index(db, "user_relations", "idx_user_relations_target_kind", ["target_uid", "kind"])
_index(db, "password_resets", "idx_password_resets_token", ["token"])
_index(db, "gists", "idx_gists_user_uid", ["user_uid"])
_index(db, "gists", "idx_gists_language", ["language"])
_index(db, "gists", "idx_gists_slug", ["slug"])
attachments = get_table("attachments")
for column, example in (
("uid", ""),
("target_type", ""),
("target_uid", ""),
("resource_type", ""),
("resource_uid", ""),
("user_uid", ""),
("original_filename", ""),
("stored_name", ""),
("directory", ""),
("file_size", 0),
("mime_type", ""),
("image_width", 0),
("image_height", 0),
("has_thumbnail", 0),
("thumbnail_name", ""),
("gitea_asset_id", 0),
("created_at", ""),
("deleted_at", ""),
):
if not attachments.has_column(column):
attachments.create_column_by_example(column, example)
_index(
db, "attachments", "idx_attachments_resource", ["resource_type", "resource_uid"]
)
_index(db, "attachments", "idx_attachments_target", ["target_type", "target_uid"])
_index(db, "attachments", "idx_attachments_user_created", ["user_uid", "created_at"])
_index(db, "reactions", "idx_reactions_target", ["target_type", "target_uid"])
_index(
db,
"reactions",
"idx_reactions_user_target",
["user_uid", "target_type", "target_uid"],
)
_index(db, "bookmarks", "idx_bookmarks_user", ["user_uid"])
_index(db, "bookmarks", "idx_bookmarks_target", ["target_type", "target_uid"])
_index(db, "polls", "idx_polls_post", ["post_uid"])
_index(db, "poll_options", "idx_poll_options_poll", ["poll_uid"])
_index(db, "poll_votes", "idx_poll_votes_poll", ["poll_uid"])
_index(db, "poll_votes", "idx_poll_votes_user", ["poll_uid", "user_uid"])
if "site_settings" in tables:
defaults = {
"site_name": "DevPlace",
"site_description": "The Developer Social Network",
"site_tagline": "Track industry shifts. Discover bold releases. Share what you are building in an open, uncensored environment.",
}
for key, value in defaults.items():
existing = db["site_settings"].find_one(key=key)
if not existing:
db["site_settings"].insert(
{"uid": f"default_{key}", "key": key, "value": value}
)
_index(db, "site_settings", "idx_site_settings_key", ["key"])
news = get_table("news")
for column, example in (
("uid", ""),
("slug", ""),
("title", ""),
("external_id", ""),
("status", ""),
("source_name", ""),
("url", ""),
("description", ""),
("content", ""),
("synced_at", ""),
("grade", 0),
("ai_grade", 0),
("show_on_landing", 0),
("featured", 0),
("author", ""),
("article_published", ""),
("image_url", ""),
("has_unique_image", 0),
("featured_locked", 0),
("landing_locked", 0),
):
if not news.has_column(column):
news.create_column_by_example(column, example)
_index(db, "news", "idx_news_external_id", ["external_id"])
_index(db, "news", "idx_news_slug", ["slug"])
_index(db, "news", "idx_news_synced_at", ["synced_at"])
_index(db, "news", "idx_news_status", ["status"])
_index(db, "news", "idx_news_featured", ["featured"])
_index(db, "news", "idx_news_landing", ["show_on_landing"])
news_images = get_table("news_images")
for column, example in (
("news_uid", ""),
("url", ""),
("alt_text", ""),
("phash", ""),
("width", 0),
("height", 0),
("is_placeholder", 0),
):
if not news_images.has_column(column):
news_images.create_column_by_example(column, example)
news_sync = get_table("news_sync")
if not news_sync.has_column("external_id"):
news_sync.create_column_by_example("external_id", "")
_index(db, "news_images", "idx_news_images_news_uid", ["news_uid"])
_index(db, "news_sync", "idx_news_sync_external_id", ["external_id"])
issue_tickets = get_table("issue_tickets")
for column, example in (
("uid", ""),
("gitea_number", 0),
("author_uid", ""),
("original_title", ""),
("original_description", ""),
("enhanced_title", ""),
("html_url", ""),
("last_status", ""),
("last_comment_count", 0),
("created_at", ""),
("updated_at", ""),
("deleted_at", ""),
("deleted_by", ""),
):
if not issue_tickets.has_column(column):
issue_tickets.create_column_by_example(column, example)
issue_comment_authors = get_table("issue_comment_authors")
for column, example in (
("uid", ""),
("gitea_comment_id", 0),
("gitea_number", 0),
("author_uid", ""),
("created_at", ""),
("deleted_at", ""),
("deleted_by", ""),
):
if not issue_comment_authors.has_column(column):
issue_comment_authors.create_column_by_example(column, example)
_index(db, "issue_tickets", "idx_issue_tickets_number", ["gitea_number"])
_index(db, "issue_tickets", "idx_issue_tickets_author", ["author_uid"])
_index(
db,
"issue_comment_authors",
"idx_issue_comment_authors_comment",
["gitea_comment_id"],
)
_index(
db, "issue_comment_authors", "idx_issue_comment_authors_number", ["gitea_number"]
)
migrate_bug_tables_to_issue_tables()
_index(db, "service_state", "idx_service_state_name", ["name"])
if "devii_conversations" in db.tables:
conversations = get_table("devii_conversations")
if not conversations.has_column("channel"):
conversations.create_column_by_example("channel", "main")
try:
db.query(
"UPDATE devii_conversations SET channel='main' WHERE channel IS NULL"
)
except Exception as e: # noqa: BLE001
logger.warning(f"Could not backfill devii_conversations.channel: {e}")
_index(
db,
"devii_conversations",
"idx_devii_conv_owner",
["owner_kind", "owner_id", "channel"],
)
_index(
db,
"devii_usage_ledger",
"idx_devii_ledger_owner_time",
["owner_kind", "owner_id", "created_at"],
)
_index(
db,
"devii_turns",
"idx_devii_turns_owner_time",
["owner_kind", "owner_id", "started_at"],
)
_index(db, "devii_tasks", "idx_devii_tasks_owner", ["owner_kind", "owner_id"])
_index(
db, "devii_tasks", "idx_devii_tasks_due", ["enabled", "status", "next_run_at"]
)
_index(db, "devii_lessons", "idx_devii_lessons_owner", ["owner_kind", "owner_id"])
_index(
db, "devii_virtual_tools", "idx_devii_vtools_owner", ["owner_kind", "owner_id"]
)
_index(
db,
"devii_virtual_tools",
"idx_devii_vtools_name",
["owner_kind", "owner_id", "name"],
)
_index(db, "devii_behavior", "idx_devii_behavior_owner", ["owner_kind", "owner_id"])
_index(db, "user_customizations", "idx_user_cust_owner", ["owner_kind", "owner_id"])
_index(
db,
"user_customizations",
"idx_user_cust_scope",
["owner_kind", "owner_id", "scope", "lang"],
)
_index(db, "gateway_usage_ledger", "idx_gw_usage_time", ["created_at"])
_index(
db,
"gateway_usage_ledger",
"idx_gw_usage_backend_time",
["backend", "created_at"],
)
_index(
db, "gateway_usage_ledger", "idx_gw_usage_model_time", ["model", "created_at"]
)
_index(
db,
"gateway_usage_ledger",
"idx_gw_usage_owner_time",
["owner_kind", "owner_id", "created_at"],
)
_index(
db,
"gateway_usage_ledger",
"idx_gw_usage_endpoint_time",
["endpoint", "created_at"],
)
_index(db, "gateway_concurrency_samples", "idx_gw_conc_time", ["created_at"])
jobs_table = get_table("jobs")
for column, example in (
("uid", ""),
("kind", ""),
("status", ""),
("owner_kind", ""),
("owner_id", ""),
("preferred_name", ""),
("payload", ""),
("result", ""),
("error", ""),
("retry_count", 0),
("created_at", ""),
("started_at", ""),
("completed_at", ""),
("updated_at", ""),
("duration_ms", 0),
("last_accessed_at", ""),
("expires_at", ""),
("bytes_in", 0),
("bytes_out", 0),
("item_count", 0),
):
if not jobs_table.has_column(column):
jobs_table.create_column_by_example(column, example)
_index(db, "jobs", "idx_jobs_kind_status", ["kind", "status"])
_index(db, "jobs", "idx_jobs_owner", ["owner_kind", "owner_id"])
_index(db, "jobs", "idx_jobs_expires", ["expires_at"])
_index(db, "project_forks", "idx_project_forks_source", ["source_project_uid"])
_index(db, "project_forks", "idx_project_forks_forked", ["forked_project_uid"])
if "instances" in db.tables:
instances = get_table("instances")
for column, example in (
("run_as_uid", ""),
("boot_language", "none"),
("boot_script", ""),
("start_on_boot", 0),
):
if not instances.has_column(column):
instances.create_column_by_example(column, example)
_index(db, "instances", "idx_instances_project", ["project_uid"])
_index(db, "instances", "idx_instances_slug", ["slug"])
_index(db, "instances", "idx_instances_name", ["name"])
_index(db, "instances", "idx_instances_state", ["desired_state", "status"])
_index(db, "instances", "idx_instances_container", ["container_id"])
_index(db, "instances", "idx_instances_ingress", ["ingress_slug"])
_index(
db,
"instance_events",
"idx_instance_events_instance",
["instance_uid", "created_at"],
)
_index(
db,
"instance_metrics",
"idx_instance_metrics_instance_ts",
["instance_uid", "ts"],
)
_index(
db,
"instance_schedules",
"idx_instance_schedules_due",
["enabled", "next_run_at"],
)
from devplacepy.services.audit import store as audit_store
audit_store.ensure_tables()
from devplacepy.services.backup import store as backup_store
backup_store.ensure_tables()
from devplacepy.services.openai_gateway import routing as gateway_routing
gateway_routing.ensure_tables()
_index(db, "audit_log", "idx_audit_created_at", ["created_at"])
_index(db, "audit_log", "idx_audit_event_key", ["event_key"])
_index(db, "audit_log", "idx_audit_category", ["category"])
_index(db, "audit_log", "idx_audit_actor_uid", ["actor_uid"])
_index(db, "audit_log", "idx_audit_actor_role", ["actor_role"])
_index(db, "audit_log", "idx_audit_origin", ["origin"])
_index(db, "audit_log", "idx_audit_target", ["target_type", "target_uid"])
_index(db, "audit_log", "idx_audit_result", ["result"])
_index(db, "audit_log_links", "idx_audit_links_audit", ["audit_uid"])
_index(db, "audit_log_links", "idx_audit_links_object", ["object_type", "object_uid"])
_index(db, "audit_log_links", "idx_audit_links_relation", ["relation"])
notification_preferences = get_table("notification_preferences")
for column, example in (
("uid", ""),
("user_uid", ""),
("notification_type", ""),
("in_app_enabled", 1),
("push_enabled", 1),
("telegram_enabled", 1),
("created_at", ""),
("updated_at", ""),
("deleted_at", ""),
("deleted_by", ""),
):
if not notification_preferences.has_column(column):
notification_preferences.create_column_by_example(column, example)
_index(db, "notification_preferences", "idx_notif_prefs_user", ["user_uid"])
_index(
db,
"notification_preferences",
"idx_notif_prefs_user_type",
["user_uid", "notification_type"],
)
correction_usage = get_table("correction_usage")
for column, example in (
("user_uid", ""),
("calls", 0),
("prompt_tokens", 0),
("completion_tokens", 0),
("total_tokens", 0),
("cost_usd", 0.0),
("upstream_latency_ms", 0.0),
("total_latency_ms", 0.0),
("updated_at", ""),
):
if not correction_usage.has_column(column):
correction_usage.create_column_by_example(column, example)
try:
if "correction_usage" in db.tables:
db.query(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user "
"ON correction_usage (user_uid)"
)
except Exception as e:
logger.warning(f"Could not create unique index on correction_usage: {e}")
modifier_usage = get_table("modifier_usage")
for column, example in (
("user_uid", ""),
("calls", 0),
("prompt_tokens", 0),
("completion_tokens", 0),
("total_tokens", 0),
("cost_usd", 0.0),
("upstream_latency_ms", 0.0),
("total_latency_ms", 0.0),
("updated_at", ""),
):
if not modifier_usage.has_column(column):
modifier_usage.create_column_by_example(column, example)
try:
if "modifier_usage" in db.tables:
db.query(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user "
"ON modifier_usage (user_uid)"
)
except Exception as e:
logger.warning(f"Could not create unique index on modifier_usage: {e}")
news_usage = get_table("news_usage")
for column, example in (
("user_uid", ""),
("calls", 0),
("prompt_tokens", 0),
("completion_tokens", 0),
("total_tokens", 0),
("cost_usd", 0.0),
("upstream_latency_ms", 0.0),
("total_latency_ms", 0.0),
("updated_at", ""),
):
if not news_usage.has_column(column):
news_usage.create_column_by_example(column, example)
try:
if "news_usage" in db.tables:
db.query(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user "
"ON news_usage (user_uid)"
)
except Exception as e:
logger.warning(f"Could not create unique index on news_usage: {e}")
issue_usage = get_table("issue_usage")
for column, example in (
("user_uid", ""),
("calls", 0),
("prompt_tokens", 0),
("completion_tokens", 0),
("total_tokens", 0),
("cost_usd", 0.0),
("upstream_latency_ms", 0.0),
("total_latency_ms", 0.0),
("updated_at", ""),
):
if not issue_usage.has_column(column):
issue_usage.create_column_by_example(column, example)
try:
if "issue_usage" in db.tables:
db.query(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user "
"ON issue_usage (user_uid)"
)
except Exception as e:
logger.warning(f"Could not create unique index on issue_usage: {e}")
seo_usage = get_table("seo_usage")
for column, example in (
("user_uid", ""),
("calls", 0),
("prompt_tokens", 0),
("completion_tokens", 0),
("total_tokens", 0),
("cost_usd", 0.0),
("upstream_latency_ms", 0.0),
("total_latency_ms", 0.0),
("updated_at", ""),
):
if not seo_usage.has_column(column):
seo_usage.create_column_by_example(column, example)
try:
if "seo_usage" in db.tables:
db.query(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user "
"ON seo_usage (user_uid)"
)
except Exception as e:
logger.warning(f"Could not create unique index on seo_usage: {e}")
seo_metadata = get_table("seo_metadata")
for column, example in (
("uid", ""),
("target_type", ""),
("target_uid", ""),
("seo_title", ""),
("seo_description", ""),
("seo_keywords", ""),
("status", ""),
("source", ""),
("generated_at", ""),
("created_at", ""),
("updated_at", ""),
):
if not seo_metadata.has_column(column):
seo_metadata.create_column_by_example(column, example)
_index(
db,
"seo_metadata",
"idx_seo_metadata_target",
["target_type", "target_uid"],
unique=True,
)
email_accounts = get_table("email_accounts")
for column, example in (
("uid", ""),
("owner_kind", ""),
("owner_id", ""),
("label", ""),
("imap_host", ""),
("imap_port", 993),
("imap_ssl", 1),
("imap_starttls", 0),
("smtp_host", ""),
("smtp_port", 587),
("smtp_ssl", 0),
("smtp_starttls", 1),
("username", ""),
("password", ""),
("from_address", ""),
("from_name", ""),
("created_at", ""),
("updated_at", ""),
("deleted_at", ""),
("deleted_by", ""),
):
if not email_accounts.has_column(column):
email_accounts.create_column_by_example(column, example)
_index(db, "email_accounts", "idx_email_accounts_owner", ["owner_kind", "owner_id"])
_index(
db,
"email_accounts",
"idx_email_accounts_label",
["owner_kind", "owner_id", "label"],
)
user_activity = get_table("user_activity")
for column, example in (
("user_uid", ""),
("action", ""),
("count", 0),
("first_at", ""),
("last_at", ""),
):
if not user_activity.has_column(column):
user_activity.create_column_by_example(column, example)
try:
if "user_activity" in db.tables:
db.query(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action "
"ON user_activity (user_uid, action)"
)
except Exception as e:
logger.warning(f"Could not create unique index on user_activity: {e}")
user_activity_seen = get_table("user_activity_seen")
for column, example in (
("user_uid", ""),
("action", ""),
("target", ""),
("created_at", ""),
):
if not user_activity_seen.has_column(column):
user_activity_seen.create_column_by_example(column, example)
try:
if "user_activity_seen" in db.tables:
db.query(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique "
"ON user_activity_seen (user_uid, action, target)"
)
except Exception as e:
logger.warning(f"Could not create unique index on user_activity_seen: {e}")
deepsearch_sessions = get_table("deepsearch_sessions")
for column, example in (
("uid", ""),
("owner_kind", ""),
("owner_id", ""),
("query", ""),
("status", ""),
("depth", 0),
("max_pages", 0),
("score", 0),
("confidence", 0.0),
("source_diversity", 0.0),
("page_count", 0),
("chunk_count", 0),
("collection", ""),
("summary", ""),
("created_at", ""),
("completed_at", ""),
("deleted_at", ""),
("deleted_by", ""),
):
if not deepsearch_sessions.has_column(column):
deepsearch_sessions.create_column_by_example(column, example)
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_owner", ["owner_kind", "owner_id"])
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_status", ["status"])
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_created", ["created_at"])
deepsearch_messages = get_table("deepsearch_messages")
for column, example in (
("uid", ""),
("session_uid", ""),
("role", ""),
("content", ""),
("citations", ""),
("created_at", ""),
("deleted_at", ""),
("deleted_by", ""),
):
if not deepsearch_messages.has_column(column):
deepsearch_messages.create_column_by_example(column, example)
_index(db, "deepsearch_messages", "idx_deepsearch_messages_session", ["session_uid"])
deepsearch_url_cache = get_table("deepsearch_url_cache")
for column, example in (
("uid", ""),
("url_hash", ""),
("url", ""),
("title", ""),
("content_hash", ""),
("status", 0),
("byte_size", 0),
("fetched_at", ""),
):
if not deepsearch_url_cache.has_column(column):
deepsearch_url_cache.create_column_by_example(column, example)
_index(db, "deepsearch_url_cache", "idx_deepsearch_url_cache_hash", ["url_hash"])
isslop_analyses = get_table("isslop_analyses")
for column, example in (
("uid", ""),
("owner_kind", ""),
("owner_id", ""),
("source_url", ""),
("source_kind", ""),
("status", ""),
("created_at", ""),
("finished_at", ""),
("content_hash", ""),
("grade", ""),
("slop_score", 0.0),
("origin_score", 0.0),
("quality_deficit_score", 0.0),
("human_percent", 0.0),
("ai_percent", 0.0),
("category", ""),
("confidence", ""),
("files_total", 0),
("files_analyzed", 0),
("error_message_text", ""),
("detected_builder", ""),
("dom_slop_score", 0.0),
("deleted_at", ""),
("deleted_by", ""),
):
if not isslop_analyses.has_column(column):
isslop_analyses.create_column_by_example(column, example)
_index(db, "isslop_analyses", "idx_isslop_analyses_owner", ["owner_kind", "owner_id", "created_at"])
_index(db, "isslop_analyses", "idx_isslop_analyses_status", ["status"])
_index(db, "isslop_analyses", "idx_isslop_analyses_hash", ["content_hash"])
isslop_events = get_table("isslop_events")
for column, example in (
("analysis_uid", ""),
("seq", 0),
("kind", ""),
("message", ""),
("payload", ""),
("created_at", ""),
):
if not isslop_events.has_column(column):
isslop_events.create_column_by_example(column, example)
_index(db, "isslop_events", "idx_isslop_events_analysis", ["analysis_uid", "seq"])
isslop_file_results = get_table("isslop_file_results")
for column, example in (
("analysis_uid", ""),
("path", ""),
("language", ""),
("lines", 0),
("origin_score", 0.0),
("quality_deficit_score", 0.0),
("category", ""),
("signals", ""),
("source", ""),
):
if not isslop_file_results.has_column(column):
isslop_file_results.create_column_by_example(column, example)
_index(db, "isslop_file_results", "idx_isslop_file_results_analysis", ["analysis_uid"])
isslop_image_results = get_table("isslop_image_results")
for column, example in (
("analysis_uid", ""),
("path", ""),
("ai_probability", 0.0),
("grade", ""),
("verdict", ""),
("image_kind", ""),
("tells", ""),
("description", ""),
("thumb", ""),
):
if not isslop_image_results.has_column(column):
isslop_image_results.create_column_by_example(column, example)
_index(db, "isslop_image_results", "idx_isslop_image_results_analysis", ["analysis_uid"])
isslop_dom_results = get_table("isslop_dom_results")
for column, example in (
("analysis_uid", ""),
("url", ""),
("detected_builder", ""),
("signal_count", 0),
("screenshot", ""),
("signals", ""),
):
if not isslop_dom_results.has_column(column):
isslop_dom_results.create_column_by_example(column, example)
_index(db, "isslop_dom_results", "idx_isslop_dom_results_analysis", ["analysis_uid"])
isslop_reports = get_table("isslop_reports")
for column, example in (
("analysis_uid", ""),
("markdown", ""),
("model_used", ""),
("generated_at", ""),
):
if not isslop_reports.has_column(column):
isslop_reports.create_column_by_example(column, example)
_index(db, "isslop_reports", "idx_isslop_reports_analysis", ["analysis_uid"], unique=True)
game_farms = get_table("game_farms")
for column, example in (
("uid", ""),
("user_uid", ""),
("coins", 0),
("xp", 0),
("level", 1),
("ci_tier", 1),
("plot_count", 0),
("total_harvests", 0),
("prestige", 0),
("streak", 0),
("last_daily_at", ""),
("perk_yield", 0),
("perk_growth", 0),
("perk_discount", 0),
("perk_xp", 0),
("stars", 0),
("legacy_autoharvest", 0),
("legacy_multiplier", 0),
("legacy_speed", 0),
("legacy_plots", 0),
("legacy_defense", 0),
("created_at", ""),
("updated_at", ""),
):
if not game_farms.has_column(column):
game_farms.create_column_by_example(column, example)
_index(db, "game_farms", "idx_game_farms_user", ["user_uid"], unique=True)
_index(db, "game_farms", "idx_game_farms_rank", ["level", "xp"])
game_steals = get_table("game_steals")
for column, example in (
("uid", ""),
("thief_uid", ""),
("owner_uid", ""),
("slot_index", 0),
("crop_key", ""),
("coins", 0),
("stolen_at", ""),
("created_at", ""),
):
if not game_steals.has_column(column):
game_steals.create_column_by_example(column, example)
_index(
db, "game_steals", "idx_game_steals_pair", ["thief_uid", "owner_uid", "stolen_at"]
)
game_quests = get_table("game_quests")
for column, example in (
("uid", ""),
("farm_uid", ""),
("user_uid", ""),
("day", ""),
("slot_index", 0),
("kind", ""),
("label", ""),
("goal", 0),
("progress", 0),
("reward_coins", 0),
("reward_xp", 0),
("claimed", 0),
("created_at", ""),
("updated_at", ""),
):
if not game_quests.has_column(column):
game_quests.create_column_by_example(column, example)
_index(db, "game_quests", "idx_game_quests_farm_day", ["farm_uid", "day"])
game_plots = get_table("game_plots")
for column, example in (
("uid", ""),
("farm_uid", ""),
("user_uid", ""),
("slot_index", 0),
("crop_key", ""),
("planted_at", ""),
("ready_at", ""),
("watered_by", "[]"),
("created_at", ""),
("updated_at", ""),
):
if not game_plots.has_column(column):
game_plots.create_column_by_example(column, example)
_index(db, "game_plots", "idx_game_plots_farm", ["farm_uid", "slot_index"])
_index(db, "posts", "idx_posts_user_created", ["user_uid", "created_at"])
_index(
db,
"posts",
"idx_posts_live_created",
["created_at"],
where="deleted_at IS NULL",
)
_index(
db,
"comments",
"idx_comments_target_created",
["target_type", "target_uid", "created_at"],
)
_index(db, "comments", "idx_comments_user_created", ["user_uid", "created_at"])
_index(db, "votes", "idx_votes_user_target", ["user_uid", "target_uid"])
_index(db, "gists", "idx_gists_user_created", ["user_uid", "created_at"])
_index(db, "projects", "idx_projects_user_created", ["user_uid", "created_at"])
_index(
db,
"notifications",
"idx_notifications_user_created",
["user_uid", "created_at"],
)
for table in db.tables:
_uid_index(db, table)
for table in SOFT_DELETE_TABLES:
ensure_soft_delete_columns(table)
_index(
db,
"comments",
"idx_comments_target_live",
["target_type", "target_uid"],
where="deleted_at IS NULL",
)
_index(
db,
"votes",
"idx_votes_target_live",
["target_uid", "value"],
where="deleted_at IS NULL",
)
_index(
db,
"reactions",
"idx_reactions_target_live",
["target_type", "target_uid"],
where="deleted_at IS NULL",
)
_index(
db,
"gists",
"idx_gists_live_created",
["created_at"],
where="deleted_at IS NULL",
)
_index(
db,
"projects",
"idx_projects_live_created",
["created_at"],
where="deleted_at IS NULL",
)
_index(
db,
"attachments",
"idx_attachments_user_created_live",
["user_uid", "created_at"],
where="deleted_at IS NULL",
)
_index(db, "seo_metadata", "idx_seo_metadata_status", ["status", "deleted_at"])
if "news" in tables:
for article in db["news"].find(status=None):
was_featured = article.get("featured", 0)
db["news"].update(
{
"uid": article["uid"],
"status": "published" if was_featured else "draft",
},
["uid"],
)
for article in db["news"].find(show_on_landing=None):
db["news"].update(
{
"uid": article["uid"],
"show_on_landing": 0,
},
["uid"],
)
for article in db["news"].find(slug=None):
from devplacepy.utils import make_combined_slug
slug = make_combined_slug(
article.get("title", "") or "news", article["uid"]
)
db["news"].update(
{
"uid": article["uid"],
"slug": slug,
},
["uid"],
)
if "news_sync" in tables:
for entry in db["news_sync"].find():
current = entry.get("status", "")
if current in ("below_threshold", ""):
db["news_sync"].update(
{
"id": entry["id"],
"status": "graded",
},
["id"],
)
if "site_settings" in tables:
news_defaults = {
"news_grade_threshold": "7",
"news_api_url": "https://news.app.molodetz.nl/api",
"news_ai_url": INTERNAL_GATEWAY_URL,
"news_ai_model": "molodetz",
}
for key, value in news_defaults.items():
existing = db["site_settings"].find_one(key=key)
if not existing:
db["site_settings"].insert(
{"uid": f"default_{key}", "key": key, "value": value}
)
upload_defaults = {
"max_upload_size_mb": "10",
"allowed_file_types": "",
"max_attachments_per_resource": "10",
}
for key, value in upload_defaults.items():
existing = db["site_settings"].find_one(key=key)
if not existing:
db["site_settings"].insert(
{"uid": f"default_{key}", "key": key, "value": value}
)
operational_defaults = {
"rate_limit_per_minute": "60",
"rate_limit_window_seconds": "60",
"news_service_interval": "3600",
"session_max_age_days": "7",
"session_remember_days": "30",
"registration_open": "1",
"maintenance_mode": "0",
"maintenance_message": "DevPlace is undergoing scheduled maintenance. Please check back shortly.",
"customization_enabled": "1",
"customization_js_enabled": "1",
"audit_log_retention_days": "90",
"docs_search_mode": "agent",
}
for key, value in operational_defaults.items():
existing = db["site_settings"].find_one(key=key)
if not existing:
db["site_settings"].insert(
{"uid": f"default_{key}", "key": key, "value": value}
)
_backfill_gamification()
backfill_api_keys()
migrate_ai_gateway_settings()
_refresh_query_planner_stats()
def _refresh_query_planner_stats() -> None:
try:
has_stats = bool(
list(db.query("SELECT name FROM sqlite_master WHERE name='sqlite_stat1'"))
)
with db:
if not has_stats:
db.query("ANALYZE")
db.query("PRAGMA optimize")
except Exception as e:
logger.warning(f"Could not refresh query planner stats: {e}")
logger.info("Database initialized")
OLD_GATEWAY_URL = "https://openai.app.molodetz.nl/v1/chat/completions"
def migrate_ai_gateway_settings() -> None:
import os
import uuid
if not get_setting("gateway_internal_key", ""):
set_setting("gateway_internal_key", str(uuid.uuid4()))
logger.info("Generated internal gateway key")
deepseek = os.environ.get("DEEPSEEK_API_KEY", "")
openrouter = os.environ.get("OPENROUTER_API_KEY", "")
if not get_setting("gateway_api_key", "") and (deepseek or openrouter):
set_setting("gateway_api_key", deepseek or openrouter)
logger.info("Migrated gateway upstream key from environment")
if not get_setting("gateway_vision_key", "") and openrouter:
set_setting("gateway_vision_key", openrouter)
logger.info("Migrated gateway vision key from environment")
for key in ("news_ai_url", "bot_api_url", "devii_ai_url"):
if get_setting(key, "") == OLD_GATEWAY_URL:
set_setting(key, INTERNAL_GATEWAY_URL)
logger.info(f"Migrated {key} to the internal gateway")
if get_setting("bot_model", "") == "deepseek-chat":
set_setting("bot_model", "molodetz")
logger.info("Migrated bot_model to molodetz")
def backfill_api_keys() -> int:
if "users" not in db.tables:
return 0
users = db["users"]
if not users.has_column("api_key"):
users.create_column_by_example("api_key", "")
if not users.has_column("cust_disable_global"):
users.create_column_by_example("cust_disable_global", 0)
if not users.has_column("cust_disable_pagetype"):
users.create_column_by_example("cust_disable_pagetype", 0)
if not users.has_column("ai_correction_enabled"):
users.create_column_by_example("ai_correction_enabled", 0)
if not users.has_column("ai_correction_prompt"):
users.create_column_by_example("ai_correction_prompt", DEFAULT_CORRECTION_PROMPT)
if not users.has_column("ai_correction_sync"):
users.create_column_by_example("ai_correction_sync", 0)
if not users.has_column("ai_modifier_enabled"):
users.create_column_by_example("ai_modifier_enabled", 1)
if not users.has_column("ai_modifier_sync"):
users.create_column_by_example("ai_modifier_sync", 1)
if not users.has_column("ai_modifier_prompt"):
users.create_column_by_example("ai_modifier_prompt", DEFAULT_MODIFIER_PROMPT)
if not users.has_column("timezone"):
users.create_column_by_example("timezone", "")
if not users.has_column("avatar_seed"):
users.create_column_by_example("avatar_seed", "")
if not users.has_column("last_seen"):
users.create_column_by_example("last_seen", "")
with db:
db.query(
"UPDATE users SET ai_modifier_enabled = 1 WHERE ai_modifier_enabled IS NULL"
)
db.query("UPDATE users SET ai_modifier_sync = 1 WHERE ai_modifier_sync IS NULL")
db.query(
"UPDATE users SET ai_modifier_prompt = :prompt "
"WHERE ai_modifier_prompt IS NULL OR ai_modifier_prompt = ''",
prompt=DEFAULT_MODIFIER_PROMPT,
)
import uuid_utils
updated = 0
for user in users.find():
if not user.get("api_key"):
users.update(
{"uid": user["uid"], "api_key": str(uuid_utils.uuid7())}, ["uid"]
)
updated += 1
return updated
def _backfill_gamification():
if "users" not in db.tables:
return
from devplacepy.utils import (
level_for_xp,
check_milestone_badges,
XP_POST,
XP_COMMENT,
XP_PROJECT,
XP_GIST,
XP_UPVOTE,
XP_FOLLOW,
)
pending = list(db["users"].find(xp=0))
if not pending:
return
xp_by_user = defaultdict(int)
def add_counts(table, column, points):
if table not in db.tables:
return
for row in db.query(
f"SELECT {column} AS uid, COUNT(*) AS c FROM {table} GROUP BY {column}"
):
if row["uid"]:
xp_by_user[row["uid"]] += row["c"] * points
add_counts("posts", "user_uid", XP_POST)
add_counts("comments", "user_uid", XP_COMMENT)
add_counts("projects", "user_uid", XP_PROJECT)
add_counts("gists", "user_uid", XP_GIST)
add_counts("follows", "following_uid", XP_FOLLOW)
if "votes" in db.tables:
for content_table, target_type in (
("posts", "post"),
("projects", "project"),
("gists", "gist"),
("comments", "comment"),
):
if content_table not in db.tables:
continue
rows = db.query(
f"SELECT c.user_uid AS uid, COUNT(*) AS c "
f"FROM votes v JOIN {content_table} c ON v.target_uid = c.uid "
f"WHERE v.target_type = :t AND v.value = 1 GROUP BY c.user_uid",
t=target_type,
)
for row in rows:
if row["uid"]:
xp_by_user[row["uid"]] += row["c"] * XP_UPVOTE
for user in pending:
xp = xp_by_user.get(user["uid"], 0)
if xp <= 0:
continue
db["users"].update(
{"uid": user["uid"], "xp": xp, "level": level_for_xp(xp)}, ["uid"]
)
_authors_cache.clear()
for user in pending:
check_milestone_badges(user["uid"])
logger.info(f"Gamification backfill processed {len(pending)} users")
Reference in New Issue View Git Blame Copy Permalink