# retoor from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger from .settings import get_setting, set_setting from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns from .ranking import _authors_cache BUG_TABLE_RENAMES = ( ("bug_tickets", "issue_tickets"), ("bug_comment_authors", "issue_comment_authors"), ) def migrate_bug_tables_to_issue_tables() -> None: for source_name, destination_name in BUG_TABLE_RENAMES: if source_name not in db.tables: continue source = db[source_name] destination = get_table(destination_name) copied = 0 for row in source.all(): payload = {key: value for key, value in row.items() if key != "id"} destination.insert_ignore(payload, ["uid"]) copied += 1 logger.info( "Migrated %s rows from %s into %s", copied, source_name, destination_name ) source.drop() logger.info("Dropped table %s after migration", source_name) def init_db(): tables = db.tables _index(db, "users", "idx_users_username", ["username"]) _index(db, "users", "idx_users_email", ["email"]) _index(db, "users", "idx_users_api_key", ["api_key"]) _index(db, "users", "idx_users_role", ["role", "created_at"]) _index(db, "users", "idx_users_last_seen", ["last_seen"]) _index(db, "users", "idx_users_created_at", ["created_at"]) _index(db, "posts", "idx_posts_user_uid", ["user_uid"]) _index(db, "posts", "idx_posts_created_at", ["created_at"]) _index(db, "posts", "idx_posts_topic", ["topic"]) _index(db, "posts", "idx_posts_slug", ["slug"]) if "posts" in tables: posts_table = get_table("posts") if not posts_table.has_column("tags"): posts_table.create_column_by_example("tags", "") if "devrant_tokens" in tables: devrant_tokens = get_table("devrant_tokens") for column, example in ( ("uid", ""), ("key", ""), ("user_uid", ""), ("user_id", 0), ("expire_time", 0), ("created_at", ""), ): if not devrant_tokens.has_column(column): devrant_tokens.create_column_by_example(column, example) _index(db, "devrant_tokens", "idx_devrant_tokens_key", ["key"]) _index(db, "devrant_tokens", "idx_devrant_tokens_user", ["user_uid"]) if "access_tokens" in tables: access_tokens = get_table("access_tokens") for column, example in ( ("uid", ""), ("token", ""), ("user_uid", ""), ("label", ""), ("expires_at", ""), ("created_at", ""), ): if not access_tokens.has_column(column): access_tokens.create_column_by_example(column, example) _index(db, "access_tokens", "idx_access_tokens_token", ["token"]) _index(db, "access_tokens", "idx_access_tokens_user", ["user_uid"]) if "telegram_pairings" in tables: telegram_pairings = get_table("telegram_pairings") for column, example in ( ("uid", ""), ("user_uid", ""), ("code_hash", ""), ("expires_at", ""), ("used", 0), ("created_at", ""), ): if not telegram_pairings.has_column(column): telegram_pairings.create_column_by_example(column, example) _index(db, "telegram_pairings", "idx_telegram_pairings_code", ["code_hash"]) _index(db, "telegram_pairings", "idx_telegram_pairings_user", ["user_uid"]) if "telegram_links" in tables: telegram_links = get_table("telegram_links") for column, example in ( ("uid", ""), ("user_uid", ""), ("chat_id", 0), ("from_id", 0), ("created_at", ""), ): if not telegram_links.has_column(column): telegram_links.create_column_by_example(column, example) _index(db, "telegram_links", "idx_telegram_links_chat", ["chat_id"]) _index(db, "telegram_links", "idx_telegram_links_user", ["user_uid"]) telegram_outbox = get_table("telegram_outbox") for column, example in ( ("uid", ""), ("user_uid", ""), ("chat_id", 0), ("text", ""), ("status", "pending"), ("attempts", 0), ("created_at", ""), ("sent_at", ""), ): if not telegram_outbox.has_column(column): telegram_outbox.create_column_by_example(column, example) _index(db, "telegram_outbox", "idx_telegram_outbox_status", ["status", "id"]) _index(db, "comments", "idx_comments_post_uid", ["post_uid"]) _index(db, "comments", "idx_comments_target", ["target_type", "target_uid"]) _index(db, "comments", "idx_comments_user_uid", ["user_uid"]) _index(db, "comments", "idx_comments_created_at", ["created_at"]) _index(db, "votes", "idx_votes_target", ["target_uid", "target_type"]) _index(db, "messages", "idx_messages_sender", ["sender_uid"]) _index(db, "messages", "idx_messages_receiver", ["receiver_uid"]) _index( db, "messages", "idx_messages_conversation", ["sender_uid", "receiver_uid"] ) _index( db, "messages", "idx_messages_conversation_rev", ["receiver_uid", "sender_uid"], ) _index(db, "notifications", "idx_notifications_user", ["user_uid"]) _index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"]) _index(db, "push_registration", "idx_push_registration_user", ["user_uid"]) _index(db, "sessions", "idx_sessions_token", ["session_token"]) projects = get_table("projects") for column, example in ( ("uid", ""), ("user_uid", ""), ("slug", ""), ("created_at", ""), ("stars", 0), ("project_type", ""), ("is_private", 0), ("read_only", 0), ("updated_at", ""), ): if not projects.has_column(column): projects.create_column_by_example(column, example) _index(db, "projects", "idx_projects_user", ["user_uid"]) _index(db, "projects", "idx_projects_slug", ["slug"]) _index(db, "projects", "idx_projects_private", ["is_private"]) _index(db, "projects", "idx_projects_stars", ["stars"]) _index(db, "projects", "idx_projects_type", ["project_type"]) project_files = get_table("project_files") for column, example in ( ("uid", ""), ("project_uid", ""), ("user_uid", ""), ("path", ""), ("name", ""), ("parent_path", ""), ("type", ""), ("content", ""), ("is_binary", 0), ("stored_name", ""), ("directory", ""), ("mime_type", ""), ("size", 0), ("created_at", ""), ("updated_at", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not project_files.has_column(column): project_files.create_column_by_example(column, example) _index(db, "project_files", "idx_project_files_path", ["project_uid", "path"]) _index( db, "project_files", "idx_project_files_parent", ["project_uid", "parent_path"] ) _index(db, "badges", "idx_badges_user", ["user_uid"]) _index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"]) _drop_index(db, "idx_follows_follower") _drop_index(db, "idx_follows_following") _index(db, "follows", "idx_follows_follower_created", ["follower_uid", "created_at"]) _index(db, "follows", "idx_follows_following_created", ["following_uid", "created_at"]) user_relations = get_table("user_relations") for column, example in ( ("uid", ""), ("user_uid", ""), ("target_uid", ""), ("kind", ""), ("created_at", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not user_relations.has_column(column): user_relations.create_column_by_example(column, example) _index(db, "user_relations", "idx_user_relations_owner_kind", ["user_uid", "kind"]) _index(db, "user_relations", "idx_user_relations_target_kind", ["target_uid", "kind"]) _index(db, "password_resets", "idx_password_resets_token", ["token"]) _index(db, "gists", "idx_gists_user_uid", ["user_uid"]) _index(db, "gists", "idx_gists_language", ["language"]) _index(db, "gists", "idx_gists_slug", ["slug"]) attachments = get_table("attachments") for column, example in ( ("uid", ""), ("target_type", ""), ("target_uid", ""), ("resource_type", ""), ("resource_uid", ""), ("user_uid", ""), ("original_filename", ""), ("stored_name", ""), ("directory", ""), ("file_size", 0), ("mime_type", ""), ("image_width", 0), ("image_height", 0), ("has_thumbnail", 0), ("thumbnail_name", ""), ("gitea_asset_id", 0), ("created_at", ""), ("deleted_at", ""), ): if not attachments.has_column(column): attachments.create_column_by_example(column, example) _index( db, "attachments", "idx_attachments_resource", ["resource_type", "resource_uid"] ) _index(db, "attachments", "idx_attachments_target", ["target_type", "target_uid"]) _index(db, "attachments", "idx_attachments_user_created", ["user_uid", "created_at"]) _index(db, "reactions", "idx_reactions_target", ["target_type", "target_uid"]) _index( db, "reactions", "idx_reactions_user_target", ["user_uid", "target_type", "target_uid"], ) _index(db, "bookmarks", "idx_bookmarks_user", ["user_uid"]) _index(db, "bookmarks", "idx_bookmarks_target", ["target_type", "target_uid"]) _index(db, "polls", "idx_polls_post", ["post_uid"]) _index(db, "poll_options", "idx_poll_options_poll", ["poll_uid"]) _index(db, "poll_votes", "idx_poll_votes_poll", ["poll_uid"]) _index(db, "poll_votes", "idx_poll_votes_user", ["poll_uid", "user_uid"]) if "site_settings" in tables: defaults = { "site_name": "DevPlace", "site_description": "The Developer Social Network", "site_tagline": "Track industry shifts. Discover bold releases. Share what you are building in an open, uncensored environment.", } for key, value in defaults.items(): existing = db["site_settings"].find_one(key=key) if not existing: db["site_settings"].insert( {"uid": f"default_{key}", "key": key, "value": value} ) _index(db, "site_settings", "idx_site_settings_key", ["key"]) news = get_table("news") for column, example in ( ("uid", ""), ("slug", ""), ("title", ""), ("external_id", ""), ("status", ""), ("source_name", ""), ("url", ""), ("description", ""), ("content", ""), ("synced_at", ""), ("grade", 0), ("ai_grade", 0), ("show_on_landing", 0), ("featured", 0), ("author", ""), ("article_published", ""), ("image_url", ""), ("has_unique_image", 0), ("featured_locked", 0), ("landing_locked", 0), ): if not news.has_column(column): news.create_column_by_example(column, example) _index(db, "news", "idx_news_external_id", ["external_id"]) _index(db, "news", "idx_news_slug", ["slug"]) _index(db, "news", "idx_news_synced_at", ["synced_at"]) _index(db, "news", "idx_news_status", ["status"]) _index(db, "news", "idx_news_featured", ["featured"]) _index(db, "news", "idx_news_landing", ["show_on_landing"]) news_images = get_table("news_images") for column, example in ( ("news_uid", ""), ("url", ""), ("alt_text", ""), ("phash", ""), ("width", 0), ("height", 0), ("is_placeholder", 0), ): if not news_images.has_column(column): news_images.create_column_by_example(column, example) news_sync = get_table("news_sync") if not news_sync.has_column("external_id"): news_sync.create_column_by_example("external_id", "") _index(db, "news_images", "idx_news_images_news_uid", ["news_uid"]) _index(db, "news_sync", "idx_news_sync_external_id", ["external_id"]) issue_tickets = get_table("issue_tickets") for column, example in ( ("uid", ""), ("gitea_number", 0), ("author_uid", ""), ("original_title", ""), ("original_description", ""), ("enhanced_title", ""), ("html_url", ""), ("last_status", ""), ("last_comment_count", 0), ("created_at", ""), ("updated_at", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not issue_tickets.has_column(column): issue_tickets.create_column_by_example(column, example) issue_comment_authors = get_table("issue_comment_authors") for column, example in ( ("uid", ""), ("gitea_comment_id", 0), ("gitea_number", 0), ("author_uid", ""), ("created_at", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not issue_comment_authors.has_column(column): issue_comment_authors.create_column_by_example(column, example) _index(db, "issue_tickets", "idx_issue_tickets_number", ["gitea_number"]) _index(db, "issue_tickets", "idx_issue_tickets_author", ["author_uid"]) _index( db, "issue_comment_authors", "idx_issue_comment_authors_comment", ["gitea_comment_id"], ) _index( db, "issue_comment_authors", "idx_issue_comment_authors_number", ["gitea_number"] ) ws_tickets = get_table("ws_tickets") for column, example in ( ("uid", ""), ("token", ""), ("user_uid", ""), ("created_at", ""), ("expires_at", ""), ("used_at", ""), ): if not ws_tickets.has_column(column): ws_tickets.create_column_by_example(column, example) _index(db, "ws_tickets", "idx_ws_tickets_token", ["token"], unique=True) _index(db, "ws_tickets", "idx_ws_tickets_expires", ["expires_at"]) migrate_bug_tables_to_issue_tables() _index(db, "service_state", "idx_service_state_name", ["name"]) if "devii_conversations" in db.tables: conversations = get_table("devii_conversations") if not conversations.has_column("channel"): conversations.create_column_by_example("channel", "main") try: with db: db.query( "UPDATE devii_conversations SET channel='main' WHERE channel IS NULL" ) except Exception as e: # noqa: BLE001 logger.warning(f"Could not backfill devii_conversations.channel: {e}") _index( db, "devii_conversations", "idx_devii_conv_owner", ["owner_kind", "owner_id", "channel"], ) _index( db, "devii_usage_ledger", "idx_devii_ledger_owner_time", ["owner_kind", "owner_id", "created_at"], ) _index( db, "devii_turns", "idx_devii_turns_owner_time", ["owner_kind", "owner_id", "started_at"], ) _index(db, "devii_tasks", "idx_devii_tasks_owner", ["owner_kind", "owner_id"]) _index( db, "devii_tasks", "idx_devii_tasks_due", ["enabled", "status", "next_run_at"] ) _index(db, "devii_lessons", "idx_devii_lessons_owner", ["owner_kind", "owner_id"]) _index(db, "devii_lessons", "idx_devii_lessons_owner_created", ["owner_kind", "owner_id", "created_at"]) _index( db, "devii_virtual_tools", "idx_devii_vtools_owner", ["owner_kind", "owner_id"] ) _index( db, "devii_virtual_tools", "idx_devii_vtools_name", ["owner_kind", "owner_id", "name"], ) _index(db, "devii_behavior", "idx_devii_behavior_owner", ["owner_kind", "owner_id"]) _index(db, "user_customizations", "idx_user_cust_owner", ["owner_kind", "owner_id"]) _index( db, "user_customizations", "idx_user_cust_scope", ["owner_kind", "owner_id", "scope", "lang"], ) _index(db, "gateway_usage_ledger", "idx_gw_usage_time", ["created_at"]) _index( db, "gateway_usage_ledger", "idx_gw_usage_backend_time", ["backend", "created_at"], ) _index( db, "gateway_usage_ledger", "idx_gw_usage_model_time", ["model", "created_at"] ) _index( db, "gateway_usage_ledger", "idx_gw_usage_owner_time", ["owner_kind", "owner_id", "created_at"], ) _index( db, "gateway_usage_ledger", "idx_gw_usage_endpoint_time", ["endpoint", "created_at"], ) _index( db, "gateway_usage_ledger", "idx_gw_usage_appref_time", ["app_reference", "created_at"], ) _index(db, "gateway_concurrency_samples", "idx_gw_conc_time", ["created_at"]) jobs_table = get_table("jobs") for column, example in ( ("uid", ""), ("kind", ""), ("status", ""), ("owner_kind", ""), ("owner_id", ""), ("preferred_name", ""), ("payload", ""), ("result", ""), ("error", ""), ("retry_count", 0), ("created_at", ""), ("started_at", ""), ("completed_at", ""), ("updated_at", ""), ("duration_ms", 0), ("last_accessed_at", ""), ("expires_at", ""), ("bytes_in", 0), ("bytes_out", 0), ("item_count", 0), ): if not jobs_table.has_column(column): jobs_table.create_column_by_example(column, example) _index(db, "jobs", "idx_jobs_kind_status", ["kind", "status"]) _index(db, "jobs", "idx_jobs_owner", ["owner_kind", "owner_id"]) _index(db, "jobs", "idx_jobs_expires", ["expires_at"]) _index(db, "project_forks", "idx_project_forks_source", ["source_project_uid"]) _index(db, "project_forks", "idx_project_forks_forked", ["forked_project_uid"]) instances = get_table("instances") for column, example in ( ("uid", ""), ("project_uid", ""), ("slug", ""), ("name", ""), ("status", ""), ("desired_state", ""), ("container_id", ""), ("ingress_slug", ""), ("ingress_port", 0), ("ports_json", ""), ("container_gateway", ""), ("run_as_uid", ""), ("boot_language", "none"), ("boot_script", ""), ("start_on_boot", 0), ): if not instances.has_column(column): instances.create_column_by_example(column, example) _index(db, "instances", "idx_instances_project", ["project_uid"]) _index(db, "instances", "idx_instances_slug", ["slug"]) _index(db, "instances", "idx_instances_name", ["name"]) _index(db, "instances", "idx_instances_state", ["desired_state", "status"]) _index(db, "instances", "idx_instances_container", ["container_id"]) _index(db, "instances", "idx_instances_ingress", ["ingress_slug"]) _index( db, "instance_events", "idx_instance_events_instance", ["instance_uid", "created_at"], ) _index( db, "instance_metrics", "idx_instance_metrics_instance_ts", ["instance_uid", "ts"], ) _index( db, "instance_schedules", "idx_instance_schedules_due", ["enabled", "next_run_at"], ) from devplacepy.services.audit import store as audit_store audit_store.ensure_tables() from devplacepy.services.backup import store as backup_store backup_store.ensure_tables() from devplacepy.services.openai_gateway import routing as gateway_routing gateway_routing.ensure_tables() from devplacepy.services.openai_gateway import quota as gateway_quota gateway_quota.ensure_tables() _index(db, "audit_log", "idx_audit_created_at", ["created_at"]) _index(db, "audit_log", "idx_audit_event_key", ["event_key"]) _index(db, "audit_log", "idx_audit_category", ["category"]) _index(db, "audit_log", "idx_audit_actor_uid", ["actor_uid"]) _index(db, "audit_log", "idx_audit_actor_role", ["actor_role"]) _index(db, "audit_log", "idx_audit_origin", ["origin"]) _index(db, "audit_log", "idx_audit_target", ["target_type", "target_uid"]) _index(db, "audit_log", "idx_audit_result", ["result"]) _index(db, "audit_log_links", "idx_audit_links_audit", ["audit_uid"]) _index(db, "audit_log_links", "idx_audit_links_object", ["object_type", "object_uid"]) _index(db, "audit_log_links", "idx_audit_links_relation", ["relation"]) notification_preferences = get_table("notification_preferences") for column, example in ( ("uid", ""), ("user_uid", ""), ("notification_type", ""), ("in_app_enabled", 1), ("push_enabled", 1), ("telegram_enabled", 1), ("created_at", ""), ("updated_at", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not notification_preferences.has_column(column): notification_preferences.create_column_by_example(column, example) _index(db, "notification_preferences", "idx_notif_prefs_user", ["user_uid"]) _index( db, "notification_preferences", "idx_notif_prefs_user_type", ["user_uid", "notification_type"], ) correction_usage = get_table("correction_usage") for column, example in ( ("user_uid", ""), ("calls", 0), ("prompt_tokens", 0), ("completion_tokens", 0), ("total_tokens", 0), ("cost_usd", 0.0), ("upstream_latency_ms", 0.0), ("total_latency_ms", 0.0), ("updated_at", ""), ): if not correction_usage.has_column(column): correction_usage.create_column_by_example(column, example) try: if "correction_usage" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_correction_usage_user " "ON correction_usage (user_uid)" ) except Exception as e: logger.warning(f"Could not create unique index on correction_usage: {e}") modifier_usage = get_table("modifier_usage") for column, example in ( ("user_uid", ""), ("calls", 0), ("prompt_tokens", 0), ("completion_tokens", 0), ("total_tokens", 0), ("cost_usd", 0.0), ("upstream_latency_ms", 0.0), ("total_latency_ms", 0.0), ("updated_at", ""), ): if not modifier_usage.has_column(column): modifier_usage.create_column_by_example(column, example) try: if "modifier_usage" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_modifier_usage_user " "ON modifier_usage (user_uid)" ) except Exception as e: logger.warning(f"Could not create unique index on modifier_usage: {e}") news_usage = get_table("news_usage") for column, example in ( ("user_uid", ""), ("calls", 0), ("prompt_tokens", 0), ("completion_tokens", 0), ("total_tokens", 0), ("cost_usd", 0.0), ("upstream_latency_ms", 0.0), ("total_latency_ms", 0.0), ("updated_at", ""), ): if not news_usage.has_column(column): news_usage.create_column_by_example(column, example) try: if "news_usage" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user " "ON news_usage (user_uid)" ) except Exception as e: logger.warning(f"Could not create unique index on news_usage: {e}") issue_usage = get_table("issue_usage") for column, example in ( ("user_uid", ""), ("calls", 0), ("prompt_tokens", 0), ("completion_tokens", 0), ("total_tokens", 0), ("cost_usd", 0.0), ("upstream_latency_ms", 0.0), ("total_latency_ms", 0.0), ("updated_at", ""), ): if not issue_usage.has_column(column): issue_usage.create_column_by_example(column, example) try: if "issue_usage" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_usage_user " "ON issue_usage (user_uid)" ) except Exception as e: logger.warning(f"Could not create unique index on issue_usage: {e}") seo_usage = get_table("seo_usage") for column, example in ( ("user_uid", ""), ("calls", 0), ("prompt_tokens", 0), ("completion_tokens", 0), ("total_tokens", 0), ("cost_usd", 0.0), ("upstream_latency_ms", 0.0), ("total_latency_ms", 0.0), ("updated_at", ""), ): if not seo_usage.has_column(column): seo_usage.create_column_by_example(column, example) try: if "seo_usage" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_seo_usage_user " "ON seo_usage (user_uid)" ) except Exception as e: logger.warning(f"Could not create unique index on seo_usage: {e}") award_usage = get_table("award_usage") for column, example in ( ("user_uid", ""), ("calls", 0), ("prompt_tokens", 0), ("completion_tokens", 0), ("total_tokens", 0), ("cost_usd", 0.0), ("upstream_latency_ms", 0.0), ("total_latency_ms", 0.0), ("updated_at", ""), ): if not award_usage.has_column(column): award_usage.create_column_by_example(column, example) try: if "award_usage" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_award_usage_user " "ON award_usage (user_uid)" ) except Exception as e: logger.warning(f"Could not create unique index on award_usage: {e}") awards = get_table("awards") for column, example in ( ("uid", ""), ("slug", ""), ("description", ""), ("giver_uid", ""), ("receiver_uid", ""), ("attachment_uid_512", ""), ("attachment_uid_256", ""), ("attachment_uid_64", ""), ("generated_at", ""), ("created_at", ""), ("job_uid", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not awards.has_column(column): awards.create_column_by_example(column, example) try: if "awards" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_awards_slug ON awards (slug)" ) db.query( "CREATE INDEX IF NOT EXISTS idx_awards_receiver_created " "ON awards (receiver_uid, created_at)" ) db.query( "CREATE INDEX IF NOT EXISTS idx_awards_giver_created " "ON awards (giver_uid, created_at)" ) db.query( "CREATE INDEX IF NOT EXISTS idx_awards_generated " "ON awards (receiver_uid, generated_at)" ) except Exception as e: logger.warning(f"Could not create awards indexes: {e}") seo_metadata = get_table("seo_metadata") for column, example in ( ("uid", ""), ("target_type", ""), ("target_uid", ""), ("seo_title", ""), ("seo_description", ""), ("seo_keywords", ""), ("status", ""), ("source", ""), ("generated_at", ""), ("created_at", ""), ("updated_at", ""), ): if not seo_metadata.has_column(column): seo_metadata.create_column_by_example(column, example) _index( db, "seo_metadata", "idx_seo_metadata_target", ["target_type", "target_uid"], unique=True, ) email_accounts = get_table("email_accounts") for column, example in ( ("uid", ""), ("owner_kind", ""), ("owner_id", ""), ("label", ""), ("imap_host", ""), ("imap_port", 993), ("imap_ssl", 1), ("imap_starttls", 0), ("smtp_host", ""), ("smtp_port", 587), ("smtp_ssl", 0), ("smtp_starttls", 1), ("username", ""), ("password", ""), ("from_address", ""), ("from_name", ""), ("created_at", ""), ("updated_at", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not email_accounts.has_column(column): email_accounts.create_column_by_example(column, example) _index(db, "email_accounts", "idx_email_accounts_owner", ["owner_kind", "owner_id"]) _index( db, "email_accounts", "idx_email_accounts_label", ["owner_kind", "owner_id", "label"], ) user_activity = get_table("user_activity") for column, example in ( ("user_uid", ""), ("action", ""), ("count", 0), ("first_at", ""), ("last_at", ""), ): if not user_activity.has_column(column): user_activity.create_column_by_example(column, example) try: if "user_activity" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_user_action " "ON user_activity (user_uid, action)" ) except Exception as e: logger.warning(f"Could not create unique index on user_activity: {e}") user_activity_seen = get_table("user_activity_seen") for column, example in ( ("user_uid", ""), ("action", ""), ("target", ""), ("created_at", ""), ): if not user_activity_seen.has_column(column): user_activity_seen.create_column_by_example(column, example) try: if "user_activity_seen" in db.tables: with db: db.query( "CREATE UNIQUE INDEX IF NOT EXISTS idx_user_activity_seen_unique " "ON user_activity_seen (user_uid, action, target)" ) except Exception as e: logger.warning(f"Could not create unique index on user_activity_seen: {e}") deepsearch_sessions = get_table("deepsearch_sessions") for column, example in ( ("uid", ""), ("owner_kind", ""), ("owner_id", ""), ("query", ""), ("status", ""), ("depth", 0), ("max_pages", 0), ("score", 0), ("confidence", 0.0), ("source_diversity", 0.0), ("page_count", 0), ("chunk_count", 0), ("collection", ""), ("summary", ""), ("created_at", ""), ("completed_at", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not deepsearch_sessions.has_column(column): deepsearch_sessions.create_column_by_example(column, example) _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_owner", ["owner_kind", "owner_id"]) _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_status", ["status"]) _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_created", ["created_at"]) deepsearch_messages = get_table("deepsearch_messages") for column, example in ( ("uid", ""), ("session_uid", ""), ("role", ""), ("content", ""), ("citations", ""), ("created_at", ""), ("deleted_at", ""), ("deleted_by", ""), ): if not deepsearch_messages.has_column(column): deepsearch_messages.create_column_by_example(column, example) _index(db, "deepsearch_messages", "idx_deepsearch_messages_session", ["session_uid"]) deepsearch_url_cache = get_table("deepsearch_url_cache") for column, example in ( ("uid", ""), ("url_hash", ""), ("url", ""), ("title", ""), ("content_hash", ""), ("status", 0), ("byte_size", 0), ("fetched_at", ""), ): if not deepsearch_url_cache.has_column(column): deepsearch_url_cache.create_column_by_example(column, example) _index(db, "deepsearch_url_cache", "idx_deepsearch_url_cache_hash", ["url_hash"]) isslop_analyses = get_table("isslop_analyses") for column, example in ( ("uid", ""), ("owner_kind", ""), ("owner_id", ""), ("source_url", ""), ("source_kind", ""), ("status", ""), ("created_at", ""), ("finished_at", ""), ("content_hash", ""), ("grade", ""), ("slop_score", 0.0), ("origin_score", 0.0), ("quality_deficit_score", 0.0), ("human_percent", 0.0), ("ai_percent", 0.0), ("category", ""), ("confidence", ""), ("files_total", 0), ("files_analyzed", 0), ("error_message_text", ""), ("detected_builder", ""), ("dom_slop_score", 0.0), ("deleted_at", ""), ("deleted_by", ""), ): if not isslop_analyses.has_column(column): isslop_analyses.create_column_by_example(column, example) _index(db, "isslop_analyses", "idx_isslop_analyses_owner", ["owner_kind", "owner_id", "created_at"]) _index(db, "isslop_analyses", "idx_isslop_analyses_status", ["status"]) _index(db, "isslop_analyses", "idx_isslop_analyses_hash", ["content_hash"]) isslop_events = get_table("isslop_events") for column, example in ( ("analysis_uid", ""), ("seq", 0), ("kind", ""), ("message", ""), ("payload", ""), ("created_at", ""), ): if not isslop_events.has_column(column): isslop_events.create_column_by_example(column, example) _index(db, "isslop_events", "idx_isslop_events_analysis", ["analysis_uid", "seq"]) isslop_file_results = get_table("isslop_file_results") for column, example in ( ("analysis_uid", ""), ("path", ""), ("language", ""), ("lines", 0), ("origin_score", 0.0), ("quality_deficit_score", 0.0), ("category", ""), ("signals", ""), ("source", ""), ): if not isslop_file_results.has_column(column): isslop_file_results.create_column_by_example(column, example) _index(db, "isslop_file_results", "idx_isslop_file_results_analysis", ["analysis_uid"]) isslop_image_results = get_table("isslop_image_results") for column, example in ( ("analysis_uid", ""), ("path", ""), ("ai_probability", 0.0), ("grade", ""), ("verdict", ""), ("image_kind", ""), ("tells", ""), ("description", ""), ("thumb", ""), ): if not isslop_image_results.has_column(column): isslop_image_results.create_column_by_example(column, example) _index(db, "isslop_image_results", "idx_isslop_image_results_analysis", ["analysis_uid"]) isslop_dom_results = get_table("isslop_dom_results") for column, example in ( ("analysis_uid", ""), ("url", ""), ("detected_builder", ""), ("signal_count", 0), ("screenshot", ""), ("signals", ""), ): if not isslop_dom_results.has_column(column): isslop_dom_results.create_column_by_example(column, example) _index(db, "isslop_dom_results", "idx_isslop_dom_results_analysis", ["analysis_uid"]) isslop_reports = get_table("isslop_reports") for column, example in ( ("analysis_uid", ""), ("markdown", ""), ("model_used", ""), ("generated_at", ""), ): if not isslop_reports.has_column(column): isslop_reports.create_column_by_example(column, example) _index(db, "isslop_reports", "idx_isslop_reports_analysis", ["analysis_uid"], unique=True) game_farms = get_table("game_farms") for column, example in ( ("uid", ""), ("user_uid", ""), ("coins", 0), ("xp", 0), ("level", 1), ("ci_tier", 1), ("plot_count", 0), ("total_harvests", 0), ("prestige", 0), ("streak", 0), ("last_daily_at", ""), ("perk_yield", 0), ("perk_growth", 0), ("perk_discount", 0), ("perk_xp", 0), ("stars", 0), ("legacy_autoharvest", 0), ("legacy_multiplier", 0), ("legacy_speed", 0), ("legacy_plots", 0), ("legacy_defense", 0), ("legacy_carryover", 0), ("last_grant_week", ""), ("prestiged_at", ""), ("mastery_points", 0), ("mastery_points_earned_total", 0), ("mastery_autoreplant", 0), ("mastery_analytics", 0), ("mastery_contracts", 0), ("lifetime_coins_earned", 0), ("lifetime_harvests", 0), ("infra_registry", 0), ("infra_canary", 0), ("infra_observability", 0), ("defense_level", 0), ("defense_last_upkeep_at", ""), ("active_title", ""), ("underdog_boost_until", ""), ("contract_boost_until", ""), ("harvests_week", 0), ("harvests_week_start", ""), ("last_kernel_harvest_prestige", 0), ("time_to_kernel_seconds", 0), ("era_coins", 0), ("era_harvests", 0), ("era_joined_at", ""), ("created_at", ""), ("updated_at", ""), ): if not game_farms.has_column(column): game_farms.create_column_by_example(column, example) _index(db, "game_farms", "idx_game_farms_user", ["user_uid"], unique=True) _index(db, "game_farms", "idx_game_farms_rank", ["level", "xp"]) game_steals = get_table("game_steals") for column, example in ( ("uid", ""), ("thief_uid", ""), ("owner_uid", ""), ("slot_index", 0), ("crop_key", ""), ("coins", 0), ("stolen_at", ""), ("created_at", ""), ): if not game_steals.has_column(column): game_steals.create_column_by_example(column, example) _index( db, "game_steals", "idx_game_steals_pair", ["thief_uid", "owner_uid", "stolen_at"] ) game_quests = get_table("game_quests") for column, example in ( ("uid", ""), ("farm_uid", ""), ("user_uid", ""), ("day", ""), ("scope", "daily"), ("slot_index", 0), ("kind", ""), ("label", ""), ("goal", 0), ("progress", 0), ("reward_coins", 0), ("reward_xp", 0), ("reward_stars", 0), ("claimed", 0), ("created_at", ""), ("updated_at", ""), ): if not game_quests.has_column(column): game_quests.create_column_by_example(column, example) with db: db.query("UPDATE game_quests SET scope = 'daily' WHERE scope IS NULL OR scope = ''") _index(db, "game_quests", "idx_game_quests_farm_day", ["farm_uid", "day"]) _index(db, "game_quests", "idx_game_quests_farm_day_scope", ["farm_uid", "day", "scope"]) game_plots = get_table("game_plots") for column, example in ( ("uid", ""), ("farm_uid", ""), ("user_uid", ""), ("slot_index", 0), ("crop_key", ""), ("planted_at", ""), ("ready_at", ""), ("watered_by", "[]"), ("created_at", ""), ("updated_at", ""), ): if not game_plots.has_column(column): game_plots.create_column_by_example(column, example) _index(db, "game_plots", "idx_game_plots_farm", ["farm_uid", "slot_index"]) game_market_ticks = get_table("game_market_ticks") for column, example in ( ("uid", ""), ("crop_key", ""), ("hour_bucket", ""), ("harvests", 0), ("updated_at", ""), ): if not game_market_ticks.has_column(column): game_market_ticks.create_column_by_example(column, example) _index( db, "game_market_ticks", "idx_game_market_ticks_bucket", ["crop_key", "hour_bucket"], unique=True, ) game_cosmetics = get_table("game_cosmetics") for column, example in ( ("uid", ""), ("user_uid", ""), ("cosmetic_key", ""), ("purchased_at", ""), ("created_at", ""), ): if not game_cosmetics.has_column(column): game_cosmetics.create_column_by_example(column, example) _index( db, "game_cosmetics", "idx_game_cosmetics_owner", ["user_uid", "cosmetic_key"], unique=True, ) game_treasury = get_table("game_treasury") for column, example in ( ("uid", ""), ("balance", 0), ("collected_total", 0), ("granted_total", 0), ("updated_at", ""), ): if not game_treasury.has_column(column): game_treasury.create_column_by_example(column, example) game_eras = get_table("game_eras") for column, example in ( ("uid", ""), ("era_number", 0), ("name", ""), ("started_at", ""), ("ends_at", ""), ("active", 0), ("created_at", ""), ): if not game_eras.has_column(column): game_eras.create_column_by_example(column, example) _index(db, "game_eras", "idx_game_eras_active", ["active"]) game_era_results = get_table("game_era_results") for column, example in ( ("uid", ""), ("era_number", 0), ("user_uid", ""), ("rank", 0), ("era_score", 0), ("era_coins_final", 0), ("reward_stars", 0), ("reward_cosmetic_key", ""), ("created_at", ""), ): if not game_era_results.has_column(column): game_era_results.create_column_by_example(column, example) _index(db, "game_era_results", "idx_game_era_results_era", ["era_number", "rank"]) _index(db, "posts", "idx_posts_user_created", ["user_uid", "created_at"]) _index( db, "posts", "idx_posts_live_created", ["created_at"], where="deleted_at IS NULL", ) _index( db, "comments", "idx_comments_target_created", ["target_type", "target_uid", "created_at"], ) _index(db, "comments", "idx_comments_user_created", ["user_uid", "created_at"]) _index(db, "votes", "idx_votes_user_target", ["user_uid", "target_uid"]) _index(db, "gists", "idx_gists_user_created", ["user_uid", "created_at"]) _index(db, "projects", "idx_projects_user_created", ["user_uid", "created_at"]) _index( db, "notifications", "idx_notifications_user_created", ["user_uid", "created_at"], ) for table in db.tables: _uid_index(db, table) for table in SOFT_DELETE_TABLES: ensure_soft_delete_columns(table) _index( db, "comments", "idx_comments_target_live", ["target_type", "target_uid"], where="deleted_at IS NULL", ) _index( db, "votes", "idx_votes_target_live", ["target_uid", "value"], where="deleted_at IS NULL", ) _index( db, "reactions", "idx_reactions_target_live", ["target_type", "target_uid"], where="deleted_at IS NULL", ) _index( db, "gists", "idx_gists_live_created", ["created_at"], where="deleted_at IS NULL", ) _index( db, "projects", "idx_projects_live_created", ["created_at"], where="deleted_at IS NULL", ) _index( db, "attachments", "idx_attachments_user_created_live", ["user_uid", "created_at"], where="deleted_at IS NULL", ) _index(db, "seo_metadata", "idx_seo_metadata_status", ["status", "deleted_at"]) if "news" in tables: for article in db["news"].find(status=None): was_featured = article.get("featured", 0) db["news"].update( { "uid": article["uid"], "status": "published" if was_featured else "draft", }, ["uid"], ) for article in db["news"].find(show_on_landing=None): db["news"].update( { "uid": article["uid"], "show_on_landing": 0, }, ["uid"], ) for article in db["news"].find(slug=None): from devplacepy.utils import make_combined_slug slug = make_combined_slug( article.get("title", "") or "news", article["uid"] ) db["news"].update( { "uid": article["uid"], "slug": slug, }, ["uid"], ) if "news_sync" in tables: for entry in db["news_sync"].find(): current = entry.get("status", "") if current in ("below_threshold", ""): db["news_sync"].update( { "id": entry["id"], "status": "graded", }, ["id"], ) if "site_settings" in tables: news_defaults = { "news_grade_threshold": "7", "news_api_url": "https://news.app.molodetz.nl/api", "news_ai_url": INTERNAL_GATEWAY_URL, "news_ai_model": "molodetz", } for key, value in news_defaults.items(): existing = db["site_settings"].find_one(key=key) if not existing: db["site_settings"].insert( {"uid": f"default_{key}", "key": key, "value": value} ) upload_defaults = { "max_upload_size_mb": "10", "allowed_file_types": "", "max_attachments_per_resource": "10", } for key, value in upload_defaults.items(): existing = db["site_settings"].find_one(key=key) if not existing: db["site_settings"].insert( {"uid": f"default_{key}", "key": key, "value": value} ) operational_defaults = { "rate_limit_per_minute": "60", "rate_limit_window_seconds": "60", "news_service_interval": "3600", "session_max_age_days": "7", "session_remember_days": "30", "registration_open": "1", "maintenance_mode": "0", "maintenance_message": "DevPlace is undergoing scheduled maintenance. Please check back shortly.", "customization_enabled": "1", "customization_js_enabled": "1", "audit_log_retention_days": "90", "statistics_tracking_enabled": "1", "docs_search_mode": "agent", "outbound_proxy_url": "", "devii_lessons_max_per_owner": "500", "devii_lessons_max_age_days": "90", } for key, value in operational_defaults.items(): existing = db["site_settings"].find_one(key=key) if not existing: db["site_settings"].insert( {"uid": f"default_{key}", "key": key, "value": value} ) with db: db.query( "CREATE TABLE IF NOT EXISTS visit_stats_hourly (" "bucket_start TEXT NOT NULL, " "page_group TEXT NOT NULL, " "referrer_group TEXT NOT NULL, " "views INTEGER NOT NULL DEFAULT 0, " "member_views INTEGER NOT NULL DEFAULT 0, " "guest_views INTEGER NOT NULL DEFAULT 0)" ) db.query( "CREATE TABLE IF NOT EXISTS visit_unique_slots (" "bucket_start TEXT NOT NULL, " "visitor_hash TEXT NOT NULL, " "page_group TEXT NOT NULL, " "user_uid TEXT)" ) _index(db, "visit_stats_hourly", "idx_visit_hourly_bucket", ["bucket_start"]) _index( db, "visit_stats_hourly", "idx_visit_hourly_page_time", ["page_group", "bucket_start"], ) _index( db, "visit_stats_hourly", "idx_visit_hourly_ref_time", ["referrer_group", "bucket_start"], ) _index( db, "visit_stats_hourly", "idx_visit_hourly_unique_row", ["bucket_start", "page_group", "referrer_group"], unique=True, ) _index(db, "visit_unique_slots", "idx_visit_unique_bucket", ["bucket_start"]) _index( db, "visit_unique_slots", "idx_visit_unique_hash", ["bucket_start", "visitor_hash"], ) _index( db, "visit_unique_slots", "idx_visit_unique_row", ["bucket_start", "visitor_hash", "page_group"], unique=True, ) _index(db, "issue_tickets", "idx_issue_tickets_created", ["created_at"]) _index(db, "devii_turns", "idx_devii_turns_started", ["started_at"]) _index(db, "instance_events", "idx_instance_events_created", ["created_at"]) _index(db, "game_steals", "idx_game_steals_stolen_at", ["stolen_at"]) _index(db, "messages", "idx_messages_created_at", ["created_at"]) _index(db, "notifications", "idx_notifications_created", ["created_at"]) _index(db, "follows", "idx_follows_created_at", ["created_at"]) _index(db, "reactions", "idx_reactions_created_at", ["created_at"]) _index(db, "votes", "idx_votes_created_at", ["created_at"]) _index(db, "bookmarks", "idx_bookmarks_created_at", ["created_at"]) _index(db, "badges", "idx_badges_created_at", ["created_at"]) _index(db, "audit_log", "idx_audit_result_created", ["result", "created_at"]) _index(db, "jobs", "idx_jobs_created_at", ["created_at"]) _index(db, "attachments", "idx_attachments_created_at", ["created_at"]) _backfill_gamification() backfill_api_keys() migrate_ai_gateway_settings() _refresh_query_planner_stats() def _refresh_query_planner_stats() -> None: try: has_stats = bool( list(db.query("SELECT name FROM sqlite_master WHERE name='sqlite_stat1'")) ) with db: if not has_stats: db.query("ANALYZE") db.query("PRAGMA optimize") except Exception as e: logger.warning(f"Could not refresh query planner stats: {e}") logger.info("Database initialized") OLD_GATEWAY_URL = "https://openai.app.molodetz.nl/v1/chat/completions" def migrate_ai_gateway_settings() -> None: import os import uuid if not get_setting("gateway_internal_key", ""): set_setting("gateway_internal_key", str(uuid.uuid4())) logger.info("Generated internal gateway key") deepseek = os.environ.get("DEEPSEEK_API_KEY", "") openrouter = os.environ.get("OPENROUTER_API_KEY", "") if not get_setting("gateway_api_key", "") and (deepseek or openrouter): set_setting("gateway_api_key", deepseek or openrouter) logger.info("Migrated gateway upstream key from environment") if not get_setting("gateway_vision_key", "") and openrouter: set_setting("gateway_vision_key", openrouter) logger.info("Migrated gateway vision key from environment") for key in ("news_ai_url", "bot_api_url", "devii_ai_url"): if get_setting(key, "") == OLD_GATEWAY_URL: set_setting(key, INTERNAL_GATEWAY_URL) logger.info(f"Migrated {key} to the internal gateway") if get_setting("bot_model", "") == "deepseek-chat": set_setting("bot_model", "molodetz") logger.info("Migrated bot_model to molodetz") from devplacepy.services.openai_gateway.routing import ( migrate_retired_image_gateway, seed_default_deepseek_routes, seed_default_image_routes, ) seed_default_deepseek_routes() seed_default_image_routes() migrate_retired_image_gateway() def backfill_api_keys() -> int: if "users" not in db.tables: return 0 users = db["users"] if not users.has_column("api_key"): users.create_column_by_example("api_key", "") if not users.has_column("cust_disable_global"): users.create_column_by_example("cust_disable_global", 0) if not users.has_column("cust_disable_pagetype"): users.create_column_by_example("cust_disable_pagetype", 0) if not users.has_column("ai_correction_enabled"): users.create_column_by_example("ai_correction_enabled", 0) if not users.has_column("ai_correction_prompt"): users.create_column_by_example("ai_correction_prompt", DEFAULT_CORRECTION_PROMPT) if not users.has_column("ai_correction_sync"): users.create_column_by_example("ai_correction_sync", 0) if not users.has_column("ai_modifier_enabled"): users.create_column_by_example("ai_modifier_enabled", 1) if not users.has_column("ai_modifier_sync"): users.create_column_by_example("ai_modifier_sync", 1) if not users.has_column("ai_modifier_prompt"): users.create_column_by_example("ai_modifier_prompt", DEFAULT_MODIFIER_PROMPT) if not users.has_column("interactions_enabled"): users.create_column_by_example("interactions_enabled", -1) if not users.has_column("timezone"): users.create_column_by_example("timezone", "") if not users.has_column("avatar_seed"): users.create_column_by_example("avatar_seed", "") if not users.has_column("last_seen"): users.create_column_by_example("last_seen", "") if not users.has_column("award_count"): users.create_column_by_example("award_count", 0) if not users.has_column("last_award_at"): users.create_column_by_example("last_award_at", "") if not users.has_column("last_award_slug"): users.create_column_by_example("last_award_slug", "") if not users.has_column("last_award_uid"): users.create_column_by_example("last_award_uid", "") with db: db.query( "UPDATE users SET ai_modifier_enabled = 1 WHERE ai_modifier_enabled IS NULL" ) db.query("UPDATE users SET ai_modifier_sync = 1 WHERE ai_modifier_sync IS NULL") db.query( "UPDATE users SET ai_modifier_prompt = :prompt " "WHERE ai_modifier_prompt IS NULL OR ai_modifier_prompt = ''", prompt=DEFAULT_MODIFIER_PROMPT, ) db.query( "UPDATE users SET interactions_enabled = -1 " "WHERE interactions_enabled IS NULL" ) import uuid_utils updated = 0 for user in users.find(): if not user.get("api_key"): users.update( {"uid": user["uid"], "api_key": str(uuid_utils.uuid7())}, ["uid"] ) updated += 1 return updated def _backfill_gamification(): if "users" not in db.tables: return from devplacepy.utils import ( level_for_xp, check_milestone_badges, XP_POST, XP_COMMENT, XP_PROJECT, XP_GIST, XP_UPVOTE, XP_FOLLOW, ) pending = list(db["users"].find(xp=0)) if not pending: return xp_by_user = defaultdict(int) def add_counts(table, column, points): if table not in db.tables: return for row in db.query( f"SELECT {column} AS uid, COUNT(*) AS c FROM {table} GROUP BY {column}" ): if row["uid"]: xp_by_user[row["uid"]] += row["c"] * points add_counts("posts", "user_uid", XP_POST) add_counts("comments", "user_uid", XP_COMMENT) add_counts("projects", "user_uid", XP_PROJECT) add_counts("gists", "user_uid", XP_GIST) add_counts("follows", "following_uid", XP_FOLLOW) if "votes" in db.tables: for content_table, target_type in ( ("posts", "post"), ("projects", "project"), ("gists", "gist"), ("comments", "comment"), ): if content_table not in db.tables: continue rows = db.query( f"SELECT c.user_uid AS uid, COUNT(*) AS c " f"FROM votes v JOIN {content_table} c ON v.target_uid = c.uid " f"WHERE v.target_type = :t AND v.value = 1 GROUP BY c.user_uid", t=target_type, ) for row in rows: if row["uid"]: xp_by_user[row["uid"]] += row["c"] * XP_UPVOTE for user in pending: xp = xp_by_user.get(user["uid"], 0) if xp <= 0: continue db["users"].update( {"uid": user["uid"], "xp": xp, "level": level_for_xp(xp)}, ["uid"] ) _authors_cache.clear() for user in pending: check_milestone_badges(user["uid"]) logger.info(f"Gamification backfill processed {len(pending)} users")