forked from retoor/devplacepy
chore: consolidate runtime data layout under single data/ root directory
Migrate all runtime artifacts (database, uploads, VAPID keys, locks, bot state, container workspaces, zip staging) from scattered locations (`var/`, `devplacepy/static/uploads/`) into a unified `data/` directory. Update `config.py` as single source of truth with `DATA_PATHS` registry and `ensure_data_dirs()`, add `devplace migrate-data` CLI command with CRC verification and idempotent relocation, adjust `.dockerignore`, `.env.example`, `.gitignore`, `Dockerfile`, `Makefile`, `README.md`, `AGENTS.md`, `CLAUDE.md`, and all import paths in `attachments.py` to reference `config.UPLOADS_DIR`/`ATTACHMENTS_DIR` instead of computing from `STATIC_DIR`.
This commit is contained in:
@@ -11,7 +11,7 @@ from PIL import Image
|
||||
from io import BytesIO
|
||||
import httpx
|
||||
from devplacepy.database import get_table, db, get_setting
|
||||
from devplacepy.config import STATIC_DIR
|
||||
from devplacepy.config import UPLOADS_DIR, ATTACHMENTS_DIR
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -22,8 +22,6 @@ REMOTE_FETCH_USER_AGENT = (
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
UPLOADS_DIR = STATIC_DIR / "uploads"
|
||||
ATTACHMENTS_DIR = UPLOADS_DIR / "attachments"
|
||||
THUMBNAIL_SIZE = (200, 200)
|
||||
THUMBNAIL_QUALITY = 80
|
||||
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff"}
|
||||
|
||||
@@ -367,6 +367,209 @@ def cmd_containers_gc_workspaces(args):
|
||||
)
|
||||
|
||||
|
||||
def _crc32(path):
|
||||
import zlib
|
||||
|
||||
crc = 0
|
||||
with open(path, "rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
crc = zlib.crc32(chunk, crc)
|
||||
return crc & 0xFFFFFFFF
|
||||
|
||||
|
||||
def _migrate_file(source, dest, dry_run, report):
|
||||
import os
|
||||
import shutil
|
||||
|
||||
if not source.exists():
|
||||
return
|
||||
if source.resolve() == dest.resolve():
|
||||
return
|
||||
size = source.stat().st_size
|
||||
if dest.exists():
|
||||
if dest.stat().st_size == size and _crc32(dest) == _crc32(source):
|
||||
report.append(("done", source, dest, size))
|
||||
if not dry_run:
|
||||
source.unlink()
|
||||
return
|
||||
report.append(("conflict", source, dest, size))
|
||||
return
|
||||
report.append(("move", source, dest, size))
|
||||
if dry_run:
|
||||
return
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dest.with_name(dest.name + ".migrating")
|
||||
shutil.copyfile(source, tmp)
|
||||
with open(tmp, "rb") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
if tmp.stat().st_size != size or _crc32(tmp) != _crc32(source):
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"verification failed copying {source} -> {dest}")
|
||||
os.replace(tmp, dest)
|
||||
source.unlink()
|
||||
|
||||
|
||||
def _prune_empty_dirs(root):
|
||||
if not root.exists():
|
||||
return
|
||||
for path in sorted(root.rglob("*"), reverse=True):
|
||||
if path.is_dir():
|
||||
try:
|
||||
path.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
root.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _migrate_tree(source, dest, dry_run, report):
|
||||
if not source.exists():
|
||||
return
|
||||
if source.resolve() == dest.resolve():
|
||||
return
|
||||
for child in sorted(source.rglob("*")):
|
||||
if child.is_file():
|
||||
_migrate_file(child, dest / child.relative_to(source), dry_run, report)
|
||||
if not dry_run:
|
||||
_prune_empty_dirs(source)
|
||||
|
||||
|
||||
def _db_is_locked(path):
|
||||
import sqlite3
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(path), timeout=0.5)
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
conn.rollback()
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
except sqlite3.OperationalError:
|
||||
return True
|
||||
|
||||
|
||||
def _checkpoint(path):
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(str(path), timeout=5)
|
||||
try:
|
||||
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _migrate_db(source, dest, dry_run, report):
|
||||
if not source.exists():
|
||||
return
|
||||
if source.resolve() == dest.resolve():
|
||||
return
|
||||
if _db_is_locked(source):
|
||||
raise RuntimeError(
|
||||
f"{source} is locked - stop the app before running migrate-data"
|
||||
)
|
||||
if not dry_run:
|
||||
_checkpoint(source)
|
||||
_migrate_file(source, dest, dry_run, report)
|
||||
for suffix in ("-wal", "-shm"):
|
||||
_migrate_file(
|
||||
source.with_name(source.name + suffix),
|
||||
dest.with_name(dest.name + suffix),
|
||||
dry_run,
|
||||
report,
|
||||
)
|
||||
|
||||
|
||||
def cmd_migrate_data(args):
|
||||
import os
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
from devplacepy import config
|
||||
|
||||
base = config.BASE_DIR
|
||||
home = Path.home()
|
||||
dry = args.dry_run
|
||||
report = []
|
||||
|
||||
config.ensure_data_dirs()
|
||||
|
||||
db_items = []
|
||||
if config.DATABASE_URL == f"sqlite:///{config.DATA_DIR / 'devplace.db'}":
|
||||
db_items.append((base / "devplace.db", config.DATA_DIR / "devplace.db"))
|
||||
else:
|
||||
print("Skipping main DB: DEVPLACE_DATABASE_URL points outside the data dir.")
|
||||
if not os.environ.get("DEVII_TASKS_DB"):
|
||||
db_items.append((base / "devii_tasks.db", config.DEVII_TASKS_DB))
|
||||
if not os.environ.get("DEVII_LESSONS_DB"):
|
||||
db_items.append((base / "devii_lessons.db", config.DEVII_LESSONS_DB))
|
||||
|
||||
file_items = [
|
||||
(base / name, config.KEYS_DIR / name)
|
||||
for name in (
|
||||
"notification-private.pem",
|
||||
"notification-private.pkcs8.pem",
|
||||
"notification-public.pem",
|
||||
)
|
||||
]
|
||||
registry_dest = config.BOT_DIR / "article_registry.json"
|
||||
registry_sources = [
|
||||
path
|
||||
for path in (
|
||||
home / ".dpbot_article_registry.json",
|
||||
base / ".dpbot_article_registry.json",
|
||||
)
|
||||
if path.exists()
|
||||
]
|
||||
registry_sources.sort(key=lambda path: path.stat().st_mtime, reverse=True)
|
||||
if registry_sources:
|
||||
file_items.append((registry_sources[0], registry_dest))
|
||||
for stale in registry_sources[1:]:
|
||||
print(f"Leaving older duplicate registry untouched: {stale}")
|
||||
|
||||
legacy_var = base / "var"
|
||||
tree_items = [
|
||||
(base / "devplacepy" / "static" / "uploads", config.UPLOADS_DIR),
|
||||
(home / ".devplace_bots", config.BOT_DIR),
|
||||
]
|
||||
for sub_name in ("container_workspaces", "zips", "zip_staging", "fork_staging"):
|
||||
tree_items.append((legacy_var / sub_name, config.DATA_PATHS[sub_name]))
|
||||
|
||||
try:
|
||||
for source, dest in db_items:
|
||||
_migrate_db(source, dest, dry, report)
|
||||
for source, dest in file_items:
|
||||
_migrate_file(source, dest, dry, report)
|
||||
for source, dest in tree_items:
|
||||
_migrate_tree(source, dest, dry, report)
|
||||
except RuntimeError as exc:
|
||||
print(f"ERROR: {exc}")
|
||||
sys.exit(1)
|
||||
|
||||
if not report:
|
||||
print("Nothing to migrate; the data directory is already consolidated.")
|
||||
return
|
||||
for status, source, dest, size in report:
|
||||
print(f" [{status}] {source} -> {dest} ({size} bytes)")
|
||||
counts = Counter(status for status, *_ in report)
|
||||
print()
|
||||
print(
|
||||
("Planned: " if dry else "Migrated: ")
|
||||
+ ", ".join(f"{count} {status}" for status, count in sorted(counts.items()))
|
||||
)
|
||||
if any(status == "conflict" for status, *_ in report):
|
||||
print(
|
||||
"Conflicts left both source and destination untouched; resolve them by hand."
|
||||
)
|
||||
if dry:
|
||||
print("Dry run - nothing changed. Re-run without --dry-run to apply.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="DevPlace admin CLI")
|
||||
sub = parser.add_subparsers(title="commands", dest="command")
|
||||
@@ -477,6 +680,17 @@ def main():
|
||||
"gc-workspaces", help="Remove workspace dirs with no instances"
|
||||
).set_defaults(func=cmd_containers_gc_workspaces)
|
||||
|
||||
migrate = sub.add_parser(
|
||||
"migrate-data",
|
||||
help="Relocate legacy runtime files into the consolidated data/ directory",
|
||||
)
|
||||
migrate.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print the source-to-destination plan without changing anything",
|
||||
)
|
||||
migrate.set_defaults(func=cmd_migrate_data)
|
||||
|
||||
args = parser.parse_args()
|
||||
if hasattr(args, "func"):
|
||||
args.func(args)
|
||||
|
||||
+49
-9
@@ -9,8 +9,29 @@ load_dotenv()
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
STATIC_DIR = BASE_DIR / "devplacepy" / "static"
|
||||
TEMPLATES_DIR = BASE_DIR / "devplacepy" / "templates"
|
||||
|
||||
# Single source of truth for every runtime/user-generated artifact. Everything the
|
||||
# app creates or modifies at runtime lives under DATA_DIR, never inside the package
|
||||
# and never served via /static. Overridable by DEVPLACE_DATA_DIR (point at a volume
|
||||
# in production). Each path below is derived from DATA_DIR; no other module computes
|
||||
# a runtime path from scratch.
|
||||
DATA_DIR = Path(environ.get("DEVPLACE_DATA_DIR", str(BASE_DIR / "data")))
|
||||
UPLOADS_DIR = DATA_DIR / "uploads"
|
||||
ATTACHMENTS_DIR = UPLOADS_DIR / "attachments"
|
||||
PROJECT_FILES_DIR = UPLOADS_DIR / "project_files"
|
||||
CONTAINER_WORKSPACES_DIR = DATA_DIR / "container_workspaces"
|
||||
ZIPS_DIR = DATA_DIR / "zips"
|
||||
ZIP_STAGING_DIR = DATA_DIR / "zip_staging"
|
||||
FORK_STAGING_DIR = DATA_DIR / "fork_staging"
|
||||
KEYS_DIR = DATA_DIR / "keys"
|
||||
BOT_DIR = DATA_DIR / "bot"
|
||||
LOCKS_DIR = DATA_DIR / "locks"
|
||||
|
||||
DEVII_TASKS_DB = DATA_DIR / "devii_tasks.db"
|
||||
DEVII_LESSONS_DB = DATA_DIR / "devii_lessons.db"
|
||||
|
||||
DATABASE_URL = environ.get(
|
||||
"DEVPLACE_DATABASE_URL", f"sqlite:///{BASE_DIR / 'devplace.db'}"
|
||||
"DEVPLACE_DATABASE_URL", f"sqlite:///{DATA_DIR / 'devplace.db'}"
|
||||
)
|
||||
SECRET_KEY = environ.get("SECRET_KEY", "devplace-secret-key-change-in-production")
|
||||
SECONDS_PER_DAY = 86400
|
||||
@@ -25,12 +46,9 @@ INTERNAL_BASE_URL = environ.get(
|
||||
INTERNAL_GATEWAY_URL = f"{INTERNAL_BASE_URL}/openai/v1/chat/completions"
|
||||
INTERNAL_MODEL = "molodetz"
|
||||
|
||||
SERVICE_LOCK_FILE = BASE_DIR / "devplace-services.lock"
|
||||
INIT_LOCK_FILE = BASE_DIR / "devplace-init.lock"
|
||||
SERVICE_LOCK_FILE = LOCKS_DIR / "devplace-services.lock"
|
||||
INIT_LOCK_FILE = LOCKS_DIR / "devplace-init.lock"
|
||||
|
||||
# Container data lives OUTSIDE the package (never served via /static, never watched by --reload).
|
||||
DATA_DIR = Path(environ.get("DEVPLACE_DATA_DIR", str(BASE_DIR / "var")))
|
||||
CONTAINER_WORKSPACES_DIR = DATA_DIR / "container_workspaces"
|
||||
# Every container instance runs this one prebuilt image (built once via `make ppy`).
|
||||
CONTAINER_IMAGE = environ.get("DEVPLACE_CONTAINER_IMAGE", "ppy:latest")
|
||||
# Override host the /p/<slug> ingress proxy dials, with the published host port,
|
||||
@@ -41,7 +59,29 @@ CONTAINER_IMAGE = environ.get("DEVPLACE_CONTAINER_IMAGE", "ppy:latest")
|
||||
# port-publishing layer (docker-proxy / iptables DNAT / loopback) entirely.
|
||||
CONTAINER_PROXY_HOST = environ.get("DEVPLACE_CONTAINER_PROXY_HOST", "").strip()
|
||||
|
||||
VAPID_PRIVATE_KEY_FILE = BASE_DIR / "notification-private.pem"
|
||||
VAPID_PRIVATE_KEY_PKCS8_FILE = BASE_DIR / "notification-private.pkcs8.pem"
|
||||
VAPID_PUBLIC_KEY_FILE = BASE_DIR / "notification-public.pem"
|
||||
VAPID_PRIVATE_KEY_FILE = KEYS_DIR / "notification-private.pem"
|
||||
VAPID_PRIVATE_KEY_PKCS8_FILE = KEYS_DIR / "notification-private.pkcs8.pem"
|
||||
VAPID_PUBLIC_KEY_FILE = KEYS_DIR / "notification-public.pem"
|
||||
VAPID_SUB = environ.get("DEVPLACE_VAPID_SUB", "mailto:retoor@molodetz.nl")
|
||||
|
||||
# Documented registry of every runtime directory. ensure_data_dirs() creates them
|
||||
# all at startup so the tree always exists before the DB, keys, locks, uploads, and
|
||||
# job staging are written.
|
||||
DATA_PATHS: dict[str, Path] = {
|
||||
"data": DATA_DIR,
|
||||
"uploads": UPLOADS_DIR,
|
||||
"attachments": ATTACHMENTS_DIR,
|
||||
"project_files": PROJECT_FILES_DIR,
|
||||
"container_workspaces": CONTAINER_WORKSPACES_DIR,
|
||||
"zips": ZIPS_DIR,
|
||||
"zip_staging": ZIP_STAGING_DIR,
|
||||
"fork_staging": FORK_STAGING_DIR,
|
||||
"keys": KEYS_DIR,
|
||||
"bot": BOT_DIR,
|
||||
"locks": LOCKS_DIR,
|
||||
}
|
||||
|
||||
|
||||
def ensure_data_dirs() -> None:
|
||||
for path in DATA_PATHS.values():
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
+10
-3
@@ -2,14 +2,21 @@
|
||||
|
||||
import dataset
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from sqlalchemy import or_
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from devplacepy.cache import TTLCache
|
||||
from devplacepy.config import DATABASE_URL, INTERNAL_GATEWAY_URL
|
||||
from devplacepy.config import DATABASE_URL, INTERNAL_GATEWAY_URL, ensure_data_dirs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ensure_data_dirs()
|
||||
if DATABASE_URL.startswith("sqlite:///"):
|
||||
_db_file = DATABASE_URL[len("sqlite:///") :]
|
||||
if _db_file and _db_file != ":memory:":
|
||||
Path(_db_file).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
db = dataset.connect(
|
||||
DATABASE_URL,
|
||||
engine_kwargs={
|
||||
@@ -1160,9 +1167,9 @@ def delete_attachments(resource_type: str, resource_uid: str) -> None:
|
||||
def _delete_attachment_file(storage_path: str) -> None:
|
||||
if not storage_path:
|
||||
return
|
||||
from devplacepy.config import STATIC_DIR
|
||||
from devplacepy.config import UPLOADS_DIR
|
||||
|
||||
file_path = STATIC_DIR / "uploads" / storage_path
|
||||
file_path = UPLOADS_DIR / storage_path
|
||||
try:
|
||||
file_path.unlink(missing_ok=True)
|
||||
parent = file_path.parent
|
||||
|
||||
+10
-2
@@ -11,7 +11,14 @@ from fastapi import FastAPI, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from devplacepy.config import STATIC_DIR, PORT, SERVICE_LOCK_FILE, INIT_LOCK_FILE
|
||||
from devplacepy.config import (
|
||||
STATIC_DIR,
|
||||
UPLOADS_DIR,
|
||||
PORT,
|
||||
SERVICE_LOCK_FILE,
|
||||
INIT_LOCK_FILE,
|
||||
ensure_data_dirs,
|
||||
)
|
||||
from devplacepy.database import (
|
||||
init_db,
|
||||
get_table,
|
||||
@@ -156,7 +163,7 @@ app = FastAPI(
|
||||
)
|
||||
app.mount(
|
||||
"/static/uploads",
|
||||
UploadStaticFiles(directory=str(STATIC_DIR / "uploads"), check_dir=False),
|
||||
UploadStaticFiles(directory=str(UPLOADS_DIR), check_dir=False),
|
||||
name="uploads",
|
||||
)
|
||||
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||
@@ -399,6 +406,7 @@ async def maintenance_middleware(request: Request, call_next):
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup():
|
||||
ensure_data_dirs()
|
||||
with init_lock():
|
||||
init_db()
|
||||
from devplacepy.push import ensure_certificates
|
||||
|
||||
@@ -6,13 +6,12 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.database import get_table, db, get_int_setting
|
||||
from devplacepy.config import STATIC_DIR
|
||||
from devplacepy.config import PROJECT_FILES_DIR
|
||||
from devplacepy.utils import generate_uid
|
||||
from devplacepy.attachments import _directory_for, _detect_mime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROJECT_FILES_DIR = STATIC_DIR / "uploads" / "project_files"
|
||||
MAX_TEXT_CHARS = 400_000
|
||||
MAX_FILES_PER_PROJECT = 5000
|
||||
MAX_PATH_LENGTH = 1024
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import random
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL, BOT_DIR
|
||||
|
||||
BASE_URL_DEFAULT = "https://pravda.education"
|
||||
API_URL_DEFAULT = INTERNAL_GATEWAY_URL
|
||||
@@ -15,8 +14,8 @@ OUTPUT_COST_PER_1M_DEFAULT = 1.10
|
||||
COST_WINDOW_SECONDS = 600
|
||||
COST_WARMUP_SECONDS = 120
|
||||
|
||||
STATE_DIR = Path.home() / ".devplace_bots"
|
||||
ARTICLE_REGISTRY_PATH = Path.home() / ".dpbot_article_registry.json"
|
||||
STATE_DIR = BOT_DIR
|
||||
ARTICLE_REGISTRY_PATH = BOT_DIR / "article_registry.json"
|
||||
|
||||
PROJECT_STATUSES = ["In Development", "Released"]
|
||||
|
||||
|
||||
@@ -6,7 +6,12 @@ import os
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
from devplacepy.config import (
|
||||
INTERNAL_GATEWAY_URL,
|
||||
INTERNAL_MODEL,
|
||||
DEVII_TASKS_DB,
|
||||
DEVII_LESSONS_DB,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_AI_URL = INTERNAL_GATEWAY_URL
|
||||
@@ -100,9 +105,9 @@ def load_settings() -> Settings:
|
||||
),
|
||||
log_level=os.environ.get("DEVII_LOG_LEVEL", "WARNING").upper(),
|
||||
log_path=os.environ.get("DEVII_LOG_PATH", "devii.log"),
|
||||
tasks_db_path=os.environ.get("DEVII_TASKS_DB", "devii_tasks.db"),
|
||||
tasks_db_path=os.environ.get("DEVII_TASKS_DB", str(DEVII_TASKS_DB)),
|
||||
scheduler_tick_seconds=float(os.environ.get("DEVII_SCHEDULER_TICK", "1.0")),
|
||||
lessons_db_path=os.environ.get("DEVII_LESSONS_DB", "devii_lessons.db"),
|
||||
lessons_db_path=os.environ.get("DEVII_LESSONS_DB", str(DEVII_LESSONS_DB)),
|
||||
delegate_max_iterations=int(
|
||||
os.environ.get(
|
||||
"DEVII_DELEGATE_MAX_ITERATIONS", DEFAULT_DELEGATE_MAX_ITERATIONS
|
||||
@@ -172,7 +177,7 @@ FIELD_RSEARCH_ENABLED = "devii_rsearch_enabled"
|
||||
FIELD_RSEARCH_URL = "devii_rsearch_url"
|
||||
FIELD_RSEARCH_TIMEOUT = "devii_rsearch_timeout"
|
||||
|
||||
LESSONS_DB_PATH = os.environ.get("DEVII_LESSONS_DB", "devii_lessons.db")
|
||||
LESSONS_DB_PATH = os.environ.get("DEVII_LESSONS_DB", str(DEVII_LESSONS_DB))
|
||||
|
||||
|
||||
def effective_daily_limit(
|
||||
|
||||
@@ -4,7 +4,7 @@ import asyncio
|
||||
import logging
|
||||
import shutil
|
||||
|
||||
from devplacepy.config import DATA_DIR
|
||||
from devplacepy.config import FORK_STAGING_DIR
|
||||
from devplacepy import project_files
|
||||
from devplacepy.content import create_content_item
|
||||
from devplacepy.database import get_table, record_fork, delete_fork_relations
|
||||
@@ -13,8 +13,6 @@ from devplacepy.utils import XP_PROJECT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
STAGING_DIR = DATA_DIR / "fork_staging"
|
||||
|
||||
|
||||
class ForkService(JobService):
|
||||
kind = "fork"
|
||||
@@ -115,7 +113,7 @@ class ForkService(JobService):
|
||||
def _copy_files(
|
||||
self, source_uid: str, new_uid: str, user: dict, job_uid: str
|
||||
) -> int:
|
||||
staging = STAGING_DIR / job_uid
|
||||
staging = FORK_STAGING_DIR / job_uid
|
||||
try:
|
||||
project_files.export_to_dir(source_uid, "", staging)
|
||||
return project_files.import_from_dir(
|
||||
@@ -133,4 +131,4 @@ class ForkService(JobService):
|
||||
logger.exception("fork rollback failed for %s", new_uid)
|
||||
|
||||
def cleanup(self, job: dict) -> None:
|
||||
shutil.rmtree(STAGING_DIR / job["uid"], ignore_errors=True)
|
||||
shutil.rmtree(FORK_STAGING_DIR / job["uid"], ignore_errors=True)
|
||||
|
||||
@@ -7,15 +7,13 @@ import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.config import BASE_DIR, DATA_DIR
|
||||
from devplacepy.config import BASE_DIR, ZIPS_DIR, ZIP_STAGING_DIR
|
||||
from devplacepy import project_files
|
||||
from devplacepy.services.jobs.base import JobService
|
||||
from devplacepy.utils import generate_uid, slugify
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ZIPS_DIR = DATA_DIR / "zips"
|
||||
STAGING_DIR = DATA_DIR / "zip_staging"
|
||||
WORKER_MODULE = "devplacepy.services.jobs.zip_worker"
|
||||
|
||||
|
||||
@@ -40,7 +38,7 @@ class ZipService(JobService):
|
||||
|
||||
source = job["payload"].get("source", {})
|
||||
uid = job["uid"]
|
||||
staging = STAGING_DIR / uid
|
||||
staging = ZIP_STAGING_DIR / uid
|
||||
project_uid = source.get("project_uid", "")
|
||||
fail_meta = {"project_uid": project_uid}
|
||||
try:
|
||||
@@ -131,4 +129,4 @@ class ZipService(JobService):
|
||||
local_path = result.get("local_path")
|
||||
if local_path:
|
||||
Path(local_path).unlink(missing_ok=True)
|
||||
shutil.rmtree(STAGING_DIR / job["uid"], ignore_errors=True)
|
||||
shutil.rmtree(ZIP_STAGING_DIR / job["uid"], ignore_errors=True)
|
||||
|
||||
@@ -19,7 +19,8 @@ cp .env.example .env
|
||||
| Variable | Default | Purpose |
|
||||
|----------|---------|---------|
|
||||
| `SECRET_KEY` | insecure placeholder | Session signing. **Change this.** |
|
||||
| `DEVPLACE_DATABASE_URL` | unset | Leave unset to share the project-root `devplace.db`. Set only to point elsewhere. |
|
||||
| `DEVPLACE_DATABASE_URL` | unset | Leave unset to share `data/devplace.db`. Set only to point elsewhere. |
|
||||
| `DEVPLACE_DATA_DIR` | `<repo>/data` | Single root for all runtime data (DB, uploads, keys, locks, bot state, staging, workspaces). Point at a volume in production. |
|
||||
| `DEVPLACE_SITE_URL` | empty | Public origin for absolute URLs (SEO, canonical, push). Empty derives from the request. |
|
||||
| `PORT` | `10500` | Host port the nginx front door binds. |
|
||||
| `NGINX_MAX_BODY_SIZE` | `50m` | nginx upload ceiling. Must be >= `max_upload_size_mb`. |
|
||||
@@ -34,7 +35,7 @@ Operational behavior (rate limits, registration, maintenance mode, upload size,
|
||||
```bash
|
||||
cp .env.example .env # set SECRET_KEY, PORT, DEVPLACE_SITE_URL
|
||||
make docker-build # build the image (installs the docker CLI)
|
||||
make docker-up # start (creates ./var, mounts the socket)
|
||||
make docker-up # start (creates ./data, mounts the socket)
|
||||
```
|
||||
|
||||
Open `http://<host>:<PORT>`. Useful targets: `make docker-logs` (tail), `make docker-down` (stop), `make docker-clean` (down).
|
||||
@@ -75,7 +76,7 @@ git checkout <previous-good-commit>
|
||||
make docker-up
|
||||
```
|
||||
|
||||
Back up `devplace.db` (and its `-wal`/`-shm`) before a risky change; the schema auto-syncs forward but is not auto-downgraded.
|
||||
Back up `data/devplace.db` (and its `-wal`/`-shm`) before a risky change; the schema auto-syncs forward but is not auto-downgraded.
|
||||
|
||||
## Bare-metal alternative
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ The nginx front door, how it serves each route, and the production-specific rule
|
||||
|
||||
## Build and configuration
|
||||
|
||||
The nginx image (`nginx/Dockerfile`) renders `nginx/nginx.conf.template` at start through `nginx/start.sh`, which substitutes a small allow-list of variables (`NGINX_CACHE_CONFIG`, `NGINX_CACHE_MAX_SIZE`, `NGINX_MAX_BODY_SIZE`) and leaves nginx runtime variables such as `$http_upgrade` untouched. The host's `devplacepy/static` directory is bind-mounted read-only at `/app/static`, so served assets and uploads always match the running code without an image rebuild.
|
||||
The nginx image (`nginx/Dockerfile`) renders `nginx/nginx.conf.template` at start through `nginx/start.sh`, which substitutes a small allow-list of variables (`NGINX_CACHE_CONFIG`, `NGINX_CACHE_MAX_SIZE`, `NGINX_MAX_BODY_SIZE`) and leaves nginx runtime variables such as `$http_upgrade` untouched. The host's `devplacepy/static` directory is bind-mounted read-only at `/app/static` for package assets, and the consolidated `<DEVPLACE_DATA_DIR>/uploads` directory is bind-mounted read-only at `/data/uploads` (the `/static/uploads/` location aliases it), so both served assets and uploads always match the running code and data without an image rebuild.
|
||||
|
||||
## Route map
|
||||
|
||||
|
||||
@@ -16,25 +16,25 @@ There is no separate database service: the data lives in SQLite files on the hos
|
||||
|
||||
The defining property of the setup: **production uses the same database and shares as the development server.**
|
||||
|
||||
The app container bind-mounts the host project directory (`./` to `/app`) and runs as the host user (`DEVPLACE_UID:DEVPLACE_GID`, default `1000:1000`). `config.py` resolves the database to an absolute path under the project root, so with no `DEVPLACE_DATABASE_URL` override the container reads and writes the **same `devplace.db`** as `make dev`. The bind mount also shares:
|
||||
The app container bind-mounts the host project directory (`./` to `/app`) and runs as the host user (`DEVPLACE_UID:DEVPLACE_GID`, default `1000:1000`). Every runtime artifact lives under one consolidated `data/` directory (`DEVPLACE_DATA_DIR`, default `<repo>/data`), resolved to an absolute path in `config.py`, so with no `DEVPLACE_DATABASE_URL` override the container reads and writes the **same `data/devplace.db`** as `make dev`. The bind mount also shares:
|
||||
|
||||
- `devplace.db` (plus its `-wal` / `-shm` companions)
|
||||
- `static/uploads/` (user attachments and images)
|
||||
- `devii_lessons.db`, `devii_tasks.db` (Devii memory and tasks)
|
||||
- `notification-*.pem` (VAPID push keys)
|
||||
- `devplace-services.lock` (background-service coordination)
|
||||
- `data/devplace.db` (plus its `-wal` / `-shm` companions)
|
||||
- `data/uploads/` (user attachments and project files)
|
||||
- `data/devii_lessons.db`, `data/devii_tasks.db` (Devii memory and tasks)
|
||||
- `data/keys/notification-*.pem` (VAPID push keys)
|
||||
- `data/locks/devplace-services.lock` (background-service coordination)
|
||||
|
||||
Because the container runs as the host user, every file it writes stays owned by the developer account, with no permission conflicts.
|
||||
|
||||
## Why this is safe
|
||||
|
||||
- **Concurrency.** SQLite runs in WAL mode with a 30s busy timeout, which allows concurrent readers and a serialized writer. The dev process and the container workers can use the file at the same time.
|
||||
- **One services owner.** The background services (news, bots, Devii hub) are guarded by an exclusive `fcntl.flock` on `devplace-services.lock`. Across every process that shares that file - dev and prod alike - exactly one acquires the lock and runs the services; the others skip them, so news is never fetched twice and bots never double-run.
|
||||
- **One services owner.** The background services (news, bots, Devii hub) are guarded by an exclusive `fcntl.flock` on `data/locks/devplace-services.lock`. Across every process that shares that file - dev and prod alike - exactly one acquires the lock and runs the services; the others skip them, so news is never fetched twice and bots never double-run.
|
||||
- **One DB, one truth.** No `DEVPLACE_DATABASE_URL` override means nothing can silently diverge to a second database.
|
||||
|
||||
## Same-host requirement
|
||||
|
||||
SQLite is a local-file database; it cannot be shared across machines. For production and development to share one `devplace.db`, they must run on the **same host and filesystem**. A second server would need its own database and a different architecture (this deployment intentionally does not do that).
|
||||
SQLite is a local-file database; it cannot be shared across machines. For production and development to share one `data/devplace.db`, they must run on the **same host and filesystem**. A second server would need its own database and a different architecture (this deployment intentionally does not do that).
|
||||
|
||||
## Code updates without rebuild
|
||||
|
||||
|
||||
Reference in New Issue
Block a user