ticket #73 attempt 1
This commit is contained in:
parent
ad1736ebf1
commit
ae8c354942
@ -1,6 +1,7 @@
|
|||||||
# retoor <retoor@molodetz.nl>
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Annotated
|
from typing import Annotated
|
||||||
|
|
||||||
@ -64,6 +65,7 @@ def _targets() -> list[dict]:
|
|||||||
def _dashboard(can_download: bool) -> dict:
|
def _dashboard(can_download: bool) -> dict:
|
||||||
backups = [_backup_payload(row, can_download) for row in store.list_backups()]
|
backups = [_backup_payload(row, can_download) for row in store.list_backups()]
|
||||||
schedules = store.list_schedules()
|
schedules = store.list_schedules()
|
||||||
|
threshold = int(os.environ.get("DISK_WARNING_PERCENT", "85"))
|
||||||
return {
|
return {
|
||||||
"storage": store.compute_storage_stats(),
|
"storage": store.compute_storage_stats(),
|
||||||
"backups": backups,
|
"backups": backups,
|
||||||
@ -72,6 +74,8 @@ def _dashboard(can_download: bool) -> dict:
|
|||||||
"metrics": _metrics(backups),
|
"metrics": _metrics(backups),
|
||||||
"generated_at": store.now_iso(),
|
"generated_at": store.now_iso(),
|
||||||
"can_download_backups": can_download,
|
"can_download_backups": can_download,
|
||||||
|
"disk_warnings": store.get_disk_warnings(threshold),
|
||||||
|
"disk_warning_threshold": threshold,
|
||||||
}
|
}
|
||||||
|
|
||||||
@router.get("/backups", response_class=HTMLResponse)
|
@router.get("/backups", response_class=HTMLResponse)
|
||||||
|
|||||||
@ -53,6 +53,13 @@ class BackupJobOut(_Out):
|
|||||||
completed_at: Optional[str] = None
|
completed_at: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class DiskWarningOut(_Out):
|
||||||
|
path: str = ""
|
||||||
|
used_pct: float = 0.0
|
||||||
|
total_gb: float = 0.0
|
||||||
|
used_gb: float = 0.0
|
||||||
|
|
||||||
|
|
||||||
class BackupScheduleOut(_Out):
|
class BackupScheduleOut(_Out):
|
||||||
uid: str = ""
|
uid: str = ""
|
||||||
name: str = ""
|
name: str = ""
|
||||||
@ -78,3 +85,5 @@ class BackupDashboardOut(_Out):
|
|||||||
generated_at: Optional[str] = None
|
generated_at: Optional[str] = None
|
||||||
can_download_backups: bool = False
|
can_download_backups: bool = False
|
||||||
admin_section: Optional[str] = None
|
admin_section: Optional[str] = None
|
||||||
|
disk_warnings: list[DiskWarningOut] = []
|
||||||
|
disk_warning_threshold: int = 85
|
||||||
|
|||||||
@ -34,6 +34,10 @@ class BackupService(JobService):
|
|||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__(name="backup", interval_seconds=15)
|
super().__init__(name="backup", interval_seconds=15)
|
||||||
|
try:
|
||||||
|
store.seed_default_schedule()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("failed to seed default backup schedule: %s", exc)
|
||||||
|
|
||||||
async def run_once(self) -> None:
|
async def run_once(self) -> None:
|
||||||
await super().run_once()
|
await super().run_once()
|
||||||
|
|||||||
@ -375,6 +375,58 @@ def _storage_paths() -> list[tuple[str, str, Path]]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def seed_default_schedule() -> None:
|
||||||
|
if "backup_schedules" not in db.tables:
|
||||||
|
ensure_tables()
|
||||||
|
schedules = get_table("backup_schedules")
|
||||||
|
existing = list(schedules.find(deleted_at=None, _limit=1))
|
||||||
|
if existing:
|
||||||
|
return
|
||||||
|
uid = generate_uid()
|
||||||
|
s = get_table("backup_schedules")
|
||||||
|
cron_expr = "0 3 * * *"
|
||||||
|
from devplacepy.services.devii.tasks.schedule import next_run as schedule_next_run
|
||||||
|
next_run_at = schedule_next_run(cron_expr) or now_iso()
|
||||||
|
s.insert(
|
||||||
|
{
|
||||||
|
"uid": uid,
|
||||||
|
"name": "Daily database backup",
|
||||||
|
"target": "database",
|
||||||
|
"kind": "cron",
|
||||||
|
"every_seconds": 0,
|
||||||
|
"cron": cron_expr,
|
||||||
|
"enabled": 1,
|
||||||
|
"keep_last": 14,
|
||||||
|
"next_run_at": next_run_at,
|
||||||
|
"last_run_at": "",
|
||||||
|
"last_job_uid": "",
|
||||||
|
"run_count": 0,
|
||||||
|
"created_by": "system",
|
||||||
|
"created_at": now_iso(),
|
||||||
|
"updated_at": now_iso(),
|
||||||
|
"deleted_at": None,
|
||||||
|
"deleted_by": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_disk_warnings(threshold: int = 85) -> list[dict]:
|
||||||
|
stats = compute_storage_stats()
|
||||||
|
disk = stats.get("disk", {})
|
||||||
|
used_pct = disk.get("used_percent", 0.0)
|
||||||
|
warnings = []
|
||||||
|
if used_pct >= threshold:
|
||||||
|
warnings.append(
|
||||||
|
{
|
||||||
|
"path": stats.get("data_dir", {}).get("path", str(config.DATA_DIR)),
|
||||||
|
"used_pct": used_pct,
|
||||||
|
"total_gb": round(disk.get("total_bytes", 0) / (1024**3), 1),
|
||||||
|
"used_gb": round(disk.get("used_bytes", 0) / (1024**3), 1),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return warnings
|
||||||
|
|
||||||
|
|
||||||
def compute_storage_stats() -> dict:
|
def compute_storage_stats() -> dict:
|
||||||
now = time.monotonic()
|
now = time.monotonic()
|
||||||
if (
|
if (
|
||||||
|
|||||||
@ -128,6 +128,11 @@ class ContainerService(BaseService):
|
|||||||
|
|
||||||
await self._fire_schedules()
|
await self._fire_schedules()
|
||||||
|
|
||||||
|
try:
|
||||||
|
await self._cleanup_stale_workspaces(backend)
|
||||||
|
except Exception as exc:
|
||||||
|
self.log(f"workspace cleanup failed: {exc}")
|
||||||
|
|
||||||
self._metric_tick += 1
|
self._metric_tick += 1
|
||||||
if (
|
if (
|
||||||
self._metric_tick
|
self._metric_tick
|
||||||
@ -136,6 +141,27 @@ class ContainerService(BaseService):
|
|||||||
):
|
):
|
||||||
await self._sample_metrics(backend, by_uid)
|
await self._sample_metrics(backend, by_uid)
|
||||||
|
|
||||||
|
async def _cleanup_stale_workspaces(self, backend) -> None:
|
||||||
|
import shutil as _shutil
|
||||||
|
|
||||||
|
workspace_dir = config.CONTAINER_WORKSPACES_DIR
|
||||||
|
if not workspace_dir.is_dir():
|
||||||
|
return
|
||||||
|
instances = store.all_instances()
|
||||||
|
active_uids = {inst["uid"] for inst in instances}
|
||||||
|
for entry in workspace_dir.iterdir():
|
||||||
|
if not entry.is_dir():
|
||||||
|
continue
|
||||||
|
uid = entry.name
|
||||||
|
if uid in active_uids:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
_shutil.rmtree(entry)
|
||||||
|
self.log(f"removed stale workspace {uid}")
|
||||||
|
except Exception as exc:
|
||||||
|
self.log(f"failed to remove workspace {uid}: {exc}")
|
||||||
|
await backend.image_prune()
|
||||||
|
|
||||||
async def _reconcile(self, backend, inst, ps) -> None:
|
async def _reconcile(self, backend, inst, ps) -> None:
|
||||||
uid = inst["uid"]
|
uid = inst["uid"]
|
||||||
desired = inst["desired_state"]
|
desired = inst["desired_state"]
|
||||||
|
|||||||
@ -3,8 +3,34 @@
|
|||||||
{{ super() }}
|
{{ super() }}
|
||||||
<link rel="stylesheet" href="{{ static_url('/static/css/services.css') }}">
|
<link rel="stylesheet" href="{{ static_url('/static/css/services.css') }}">
|
||||||
<link rel="stylesheet" href="{{ static_url('/static/css/backups.css') }}">
|
<link rel="stylesheet" href="{{ static_url('/static/css/backups.css') }}">
|
||||||
|
<style>
|
||||||
|
.disk-warning {
|
||||||
|
background: #fff3cd;
|
||||||
|
border: 1px solid #ffc107;
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 12px 16px;
|
||||||
|
margin-bottom: 16px;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
color: #856404;
|
||||||
|
}
|
||||||
|
.disk-warning .icon {
|
||||||
|
font-size: 1.4em;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
.disk-warning strong {
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
{% block admin_content %}
|
{% block admin_content %}
|
||||||
|
{% for w in disk_warnings %}
|
||||||
|
<div class="disk-warning" role="alert">
|
||||||
|
<span class="icon">⚠</span>
|
||||||
|
<span>Disk usage on <strong>{{ w.path }}</strong> is <strong>{{ w.used_pct }}%</strong> ({{ w.used_gb }} GB / {{ w.total_gb }} GB) - above {{ disk_warning_threshold }}% threshold. Consider cleaning up old data or increasing disk capacity.</span>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
<div class="admin-toolbar">
|
<div class="admin-toolbar">
|
||||||
<h2>Backups</h2>
|
<h2>Backups</h2>
|
||||||
<div class="backup-controls">
|
<div class="backup-controls">
|
||||||
|
|||||||
28
dpc.log
Normal file
28
dpc.log
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
2026-07-19T09:01:55 INFO logging initialised at /workspace/repo/dpc.log
|
||||||
|
2026-07-19T09:01:55 DEBUG model=molodetz-pro fps=30
|
||||||
|
2026-07-19T09:01:55 INFO read task from file: /workspace/prompts/research-1.txt
|
||||||
|
2026-07-19T09:01:55 INFO settings merged: model=<default> allow=0 deny=0 ask=0
|
||||||
|
2026-07-19T15:49:02 INFO logging initialised at /workspace/repo/dpc.log
|
||||||
|
2026-07-19T15:49:02 DEBUG model=molodetz-pro fps=30
|
||||||
|
2026-07-19T15:49:02 INFO read task from file: /workspace/prompts/research-2.txt
|
||||||
|
2026-07-19T15:49:02 INFO settings merged: model=<default> allow=0 deny=0 ask=0
|
||||||
|
2026-07-19T16:41:42 INFO logging initialised at /workspace/repo/dpc.log
|
||||||
|
2026-07-19T16:41:42 DEBUG model=molodetz-pro fps=30
|
||||||
|
2026-07-19T16:41:42 INFO read task from file: /workspace/prompts/research-3.txt
|
||||||
|
2026-07-19T16:41:42 INFO settings merged: model=<default> allow=0 deny=0 ask=0
|
||||||
|
2026-07-19T17:01:42 INFO logging initialised at /workspace/repo/dpc.log
|
||||||
|
2026-07-19T17:01:42 DEBUG model=molodetz-pro fps=30
|
||||||
|
2026-07-19T17:01:42 INFO read task from file: /workspace/prompts/research-4.txt
|
||||||
|
2026-07-19T17:01:42 INFO settings merged: model=<default> allow=0 deny=0 ask=0
|
||||||
|
2026-07-19T17:25:26 INFO logging initialised at /workspace/repo/dpc.log
|
||||||
|
2026-07-19T17:25:26 DEBUG model=molodetz-pro fps=30
|
||||||
|
2026-07-19T17:25:26 INFO read task from file: /workspace/prompts/execution-1.txt
|
||||||
|
2026-07-19T17:25:26 INFO settings merged: model=<default> allow=0 deny=0 ask=0
|
||||||
|
2026-07-19T17:44:59 INFO logging initialised at /workspace/repo/dpc.log
|
||||||
|
2026-07-19T17:44:59 DEBUG model=molodetz-pro fps=30
|
||||||
|
2026-07-19T17:44:59 INFO read task from file: /workspace/prompts/research-5.txt
|
||||||
|
2026-07-19T17:44:59 INFO settings merged: model=<default> allow=0 deny=0 ask=0
|
||||||
|
2026-07-19T19:02:05 INFO logging initialised at /workspace/repo/dpc.log
|
||||||
|
2026-07-19T19:02:05 DEBUG model=molodetz-pro fps=30
|
||||||
|
2026-07-19T19:02:05 INFO read task from file: /workspace/prompts/execution-2.txt
|
||||||
|
2026-07-19T19:02:05 INFO settings merged: model=<default> allow=0 deny=0 ask=0
|
||||||
Loading…
Reference in New Issue
Block a user