forked from retoor/devplacepy
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6970f8a270 |
File diff suppressed because one or more lines are too long
@@ -18,6 +18,7 @@ NOTIFICATION_TYPES = [
|
|||||||
{"key": "reminder", "label": "Reminders", "description": "A reminder or scheduled task you asked Devii to run fires"},
|
{"key": "reminder", "label": "Reminders", "description": "A reminder or scheduled task you asked Devii to run fires"},
|
||||||
{"key": "harvest_stolen", "label": "Farm raids", "description": "Someone steals a ready build from your Code Farm"},
|
{"key": "harvest_stolen", "label": "Farm raids", "description": "Someone steals a ready build from your Code Farm"},
|
||||||
{"key": "award", "label": "Awards", "description": "Someone gives you an award on your profile"},
|
{"key": "award", "label": "Awards", "description": "Someone gives you an award on your profile"},
|
||||||
|
{"key": "admin_alert", "label": "Admin alerts", "description": "System alerts requiring admin attention"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from devplacepy.utils import (
|
|||||||
require_user,
|
require_user,
|
||||||
)
|
)
|
||||||
from devplacepy.services.audit import record as audit
|
from devplacepy.services.audit import record as audit
|
||||||
|
from devplacepy.services.devii.quota import notify_admins_quota_blocked
|
||||||
|
|
||||||
logger = logging.getLogger("devii.router")
|
logger = logging.getLogger("devii.router")
|
||||||
|
|
||||||
@@ -279,6 +280,7 @@ async def devii_ws(websocket: WebSocket):
|
|||||||
summary=f"AI request by {username or owner_id} blocked - 24h quota reached",
|
summary=f"AI request by {username or owner_id} blocked - 24h quota reached",
|
||||||
metadata={"limit_usd": limit, "owner_kind": owner_kind},
|
metadata={"limit_usd": limit, "owner_kind": owner_kind},
|
||||||
)
|
)
|
||||||
|
notify_admins_quota_blocked(owner_kind, owner_id, username)
|
||||||
await websocket.send_json(
|
await websocket.send_json(
|
||||||
{
|
{
|
||||||
"type": "error",
|
"type": "error",
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from devplacepy.database import get_table
|
||||||
|
from devplacepy.utils.notifications import create_notification
|
||||||
|
|
||||||
|
logger = logging.getLogger("devii.quota")
|
||||||
|
|
||||||
|
ADMIN_ALERT_TYPE = "admin_alert"
|
||||||
|
|
||||||
|
|
||||||
|
def notify_admins_quota_blocked(owner_kind: str, owner_id: str, username: str) -> None:
|
||||||
|
if owner_kind != "user":
|
||||||
|
return
|
||||||
|
admins = get_table("users").find(role="Admin")
|
||||||
|
for admin in admins:
|
||||||
|
if admin["uid"] == owner_id:
|
||||||
|
continue
|
||||||
|
create_notification(
|
||||||
|
admin["uid"],
|
||||||
|
ADMIN_ALERT_TYPE,
|
||||||
|
f"User {username} blocked - 24h AI quota reached",
|
||||||
|
owner_id,
|
||||||
|
"/admin/users",
|
||||||
|
)
|
||||||
@@ -18,9 +18,7 @@ from devplacepy.services.openai_gateway.usage import (
|
|||||||
usage_metric_cards,
|
usage_metric_cards,
|
||||||
)
|
)
|
||||||
from devplacepy.utils import generate_uid, make_combined_slug
|
from devplacepy.utils import generate_uid, make_combined_slug
|
||||||
from devplacepy.utils.notifications import create_notification
|
|
||||||
from devplacepy.services.audit import record as audit
|
from devplacepy.services.audit import record as audit
|
||||||
from devplacepy.services.openai_gateway.reliability import retry_send
|
|
||||||
from devplacepy.services.seo_meta import schedule_seo_meta
|
from devplacepy.services.seo_meta import schedule_seo_meta
|
||||||
|
|
||||||
from . import _get_ai_key
|
from . import _get_ai_key
|
||||||
@@ -153,32 +151,6 @@ class NewsService(BaseService):
|
|||||||
),
|
),
|
||||||
group="AI formatting",
|
group="AI formatting",
|
||||||
),
|
),
|
||||||
ConfigField(
|
|
||||||
"news_fetch_retries",
|
|
||||||
"News API fetch retries",
|
|
||||||
type="int",
|
|
||||||
default=3,
|
|
||||||
minimum=0,
|
|
||||||
maximum=10,
|
|
||||||
help=(
|
|
||||||
"How many times to retry the upstream news API on failure. "
|
|
||||||
"Each retry waits longer (linear backoff). Set 0 for no retries."
|
|
||||||
),
|
|
||||||
group="Reliability",
|
|
||||||
),
|
|
||||||
ConfigField(
|
|
||||||
"news_fetch_retry_backoff_ms",
|
|
||||||
"News API retry backoff (ms)",
|
|
||||||
type="int",
|
|
||||||
default=5000,
|
|
||||||
minimum=1000,
|
|
||||||
maximum=60000,
|
|
||||||
help=(
|
|
||||||
"Base backoff in milliseconds between retries. The actual "
|
|
||||||
"delay is backoff * attempt number."
|
|
||||||
),
|
|
||||||
group="Reliability",
|
|
||||||
),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
@@ -193,38 +165,13 @@ class NewsService(BaseService):
|
|||||||
format_enabled = config["news_format_enabled"]
|
format_enabled = config["news_format_enabled"]
|
||||||
|
|
||||||
self.log(f"Fetching news from {api_url}")
|
self.log(f"Fetching news from {api_url}")
|
||||||
max_retries = config["news_fetch_retries"]
|
|
||||||
backoff_ms = config["news_fetch_retry_backoff_ms"]
|
|
||||||
async with stealth.stealth_async_client(timeout=30.0) as client:
|
async with stealth.stealth_async_client(timeout=30.0) as client:
|
||||||
try:
|
try:
|
||||||
resp, exc, attempts = await retry_send(
|
resp = await client.get(api_url)
|
||||||
do_call=lambda: client.get(api_url),
|
resp.raise_for_status()
|
||||||
max_retries=max_retries,
|
|
||||||
backoff_ms=backoff_ms,
|
|
||||||
log=lambda msg: self.log(msg),
|
|
||||||
)
|
|
||||||
if exc is not None:
|
|
||||||
raise exc
|
|
||||||
if resp is None or resp.status_code >= 400:
|
|
||||||
raise httpx.HTTPError(f"status {resp.status_code if resp else 0}")
|
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log(f"Failed to fetch news API after {attempts} attempts: {e}")
|
self.log(f"Failed to fetch news API: {e}")
|
||||||
audit.record_system(
|
|
||||||
"news.service.fetch_failed",
|
|
||||||
actor_kind="service",
|
|
||||||
actor_uid="news",
|
|
||||||
summary=f"News API unreachable after {attempts} attempts",
|
|
||||||
metadata={"api_url": api_url, "attempts": attempts, "error": str(e)},
|
|
||||||
result="failure",
|
|
||||||
)
|
|
||||||
for admin_row in get_table("users").find(role="Admin"):
|
|
||||||
create_notification(
|
|
||||||
admin_row["uid"],
|
|
||||||
"system",
|
|
||||||
f"News API unreachable after {attempts} attempts",
|
|
||||||
related_uid="",
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
articles = data.get("articles", [])
|
articles = data.get("articles", [])
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from typing import Any
|
|||||||
|
|
||||||
from devplacepy.database import get_table
|
from devplacepy.database import get_table
|
||||||
from devplacepy.services.audit import record as audit
|
from devplacepy.services.audit import record as audit
|
||||||
|
from devplacepy.services.devii.quota import notify_admins_quota_blocked
|
||||||
from devplacepy.services.manager import service_manager
|
from devplacepy.services.manager import service_manager
|
||||||
from devplacepy.utils import is_admin, is_primary_admin
|
from devplacepy.utils import is_admin, is_primary_admin
|
||||||
|
|
||||||
@@ -400,6 +401,7 @@ class TelegramBridge:
|
|||||||
summary=f"Telegram AI request by {user.get('username', owner_id)} blocked - 24h quota reached",
|
summary=f"Telegram AI request by {user.get('username', owner_id)} blocked - 24h quota reached",
|
||||||
metadata={"limit_usd": limit},
|
metadata={"limit_usd": limit},
|
||||||
)
|
)
|
||||||
|
notify_admins_quota_blocked("user", owner_id, user.get("username", ""))
|
||||||
await self._service.send(
|
await self._service.send(
|
||||||
chat_id, "Your daily AI quota is reached (100%). Please try again later."
|
chat_id, "Your daily AI quota is reached (100%). Please try again later."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -67,14 +67,9 @@ class FakeClient_news_service:
|
|||||||
grade = "9" if "HighArticle" in prompt else "3"
|
grade = "9" if "HighArticle" in prompt else "3"
|
||||||
return FakeResp_news_service(json_data={"choices": [{"message": {"content": grade}}]})
|
return FakeResp_news_service(json_data={"choices": [{"message": {"content": grade}}]})
|
||||||
class FailingApiClient(FakeClient_news_service):
|
class FailingApiClient(FakeClient_news_service):
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
super().__init__(*args, **kwargs)
|
|
||||||
self.call_count = 0
|
|
||||||
|
|
||||||
async def get(self, url, timeout=None):
|
async def get(self, url, timeout=None):
|
||||||
self.call_count += 1
|
|
||||||
if url == API_URL:
|
if url == API_URL:
|
||||||
raise httpx.RequestError("api down")
|
raise httpx.HTTPError("api down")
|
||||||
return FakeResp_news_service(text="")
|
return FakeResp_news_service(text="")
|
||||||
GATEWAY_HEADERS = {
|
GATEWAY_HEADERS = {
|
||||||
"X-Gateway-Cost-USD": "0.00010000",
|
"X-Gateway-Cost-USD": "0.00010000",
|
||||||
@@ -103,7 +98,7 @@ class UsageClient(FakeClient_news_service):
|
|||||||
json_data={"choices": [{"message": {"content": grade}}]},
|
json_data={"choices": [{"message": {"content": grade}}]},
|
||||||
headers=GATEWAY_HEADERS,
|
headers=GATEWAY_HEADERS,
|
||||||
)
|
)
|
||||||
def _settings_stub(threshold="7", retries="3", backoff="5"):
|
def _settings_stub(threshold="7"):
|
||||||
def fake_get_setting(key, default=None):
|
def fake_get_setting(key, default=None):
|
||||||
return {
|
return {
|
||||||
"news_api_url": API_URL,
|
"news_api_url": API_URL,
|
||||||
@@ -111,8 +106,6 @@ def _settings_stub(threshold="7", retries="3", backoff="5"):
|
|||||||
"news_ai_model": "test-model",
|
"news_ai_model": "test-model",
|
||||||
"news_grade_threshold": threshold,
|
"news_grade_threshold": threshold,
|
||||||
"news_ai_key": "",
|
"news_ai_key": "",
|
||||||
"news_fetch_retries": retries,
|
|
||||||
"news_fetch_retry_backoff_ms": backoff,
|
|
||||||
}.get(key, default)
|
}.get(key, default)
|
||||||
|
|
||||||
return fake_get_setting
|
return fake_get_setting
|
||||||
@@ -178,47 +171,12 @@ def test_grade_article_unparseable_returns_none(local_db, monkeypatch):
|
|||||||
|
|
||||||
|
|
||||||
def test_run_once_handles_api_failure(local_db, monkeypatch):
|
def test_run_once_handles_api_failure(local_db, monkeypatch):
|
||||||
admin_uid = generate_uid()
|
|
||||||
get_table("users").insert({
|
|
||||||
"uid": admin_uid,
|
|
||||||
"username": "testadmin",
|
|
||||||
"role": "Admin",
|
|
||||||
"email": "admin@test.test",
|
|
||||||
"password": "hash",
|
|
||||||
"api_key": generate_uid(),
|
|
||||||
"created_at": "2025-01-01T00:00:00Z",
|
|
||||||
})
|
|
||||||
monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
|
monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
|
||||||
monkeypatch.setattr(base_mod, "get_setting", _settings_stub())
|
monkeypatch.setattr(base_mod, "get_setting", _settings_stub())
|
||||||
failing_client = FailingApiClient([])
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
news_mod.stealth, "stealth_async_client",
|
news_mod.httpx, "AsyncClient", lambda *a, **k: FailingApiClient([])
|
||||||
lambda *a, **k: failing_client,
|
|
||||||
)
|
|
||||||
notifications = []
|
|
||||||
monkeypatch.setattr(
|
|
||||||
news_mod, "create_notification",
|
|
||||||
lambda user_uid, notification_type, message, related_uid, target_url=None: (
|
|
||||||
notifications.append((user_uid, message))
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
run_async(NewsService().run_once())
|
run_async(NewsService().run_once())
|
||||||
assert failing_client.call_count == 4
|
|
||||||
assert len(notifications) == 1
|
|
||||||
assert notifications[0][0] == admin_uid
|
|
||||||
assert "News API unreachable" in notifications[0][1]
|
|
||||||
|
|
||||||
|
|
||||||
def test_run_once_handles_api_failure_zero_retries(local_db, monkeypatch):
|
|
||||||
monkeypatch.setattr(news_mod, "get_setting", _settings_stub(retries="0"))
|
|
||||||
monkeypatch.setattr(base_mod, "get_setting", _settings_stub(retries="0"))
|
|
||||||
failing_client = FailingApiClient([])
|
|
||||||
monkeypatch.setattr(
|
|
||||||
news_mod.stealth, "stealth_async_client",
|
|
||||||
lambda *a, **k: failing_client,
|
|
||||||
)
|
|
||||||
run_async(NewsService().run_once())
|
|
||||||
assert failing_client.call_count == 1
|
|
||||||
|
|
||||||
|
|
||||||
def test_run_once_updates_existing_news_row(local_db, monkeypatch):
|
def test_run_once_updates_existing_news_row(local_db, monkeypatch):
|
||||||
|
|||||||
Reference in New Issue
Block a user