forked from retoor/devplacepy
feat: add issue_usage tracking and metrics for AI ticket enhancement and planning
Add `issue_usage` table with columns for user_uid, tokens, cost, and latency metrics, including a unique index on user_uid. Wire `accumulate_usage` into `enhance_ticket` and `generate_plan` to capture per-request AI usage, persist totals via `add_issue_usage` in both `IssueCreateService` and `PlanningReportService`, and expose aggregated usage as metric cards through `IssueTrackerService.collect_metrics`. Update planning API docs summary to reflect the new phased implementation document format.
This commit is contained in:
@@ -3,6 +3,8 @@
|
||||
import asyncio
|
||||
import pytest
|
||||
from devplacepy.database import (
|
||||
add_issue_usage,
|
||||
get_issue_usage,
|
||||
get_table,
|
||||
init_db,
|
||||
refresh_snapshot,
|
||||
@@ -12,10 +14,56 @@ from devplacepy.services.gitea import runtime, store
|
||||
from devplacepy.services.gitea.config import gitea_config
|
||||
from devplacepy.services.gitea.enhance import enhance_ticket
|
||||
from devplacepy.services.gitea.fake import FakeGiteaClient
|
||||
from devplacepy.services.gitea.planning import generate_plan
|
||||
from devplacepy.services.gitea.service import IssueTrackerService
|
||||
from devplacepy.services.jobs import queue
|
||||
from devplacepy.services.jobs.issue_create_service import IssueCreateService
|
||||
from devplacepy.services.openai_gateway.usage import new_usage_totals
|
||||
from tests.conftest import run_async
|
||||
|
||||
_GATEWAY_HEADERS = {
|
||||
"X-Gateway-Cost-USD": "0.00010000",
|
||||
"X-Gateway-Model": "molodetz",
|
||||
"X-Gateway-Prompt-Tokens": "500",
|
||||
"X-Gateway-Completion-Tokens": "100",
|
||||
"X-Gateway-Total-Tokens": "600",
|
||||
"X-Gateway-Upstream-Latency-Ms": "400",
|
||||
"X-Gateway-Total-Latency-Ms": "450",
|
||||
}
|
||||
|
||||
|
||||
class _FakeGatewayResp:
|
||||
def __init__(self, content, headers):
|
||||
self._content = content
|
||||
self.headers = headers
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return {"choices": [{"message": {"content": self._content}}]}
|
||||
|
||||
|
||||
class _FakeGatewayClient:
|
||||
def __init__(self, content, headers):
|
||||
self._content = content
|
||||
self._headers = headers
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
async def post(self, *args, **kwargs):
|
||||
return _FakeGatewayResp(self._content, self._headers)
|
||||
|
||||
|
||||
def _fake_gateway(content):
|
||||
def factory(*args, **kwargs):
|
||||
return _FakeGatewayClient(content, _GATEWAY_HEADERS)
|
||||
|
||||
return factory
|
||||
_counter_issues_gitea = [0]
|
||||
@pytest.fixture(autouse=True)
|
||||
def _init_db_issues_gitea():
|
||||
@@ -228,3 +276,117 @@ def test_poller_notifies_on_status_change(gitea_env):
|
||||
refresh_snapshot()
|
||||
assert len(_unread(uid)) == before + 1
|
||||
assert store.get_ticket(number)["last_status"] == "closed"
|
||||
|
||||
|
||||
def test_planning_empty_when_no_issues(gitea_env):
|
||||
markdown, ai_used = run_async(generate_plan([], gitea_config()))
|
||||
assert ai_used is False
|
||||
assert markdown.startswith("# Open Tickets Implementation Plan")
|
||||
assert "no open tickets" in markdown.lower()
|
||||
|
||||
|
||||
def test_planning_fallback_is_phased_with_verbatim_body(gitea_env):
|
||||
issues = [
|
||||
{
|
||||
"number": 41,
|
||||
"title": "Broken functionality",
|
||||
"labels": [{"name": "bug"}],
|
||||
"body": "Something is broken.\nStep one fails.",
|
||||
},
|
||||
{
|
||||
"number": 7,
|
||||
"title": "Show costs",
|
||||
"labels": [{"name": "feature"}],
|
||||
"body": "Display research costs for admins.",
|
||||
},
|
||||
]
|
||||
totals = new_usage_totals()
|
||||
markdown, ai_used = run_async(generate_plan(issues, gitea_config(), totals))
|
||||
assert ai_used is False
|
||||
assert "## Execution Order" in markdown
|
||||
assert "1. #41 - Broken functionality" in markdown
|
||||
assert "## Phase 1: bug" in markdown
|
||||
assert "## Phase 2: feature" in markdown
|
||||
assert "### #41 Broken functionality" in markdown
|
||||
assert "> Something is broken." in markdown
|
||||
assert "> Step one fails." in markdown
|
||||
assert totals["calls"] == 0
|
||||
|
||||
|
||||
def test_planning_meters_usage_via_gateway(gitea_env, monkeypatch):
|
||||
set_setting("issue_ai_enhance", "1")
|
||||
refresh_snapshot()
|
||||
monkeypatch.setattr(
|
||||
"devplacepy.stealth.stealth_async_client",
|
||||
_fake_gateway("# Open Tickets Implementation Plan\n\nDone."),
|
||||
)
|
||||
totals = new_usage_totals()
|
||||
issues = [{"number": 1, "title": "t", "labels": [], "body": "b"}]
|
||||
markdown, ai_used = run_async(generate_plan(issues, gitea_config(), totals))
|
||||
assert ai_used is True
|
||||
assert markdown == "# Open Tickets Implementation Plan\n\nDone."
|
||||
assert totals["calls"] == 1
|
||||
assert totals["total_tokens"] == 600
|
||||
|
||||
|
||||
def test_enhance_meters_usage_via_gateway(gitea_env, monkeypatch):
|
||||
set_setting("issue_ai_enhance", "1")
|
||||
refresh_snapshot()
|
||||
monkeypatch.setattr(
|
||||
"devplacepy.stealth.stealth_async_client",
|
||||
_fake_gateway('{"title": "Fix login", "body": "## Summary\\nFix it."}'),
|
||||
)
|
||||
totals = new_usage_totals()
|
||||
result = run_async(enhance_ticket("login", "broken", gitea_config(), totals))
|
||||
assert result.enhanced is True
|
||||
assert result.title == "Fix login"
|
||||
assert totals["calls"] == 1
|
||||
assert abs(totals["cost_usd"] - 0.0001) < 1e-9
|
||||
|
||||
|
||||
def test_enhance_fallback_does_not_meter(gitea_env):
|
||||
totals = new_usage_totals()
|
||||
result = run_async(enhance_ticket("Title", "Body", gitea_config(), totals))
|
||||
assert result.enhanced is False
|
||||
assert totals["calls"] == 0
|
||||
|
||||
|
||||
def test_create_job_meters_usage(gitea_env, monkeypatch):
|
||||
get_table("issue_usage").delete()
|
||||
set_setting("issue_ai_enhance", "1")
|
||||
refresh_snapshot()
|
||||
monkeypatch.setattr(
|
||||
"devplacepy.stealth.stealth_async_client",
|
||||
_fake_gateway('{"title": "Login broken", "body": "## Summary\\nbroken."}'),
|
||||
)
|
||||
uid, _ = _make_user_issues_gitea()
|
||||
job_uid = queue.enqueue(
|
||||
"issue_create",
|
||||
{"author_uid": uid, "title": "login", "description": "cannot log in"},
|
||||
"user",
|
||||
uid,
|
||||
"login",
|
||||
)
|
||||
_drive_jobs()
|
||||
assert queue.get_job(job_uid)["status"] == "done"
|
||||
assert get_issue_usage()["calls"] == 1
|
||||
|
||||
|
||||
def test_issue_tracker_collect_metrics(gitea_env):
|
||||
get_table("issue_usage").delete()
|
||||
add_issue_usage(
|
||||
{
|
||||
"calls": 3,
|
||||
"prompt_tokens": 900,
|
||||
"completion_tokens": 300,
|
||||
"total_tokens": 1200,
|
||||
"cost_usd": 0.003,
|
||||
"upstream_latency_ms": 1200.0,
|
||||
"total_latency_ms": 1500.0,
|
||||
}
|
||||
)
|
||||
metrics = IssueTrackerService().collect_metrics()
|
||||
by_label = {card["label"]: card["value"] for card in metrics["stats"]}
|
||||
assert by_label["AI calls"] == 3
|
||||
assert by_label["Total tokens"] == 1200
|
||||
assert by_label["Total cost"] == "$0.0030"
|
||||
|
||||
Reference in New Issue
Block a user