feat: add issue_usage tracking and metrics for AI ticket enhancement and planning

Add `issue_usage` table with columns for user_uid, tokens, cost, and latency metrics, including a unique index on user_uid. Wire `accumulate_usage` into `enhance_ticket` and `generate_plan` to capture per-request AI usage, persist totals via `add_issue_usage` in both `IssueCreateService` and `PlanningReportService`, and expose aggregated usage as metric cards through `IssueTrackerService.collect_metrics`. Update planning API docs summary to reflect the new phased implementation document format.
This commit is contained in:
2026-06-19 11:59:40 +00:00
parent 7bbaf51450
commit ff3cbbbfe2
16 changed files with 503 additions and 122 deletions
+162
View File
@@ -3,6 +3,8 @@
import asyncio
import pytest
from devplacepy.database import (
add_issue_usage,
get_issue_usage,
get_table,
init_db,
refresh_snapshot,
@@ -12,10 +14,56 @@ from devplacepy.services.gitea import runtime, store
from devplacepy.services.gitea.config import gitea_config
from devplacepy.services.gitea.enhance import enhance_ticket
from devplacepy.services.gitea.fake import FakeGiteaClient
from devplacepy.services.gitea.planning import generate_plan
from devplacepy.services.gitea.service import IssueTrackerService
from devplacepy.services.jobs import queue
from devplacepy.services.jobs.issue_create_service import IssueCreateService
from devplacepy.services.openai_gateway.usage import new_usage_totals
from tests.conftest import run_async
_GATEWAY_HEADERS = {
"X-Gateway-Cost-USD": "0.00010000",
"X-Gateway-Model": "molodetz",
"X-Gateway-Prompt-Tokens": "500",
"X-Gateway-Completion-Tokens": "100",
"X-Gateway-Total-Tokens": "600",
"X-Gateway-Upstream-Latency-Ms": "400",
"X-Gateway-Total-Latency-Ms": "450",
}
class _FakeGatewayResp:
def __init__(self, content, headers):
self._content = content
self.headers = headers
def raise_for_status(self):
return None
def json(self):
return {"choices": [{"message": {"content": self._content}}]}
class _FakeGatewayClient:
def __init__(self, content, headers):
self._content = content
self._headers = headers
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
return False
async def post(self, *args, **kwargs):
return _FakeGatewayResp(self._content, self._headers)
def _fake_gateway(content):
def factory(*args, **kwargs):
return _FakeGatewayClient(content, _GATEWAY_HEADERS)
return factory
_counter_issues_gitea = [0]
@pytest.fixture(autouse=True)
def _init_db_issues_gitea():
@@ -228,3 +276,117 @@ def test_poller_notifies_on_status_change(gitea_env):
refresh_snapshot()
assert len(_unread(uid)) == before + 1
assert store.get_ticket(number)["last_status"] == "closed"
def test_planning_empty_when_no_issues(gitea_env):
markdown, ai_used = run_async(generate_plan([], gitea_config()))
assert ai_used is False
assert markdown.startswith("# Open Tickets Implementation Plan")
assert "no open tickets" in markdown.lower()
def test_planning_fallback_is_phased_with_verbatim_body(gitea_env):
issues = [
{
"number": 41,
"title": "Broken functionality",
"labels": [{"name": "bug"}],
"body": "Something is broken.\nStep one fails.",
},
{
"number": 7,
"title": "Show costs",
"labels": [{"name": "feature"}],
"body": "Display research costs for admins.",
},
]
totals = new_usage_totals()
markdown, ai_used = run_async(generate_plan(issues, gitea_config(), totals))
assert ai_used is False
assert "## Execution Order" in markdown
assert "1. #41 - Broken functionality" in markdown
assert "## Phase 1: bug" in markdown
assert "## Phase 2: feature" in markdown
assert "### #41 Broken functionality" in markdown
assert "> Something is broken." in markdown
assert "> Step one fails." in markdown
assert totals["calls"] == 0
def test_planning_meters_usage_via_gateway(gitea_env, monkeypatch):
set_setting("issue_ai_enhance", "1")
refresh_snapshot()
monkeypatch.setattr(
"devplacepy.stealth.stealth_async_client",
_fake_gateway("# Open Tickets Implementation Plan\n\nDone."),
)
totals = new_usage_totals()
issues = [{"number": 1, "title": "t", "labels": [], "body": "b"}]
markdown, ai_used = run_async(generate_plan(issues, gitea_config(), totals))
assert ai_used is True
assert markdown == "# Open Tickets Implementation Plan\n\nDone."
assert totals["calls"] == 1
assert totals["total_tokens"] == 600
def test_enhance_meters_usage_via_gateway(gitea_env, monkeypatch):
set_setting("issue_ai_enhance", "1")
refresh_snapshot()
monkeypatch.setattr(
"devplacepy.stealth.stealth_async_client",
_fake_gateway('{"title": "Fix login", "body": "## Summary\\nFix it."}'),
)
totals = new_usage_totals()
result = run_async(enhance_ticket("login", "broken", gitea_config(), totals))
assert result.enhanced is True
assert result.title == "Fix login"
assert totals["calls"] == 1
assert abs(totals["cost_usd"] - 0.0001) < 1e-9
def test_enhance_fallback_does_not_meter(gitea_env):
totals = new_usage_totals()
result = run_async(enhance_ticket("Title", "Body", gitea_config(), totals))
assert result.enhanced is False
assert totals["calls"] == 0
def test_create_job_meters_usage(gitea_env, monkeypatch):
get_table("issue_usage").delete()
set_setting("issue_ai_enhance", "1")
refresh_snapshot()
monkeypatch.setattr(
"devplacepy.stealth.stealth_async_client",
_fake_gateway('{"title": "Login broken", "body": "## Summary\\nbroken."}'),
)
uid, _ = _make_user_issues_gitea()
job_uid = queue.enqueue(
"issue_create",
{"author_uid": uid, "title": "login", "description": "cannot log in"},
"user",
uid,
"login",
)
_drive_jobs()
assert queue.get_job(job_uid)["status"] == "done"
assert get_issue_usage()["calls"] == 1
def test_issue_tracker_collect_metrics(gitea_env):
get_table("issue_usage").delete()
add_issue_usage(
{
"calls": 3,
"prompt_tokens": 900,
"completion_tokens": 300,
"total_tokens": 1200,
"cost_usd": 0.003,
"upstream_latency_ms": 1200.0,
"total_latency_ms": 1500.0,
}
)
metrics = IssueTrackerService().collect_metrics()
by_label = {card["label"]: card["value"] for card in metrics["stats"]}
assert by_label["AI calls"] == 3
assert by_label["Total tokens"] == 1200
assert by_label["Total cost"] == "$0.0030"