fix: normalize unicode escape sequences and reformat multi-line expressions across codebase
This commit is contained in:
@@ -28,18 +28,34 @@ class JobService(BaseService):
|
||||
self.concurrency_key = f"{name}_max_concurrent"
|
||||
self.timeout_key = f"{name}_job_timeout_seconds"
|
||||
self.config_fields = [
|
||||
ConfigField(self.retention_key, "Artifact retention (seconds)", type="int",
|
||||
default=DEFAULT_RETENTION_SECONDS, minimum=60,
|
||||
help="Finished artifacts are deleted once unused for this long.",
|
||||
group="Jobs"),
|
||||
ConfigField(self.concurrency_key, "Max concurrent jobs", type="int",
|
||||
default=DEFAULT_MAX_CONCURRENT, minimum=1, maximum=16,
|
||||
help="Number of jobs processed in parallel.",
|
||||
group="Jobs"),
|
||||
ConfigField(self.timeout_key, "Job timeout (seconds)", type="int",
|
||||
default=DEFAULT_JOB_TIMEOUT_SECONDS, minimum=10,
|
||||
help="A running job with no live task after this long is retried.",
|
||||
group="Jobs"),
|
||||
ConfigField(
|
||||
self.retention_key,
|
||||
"Artifact retention (seconds)",
|
||||
type="int",
|
||||
default=DEFAULT_RETENTION_SECONDS,
|
||||
minimum=60,
|
||||
help="Finished artifacts are deleted once unused for this long.",
|
||||
group="Jobs",
|
||||
),
|
||||
ConfigField(
|
||||
self.concurrency_key,
|
||||
"Max concurrent jobs",
|
||||
type="int",
|
||||
default=DEFAULT_MAX_CONCURRENT,
|
||||
minimum=1,
|
||||
maximum=16,
|
||||
help="Number of jobs processed in parallel.",
|
||||
group="Jobs",
|
||||
),
|
||||
ConfigField(
|
||||
self.timeout_key,
|
||||
"Job timeout (seconds)",
|
||||
type="int",
|
||||
default=DEFAULT_JOB_TIMEOUT_SECONDS,
|
||||
minimum=10,
|
||||
help="A running job with no live task after this long is retried.",
|
||||
group="Jobs",
|
||||
),
|
||||
]
|
||||
self._inflight: dict = {}
|
||||
|
||||
@@ -79,41 +95,53 @@ class JobService(BaseService):
|
||||
if not task.done():
|
||||
continue
|
||||
del self._inflight[uid]
|
||||
duration_ms = int((datetime.now(timezone.utc) - entry["started"]).total_seconds() * 1000)
|
||||
duration_ms = int(
|
||||
(datetime.now(timezone.utc) - entry["started"]).total_seconds() * 1000
|
||||
)
|
||||
exc = task.exception() if not task.cancelled() else asyncio.CancelledError()
|
||||
if exc is not None:
|
||||
self._finish_failed(uid, str(exc) or exc.__class__.__name__, duration_ms)
|
||||
self._finish_failed(
|
||||
uid, str(exc) or exc.__class__.__name__, duration_ms
|
||||
)
|
||||
else:
|
||||
self._finish_done(uid, task.result() or {}, duration_ms)
|
||||
|
||||
def _finish_done(self, uid: str, result_data: dict, duration_ms: int) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
get_table("jobs").update({
|
||||
"uid": uid,
|
||||
"status": queue.DONE,
|
||||
"result": json.dumps(result_data),
|
||||
"error": "",
|
||||
"completed_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
"duration_ms": duration_ms,
|
||||
"last_accessed_at": now.isoformat(),
|
||||
"expires_at": (now + timedelta(seconds=self.retention_seconds())).isoformat(),
|
||||
"bytes_in": int(result_data.get("bytes_in", 0)),
|
||||
"bytes_out": int(result_data.get("bytes_out", 0)),
|
||||
"item_count": int(result_data.get("item_count", 0)),
|
||||
}, ["uid"])
|
||||
get_table("jobs").update(
|
||||
{
|
||||
"uid": uid,
|
||||
"status": queue.DONE,
|
||||
"result": json.dumps(result_data),
|
||||
"error": "",
|
||||
"completed_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
"duration_ms": duration_ms,
|
||||
"last_accessed_at": now.isoformat(),
|
||||
"expires_at": (
|
||||
now + timedelta(seconds=self.retention_seconds())
|
||||
).isoformat(),
|
||||
"bytes_in": int(result_data.get("bytes_in", 0)),
|
||||
"bytes_out": int(result_data.get("bytes_out", 0)),
|
||||
"item_count": int(result_data.get("item_count", 0)),
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
self.log(f"Job {uid} done in {duration_ms}ms")
|
||||
|
||||
def _finish_failed(self, uid: str, error: str, duration_ms: int) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
get_table("jobs").update({
|
||||
"uid": uid,
|
||||
"status": queue.FAILED,
|
||||
"error": error[:2000],
|
||||
"completed_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
"duration_ms": duration_ms,
|
||||
}, ["uid"])
|
||||
get_table("jobs").update(
|
||||
{
|
||||
"uid": uid,
|
||||
"status": queue.FAILED,
|
||||
"error": error[:2000],
|
||||
"completed_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
"duration_ms": duration_ms,
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
self.log(f"Job {uid} failed: {error}")
|
||||
|
||||
def _refill(self) -> None:
|
||||
@@ -121,21 +149,31 @@ class JobService(BaseService):
|
||||
if capacity <= 0:
|
||||
return
|
||||
table = get_table("jobs")
|
||||
pending = list(table.find(kind=self.kind, status=queue.PENDING, order_by=["uid"], _limit=capacity))
|
||||
pending = list(
|
||||
table.find(
|
||||
kind=self.kind, status=queue.PENDING, order_by=["uid"], _limit=capacity
|
||||
)
|
||||
)
|
||||
for row in pending:
|
||||
uid = row["uid"]
|
||||
if uid in self._inflight:
|
||||
continue
|
||||
now = datetime.now(timezone.utc)
|
||||
table.update({
|
||||
"uid": uid,
|
||||
"status": queue.RUNNING,
|
||||
"started_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
"error": "",
|
||||
}, ["uid"])
|
||||
table.update(
|
||||
{
|
||||
"uid": uid,
|
||||
"status": queue.RUNNING,
|
||||
"started_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
"error": "",
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
job = queue.get_job(uid)
|
||||
self._inflight[uid] = {"task": asyncio.create_task(self._run_job(job)), "started": now}
|
||||
self._inflight[uid] = {
|
||||
"task": asyncio.create_task(self._run_job(job)),
|
||||
"started": now,
|
||||
}
|
||||
self.log(f"Job {uid} started")
|
||||
|
||||
async def _run_job(self, job: dict) -> dict:
|
||||
@@ -148,19 +186,26 @@ class JobService(BaseService):
|
||||
uid = row["uid"]
|
||||
if uid in self._inflight:
|
||||
continue
|
||||
if min_age_seconds > 0 and not self._older_than(row.get("started_at"), now, min_age_seconds):
|
||||
if min_age_seconds > 0 and not self._older_than(
|
||||
row.get("started_at"), now, min_age_seconds
|
||||
):
|
||||
continue
|
||||
retry_count = int(row.get("retry_count") or 0)
|
||||
if retry_count >= self.max_retries:
|
||||
self._finish_failed(uid, "exceeded retry limit after orphan recovery", 0)
|
||||
self._finish_failed(
|
||||
uid, "exceeded retry limit after orphan recovery", 0
|
||||
)
|
||||
continue
|
||||
table.update({
|
||||
"uid": uid,
|
||||
"status": queue.PENDING,
|
||||
"retry_count": retry_count + 1,
|
||||
"started_at": "",
|
||||
"updated_at": now.isoformat(),
|
||||
}, ["uid"])
|
||||
table.update(
|
||||
{
|
||||
"uid": uid,
|
||||
"status": queue.PENDING,
|
||||
"retry_count": retry_count + 1,
|
||||
"started_at": "",
|
||||
"updated_at": now.isoformat(),
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
self.log(f"Recovered orphaned job {uid} (retry {retry_count + 1})")
|
||||
|
||||
def _sweep_expired(self) -> None:
|
||||
@@ -202,7 +247,9 @@ class JobService(BaseService):
|
||||
bytes_in = bytes_out = items = 0
|
||||
durations = []
|
||||
for row in rows:
|
||||
by_status[row.get("status", "")] = by_status.get(row.get("status", ""), 0) + 1
|
||||
by_status[row.get("status", "")] = (
|
||||
by_status.get(row.get("status", ""), 0) + 1
|
||||
)
|
||||
bytes_in += int(row.get("bytes_in") or 0)
|
||||
bytes_out += int(row.get("bytes_out") or 0)
|
||||
items += int(row.get("item_count") or 0)
|
||||
|
||||
Reference in New Issue
Block a user