DevPlace CI / test (push) Failing after 3h10m39s
Comment and post layouts stay full width on small screens. Secondary actions live in the More menu. Notifications can be grouped and throttled, and Enter can submit when that profile setting is on. Emulated-tool models accept large catalogs, with reserved names aliased on the wire and the catalog cached per owner. A dropped service-worker navigation is retried once. The test server waits 120s to finish importing before it is stopped.
545 lines
19 KiB
Python
545 lines
19 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import uuid
|
|
from collections import OrderedDict
|
|
from typing import Optional
|
|
|
|
TOOLS_MODE_NATIVE = "native"
|
|
TOOLS_MODE_EMULATED = "emulated"
|
|
TOOLS_MODES = (TOOLS_MODE_NATIVE, TOOLS_MODE_EMULATED)
|
|
|
|
# There is deliberately no cap on the tool count or the catalog size: a
|
|
# client may declare any number of tools and the full catalog is always
|
|
# injected. Servability is bounded downstream, not here - the gateway's
|
|
# context-window precheck routes an oversized request to a fallback model
|
|
# before dispatch, and the upstream's own context limit is relayed when
|
|
# nothing can serve it. A count/size rejection here would make large
|
|
# catalogs (Devii alone declares hundreds of tools) unservable everywhere.
|
|
MAX_CALLS_PER_TURN = 8
|
|
MAX_STOP_SEQUENCES = 4
|
|
|
|
CALL_OPEN = "<tool_call>"
|
|
CALL_CLOSE = "</tool_call>"
|
|
RESPONSE_OPEN = "<tool_response>"
|
|
RESPONSE_CLOSE = "</tool_response>"
|
|
|
|
_CALL_BLOCK = re.compile(
|
|
re.escape(CALL_OPEN) + r"(.*?)(?:" + re.escape(CALL_CLOSE) + r"|\Z)", re.DOTALL
|
|
)
|
|
_FENCE = re.compile(r"^```[a-zA-Z0-9_-]*\s*\n?(.*?)\n?```$", re.DOTALL)
|
|
|
|
REMOVED_PAYLOAD_KEYS = (
|
|
"tools",
|
|
"tool_choice",
|
|
"parallel_tool_calls",
|
|
"functions",
|
|
"function_call",
|
|
)
|
|
|
|
|
|
class ToolEmulationError(ValueError):
|
|
pass
|
|
|
|
|
|
def emulation_enabled(cfg: dict) -> bool:
|
|
return (cfg.get("gateway_tools_mode") or TOOLS_MODE_NATIVE) == TOOLS_MODE_EMULATED
|
|
|
|
|
|
def _function_specs(payload: dict) -> list[dict]:
|
|
tools = payload.get("tools")
|
|
if not tools:
|
|
return []
|
|
if not isinstance(tools, list):
|
|
raise ToolEmulationError("tools must be a list")
|
|
specs = []
|
|
for tool in tools:
|
|
if not isinstance(tool, dict) or tool.get("type") != "function":
|
|
raise ToolEmulationError(
|
|
"an emulated-tools model supports only tools of type 'function'"
|
|
)
|
|
function = tool.get("function")
|
|
if not isinstance(function, dict) or not str(function.get("name") or "").strip():
|
|
raise ToolEmulationError("every tool needs function.name")
|
|
specs.append(function)
|
|
return specs
|
|
|
|
|
|
def declared_tools(payload: dict) -> dict[str, dict]:
|
|
return {str(spec["name"]): spec for spec in _function_specs(payload)}
|
|
|
|
|
|
def parallel_allowed(payload: dict) -> bool:
|
|
return payload.get("parallel_tool_calls") is not False
|
|
|
|
|
|
def _choice_directive(payload: dict) -> tuple[bool, str]:
|
|
choice = payload.get("tool_choice")
|
|
if choice == "none":
|
|
return False, ""
|
|
if choice == "required":
|
|
return True, "You MUST call at least one tool in this reply."
|
|
if isinstance(choice, dict):
|
|
name = str((choice.get("function") or {}).get("name") or "").strip()
|
|
if name:
|
|
return True, f'You MUST call the tool "{name}" in this reply.'
|
|
return True, ""
|
|
|
|
|
|
def _catalog_text(specs: list[dict]) -> str:
|
|
lines = [json.dumps({"type": "function", "function": spec}, ensure_ascii=False) for spec in specs]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def build_instruction(payload: dict, specs: list[dict]) -> str:
|
|
include, directive = _choice_directive(payload)
|
|
if not include or not specs:
|
|
return ""
|
|
parts = [
|
|
"You can call tools. The available tools are listed below inside "
|
|
"<tools></tools>, one JSON object per line.",
|
|
"<tools>\n" + _catalog_text(specs) + "\n</tools>",
|
|
"To call a tool, reply with one block per call, in exactly this format:\n"
|
|
f"{CALL_OPEN}\n"
|
|
'{"name": "<tool name>", "arguments": {<arguments object>}}\n'
|
|
f"{CALL_CLOSE}",
|
|
"Rules: use only the tool names listed above. The arguments must be a "
|
|
"valid JSON object that matches the tool's parameter schema. After the "
|
|
"call blocks, stop: the results arrive in the next message as "
|
|
f"{RESPONSE_OPEN} blocks. Never write {RESPONSE_OPEN} yourself. When no "
|
|
"tool is needed, answer normally without any call block.",
|
|
]
|
|
if not parallel_allowed(payload):
|
|
parts.append("Emit at most one tool call per reply.")
|
|
if directive:
|
|
parts.append(directive)
|
|
return "\n\n".join(parts)
|
|
|
|
|
|
def _text_of(content) -> str:
|
|
if content is None:
|
|
return ""
|
|
if isinstance(content, str):
|
|
return content
|
|
if isinstance(content, list):
|
|
return "\n".join(
|
|
str(part.get("text", ""))
|
|
for part in content
|
|
if isinstance(part, dict) and part.get("type") == "text"
|
|
)
|
|
return str(content)
|
|
|
|
|
|
def _arguments_object(raw):
|
|
if isinstance(raw, dict):
|
|
return raw
|
|
if isinstance(raw, str):
|
|
try:
|
|
parsed = json.loads(raw) if raw.strip() else {}
|
|
except ValueError:
|
|
return raw
|
|
return parsed
|
|
return {}
|
|
|
|
|
|
def _call_block(name: str, arguments) -> str:
|
|
body = json.dumps({"name": name, "arguments": arguments}, ensure_ascii=False)
|
|
return f"{CALL_OPEN}\n{body}\n{CALL_CLOSE}"
|
|
|
|
|
|
def _response_block(call_id: str, name: str, content) -> str:
|
|
body = json.dumps(
|
|
{"tool_call_id": call_id, "name": name, "content": _text_of(content)},
|
|
ensure_ascii=False,
|
|
)
|
|
return f"{RESPONSE_OPEN}\n{body}\n{RESPONSE_CLOSE}"
|
|
|
|
|
|
def _rewrite_history(messages: list) -> list:
|
|
names: dict[str, str] = {}
|
|
rewritten: list = []
|
|
for message in messages:
|
|
if not isinstance(message, dict):
|
|
rewritten.append(message)
|
|
continue
|
|
role = message.get("role")
|
|
if role == "assistant" and message.get("tool_calls"):
|
|
blocks = []
|
|
for call in message["tool_calls"]:
|
|
function = (call or {}).get("function") or {}
|
|
name = str(function.get("name") or "")
|
|
call_id = str((call or {}).get("id") or "")
|
|
if call_id:
|
|
names[call_id] = name
|
|
blocks.append(_call_block(name, _arguments_object(function.get("arguments"))))
|
|
text = _text_of(message.get("content")).strip()
|
|
converted = {k: v for k, v in message.items() if k != "tool_calls"}
|
|
converted["content"] = "\n".join(([text] if text else []) + blocks)
|
|
rewritten.append(converted)
|
|
elif role in ("tool", "function"):
|
|
call_id = str(message.get("tool_call_id") or "")
|
|
name = str(message.get("name") or names.get(call_id) or "")
|
|
block = _response_block(call_id, name, message.get("content"))
|
|
previous = rewritten[-1] if rewritten else None
|
|
if (
|
|
isinstance(previous, dict)
|
|
and previous.get("_emulated_tool_results")
|
|
):
|
|
previous["content"] += "\n" + block
|
|
else:
|
|
rewritten.append(
|
|
{"role": "user", "content": block, "_emulated_tool_results": True}
|
|
)
|
|
else:
|
|
rewritten.append(message)
|
|
for message in rewritten:
|
|
if isinstance(message, dict):
|
|
message.pop("_emulated_tool_results", None)
|
|
return rewritten
|
|
|
|
|
|
def _with_system_instruction(messages: list, instruction: str) -> list:
|
|
if not instruction:
|
|
return messages
|
|
if messages and isinstance(messages[0], dict) and messages[0].get("role") == "system":
|
|
first = dict(messages[0])
|
|
content = first.get("content")
|
|
if isinstance(content, list):
|
|
first["content"] = list(content) + [{"type": "text", "text": instruction}]
|
|
else:
|
|
existing = _text_of(content)
|
|
first["content"] = (existing + "\n\n" if existing else "") + instruction
|
|
return [first] + messages[1:]
|
|
return [{"role": "system", "content": instruction}] + messages
|
|
|
|
|
|
def _with_stop(payload: dict) -> None:
|
|
stop = payload.get("stop")
|
|
sequences = [stop] if isinstance(stop, str) else list(stop or [])
|
|
if RESPONSE_OPEN in sequences or len(sequences) >= MAX_STOP_SEQUENCES:
|
|
return
|
|
payload["stop"] = sequences + [RESPONSE_OPEN]
|
|
|
|
|
|
class CatalogCache:
|
|
MAX_ENTRIES = 64
|
|
MAX_CHARS = 2_000_000
|
|
|
|
def __init__(self) -> None:
|
|
self._entries: OrderedDict[tuple, tuple[str, dict, bool]] = OrderedDict()
|
|
self._chars = 0
|
|
|
|
@staticmethod
|
|
def fingerprint(payload: dict) -> Optional[str]:
|
|
try:
|
|
canonical = json.dumps(
|
|
{
|
|
"tools": payload.get("tools"),
|
|
"tool_choice": payload.get("tool_choice"),
|
|
"parallel_tool_calls": payload.get("parallel_tool_calls"),
|
|
},
|
|
sort_keys=True,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
default=str,
|
|
)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
|
|
def lookup(
|
|
self, payload: dict, owner: Optional[tuple] = None
|
|
) -> Optional[tuple[str, dict, bool]]:
|
|
digest = self.fingerprint(payload)
|
|
if digest is None:
|
|
return None
|
|
key = ((owner or (None, None)), digest)
|
|
try:
|
|
found = self._entries.pop(key)
|
|
except KeyError:
|
|
return None
|
|
self._entries[key] = found
|
|
return found
|
|
|
|
def store(
|
|
self,
|
|
payload: dict,
|
|
owner: Optional[tuple],
|
|
instruction: str,
|
|
declared: dict,
|
|
allow_parallel: bool,
|
|
) -> None:
|
|
digest = self.fingerprint(payload)
|
|
if digest is None:
|
|
return
|
|
key = ((owner or (None, None)), digest)
|
|
previous = self._entries.pop(key, None)
|
|
if previous is not None:
|
|
self._chars -= len(previous[0])
|
|
self._entries[key] = (instruction, declared, allow_parallel)
|
|
self._chars += len(instruction)
|
|
while len(self._entries) > self.MAX_ENTRIES or self._chars > self.MAX_CHARS:
|
|
_, dropped = self._entries.popitem(last=False)
|
|
self._chars -= len(dropped[0])
|
|
|
|
|
|
def _cached_parts(
|
|
payload: dict,
|
|
specs: list[dict],
|
|
catalog_cache: Optional[CatalogCache],
|
|
owner: Optional[tuple],
|
|
) -> tuple[str, dict, bool]:
|
|
allow_parallel = parallel_allowed(payload)
|
|
if catalog_cache is not None:
|
|
found = catalog_cache.lookup(payload, owner)
|
|
if found is not None:
|
|
return found
|
|
declared = {str(spec["name"]): spec for spec in specs}
|
|
instruction = build_instruction(payload, specs)
|
|
if catalog_cache is not None:
|
|
catalog_cache.store(payload, owner, instruction, declared, allow_parallel)
|
|
return instruction, declared, allow_parallel
|
|
|
|
|
|
def emulate_request(
|
|
payload: dict,
|
|
catalog_cache: Optional[CatalogCache] = None,
|
|
owner: Optional[tuple] = None,
|
|
) -> dict:
|
|
specs = _function_specs(payload)
|
|
history = payload.get("messages") or []
|
|
has_tool_history = any(
|
|
isinstance(m, dict) and (m.get("role") in ("tool", "function") or m.get("tool_calls"))
|
|
for m in history
|
|
)
|
|
if not specs and not has_tool_history:
|
|
return payload
|
|
instruction, _, _ = _cached_parts(payload, specs, catalog_cache, owner)
|
|
converted = {k: copy.copy(v) for k, v in payload.items() if k not in REMOVED_PAYLOAD_KEYS}
|
|
messages = _rewrite_history(list(history))
|
|
converted["messages"] = _with_system_instruction(messages, instruction)
|
|
if specs:
|
|
_with_stop(converted)
|
|
return converted
|
|
|
|
|
|
def cached_emulated_tools(
|
|
payload: dict,
|
|
catalog_cache: Optional[CatalogCache] = None,
|
|
owner: Optional[tuple] = None,
|
|
) -> Optional[tuple[dict, bool]]:
|
|
if catalog_cache is not None:
|
|
found = catalog_cache.lookup(payload, owner)
|
|
if found is not None:
|
|
_, declared, allow_parallel = found
|
|
if not declared:
|
|
return None
|
|
return declared, allow_parallel
|
|
declared = declared_tools(payload)
|
|
if not declared:
|
|
return None
|
|
return declared, parallel_allowed(payload)
|
|
|
|
|
|
def _strip_fence(text: str) -> str:
|
|
stripped = text.strip()
|
|
match = _FENCE.match(stripped)
|
|
return match.group(1).strip() if match else stripped
|
|
|
|
|
|
def _call_objects(raw: str) -> list:
|
|
try:
|
|
parsed = json.loads(_strip_fence(raw))
|
|
except ValueError:
|
|
return []
|
|
items = parsed if isinstance(parsed, list) else [parsed]
|
|
return [item for item in items if isinstance(item, dict)]
|
|
|
|
|
|
def _normalize_call(item: dict, declared: dict[str, dict]) -> tuple[Optional[dict], Optional[str]]:
|
|
function = item.get("function") if isinstance(item.get("function"), dict) else item
|
|
name = str(function.get("name") or "").strip()
|
|
if name not in declared:
|
|
return None, f"unknown tool {name!r}"
|
|
arguments = function.get("arguments", function.get("parameters", {}))
|
|
if isinstance(arguments, str):
|
|
try:
|
|
arguments = json.loads(arguments) if arguments.strip() else {}
|
|
except ValueError:
|
|
return None, f"arguments of {name!r} are not valid JSON"
|
|
if not isinstance(arguments, dict):
|
|
return None, f"arguments of {name!r} are not an object"
|
|
schema = declared[name].get("parameters")
|
|
required = schema.get("required") if isinstance(schema, dict) else None
|
|
missing = [key for key in (required or []) if key not in arguments]
|
|
if missing:
|
|
return None, f"call to {name!r} misses required arguments: {', '.join(missing)}"
|
|
return (
|
|
{
|
|
"id": "call_" + uuid.uuid4().hex[:24],
|
|
"type": "function",
|
|
"function": {
|
|
"name": name,
|
|
"arguments": json.dumps(arguments, ensure_ascii=False),
|
|
},
|
|
},
|
|
None,
|
|
)
|
|
|
|
|
|
def parse_tool_calls(
|
|
text: str, declared: dict[str, dict], allow_parallel: bool = True
|
|
) -> tuple[str, list[dict], list[str]]:
|
|
if CALL_OPEN not in (text or ""):
|
|
return text or "", [], []
|
|
calls: list[dict] = []
|
|
warnings: list[str] = []
|
|
kept: list[str] = []
|
|
cursor = 0
|
|
for match in _CALL_BLOCK.finditer(text):
|
|
kept.append(text[cursor : match.start()])
|
|
cursor = match.end()
|
|
items = _call_objects(match.group(1))
|
|
if not items:
|
|
warnings.append("a tool_call block did not contain valid JSON")
|
|
kept.append(match.group(0))
|
|
continue
|
|
for item in items:
|
|
call, problem = _normalize_call(item, declared)
|
|
if call is None:
|
|
warnings.append(problem or "invalid tool call")
|
|
kept.append(match.group(0))
|
|
continue
|
|
calls.append(call)
|
|
kept.append(text[cursor:])
|
|
if not allow_parallel and len(calls) > 1:
|
|
warnings.append(f"parallel calls disabled: dropped {len(calls) - 1} extra call(s)")
|
|
calls = calls[:1]
|
|
if len(calls) > MAX_CALLS_PER_TURN:
|
|
warnings.append(f"dropped {len(calls) - MAX_CALLS_PER_TURN} call(s) over the per-turn cap")
|
|
calls = calls[:MAX_CALLS_PER_TURN]
|
|
return "".join(kept).strip(), calls, warnings
|
|
|
|
|
|
def convert_completion(
|
|
completion: dict, declared: dict[str, dict], allow_parallel: bool = True
|
|
) -> tuple[dict, list[str]]:
|
|
warnings: list[str] = []
|
|
for choice in completion.get("choices") or []:
|
|
message = choice.get("message") if isinstance(choice, dict) else None
|
|
if not isinstance(message, dict) or message.get("tool_calls"):
|
|
continue
|
|
text = message.get("content")
|
|
if not isinstance(text, str):
|
|
continue
|
|
remaining, calls, found = parse_tool_calls(text, declared, allow_parallel)
|
|
warnings.extend(found)
|
|
if calls:
|
|
message["tool_calls"] = calls
|
|
message["content"] = remaining or None
|
|
choice["finish_reason"] = "tool_calls"
|
|
return completion, warnings
|
|
|
|
|
|
def _holdback(text: str) -> int:
|
|
longest = min(len(CALL_OPEN) - 1, len(text))
|
|
for size in range(longest, 0, -1):
|
|
if CALL_OPEN.startswith(text[-size:]):
|
|
return size
|
|
return 0
|
|
|
|
|
|
class ToolCallStream:
|
|
def __init__(self, declared: dict[str, dict], allow_parallel: bool = True):
|
|
self.declared = declared
|
|
self.allow_parallel = allow_parallel
|
|
self.warnings: list[str] = []
|
|
self._base: dict = {}
|
|
self._pending = ""
|
|
self._call_text = ""
|
|
self._in_call = False
|
|
self._flushed = False
|
|
self._saw_calls = False
|
|
|
|
def _chunk(self, delta: dict, finish_reason=None) -> dict:
|
|
chunk = {k: v for k, v in self._base.items() if k not in ("choices", "usage")}
|
|
chunk["choices"] = [{"index": 0, "delta": delta, "finish_reason": finish_reason}]
|
|
return chunk
|
|
|
|
def _content_chunk(self, text: str) -> list[dict]:
|
|
return [self._chunk({"content": text})] if text else []
|
|
|
|
def _consume(self, text: str) -> list[dict]:
|
|
if self._in_call:
|
|
self._call_text += text
|
|
return []
|
|
self._pending += text
|
|
index = self._pending.find(CALL_OPEN)
|
|
if index >= 0:
|
|
out = self._content_chunk(self._pending[:index])
|
|
self._call_text = self._pending[index:]
|
|
self._pending = ""
|
|
self._in_call = True
|
|
return out
|
|
keep = _holdback(self._pending)
|
|
emit = self._pending[: len(self._pending) - keep]
|
|
self._pending = self._pending[len(self._pending) - keep :]
|
|
return self._content_chunk(emit)
|
|
|
|
def flush(self) -> list[dict]:
|
|
if self._flushed:
|
|
return []
|
|
self._flushed = True
|
|
if not self._in_call:
|
|
text, self._pending = self._pending, ""
|
|
return self._content_chunk(text)
|
|
remaining, calls, found = parse_tool_calls(
|
|
self._call_text, self.declared, self.allow_parallel
|
|
)
|
|
self.warnings.extend(found)
|
|
out = self._content_chunk(remaining)
|
|
if calls:
|
|
self._saw_calls = True
|
|
out.append(
|
|
self._chunk(
|
|
{
|
|
"tool_calls": [
|
|
{"index": i, **call} for i, call in enumerate(calls)
|
|
]
|
|
}
|
|
)
|
|
)
|
|
return out
|
|
|
|
def feed(self, chunk: dict) -> list[dict]:
|
|
choices = chunk.get("choices") or []
|
|
if not choices:
|
|
return [chunk]
|
|
self._base = chunk
|
|
choice = choices[0]
|
|
delta = choice.get("delta") or {}
|
|
out: list[dict] = []
|
|
text = delta.get("content")
|
|
passthrough = {k: v for k, v in delta.items() if k != "content"}
|
|
if isinstance(text, str) and text:
|
|
out.extend(self._consume(text))
|
|
finish = choice.get("finish_reason")
|
|
if passthrough:
|
|
out.append(self._chunk(passthrough))
|
|
if finish is not None:
|
|
out.extend(self.flush())
|
|
out.append(
|
|
self._chunk(
|
|
{}, "tool_calls" if self._saw_calls and finish == "stop" else finish
|
|
)
|
|
)
|
|
if chunk.get("usage"):
|
|
out.append({**{k: v for k, v in chunk.items() if k != "choices"}, "choices": []})
|
|
return out
|
|
|
|
def finish(self) -> list[dict]:
|
|
return self.flush()
|