Files
devplacepy/devplacepy/services/openai_gateway/tool_emulation.py
T
retoor 8a48e3ef85
DevPlace CI / test (push) Failing after 3h10m39s
Keep composers usable, group actions and notifications, and drop the tool-count cap
Comment and post layouts stay full width on small screens. Secondary actions live in the More menu. Notifications can be grouped and throttled, and Enter can submit when that profile setting is on. Emulated-tool models accept large catalogs, with reserved names aliased on the wire and the catalog cached per owner. A dropped service-worker navigation is retried once. The test server waits 120s to finish importing before it is stopped.
2026-11-12 15:29:27 +01:00

545 lines
19 KiB
Python

# retoor <retoor@molodetz.nl>
import copy
import hashlib
import json
import re
import uuid
from collections import OrderedDict
from typing import Optional
TOOLS_MODE_NATIVE = "native"
TOOLS_MODE_EMULATED = "emulated"
TOOLS_MODES = (TOOLS_MODE_NATIVE, TOOLS_MODE_EMULATED)
# There is deliberately no cap on the tool count or the catalog size: a
# client may declare any number of tools and the full catalog is always
# injected. Servability is bounded downstream, not here - the gateway's
# context-window precheck routes an oversized request to a fallback model
# before dispatch, and the upstream's own context limit is relayed when
# nothing can serve it. A count/size rejection here would make large
# catalogs (Devii alone declares hundreds of tools) unservable everywhere.
MAX_CALLS_PER_TURN = 8
MAX_STOP_SEQUENCES = 4
CALL_OPEN = "<tool_call>"
CALL_CLOSE = "</tool_call>"
RESPONSE_OPEN = "<tool_response>"
RESPONSE_CLOSE = "</tool_response>"
_CALL_BLOCK = re.compile(
re.escape(CALL_OPEN) + r"(.*?)(?:" + re.escape(CALL_CLOSE) + r"|\Z)", re.DOTALL
)
_FENCE = re.compile(r"^```[a-zA-Z0-9_-]*\s*\n?(.*?)\n?```$", re.DOTALL)
REMOVED_PAYLOAD_KEYS = (
"tools",
"tool_choice",
"parallel_tool_calls",
"functions",
"function_call",
)
class ToolEmulationError(ValueError):
pass
def emulation_enabled(cfg: dict) -> bool:
return (cfg.get("gateway_tools_mode") or TOOLS_MODE_NATIVE) == TOOLS_MODE_EMULATED
def _function_specs(payload: dict) -> list[dict]:
tools = payload.get("tools")
if not tools:
return []
if not isinstance(tools, list):
raise ToolEmulationError("tools must be a list")
specs = []
for tool in tools:
if not isinstance(tool, dict) or tool.get("type") != "function":
raise ToolEmulationError(
"an emulated-tools model supports only tools of type 'function'"
)
function = tool.get("function")
if not isinstance(function, dict) or not str(function.get("name") or "").strip():
raise ToolEmulationError("every tool needs function.name")
specs.append(function)
return specs
def declared_tools(payload: dict) -> dict[str, dict]:
return {str(spec["name"]): spec for spec in _function_specs(payload)}
def parallel_allowed(payload: dict) -> bool:
return payload.get("parallel_tool_calls") is not False
def _choice_directive(payload: dict) -> tuple[bool, str]:
choice = payload.get("tool_choice")
if choice == "none":
return False, ""
if choice == "required":
return True, "You MUST call at least one tool in this reply."
if isinstance(choice, dict):
name = str((choice.get("function") or {}).get("name") or "").strip()
if name:
return True, f'You MUST call the tool "{name}" in this reply.'
return True, ""
def _catalog_text(specs: list[dict]) -> str:
lines = [json.dumps({"type": "function", "function": spec}, ensure_ascii=False) for spec in specs]
return "\n".join(lines)
def build_instruction(payload: dict, specs: list[dict]) -> str:
include, directive = _choice_directive(payload)
if not include or not specs:
return ""
parts = [
"You can call tools. The available tools are listed below inside "
"<tools></tools>, one JSON object per line.",
"<tools>\n" + _catalog_text(specs) + "\n</tools>",
"To call a tool, reply with one block per call, in exactly this format:\n"
f"{CALL_OPEN}\n"
'{"name": "<tool name>", "arguments": {<arguments object>}}\n'
f"{CALL_CLOSE}",
"Rules: use only the tool names listed above. The arguments must be a "
"valid JSON object that matches the tool's parameter schema. After the "
"call blocks, stop: the results arrive in the next message as "
f"{RESPONSE_OPEN} blocks. Never write {RESPONSE_OPEN} yourself. When no "
"tool is needed, answer normally without any call block.",
]
if not parallel_allowed(payload):
parts.append("Emit at most one tool call per reply.")
if directive:
parts.append(directive)
return "\n\n".join(parts)
def _text_of(content) -> str:
if content is None:
return ""
if isinstance(content, str):
return content
if isinstance(content, list):
return "\n".join(
str(part.get("text", ""))
for part in content
if isinstance(part, dict) and part.get("type") == "text"
)
return str(content)
def _arguments_object(raw):
if isinstance(raw, dict):
return raw
if isinstance(raw, str):
try:
parsed = json.loads(raw) if raw.strip() else {}
except ValueError:
return raw
return parsed
return {}
def _call_block(name: str, arguments) -> str:
body = json.dumps({"name": name, "arguments": arguments}, ensure_ascii=False)
return f"{CALL_OPEN}\n{body}\n{CALL_CLOSE}"
def _response_block(call_id: str, name: str, content) -> str:
body = json.dumps(
{"tool_call_id": call_id, "name": name, "content": _text_of(content)},
ensure_ascii=False,
)
return f"{RESPONSE_OPEN}\n{body}\n{RESPONSE_CLOSE}"
def _rewrite_history(messages: list) -> list:
names: dict[str, str] = {}
rewritten: list = []
for message in messages:
if not isinstance(message, dict):
rewritten.append(message)
continue
role = message.get("role")
if role == "assistant" and message.get("tool_calls"):
blocks = []
for call in message["tool_calls"]:
function = (call or {}).get("function") or {}
name = str(function.get("name") or "")
call_id = str((call or {}).get("id") or "")
if call_id:
names[call_id] = name
blocks.append(_call_block(name, _arguments_object(function.get("arguments"))))
text = _text_of(message.get("content")).strip()
converted = {k: v for k, v in message.items() if k != "tool_calls"}
converted["content"] = "\n".join(([text] if text else []) + blocks)
rewritten.append(converted)
elif role in ("tool", "function"):
call_id = str(message.get("tool_call_id") or "")
name = str(message.get("name") or names.get(call_id) or "")
block = _response_block(call_id, name, message.get("content"))
previous = rewritten[-1] if rewritten else None
if (
isinstance(previous, dict)
and previous.get("_emulated_tool_results")
):
previous["content"] += "\n" + block
else:
rewritten.append(
{"role": "user", "content": block, "_emulated_tool_results": True}
)
else:
rewritten.append(message)
for message in rewritten:
if isinstance(message, dict):
message.pop("_emulated_tool_results", None)
return rewritten
def _with_system_instruction(messages: list, instruction: str) -> list:
if not instruction:
return messages
if messages and isinstance(messages[0], dict) and messages[0].get("role") == "system":
first = dict(messages[0])
content = first.get("content")
if isinstance(content, list):
first["content"] = list(content) + [{"type": "text", "text": instruction}]
else:
existing = _text_of(content)
first["content"] = (existing + "\n\n" if existing else "") + instruction
return [first] + messages[1:]
return [{"role": "system", "content": instruction}] + messages
def _with_stop(payload: dict) -> None:
stop = payload.get("stop")
sequences = [stop] if isinstance(stop, str) else list(stop or [])
if RESPONSE_OPEN in sequences or len(sequences) >= MAX_STOP_SEQUENCES:
return
payload["stop"] = sequences + [RESPONSE_OPEN]
class CatalogCache:
MAX_ENTRIES = 64
MAX_CHARS = 2_000_000
def __init__(self) -> None:
self._entries: OrderedDict[tuple, tuple[str, dict, bool]] = OrderedDict()
self._chars = 0
@staticmethod
def fingerprint(payload: dict) -> Optional[str]:
try:
canonical = json.dumps(
{
"tools": payload.get("tools"),
"tool_choice": payload.get("tool_choice"),
"parallel_tool_calls": payload.get("parallel_tool_calls"),
},
sort_keys=True,
ensure_ascii=False,
separators=(",", ":"),
default=str,
)
except (TypeError, ValueError):
return None
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def lookup(
self, payload: dict, owner: Optional[tuple] = None
) -> Optional[tuple[str, dict, bool]]:
digest = self.fingerprint(payload)
if digest is None:
return None
key = ((owner or (None, None)), digest)
try:
found = self._entries.pop(key)
except KeyError:
return None
self._entries[key] = found
return found
def store(
self,
payload: dict,
owner: Optional[tuple],
instruction: str,
declared: dict,
allow_parallel: bool,
) -> None:
digest = self.fingerprint(payload)
if digest is None:
return
key = ((owner or (None, None)), digest)
previous = self._entries.pop(key, None)
if previous is not None:
self._chars -= len(previous[0])
self._entries[key] = (instruction, declared, allow_parallel)
self._chars += len(instruction)
while len(self._entries) > self.MAX_ENTRIES or self._chars > self.MAX_CHARS:
_, dropped = self._entries.popitem(last=False)
self._chars -= len(dropped[0])
def _cached_parts(
payload: dict,
specs: list[dict],
catalog_cache: Optional[CatalogCache],
owner: Optional[tuple],
) -> tuple[str, dict, bool]:
allow_parallel = parallel_allowed(payload)
if catalog_cache is not None:
found = catalog_cache.lookup(payload, owner)
if found is not None:
return found
declared = {str(spec["name"]): spec for spec in specs}
instruction = build_instruction(payload, specs)
if catalog_cache is not None:
catalog_cache.store(payload, owner, instruction, declared, allow_parallel)
return instruction, declared, allow_parallel
def emulate_request(
payload: dict,
catalog_cache: Optional[CatalogCache] = None,
owner: Optional[tuple] = None,
) -> dict:
specs = _function_specs(payload)
history = payload.get("messages") or []
has_tool_history = any(
isinstance(m, dict) and (m.get("role") in ("tool", "function") or m.get("tool_calls"))
for m in history
)
if not specs and not has_tool_history:
return payload
instruction, _, _ = _cached_parts(payload, specs, catalog_cache, owner)
converted = {k: copy.copy(v) for k, v in payload.items() if k not in REMOVED_PAYLOAD_KEYS}
messages = _rewrite_history(list(history))
converted["messages"] = _with_system_instruction(messages, instruction)
if specs:
_with_stop(converted)
return converted
def cached_emulated_tools(
payload: dict,
catalog_cache: Optional[CatalogCache] = None,
owner: Optional[tuple] = None,
) -> Optional[tuple[dict, bool]]:
if catalog_cache is not None:
found = catalog_cache.lookup(payload, owner)
if found is not None:
_, declared, allow_parallel = found
if not declared:
return None
return declared, allow_parallel
declared = declared_tools(payload)
if not declared:
return None
return declared, parallel_allowed(payload)
def _strip_fence(text: str) -> str:
stripped = text.strip()
match = _FENCE.match(stripped)
return match.group(1).strip() if match else stripped
def _call_objects(raw: str) -> list:
try:
parsed = json.loads(_strip_fence(raw))
except ValueError:
return []
items = parsed if isinstance(parsed, list) else [parsed]
return [item for item in items if isinstance(item, dict)]
def _normalize_call(item: dict, declared: dict[str, dict]) -> tuple[Optional[dict], Optional[str]]:
function = item.get("function") if isinstance(item.get("function"), dict) else item
name = str(function.get("name") or "").strip()
if name not in declared:
return None, f"unknown tool {name!r}"
arguments = function.get("arguments", function.get("parameters", {}))
if isinstance(arguments, str):
try:
arguments = json.loads(arguments) if arguments.strip() else {}
except ValueError:
return None, f"arguments of {name!r} are not valid JSON"
if not isinstance(arguments, dict):
return None, f"arguments of {name!r} are not an object"
schema = declared[name].get("parameters")
required = schema.get("required") if isinstance(schema, dict) else None
missing = [key for key in (required or []) if key not in arguments]
if missing:
return None, f"call to {name!r} misses required arguments: {', '.join(missing)}"
return (
{
"id": "call_" + uuid.uuid4().hex[:24],
"type": "function",
"function": {
"name": name,
"arguments": json.dumps(arguments, ensure_ascii=False),
},
},
None,
)
def parse_tool_calls(
text: str, declared: dict[str, dict], allow_parallel: bool = True
) -> tuple[str, list[dict], list[str]]:
if CALL_OPEN not in (text or ""):
return text or "", [], []
calls: list[dict] = []
warnings: list[str] = []
kept: list[str] = []
cursor = 0
for match in _CALL_BLOCK.finditer(text):
kept.append(text[cursor : match.start()])
cursor = match.end()
items = _call_objects(match.group(1))
if not items:
warnings.append("a tool_call block did not contain valid JSON")
kept.append(match.group(0))
continue
for item in items:
call, problem = _normalize_call(item, declared)
if call is None:
warnings.append(problem or "invalid tool call")
kept.append(match.group(0))
continue
calls.append(call)
kept.append(text[cursor:])
if not allow_parallel and len(calls) > 1:
warnings.append(f"parallel calls disabled: dropped {len(calls) - 1} extra call(s)")
calls = calls[:1]
if len(calls) > MAX_CALLS_PER_TURN:
warnings.append(f"dropped {len(calls) - MAX_CALLS_PER_TURN} call(s) over the per-turn cap")
calls = calls[:MAX_CALLS_PER_TURN]
return "".join(kept).strip(), calls, warnings
def convert_completion(
completion: dict, declared: dict[str, dict], allow_parallel: bool = True
) -> tuple[dict, list[str]]:
warnings: list[str] = []
for choice in completion.get("choices") or []:
message = choice.get("message") if isinstance(choice, dict) else None
if not isinstance(message, dict) or message.get("tool_calls"):
continue
text = message.get("content")
if not isinstance(text, str):
continue
remaining, calls, found = parse_tool_calls(text, declared, allow_parallel)
warnings.extend(found)
if calls:
message["tool_calls"] = calls
message["content"] = remaining or None
choice["finish_reason"] = "tool_calls"
return completion, warnings
def _holdback(text: str) -> int:
longest = min(len(CALL_OPEN) - 1, len(text))
for size in range(longest, 0, -1):
if CALL_OPEN.startswith(text[-size:]):
return size
return 0
class ToolCallStream:
def __init__(self, declared: dict[str, dict], allow_parallel: bool = True):
self.declared = declared
self.allow_parallel = allow_parallel
self.warnings: list[str] = []
self._base: dict = {}
self._pending = ""
self._call_text = ""
self._in_call = False
self._flushed = False
self._saw_calls = False
def _chunk(self, delta: dict, finish_reason=None) -> dict:
chunk = {k: v for k, v in self._base.items() if k not in ("choices", "usage")}
chunk["choices"] = [{"index": 0, "delta": delta, "finish_reason": finish_reason}]
return chunk
def _content_chunk(self, text: str) -> list[dict]:
return [self._chunk({"content": text})] if text else []
def _consume(self, text: str) -> list[dict]:
if self._in_call:
self._call_text += text
return []
self._pending += text
index = self._pending.find(CALL_OPEN)
if index >= 0:
out = self._content_chunk(self._pending[:index])
self._call_text = self._pending[index:]
self._pending = ""
self._in_call = True
return out
keep = _holdback(self._pending)
emit = self._pending[: len(self._pending) - keep]
self._pending = self._pending[len(self._pending) - keep :]
return self._content_chunk(emit)
def flush(self) -> list[dict]:
if self._flushed:
return []
self._flushed = True
if not self._in_call:
text, self._pending = self._pending, ""
return self._content_chunk(text)
remaining, calls, found = parse_tool_calls(
self._call_text, self.declared, self.allow_parallel
)
self.warnings.extend(found)
out = self._content_chunk(remaining)
if calls:
self._saw_calls = True
out.append(
self._chunk(
{
"tool_calls": [
{"index": i, **call} for i, call in enumerate(calls)
]
}
)
)
return out
def feed(self, chunk: dict) -> list[dict]:
choices = chunk.get("choices") or []
if not choices:
return [chunk]
self._base = chunk
choice = choices[0]
delta = choice.get("delta") or {}
out: list[dict] = []
text = delta.get("content")
passthrough = {k: v for k, v in delta.items() if k != "content"}
if isinstance(text, str) and text:
out.extend(self._consume(text))
finish = choice.get("finish_reason")
if passthrough:
out.append(self._chunk(passthrough))
if finish is not None:
out.extend(self.flush())
out.append(
self._chunk(
{}, "tool_calls" if self._saw_calls and finish == "stop" else finish
)
)
if chunk.get("usage"):
out.append({**{k: v for k, v in chunk.items() if k != "choices"}, "choices": []})
return out
def finish(self) -> list[dict]:
return self.flush()