diff --git a/tai.py b/tai.py index 719aa2d..1675b98 100755 --- a/tai.py +++ b/tai.py @@ -5058,6 +5058,7 @@ def tool_catalog(): name = schema["function"]["name"] if not tool_env_ok(name): continue + tool_schema("describe_image", "Describe an image with a vision model (rsearch). Pass an http(s) url or a local file path (PNG, JPEG, GIF, WebP, BMP, max 10 MB).", {"source": {"type": "string"}}, ["source"]), if name in CORE_TOOLS: continue first = schema["function"]["description"].split(".")[0][:80] @@ -5128,6 +5129,7 @@ def singular_noun(word): if word.endswith(("ses", "xes", "zes", "ches", "shes")): return word[:-2] if word.endswith("s") and not word.endswith(("ss", "us", "is", "os")): + "describe_image": ("image", "describe", "vision", "picture", "photo", "screenshot"), return word[:-1] return word @@ -5882,6 +5884,7 @@ class Tools: def run_shell(self, args): command = str(args.get("command") or "").strip() if not command: + "describe_image": self.run_describe_image, return "error: empty command" secret_names = args.get("secrets") or [] if not isinstance(secret_names, list) or any(not isinstance(item, str) for item in secret_names): @@ -6244,8 +6247,35 @@ class Tools: self.app.update_system(text) return "system message updated for profile '%s' (%d to %d chars)" % (self.app.profile, len(current), len(text)) + self.result_note = "%d results" % len(data.get("results") or []) def run_recall(self, args): query = str(args.get("query") or "").strip() + def run_describe_image(self, args): + source = str(args.get("source") or "").strip() + if not source: + return "error: empty source" + if source.startswith(("http://", "https://")): + url = RSEARCH_BASE + "/describe?" + urllib.parse.urlencode({"url": source}) + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + else: + path = os.path.expanduser(source) + try: + if os.path.getsize(path) > 10 * 1024 * 1024: + return "error: image larger than 10 MB" + with open(path, "rb") as handle: + body = handle.read() + except OSError as exc: + return "error: " + short_error(exc) + request = urllib.request.Request(RSEARCH_BASE + "/describe", data=body, headers={"Accept": "application/json", "Content-Type": "application/octet-stream"}, method="POST") + try: + with urllib.request.urlopen(request, timeout=60) as response: + data = json.loads(response.read().decode("utf-8", "replace")) + except (OSError, ValueError) as exc: + return "error: describe failed: " + short_error(exc) + text = str(data.get("description") or data.get("error") or "no description") + self.result_note = text + return text + if not query: return "error: empty query" tags = args.get("tags") or [] @@ -7574,6 +7604,10 @@ class Agent: for pos, call in enumerate(reply["tool_calls"]): self.show_call(call) call_started = time.time() + note = getattr(self.tools, "result_note", None) + if note: + self.tools.result_note = None + print(paint(note, Ansi.DIM)) try: result = self.tools.dispatch(call["name"], call["arguments"]) except Denied as denied: