server.py

@@ -157,7 +157,6 @@ async def websocket_handler(request):
     return ws
 
 async def http_handler(request):
-    print("AAAAA")
     request_id = str(uuid.uuid4())
     data = None
     try:
@@ -184,75 +183,6 @@ async def http_handler(request):
     await resp.write_eof()
     return resp
 
-async def openai_http_handler(request):
-    request_id = str(uuid.uuid4())
-    openai_request_data = None
-    try:
-        openai_request_data = await request.json()
-    except ValueError:
-        return web.Response(status=400)
-
-    # Translate OpenAI request to Ollama request
-    ollama_request_data = {
-        "model": openai_request_data.get("model"),
-        "messages": openai_request_data.get("messages"),
-        "stream": openai_request_data.get("stream", False),
-        "options": {}
-    }
-
-    if "temperature" in openai_request_data:
-        ollama_request_data["options"]["temperature"] = openai_request_data["temperature"]
-    if "max_tokens" in openai_request_data:
-        ollama_request_data["options"]["num_predict"] = openai_request_data["max_tokens"]
-    # Add more OpenAI to Ollama parameter mappings here as needed
-
-    resp = web.StreamResponse(headers={'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Transfer-Encoding': 'chunked'})
-    await resp.prepare(request)
-    import json
-    try:
-        completion_id = "chatcmpl-" + str(uuid.uuid4()).replace("-", "")[:24] # Generate a unique ID
-        created_time = int(asyncio.get_event_loop().time()) # Unix timestamp
-
-        async for ollama_result in server_manager.forward_to_websocket(request_id, ollama_request_data, path="/api/chat"):
-            openai_response_chunk = {
-                "id": completion_id,
-                "object": "chat.completion.chunk",
-                "created": created_time,
-                "model": ollama_result.get("model", openai_request_data.get("model")),
-                "choices": [
-                    {
-                        "index": 0,
-                        "delta": {},
-                        "logprobs": None,
-                        "finish_reason": None
-                    }
-                ]
-            }
-
-            if "message" in ollama_result and "content" in ollama_result["message"]:
-                openai_response_chunk["choices"][0]["delta"]["content"] = ollama_result["message"]["content"]
-            elif "role" in ollama_result["message"]: # First chunk might have role
-                 openai_response_chunk["choices"][0]["delta"]["role"] = ollama_result["message"]["role"]
-
-
-            if ollama_result.get("done"):
-                openai_response_chunk["choices"][0]["finish_reason"] = "stop" # Or "length", etc.
-                await resp.write(f"data: {json.dumps(openai_response_chunk)}\n\n".encode())
-                await resp.write(b"data: [DONE]\n\n")
-            else:
-                await resp.write(f"data: {json.dumps(openai_response_chunk)}\n\n".encode())
-
-    except NoServerFoundException:
-        await resp.write(f"data: {json.dumps(dict(error='No server with that model found.', available=server_manager.get_models()))}\n\n".encode())
-        await resp.write(b"data: [DONE]\n\n")
-    except Exception as e:
-        logging.error(f"Error in openai_http_handler: {e}")
-        await resp.write(f"data: {json.dumps(dict(error=str(e)))}\n\n".encode())
-        await resp.write(b"data: [DONE]\n\n")
-
-    await resp.write_eof()
-    return resp
-
 async def index_handler(request):
     index_template = pathlib.Path("index.html").read_text()
     client_py = pathlib.Path("client.py").read_text()
@@ -268,12 +198,7 @@ async def models_handler(self):
     response_json = json.dumps(server_manager.get_models(),indent=2)
     return web.Response(text=response_json,content_type="application/json")
 
-import logging
-logging.basicConfig(
-    level=logging.DEBUG
-)
-
-app = web.Application(debug=True)
+app = web.Application()
 
 app.router.add_get("/", index_handler)
 app.router.add_route('GET', '/publish', websocket_handler)
@@ -281,7 +206,6 @@ app.router.add_route('POST', '/api/chat', http_handler)
 app.router.add_route('POST', '/v1/chat', http_handler)
 app.router.add_route('POST', '/v1/completions', http_handler)
 app.router.add_route('POST', '/v1/chat/completions', http_handler)
-app.router.add_route('POST', '/v1/chat/completions', openai_http_handler)
 app.router.add_route('GET', '/models', models_handler)
 app.router.add_route('GET', '/v1/models', models_handler)
 app.router.add_route('*', '/{tail:.*}', not_found_handler)

test.py

@@ -1,87 +1,52 @@
-import asyncio
-import aiohttp
-import json
-import uuid
+from ollama import Client
+client = Client(
+  host='https://ollama.molodetz.nl/',
+  headers={'x-some-header': 'some-value'}
+)
 
-# Configuration for the local server
-LOCAL_SERVER_URL = "http://localhost:1984"
+def times_two(nr_1: int) -> int:
+    return nr_1 * 2
 
-async def test_openai_chat_completions():
-    print("--- Starting OpenAI Chat Completions Test ---")
-    session = aiohttp.ClientSession()
+available_functions = {
+    'times_two': times_two
+}
 
-    try:
-        # OpenAI-style request payload
-        payload = {
-            "model": "qwen2.5:3b", # Assuming this model is available on an Ollama instance connected to the server
-            "messages": [
-                {"role": "user", "content": "Tell me a short story about a brave knight."}
-            ],
-            "stream": True,
-            "temperature": 0.7,
-            "max_tokens": 50
-        }
+messages = []
 
-        openai_endpoint = f"{LOCAL_SERVER_URL}/v1/chat/completions"
-        print(f"Sending request to: {openai_endpoint}")
-        print(f"Payload: {json.dumps(payload, indent=2)}")
 
-        async with session.post(openai_endpoint, json=payload) as response:
-            print(f"Response status: {response.status}")
-            assert response.status == 200, f"Expected status 200, got {response.status}"
+def chat_stream(message):
+    if message:
+        messages.append({'role': 'user', 'content': message})
+    content = ''
+    for response in client.chat(model='qwen2.5-coder:0.5b', messages=messages, stream=True):
+        content += response.message.content
+        print(response.message.content, end='', flush=True)
+    messages.append({'role': 'assistant', 'content': content})
+    print("")
 
-            response_data = ""
-            async for chunk in response.content.iter_any():
-                chunk_str = chunk.decode('utf-8')
-                # print(f"Received chunk: {chunk_str.strip()}")
-                
-                # Split by 'data: ' and process each JSON object
-                for line in chunk_str.splitlines():
-                    if line.startswith("data: "):
-                        json_str = line[len("data: "):].strip()
-                        if json_str == "[DONE]":
-                            print("Received [DONE] signal.")
-                            continue
 
-                        try:
-                            data = json.loads(json_str)
-                            # Basic assertions for OpenAI streaming format
-                            assert "id" in data
-                            assert "object" in data
-                            assert data["object"] == "chat.completion.chunk"
-                            assert "choices" in data
-                            assert isinstance(data["choices"], list)
-                            assert len(data["choices"]) > 0
-                            assert "delta" in data["choices"][0]
-                            
-                            if "content" in data["choices"][0]["delta"]:
-                                response_data += data["choices"][0]["delta"]["content"]
+def chat(message, stream=False):
+    if stream:
+        return chat_stream(message)
+    if message:
+        messages.append({'role': 'user', 'content': message})
+    response = client.chat(model='qwen2.5:3b', messages=messages,
+    tools=[times_two])
+    if response.message.tool_calls:
+        for tool in response.message.tool_calls:
+            if function_to_call := available_functions.get(tool.function.name):
+                print('Calling function:', tool.function.name)
+                print('Arguments:', tool.function.arguments)
+                output = function_to_call(**tool.function.arguments)
+                print('Function output:', output)
+            else:
+                print('Function', tool.function.name, 'not found')
 
-                        except json.JSONDecodeError as e:
-                            print(f"JSON Decode Error: {e} in chunk: {json_str}")
-                            assert False, f"Invalid JSON received: {json_str}"
-                    elif line.strip(): # Handle potential non-data lines, though unlikely for SSE
-                        print(f"Non-data line received: {line.strip()}")
+            if response.message.tool_calls:
+                messages.append(response.message)
+                messages.append({'role': 'tool', 'content': str(output), 'name': tool.function.name})
+                return chat(None)
+    return response.message.content
 
-            print(f"\nFull response content:\n{response_data}")
-            assert len(response_data) > 0, "No content received in OpenAI response."
-            print("--- OpenAI Chat Completions Test Passed ---")
-
-    except aiohttp.ClientError as e:
-        print(f"Client error during test: {e}")
-        assert False, f"Client error: {e}"
-    except AssertionError as e:
-        print(f"Assertion failed: {e}")
-        assert False, f"Test failed: {e}"
-    except Exception as e:
-        print(f"An unexpected error occurred during test: {e}")
-        assert False, f"Unexpected error: {e}"
-    finally:
-        await session.close()
-        print("--- OpenAI Chat Completions Test Finished ---")
-
-async def main():
-    await test_openai_chat_completions()
-
-if __name__ == '__main__':
-    asyncio.run(main())
+while True:
+    chat_stream("A farmer and a sheep are standing on one side of a river. There is a boat with enough room for one human and one animal. How can the farmer get across the river with the sheep in the fewest number of trips?")