Initial commit.

Browse Source
This commit is contained in:
retoor 2025-04-01 10:56:11 +02:00
parent 85f954249a
commit 4768c9d385
5 changed files with 33 additions and 86 deletions

2
.gitignore vendored
View File

@ -1 +1,3 @@
.venv .venv
prompt.txt

6
client.py
View File

@ -1,5 +1,3 @@
# This script requires aiohttp Python library to function.
import asyncio import asyncio
import aiohttp import aiohttp
import json import json
@ -7,11 +5,9 @@ import logging
import argparse import argparse
from urllib.parse import urlparse, urlunparse from urllib.parse import urlparse, urlunparse
# Default values
DEFAULT_CONCURRENCY = 4 DEFAULT_CONCURRENCY = 4
DEFAULT_OLLAMA_URL = 'https://localhost:11434' DEFAULT_OLLAMA_URL = 'https://localhost:11434'
# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
async def websocket_client(url: str, ollama_url: str) -> None: async def websocket_client(url: str, ollama_url: str) -> None:
@ -64,7 +60,7 @@ async def main(concurrency: int, ollama_url: str) -> None:
await asyncio.gather(*tasks) await asyncio.gather(*tasks)
except Exception as e: except Exception as e:
logging.error(f"Connection error: {e}") logging.error(f"Connection error: {e}")
await asyncio.sleep(1) # Wait before retrying await asyncio.sleep(1)
def validate_url(url: str) -> bool: def validate_url(url: str) -> bool:
parsed = urlparse(url) parsed = urlparse(url)

24
index.html
View File

@ -4,22 +4,15 @@
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="color-scheme" content="light dark"> <meta name="color-scheme" content="light dark">
<link <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@2/css/pico.yellow.min.css">
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/@picocss/pico@2/css/pico.yellow.min.css"
>
<title>Ollama Crowd-Funded Server</title> <title>Ollama Crowd-Funded Server</title>
</head> </head>
<body> <body>
<main class="container"> <main class="container">
<h1>Ollama Crowd-Funded Server</h1> <h1>Ollama Crowd-Funded Server</h1>
<p> <p>Welcome to the Ollama Crowd-Funded Server. You can use this URL with the official Ollama JavaScript or Python clients to communicate with an Ollama server. The Ollama servers are generously provided by individuals.</p>
Welcome to the Ollama Crowd-Funded Server. You can use this URL with the official Ollama JavaScript or Python clients to communicate with an Ollama server. The Ollama servers are generously provided by individuals. <h2>Using this Ollama Server</h2>
</p> <p>Simply use the original client! The only difference is the URL.</p>
<h2>Using this Ollama Server</h2>
<p>
Simply use the original client! The only difference is the URL.
</p>
<code> <code>
<pre> <pre>
from ollama import Client from ollama import Client
@ -45,13 +38,8 @@ while True:
</pre> </pre>
</code> </code>
<h2>Donate Your Resources</h2> <h2>Donate Your Resources</h2>
<p> <p>You can contribute your resources to the server by using the following script:</p>
You can contribute your resources to the server by using the following script: <code><pre>#client.py</pre></code>
</p>
<code><pre>
#client.py
</pre>
</code>
</main> </main>
</body> </body>
</html> </html>

72
server.py
View File

@ -1,63 +1,34 @@
# Written by retoor@molodetz.nl
# This code creates a server using asyncio and aiohttp that manages websocket and HTTP connections to forward messages between them.
# Used Imports: asyncio, aiohttp
# The MIT License (MIT)
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import asyncio import asyncio
import aiohttp import aiohttp
from aiohttp import web from aiohttp import web
import uuid import uuid
import pathlib import pathlib
class OllamaServer: class OllamaServer:
def __init__(self,ws,models): def __init__(self, ws, models):
self.ws = ws self.ws = ws
self.queues = {} self.queues = {}
self.models = models self.models = models
print("New OllamaServer created")
print(self.model_names)
@property @property
def model_names(self): def model_names(self):
return [model['name'] for model in self.models] return [model['name'] for model in self.models]
async def forward_to_http(self, request_id, message): async def forward_to_http(self, request_id, message):
if not request_id in self.queues: if request_id not in self.queues:
self.queues[request_id] = asyncio.Queue() self.queues[request_id] = asyncio.Queue()
await self.queues[request_id].put(message) await self.queues[request_id].put(message)
async def forward_to_websocket(self, request_id, message,path): async def forward_to_websocket(self, request_id, message, path):
self.queues[request_id] = asyncio.Queue() self.queues[request_id] = asyncio.Queue()
await self.ws.send_json(dict(request_id=request_id, data=message,path=path)) await self.ws.send_json(dict(request_id=request_id, data=message, path=path))
while True: while True:
chunk = await self.queues[request_id].get() chunk = await self.queues[request_id].get()
yield chunk yield chunk
if chunk['done']: if chunk['done']:
break break
async def serve(self): async def serve(self):
async for msg in self.ws: async for msg in self.ws:
if msg.type == web.WSMsgType.TEXT: if msg.type == web.WSMsgType.TEXT:
@ -66,7 +37,7 @@ class OllamaServer:
await self.forward_to_http(request_id, data['data']) await self.forward_to_http(request_id, data['data'])
elif msg.type == web.WSMsgType.ERROR: elif msg.type == web.WSMsgType.ERROR:
break break
class ServerManager: class ServerManager:
def __init__(self): def __init__(self):
self.servers = [] self.servers = []
@ -77,26 +48,24 @@ class ServerManager:
def remove_server(self, server): def remove_server(self, server):
self.servers.remove(server) self.servers.remove(server)
async def forward_to_websocket(self, request_id, message,path): async def forward_to_websocket(self, request_id, message, path):
try: try:
server = self.servers.pop(0) server = self.servers.pop(0)
self.servers.append(server) self.servers.append(server)
async for msg in server.forward_to_websocket(request_id, message,path): async for msg in server.forward_to_websocket(request_id, message, path):
yield msg yield msg
except: except:
raise raise
server_manager = ServerManager() server_manager = ServerManager()
async def websocket_handler(request): async def websocket_handler(request):
ws = web.WebSocketResponse() ws = web.WebSocketResponse()
await ws.prepare(request) await ws.prepare(request)
models = await ws.receive_json() models = await ws.receive_json()
server = OllamaServer(ws,models['models']) server = OllamaServer(ws, models['models'])
server_manager.add_server(server) server_manager.add_server(server)
async for msg in ws: async for msg in ws:
@ -109,7 +78,6 @@ async def websocket_handler(request):
server_manager.remove_server(server) server_manager.remove_server(server)
return ws return ws
async def http_handler(request): async def http_handler(request):
request_id = str(uuid.uuid4()) request_id = str(uuid.uuid4())
data = None data = None
@ -118,11 +86,11 @@ async def http_handler(request):
except ValueError: except ValueError:
return web.Response(status=400) return web.Response(status=400)
resp = web.StreamResponse(headers={'Content-Type': 'application/x-ndjson','Transfer-Encoding': 'chunked'}) resp = web.StreamResponse(headers={'Content-Type': 'application/x-ndjson', 'Transfer-Encoding': 'chunked'})
await resp.prepare(request) await resp.prepare(request)
import json import json
async for result in server_manager.forward_to_websocket(request_id, data,path=request.path): async for result in server_manager.forward_to_websocket(request_id, data, path=request.path):
await resp.write(json.dumps(result).encode() + b'\n') await resp.write(json.dumps(result).encode() + b'\n')
await resp.write_eof() await resp.write_eof()
return resp return resp
@ -139,4 +107,4 @@ app.router.add_route('GET', '/publish', websocket_handler)
app.router.add_route('POST', '/api/chat', http_handler) app.router.add_route('POST', '/api/chat', http_handler)
if __name__ == '__main__': if __name__ == '__main__':
web.run_app(app, port=8080) web.run_app(app, port=1984)

15
test.py
View File

@ -1,6 +1,5 @@
from ollama import Client from ollama import Client
client = Client( client = Client(
#host="https://ollama.molodetz.nl",
host='http://localhost:8080', host='http://localhost:8080',
headers={'x-some-header': 'some-value'} headers={'x-some-header': 'some-value'}
) )
@ -19,13 +18,14 @@ def chat_stream(message):
if message: if message:
messages.append({'role': 'user', 'content': message}) messages.append({'role': 'user', 'content': message})
content = '' content = ''
for response in client.chat(model='qwen2.5-coder:0.5b', messages=messages,stream=True): for response in client.chat(model='qwen2.5-coder:0.5b', messages=messages, stream=True):
content += response.message.content content += response.message.content
print(response.message.content,end='',flush=True) print(response.message.content, end='', flush=True)
messages.append({'role': 'assistant', 'content': content}) messages.append({'role': 'assistant', 'content': content})
print("") print("")
def chat(message,stream=False):
def chat(message, stream=False):
if stream: if stream:
return chat_stream(message) return chat_stream(message)
if message: if message:
@ -33,7 +33,6 @@ def chat(message,stream=False):
response = client.chat(model='qwen2.5:3b', messages=messages, response = client.chat(model='qwen2.5:3b', messages=messages,
tools=[times_two]) tools=[times_two])
if response.message.tool_calls: if response.message.tool_calls:
# There may be multiple tool calls in the response
for tool in response.message.tool_calls: for tool in response.message.tool_calls:
if function_to_call := available_functions.get(tool.function.name): if function_to_call := available_functions.get(tool.function.name):
print('Calling function:', tool.function.name) print('Calling function:', tool.function.name)
@ -43,17 +42,11 @@ def chat(message,stream=False):
else: else:
print('Function', tool.function.name, 'not found') print('Function', tool.function.name, 'not found')
# Only needed to chat with the model using the tool call results
if response.message.tool_calls: if response.message.tool_calls:
# Add the function response to messages for the model to use
messages.append(response.message) messages.append(response.message)
messages.append({'role': 'tool', 'content': str(output), 'name': tool.function.name}) messages.append({'role': 'tool', 'content': str(output), 'name': tool.function.name})
# Get final response from model with function outputs
return chat(None) return chat(None)
return response.message.content return response.message.content
while True: while True:
chat_stream("A farmer and a sheep are standing on one side of a river. There is a boat with enough room for one human and one animal. How can the farmer get across the river with the sheep in the fewest number of trips?") chat_stream("A farmer and a sheep are standing on one side of a river. There is a boat with enough room for one human and one animal. How can the farmer get across the river with the sheep in the fewest number of trips?")