Initial commit.

Browse Source
This commit is contained in:
retoor 2025-04-01 10:33:50 +02:00
commit 85f954249a
5 changed files with 345 additions and 0 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.venv

86
client.py Normal file
View File

@ -0,0 +1,86 @@
# This script requires aiohttp Python library to function.
import asyncio
import aiohttp
import json
import logging
import argparse
from urllib.parse import urlparse, urlunparse
# Default values
DEFAULT_CONCURRENCY = 4
DEFAULT_OLLAMA_URL = 'https://localhost:11434'
# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
async def websocket_client(url: str, ollama_url: str) -> None:
async with aiohttp.ClientSession() as session:
try:
async with session.ws_connect(f'{url}/publish') as ws:
async with session.get(f'{ollama_url}/api/tags') as response:
if response.status != 200:
logging.error(f"Failed to fetch models: {response.status}")
return
models = await response.json()
await ws.send_json(models)
async for msg in ws:
if msg.type == aiohttp.WSMsgType.TEXT:
data = msg.json()
logging.info(f"Received data: {data}")
request_id = data['request_id']
api_url = urlunparse(urlparse(ollama_url)._replace(path=data['path']))
async with session.post(api_url, json=data['data']) as response:
if response.status != 200:
logging.error(f"Failed to post data: {response.status}")
continue
logging.info(f"Streaming response.")
async for msg in response.content:
msg = json.loads(msg.decode('utf-8'))
await ws.send_json(dict(
request_id=request_id,
data=msg
))
logging.info(f"Response complete.")
elif msg.type == aiohttp.WSMsgType.ERROR:
logging.error("WebSocket error occurred.")
break
except aiohttp.ClientError as e:
logging.error(f"Client error occurred: {e}")
except Exception as e:
logging.error(f"An unexpected error occurred: {e}")
async def main(concurrency: int, ollama_url: str) -> None:
url = 'https://ollama.molodetz.nl'
tasks = []
for _ in range(concurrency):
tasks.append(websocket_client(url, ollama_url))
while True:
try:
await asyncio.gather(*tasks)
except Exception as e:
logging.error(f"Connection error: {e}")
await asyncio.sleep(1) # Wait before retrying
def validate_url(url: str) -> bool:
parsed = urlparse(url)
return all([parsed.scheme, parsed.netloc])
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='WebSocket Client for Ollama API')
parser.add_argument('--concurrency', type=int, default=DEFAULT_CONCURRENCY,
help='Number of concurrent WebSocket connections (default: 4)')
parser.add_argument('--ollama_url', type=str, default=DEFAULT_OLLAMA_URL,
help='Ollama API URL (default: http://localhost:11434)')
args = parser.parse_args()
if not validate_url(args.ollama_url):
logging.error(f"Invalid Ollama URL: {args.ollama_url}")
exit(1)
asyncio.run(main(args.concurrency, args.ollama_url))

57
index.html Normal file
View File

@ -0,0 +1,57 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="color-scheme" content="light dark">
<link
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/@picocss/pico@2/css/pico.yellow.min.css"
>
<title>Ollama Crowd-Funded Server</title>
</head>
<body>
<main class="container">
<h1>Ollama Crowd-Funded Server</h1>
<p>
Welcome to the Ollama Crowd-Funded Server. You can use this URL with the official Ollama JavaScript or Python clients to communicate with an Ollama server. The Ollama servers are generously provided by individuals.
</p>
<h2>Using this Ollama Server</h2>
<p>
Simply use the original client! The only difference is the URL.
</p>
<code>
<pre>
from ollama import Client
client = Client(
host="https://ollama.molodetz.nl"
)
messages = []
def chat(message):
if message:
messages.append({'role': 'user', 'content': message})
content = ''
for response in client.chat(model='qwen2.5-coder:0.5b', messages=messages, stream=True):
content += response.message.content
print(response.message.content, end='', flush=True)
messages.append({'role': 'assistant', 'content': content})
print("")
while True:
message = input("You: ")
chat(message)
</pre>
</code>
<h2>Donate Your Resources</h2>
<p>
You can contribute your resources to the server by using the following script:
</p>
<code><pre>
#client.py
</pre>
</code>
</main>
</body>
</html>

142
server.py Normal file
View File

@ -0,0 +1,142 @@
# Written by retoor@molodetz.nl
# This code creates a server using asyncio and aiohttp that manages websocket and HTTP connections to forward messages between them.
# Used Imports: asyncio, aiohttp
# The MIT License (MIT)
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import asyncio
import aiohttp
from aiohttp import web
import uuid
import pathlib
class OllamaServer:
def __init__(self,ws,models):
self.ws = ws
self.queues = {}
self.models = models
print("New OllamaServer created")
print(self.model_names)
@property
def model_names(self):
return [model['name'] for model in self.models]
async def forward_to_http(self, request_id, message):
if not request_id in self.queues:
self.queues[request_id] = asyncio.Queue()
await self.queues[request_id].put(message)
async def forward_to_websocket(self, request_id, message,path):
self.queues[request_id] = asyncio.Queue()
await self.ws.send_json(dict(request_id=request_id, data=message,path=path))
while True:
chunk = await self.queues[request_id].get()
yield chunk
if chunk['done']:
break
async def serve(self):
async for msg in self.ws:
if msg.type == web.WSMsgType.TEXT:
data = msg.json()
request_id = data['request_id']
await self.forward_to_http(request_id, data['data'])
elif msg.type == web.WSMsgType.ERROR:
break
class ServerManager:
def __init__(self):
self.servers = []
def add_server(self, server):
self.servers.append(server)
def remove_server(self, server):
self.servers.remove(server)
async def forward_to_websocket(self, request_id, message,path):
try:
server = self.servers.pop(0)
self.servers.append(server)
async for msg in server.forward_to_websocket(request_id, message,path):
yield msg
except:
raise
server_manager = ServerManager()
async def websocket_handler(request):
ws = web.WebSocketResponse()
await ws.prepare(request)
models = await ws.receive_json()
server = OllamaServer(ws,models['models'])
server_manager.add_server(server)
async for msg in ws:
if msg.type == web.WSMsgType.TEXT:
data = msg.json()
await server.forward_to_http(data['request_id'], data['data'])
elif msg.type == web.WSMsgType.ERROR:
print(f'WebSocket connection closed with exception: {ws.exception()}')
server_manager.remove_server(server)
return ws
async def http_handler(request):
request_id = str(uuid.uuid4())
data = None
try:
data = await request.json()
except ValueError:
return web.Response(status=400)
resp = web.StreamResponse(headers={'Content-Type': 'application/x-ndjson','Transfer-Encoding': 'chunked'})
await resp.prepare(request)
import json
async for result in server_manager.forward_to_websocket(request_id, data,path=request.path):
await resp.write(json.dumps(result).encode() + b'\n')
await resp.write_eof()
return resp
async def index_handler(request):
index_template = pathlib.Path("index.html").read_text()
client_py = pathlib.Path("client.py").read_text()
index_template = index_template.replace("#client.py", client_py)
return web.Response(text=index_template, content_type="text/html")
app = web.Application()
app.router.add_get("/", index_handler)
app.router.add_route('GET', '/publish', websocket_handler)
app.router.add_route('POST', '/api/chat', http_handler)
if __name__ == '__main__':
web.run_app(app, port=8080)

59
test.py Normal file
View File

@ -0,0 +1,59 @@
from ollama import Client
client = Client(
#host="https://ollama.molodetz.nl",
host='http://localhost:8080',
headers={'x-some-header': 'some-value'}
)
def times_two(nr_1: int) -> int:
return nr_1 * 2
available_functions = {
'times_two': times_two
}
messages = []
def chat_stream(message):
if message:
messages.append({'role': 'user', 'content': message})
content = ''
for response in client.chat(model='qwen2.5-coder:0.5b', messages=messages,stream=True):
content += response.message.content
print(response.message.content,end='',flush=True)
messages.append({'role': 'assistant', 'content': content})
print("")
def chat(message,stream=False):
if stream:
return chat_stream(message)
if message:
messages.append({'role': 'user', 'content': message})
response = client.chat(model='qwen2.5:3b', messages=messages,
tools=[times_two])
if response.message.tool_calls:
# There may be multiple tool calls in the response
for tool in response.message.tool_calls:
if function_to_call := available_functions.get(tool.function.name):
print('Calling function:', tool.function.name)
print('Arguments:', tool.function.arguments)
output = function_to_call(**tool.function.arguments)
print('Function output:', output)
else:
print('Function', tool.function.name, 'not found')
# Only needed to chat with the model using the tool call results
if response.message.tool_calls:
# Add the function response to messages for the model to use
messages.append(response.message)
messages.append({'role': 'tool', 'content': str(output), 'name': tool.function.name})
# Get final response from model with function outputs
return chat(None)
return response.message.content
while True:
chat_stream("A farmer and a sheep are standing on one side of a river. There is a boat with enough room for one human and one animal. How can the farmer get across the river with the sheep in the fewest number of trips?")