rp/pr/tools/web.py

import json
import urllib.error
import urllib.parse
import urllib.request


def http_fetch(url, headers=None):
    """Fetch content from an HTTP URL.

    Args:
        url: The URL to fetch.
        headers: Optional HTTP headers.

    Returns:
        Dict with status and content.
    """
    try:
        req = urllib.request.Request(url)
        if headers:
            for key, value in headers.items():
                req.add_header(key, value)

        with urllib.request.urlopen(req) as response:
            content = response.read().decode("utf-8")
            return {"status": "success", "content": content[:10000]}
    except Exception as e:
        return {"status": "error", "error": str(e)}


def _perform_search(base_url, query, params=None):
    try:
        full_url = f"https://static.molodetz.nl/search.cgi?query={query}"

        with urllib.request.urlopen(full_url) as response:
            content = response.read().decode("utf-8")
            return {"status": "success", "content": json.loads(content)}
    except Exception as e:
        return {"status": "error", "error": str(e)}


def web_search(query):
    """Perform a web search.

    Args:
        query: Search query.

    Returns:
        Dict with status and search results.
    """
    base_url = "https://search.molodetz.nl/search"
    return _perform_search(base_url, query)


def web_search_news(query):
    """Perform a web search for news.

    Args:
        query: Search query for news.

    Returns:
        Dict with status and news search results.
    """
    base_url = "https://search.molodetz.nl/search"
    return _perform_search(base_url, query)