retoor/rp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: bump project version to 1.28.0 across all configuration files and package manifests

Browse Source
This commit is contained in:
retoor
2025-11-08 01:11:31 +00:00
parent abbfd2c6ef
commit 2e0a1e4928
53 changed files with 369 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rp/tools/web.py
+69 -49
View File
@@ -1,14 +1,7 @@
import imghdr
import json
import random
import urllib.error
import urllib.parse
import urllib.request
import json
import urllib.parse
import urllib.request
import requests
from typing import Optional, Dict, Any
# Realistic User-Agents
USER_AGENTS = [
@@ -24,6 +17,7 @@ USER_AGENTS = [
"Mozilla/5.0 (Android 11; Mobile; rv:68.0) Gecko/68.0 Firefox/88.0",
]
def get_default_headers():
"""Get default realistic headers with variations."""
accept_languages = [
@@ -50,7 +44,7 @@ def get_default_headers():
return headers
def http_fetch(url, headers=None):
def http_fetch(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
"""Fetch content from an HTTP URL.
Args:
@@ -61,19 +55,27 @@ def http_fetch(url, headers=None):
Dict with status and content.
"""
try:
request = urllib.request.Request(url)
default_headers = get_default_headers()
if headers:
default_headers.update(headers)
for header_key, header_value in default_headers.items():
request.add_header(header_key, header_value)
with urllib.request.urlopen(request) as response:
content = response.read().decode("utf-8")
return {"status": "success", "content": content[:10000]}
except Exception as exception:
return {"status": "error", "error": str(exception)}
def download_to_file(source_url, destination_path, headers=None):
response = requests.get(url, headers=default_headers, timeout=30)
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
content_type = response.headers.get("Content-Type", "").lower()
if "text" in content_type or "json" in content_type or "xml" in content_type:
content = response.text
return {"status": "success", "content": content[:10000]}
else:
content = response.content
return {"status": "success", "content": content}
except requests.exceptions.RequestException as e:
return {"status": "error", "error": str(e)}
def download_to_file(
source_url: str, destination_path: str, headers: Optional[Dict[str, str]] = None
) -> Dict[str, Any]:
"""Download content from an HTTP URL to a file.
Args:
@@ -87,45 +89,63 @@ def download_to_file(source_url, destination_path, headers=None):
This function can be used for binary files like images as well.
"""
try:
request = urllib.request.Request(source_url)
default_headers = get_default_headers()
if headers:
default_headers.update(headers)
for header_key, header_value in default_headers.items():
request.add_header(header_key, header_value)
with urllib.request.urlopen(request) as response:
content = response.read()
with open(destination_path, 'wb') as file:
file.write(content)
content_type = response.headers.get('Content-Type', '').lower()
if content_type.startswith('image/'):
img_type = imghdr.what(destination_path)
if img_type is None:
return {"status": "success", "downloaded_from": source_url, "downloaded_to": destination_path, "is_valid_image": False, "warning": "Downloaded content is not a valid image, consider finding a different source."}
else:
return {"status": "success", "downloaded_from": source_url, "downloaded_to": destination_path, "is_valid_image": True}
response = requests.get(source_url, headers=default_headers, stream=True, timeout=60)
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
with open(destination_path, "wb") as file:
for chunk in response.iter_content(chunk_size=8192):
file.write(chunk)
content_type = response.headers.get("Content-Type", "").lower()
if content_type.startswith("image/"):
img_type = imghdr.what(destination_path)
if img_type is None:
return {
"status": "success",
"downloaded_from": source_url,
"downloaded_to": destination_path,
"is_valid_image": False,
"warning": "Downloaded content is not a valid image, consider finding a different source.",
}
else:
return {"status": "success", "downloaded_from": source_url, "downloaded_to": destination_path}
except Exception as exception:
return {"status": "error", "error": str(exception)}
return {
"status": "success",
"downloaded_from": source_url,
"downloaded_to": destination_path,
"is_valid_image": True,
}
else:
return {
"status": "success",
"downloaded_from": source_url,
"downloaded_to": destination_path,
}
except requests.exceptions.RequestException as e:
return {"status": "error", "error": str(e)}
def _perform_search(base_url, query, params=None):
def _perform_search(
base_url: str, query: str, params: Optional[Dict[str, str]] = None
) -> Dict[str, Any]:
try:
encoded_query = urllib.parse.quote(query)
full_url = f"{base_url}?query={encoded_query}"
request = urllib.request.Request(full_url)
default_headers = get_default_headers()
for header_key, header_value in default_headers.items():
request.add_header(header_key, header_value)
with urllib.request.urlopen(request) as response:
content = response.read().decode("utf-8")
return {"status": "success", "content": json.loads(content)}
except Exception as exception:
return {"status": "error", "error": str(exception)}
search_params = {"query": query}
if params:
search_params.update(params)
response = requests.get(base_url, headers=default_headers, params=search_params, timeout=30)
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
return {"status": "success", "content": response.json()}
except requests.exceptions.RequestException as e:
return {"status": "error", "error": str(e)}
def web_search(query):
def web_search(query: str) -> Dict[str, Any]:
"""Perform a web search.
Args:
@@ -138,7 +158,7 @@ def web_search(query):
return _perform_search(base_url, query)
def web_search_news(query):
def web_search_news(query: str) -> Dict[str, Any]:
"""Perform a web search for news.
Args: