feat: rename assistant identifier to "rp" across config and runtime references
This commit is contained in:
+775
-61
@@ -1,9 +1,24 @@
|
||||
import base64
|
||||
import imghdr
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
import requests
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from rp.core.operations import Validator, ValidationError
|
||||
|
||||
logger = logging.getLogger("rp")
|
||||
|
||||
NETWORK_TRANSIENT_ERRORS = (
|
||||
requests.exceptions.ConnectionError,
|
||||
requests.exceptions.Timeout,
|
||||
requests.exceptions.ChunkedEncodingError,
|
||||
)
|
||||
|
||||
MAX_RETRIES = 3
|
||||
BASE_DELAY = 1.0
|
||||
|
||||
# Realistic User-Agents
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
||||
@@ -46,7 +61,7 @@ def get_default_headers():
|
||||
|
||||
|
||||
def http_fetch(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
|
||||
"""Fetch content from an HTTP URL.
|
||||
"""Fetch content from an HTTP URL with automatic retry.
|
||||
|
||||
Args:
|
||||
url: The URL to fetch.
|
||||
@@ -56,42 +71,64 @@ def http_fetch(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str,
|
||||
Dict with status and content.
|
||||
"""
|
||||
try:
|
||||
default_headers = get_default_headers()
|
||||
if headers:
|
||||
default_headers.update(headers)
|
||||
|
||||
response = requests.get(url, headers=default_headers, timeout=30)
|
||||
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
|
||||
|
||||
content_type = response.headers.get("Content-Type", "").lower()
|
||||
if "text" in content_type or "json" in content_type or "xml" in content_type:
|
||||
content = response.text
|
||||
return {"status": "success", "content": content[:10000]}
|
||||
else:
|
||||
content = response.content
|
||||
content_length = len(content)
|
||||
if content_length > 10000:
|
||||
return {
|
||||
"status": "success",
|
||||
"content_type": content_type,
|
||||
"size_bytes": content_length,
|
||||
"message": f"Binary content ({content_length} bytes). Use download_to_file to save it.",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "success",
|
||||
"content_type": content_type,
|
||||
"size_bytes": content_length,
|
||||
"content_base64": base64.b64encode(content).decode("utf-8"),
|
||||
}
|
||||
except requests.exceptions.RequestException as e:
|
||||
url = Validator.string(url, "url", min_length=1, max_length=8192)
|
||||
except ValidationError as e:
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
last_error = None
|
||||
|
||||
for attempt in range(MAX_RETRIES):
|
||||
try:
|
||||
default_headers = get_default_headers()
|
||||
if headers:
|
||||
default_headers.update(headers)
|
||||
|
||||
response = requests.get(url, headers=default_headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
content_type = response.headers.get("Content-Type", "").lower()
|
||||
if "text" in content_type or "json" in content_type or "xml" in content_type:
|
||||
content = response.text
|
||||
return {"status": "success", "content": content[:10000]}
|
||||
else:
|
||||
content = response.content
|
||||
content_length = len(content)
|
||||
if content_length > 10000:
|
||||
return {
|
||||
"status": "success",
|
||||
"content_type": content_type,
|
||||
"size_bytes": content_length,
|
||||
"message": f"Binary content ({content_length} bytes). Use download_to_file to save it.",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "success",
|
||||
"content_type": content_type,
|
||||
"size_bytes": content_length,
|
||||
"content_base64": base64.b64encode(content).decode("utf-8"),
|
||||
}
|
||||
|
||||
except NETWORK_TRANSIENT_ERRORS as e:
|
||||
last_error = e
|
||||
if attempt < MAX_RETRIES - 1:
|
||||
delay = BASE_DELAY * (2 ** attempt)
|
||||
logger.warning(f"http_fetch attempt {attempt + 1} failed: {e}. Retrying in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
return {"status": "error", "error": f"HTTP error: {e}"}
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
return {"status": "error", "error": f"Failed after {MAX_RETRIES} retries: {last_error}"}
|
||||
|
||||
|
||||
def download_to_file(
|
||||
source_url: str, destination_path: str, headers: Optional[Dict[str, str]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Download content from an HTTP URL to a file.
|
||||
"""Download content from an HTTP URL to a file with retry and safe write.
|
||||
|
||||
Args:
|
||||
source_url: The URL to download from.
|
||||
@@ -100,47 +137,88 @@ def download_to_file(
|
||||
|
||||
Returns:
|
||||
Dict with status, downloaded_from, and downloaded_to on success, or status and error on failure.
|
||||
|
||||
This function can be used for binary files like images as well.
|
||||
"""
|
||||
import os
|
||||
|
||||
try:
|
||||
default_headers = get_default_headers()
|
||||
if headers:
|
||||
default_headers.update(headers)
|
||||
source_url = Validator.string(source_url, "source_url", min_length=1, max_length=8192)
|
||||
destination_path = Validator.string(destination_path, "destination_path", min_length=1, max_length=4096)
|
||||
except ValidationError as e:
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
response = requests.get(source_url, headers=default_headers, stream=True, timeout=60)
|
||||
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
|
||||
temp_path = destination_path + ".download"
|
||||
last_error = None
|
||||
|
||||
with open(destination_path, "wb") as file:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
file.write(chunk)
|
||||
for attempt in range(MAX_RETRIES):
|
||||
try:
|
||||
default_headers = get_default_headers()
|
||||
if headers:
|
||||
default_headers.update(headers)
|
||||
|
||||
content_type = response.headers.get("Content-Type", "").lower()
|
||||
if content_type.startswith("image/"):
|
||||
img_type = imghdr.what(destination_path)
|
||||
if img_type is None:
|
||||
return {
|
||||
"status": "success",
|
||||
"downloaded_from": source_url,
|
||||
"downloaded_to": destination_path,
|
||||
"is_valid_image": False,
|
||||
"warning": "Downloaded content is not a valid image, consider finding a different source.",
|
||||
}
|
||||
response = requests.get(source_url, headers=default_headers, stream=True, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
with open(temp_path, "wb") as file:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
file.write(chunk)
|
||||
|
||||
os.replace(temp_path, destination_path)
|
||||
|
||||
content_type = response.headers.get("Content-Type", "").lower()
|
||||
if content_type.startswith("image/"):
|
||||
img_type = imghdr.what(destination_path)
|
||||
if img_type is None:
|
||||
return {
|
||||
"status": "success",
|
||||
"downloaded_from": source_url,
|
||||
"downloaded_to": destination_path,
|
||||
"is_valid_image": False,
|
||||
"warning": "Downloaded content is not a valid image, consider finding a different source.",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "success",
|
||||
"downloaded_from": source_url,
|
||||
"downloaded_to": destination_path,
|
||||
"is_valid_image": True,
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "success",
|
||||
"downloaded_from": source_url,
|
||||
"downloaded_to": destination_path,
|
||||
"is_valid_image": True,
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "success",
|
||||
"downloaded_from": source_url,
|
||||
"downloaded_to": destination_path,
|
||||
}
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
except NETWORK_TRANSIENT_ERRORS as e:
|
||||
last_error = e
|
||||
if attempt < MAX_RETRIES - 1:
|
||||
delay = BASE_DELAY * (2 ** attempt)
|
||||
logger.warning(f"download_to_file attempt {attempt + 1} failed: {e}. Retrying in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
return {"status": "error", "error": f"HTTP error: {e}"}
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
except Exception as e:
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
if os.path.exists(temp_path):
|
||||
try:
|
||||
os.remove(temp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return {"status": "error", "error": f"Failed after {MAX_RETRIES} retries: {last_error}"}
|
||||
|
||||
|
||||
def _perform_search(
|
||||
@@ -188,3 +266,639 @@ def web_search_news(query: str) -> Dict[str, Any]:
|
||||
"""
|
||||
base_url = "https://static.molodetz.nl/search.cgi"
|
||||
return _perform_search(base_url, query)
|
||||
|
||||
|
||||
def scrape_images(
|
||||
url: str,
|
||||
destination_dir: str,
|
||||
full_size: bool = True,
|
||||
extensions: Optional[str] = None,
|
||||
min_size_kb: int = 0,
|
||||
max_size_kb: int = 0,
|
||||
max_workers: int = 5,
|
||||
extract_captions: bool = False,
|
||||
log_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Scrape and download all images from a webpage with filtering and concurrent downloads.
|
||||
|
||||
Args:
|
||||
url: The webpage URL to scrape for images.
|
||||
destination_dir: Directory to save downloaded images.
|
||||
full_size: If True, attempt to find full-size image URLs instead of thumbnails.
|
||||
extensions: Comma-separated list of extensions to filter (e.g., "jpg,png,gif"). Default: all images.
|
||||
min_size_kb: Minimum file size in KB to keep (0 for no minimum).
|
||||
max_size_kb: Maximum file size in KB to keep (0 for no maximum).
|
||||
max_workers: Number of concurrent download threads.
|
||||
extract_captions: If True, extract alt text and captions.
|
||||
log_file: Path to CSV log file for metadata.
|
||||
|
||||
Returns:
|
||||
Dict with status, downloaded files list, and any errors.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import csv
|
||||
from urllib.parse import urljoin, urlparse, unquote
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
allowed_extensions = None
|
||||
if extensions:
|
||||
allowed_extensions = [ext.strip().lower().lstrip('.') for ext in extensions.split(',')]
|
||||
|
||||
min_size_bytes = min_size_kb * 1024
|
||||
max_size_bytes = max_size_kb * 1024 if max_size_kb > 0 else float('inf')
|
||||
|
||||
def normalize_url(base_url: str, img_url: str) -> str:
|
||||
if img_url.startswith(('http://', 'https://')):
|
||||
return img_url
|
||||
return urljoin(base_url, img_url)
|
||||
|
||||
def extract_filename(img_url: str) -> str:
|
||||
parsed = urlparse(img_url)
|
||||
path = unquote(parsed.path)
|
||||
filename = os.path.basename(path)
|
||||
filename = re.sub(r'\?.*$', '', filename)
|
||||
filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
|
||||
return filename if filename else f"image_{hash(img_url) % 10000}.jpg"
|
||||
|
||||
def is_valid_image_url(img_url: str) -> bool:
|
||||
lower_url = img_url.lower()
|
||||
img_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.svg', '.ico')
|
||||
if any(ext in lower_url for ext in img_extensions):
|
||||
return True
|
||||
if '_media' in lower_url or '/images/' in lower_url or '/img/' in lower_url:
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_full_size_url(img_url: str, base_url: str) -> str:
|
||||
clean_url = re.sub(r'\?w=\d+.*$', '', img_url)
|
||||
clean_url = re.sub(r'\?.*tok=[a-f0-9]+.*$', '', clean_url)
|
||||
clean_url = re.sub(r'&w=\d+&h=\d+', '', clean_url)
|
||||
clean_url = re.sub(r'[?&]cache=.*$', '', clean_url)
|
||||
if '_media' in clean_url:
|
||||
clean_url = clean_url.replace('%3A', ':').replace('%2F', '/')
|
||||
return clean_url
|
||||
|
||||
def extract_dokuwiki_images(html: str, base_url: str) -> list:
|
||||
images = []
|
||||
media_patterns = [
|
||||
r'href=["\']([^"\']*/_media/[^"\']+)["\']',
|
||||
r'href=["\']([^"\']*/_detail/[^"\']+)["\']',
|
||||
r'src=["\']([^"\']*/_media/[^"\']+)["\']',
|
||||
]
|
||||
for pattern in media_patterns:
|
||||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||||
for match in matches:
|
||||
full_url = normalize_url(base_url, match)
|
||||
if '_detail' in full_url:
|
||||
full_url = full_url.replace('/_detail/', '/_media/')
|
||||
full_url = re.sub(r'\?id=[^&]*', '', full_url)
|
||||
full_url = re.sub(r'&.*$', '', full_url)
|
||||
images.append({"url": full_url, "caption": ""})
|
||||
return images
|
||||
|
||||
def extract_standard_images(html: str, base_url: str) -> list:
|
||||
images = []
|
||||
img_pattern = r'<img[^>]+src=["\']([^"\']+)["\'][^>]*(?:alt=["\']([^"\']*)["\'])?[^>]*>'
|
||||
alt_pattern = r'<img[^>]*alt=["\']([^"\']*)["\'][^>]*src=["\']([^"\']+)["\'][^>]*>'
|
||||
|
||||
for match in re.finditer(img_pattern, html, re.IGNORECASE):
|
||||
src = match.group(1)
|
||||
alt = match.group(2) or ""
|
||||
if is_valid_image_url(src):
|
||||
images.append({"url": normalize_url(base_url, src), "caption": alt})
|
||||
|
||||
for match in re.finditer(alt_pattern, html, re.IGNORECASE):
|
||||
alt = match.group(1) or ""
|
||||
src = match.group(2)
|
||||
if is_valid_image_url(src):
|
||||
existing = [i for i in images if i["url"] == normalize_url(base_url, src)]
|
||||
if not existing:
|
||||
images.append({"url": normalize_url(base_url, src), "caption": alt})
|
||||
|
||||
link_patterns = [
|
||||
r'<a[^>]+href=["\']([^"\']+\.(?:jpg|jpeg|png|gif|webp))["\']',
|
||||
r'srcset=["\']([^"\',\s]+)',
|
||||
r'data-src=["\']([^"\']+)["\']',
|
||||
]
|
||||
for pattern in link_patterns:
|
||||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||||
for match in matches:
|
||||
if is_valid_image_url(match):
|
||||
url = normalize_url(base_url, match)
|
||||
existing = [i for i in images if i["url"] == url]
|
||||
if not existing:
|
||||
images.append({"url": url, "caption": ""})
|
||||
return images
|
||||
|
||||
def download_image(img_data: dict, dest_dir: str, headers: dict) -> dict:
|
||||
img_url = img_data["url"]
|
||||
caption = img_data.get("caption", "")
|
||||
filename = extract_filename(img_url)
|
||||
if not filename:
|
||||
return {"status": "error", "url": img_url, "error": "Could not extract filename"}
|
||||
|
||||
dest_path = os.path.join(dest_dir, filename)
|
||||
|
||||
if os.path.exists(dest_path):
|
||||
return {"status": "skipped", "url": img_url, "path": dest_path, "reason": "already exists"}
|
||||
|
||||
try:
|
||||
head_response = requests.head(img_url, headers=headers, timeout=10, allow_redirects=True)
|
||||
content_length = int(head_response.headers.get('Content-Length', 0))
|
||||
|
||||
if content_length > 0:
|
||||
if content_length < min_size_bytes:
|
||||
return {"status": "skipped", "url": img_url, "reason": f"Too small ({content_length} bytes)"}
|
||||
if content_length > max_size_bytes:
|
||||
return {"status": "skipped", "url": img_url, "reason": f"Too large ({content_length} bytes)"}
|
||||
|
||||
img_response = requests.get(img_url, headers=headers, stream=True, timeout=30)
|
||||
img_response.raise_for_status()
|
||||
|
||||
content_type = img_response.headers.get('Content-Type', '').lower()
|
||||
if 'text/html' in content_type:
|
||||
return {"status": "error", "url": img_url, "error": "URL returned HTML instead of image"}
|
||||
|
||||
with open(dest_path, 'wb') as f:
|
||||
for chunk in img_response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
file_size = os.path.getsize(dest_path)
|
||||
|
||||
if file_size < min_size_bytes:
|
||||
os.remove(dest_path)
|
||||
return {"status": "skipped", "url": img_url, "reason": f"File too small ({file_size} bytes)"}
|
||||
|
||||
if file_size > max_size_bytes:
|
||||
os.remove(dest_path)
|
||||
return {"status": "skipped", "url": img_url, "reason": f"File too large ({file_size} bytes)"}
|
||||
|
||||
if file_size < 100:
|
||||
os.remove(dest_path)
|
||||
return {"status": "error", "url": img_url, "error": f"File too small ({file_size} bytes)"}
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"url": img_url,
|
||||
"path": dest_path,
|
||||
"size": file_size,
|
||||
"caption": caption
|
||||
}
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {"status": "error", "url": img_url, "error": str(e)}
|
||||
|
||||
try:
|
||||
os.makedirs(destination_dir, exist_ok=True)
|
||||
|
||||
default_headers = get_default_headers()
|
||||
response = requests.get(url, headers=default_headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
html = response.text
|
||||
|
||||
image_data = []
|
||||
dokuwiki_images = extract_dokuwiki_images(html, url)
|
||||
image_data.extend(dokuwiki_images)
|
||||
standard_images = extract_standard_images(html, url)
|
||||
|
||||
existing_urls = {i["url"] for i in image_data}
|
||||
for img in standard_images:
|
||||
if img["url"] not in existing_urls:
|
||||
image_data.append(img)
|
||||
existing_urls.add(img["url"])
|
||||
|
||||
if full_size:
|
||||
for img in image_data:
|
||||
img["url"] = get_full_size_url(img["url"], url)
|
||||
|
||||
if allowed_extensions:
|
||||
filtered = []
|
||||
for img in image_data:
|
||||
lower_url = img["url"].lower()
|
||||
if any(lower_url.endswith('.' + ext) or ('.' + ext + '?') in lower_url or ('.' + ext + '&') in lower_url for ext in allowed_extensions):
|
||||
filtered.append(img)
|
||||
image_data = filtered
|
||||
|
||||
downloaded = []
|
||||
errors = []
|
||||
skipped = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {
|
||||
executor.submit(download_image, img, destination_dir, default_headers): img
|
||||
for img in image_data
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result["status"] == "success":
|
||||
downloaded.append(result)
|
||||
elif result["status"] == "skipped":
|
||||
skipped.append(result)
|
||||
else:
|
||||
errors.append(result)
|
||||
|
||||
if log_file and downloaded:
|
||||
log_path = os.path.expanduser(log_file)
|
||||
os.makedirs(os.path.dirname(log_path) if os.path.dirname(log_path) else '.', exist_ok=True)
|
||||
with open(log_path, 'w', newline='') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=['filename', 'url', 'size', 'caption'])
|
||||
writer.writeheader()
|
||||
for item in downloaded:
|
||||
writer.writerow({
|
||||
'filename': os.path.basename(item['path']),
|
||||
'url': item['url'],
|
||||
'size': item['size'],
|
||||
'caption': item.get('caption', '')
|
||||
})
|
||||
|
||||
captions_text = ""
|
||||
if extract_captions and downloaded:
|
||||
caption_lines = []
|
||||
for item in downloaded:
|
||||
if item.get('caption'):
|
||||
caption_lines.append(f"{os.path.basename(item['path'])}: {item['caption']}")
|
||||
captions_text = "\n".join(caption_lines)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"source_url": url,
|
||||
"destination_dir": destination_dir,
|
||||
"total_found": len(image_data),
|
||||
"downloaded": len(downloaded),
|
||||
"skipped": len(skipped),
|
||||
"errors": len(errors),
|
||||
"files": downloaded[:20],
|
||||
"skipped_files": skipped[:5] if skipped else [],
|
||||
"error_details": errors[:5] if errors else [],
|
||||
"captions": captions_text if extract_captions else None,
|
||||
"log_file": log_file if log_file else None
|
||||
}
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {"status": "error", "error": f"Failed to fetch page: {str(e)}"}
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
|
||||
def crawl_and_download(
|
||||
start_url: str,
|
||||
destination_dir: str,
|
||||
resource_pattern: str = r'\.(jpg|jpeg|png|gif|svg|pdf|mp3|mp4)$',
|
||||
max_pages: int = 10,
|
||||
follow_links: bool = True,
|
||||
link_pattern: Optional[str] = None,
|
||||
min_size_kb: int = 0,
|
||||
max_size_kb: int = 0,
|
||||
max_workers: int = 5,
|
||||
log_file: Optional[str] = None,
|
||||
extract_metadata: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""Crawl website pages and download matching resources with metadata extraction.
|
||||
|
||||
Args:
|
||||
start_url: Starting URL to crawl.
|
||||
destination_dir: Directory to save downloaded resources.
|
||||
resource_pattern: Regex pattern for resource URLs to download.
|
||||
max_pages: Maximum number of pages to crawl.
|
||||
follow_links: If True, follow links to other pages on same domain.
|
||||
link_pattern: Regex pattern for links to follow (e.g., "page=\\d+" for pagination).
|
||||
min_size_kb: Minimum file size in KB to download.
|
||||
max_size_kb: Maximum file size in KB to download (0 for unlimited).
|
||||
max_workers: Number of concurrent download threads.
|
||||
log_file: Path to CSV log file for metadata.
|
||||
extract_metadata: If True, extract additional metadata (title, description, etc.).
|
||||
|
||||
Returns:
|
||||
Dict with status, pages crawled, resources downloaded, and metadata.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import csv
|
||||
from urllib.parse import urljoin, urlparse, unquote
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from collections import deque
|
||||
|
||||
min_size_bytes = min_size_kb * 1024
|
||||
max_size_bytes = max_size_kb * 1024 if max_size_kb > 0 else float('inf')
|
||||
|
||||
results = {
|
||||
"status": "success",
|
||||
"start_url": start_url,
|
||||
"destination_dir": destination_dir,
|
||||
"pages_crawled": 0,
|
||||
"resources_found": 0,
|
||||
"downloaded": [],
|
||||
"skipped": [],
|
||||
"errors": [],
|
||||
"metadata": []
|
||||
}
|
||||
|
||||
visited_pages = set()
|
||||
visited_resources = set()
|
||||
pages_queue = deque([start_url])
|
||||
resources_to_download = []
|
||||
|
||||
parsed_start = urlparse(start_url)
|
||||
base_domain = parsed_start.netloc
|
||||
|
||||
def extract_filename(resource_url: str) -> str:
|
||||
parsed = urlparse(resource_url)
|
||||
path = unquote(parsed.path)
|
||||
filename = os.path.basename(path)
|
||||
filename = re.sub(r'\?.*$', '', filename)
|
||||
filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
|
||||
return filename if filename else f"resource_{hash(resource_url) % 100000}"
|
||||
|
||||
def extract_page_metadata(html: str, page_url: str) -> dict:
|
||||
metadata = {"url": page_url, "title": "", "description": "", "creator": ""}
|
||||
|
||||
title_match = re.search(r'<title[^>]*>([^<]+)</title>', html, re.IGNORECASE)
|
||||
if title_match:
|
||||
metadata["title"] = title_match.group(1).strip()
|
||||
|
||||
desc_match = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
||||
if not desc_match:
|
||||
desc_match = re.search(r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+name=["\']description["\']', html, re.IGNORECASE)
|
||||
if desc_match:
|
||||
metadata["description"] = desc_match.group(1).strip()
|
||||
|
||||
author_match = re.search(r'<meta[^>]+name=["\']author["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
||||
if author_match:
|
||||
metadata["creator"] = author_match.group(1).strip()
|
||||
|
||||
return metadata
|
||||
|
||||
def download_resource(resource_url: str, dest_dir: str, headers: dict) -> dict:
|
||||
filename = extract_filename(resource_url)
|
||||
dest_path = os.path.join(dest_dir, filename)
|
||||
|
||||
if os.path.exists(dest_path):
|
||||
return {"status": "skipped", "url": resource_url, "path": dest_path, "reason": "already exists"}
|
||||
|
||||
try:
|
||||
head_response = requests.head(resource_url, headers=headers, timeout=10, allow_redirects=True)
|
||||
content_length = int(head_response.headers.get('Content-Length', 0))
|
||||
|
||||
if content_length > 0:
|
||||
if content_length < min_size_bytes:
|
||||
return {"status": "skipped", "url": resource_url, "reason": f"Too small ({content_length} bytes)"}
|
||||
if content_length > max_size_bytes:
|
||||
return {"status": "skipped", "url": resource_url, "reason": f"Too large ({content_length} bytes)"}
|
||||
|
||||
response = requests.get(resource_url, headers=headers, stream=True, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
content_type = response.headers.get('Content-Type', '').lower()
|
||||
if 'text/html' in content_type:
|
||||
return {"status": "error", "url": resource_url, "error": "URL returned HTML"}
|
||||
|
||||
with open(dest_path, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
file_size = os.path.getsize(dest_path)
|
||||
|
||||
if file_size < min_size_bytes:
|
||||
os.remove(dest_path)
|
||||
return {"status": "skipped", "url": resource_url, "reason": f"File too small ({file_size} bytes)"}
|
||||
|
||||
if file_size > max_size_bytes:
|
||||
os.remove(dest_path)
|
||||
return {"status": "skipped", "url": resource_url, "reason": f"File too large ({file_size} bytes)"}
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"url": resource_url,
|
||||
"path": dest_path,
|
||||
"filename": filename,
|
||||
"size": file_size
|
||||
}
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {"status": "error", "url": resource_url, "error": str(e)}
|
||||
|
||||
try:
|
||||
os.makedirs(destination_dir, exist_ok=True)
|
||||
default_headers = get_default_headers()
|
||||
|
||||
while pages_queue and len(visited_pages) < max_pages:
|
||||
current_url = pages_queue.popleft()
|
||||
|
||||
if current_url in visited_pages:
|
||||
continue
|
||||
|
||||
visited_pages.add(current_url)
|
||||
|
||||
try:
|
||||
response = requests.get(current_url, headers=default_headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
html = response.text
|
||||
results["pages_crawled"] += 1
|
||||
|
||||
if extract_metadata:
|
||||
page_meta = extract_page_metadata(html, current_url)
|
||||
results["metadata"].append(page_meta)
|
||||
|
||||
resource_urls = re.findall(r'(?:href|src)=["\']([^"\']+)["\']', html)
|
||||
for res_url in resource_urls:
|
||||
full_url = urljoin(current_url, res_url)
|
||||
if re.search(resource_pattern, full_url, re.IGNORECASE):
|
||||
if full_url not in visited_resources:
|
||||
visited_resources.add(full_url)
|
||||
resources_to_download.append(full_url)
|
||||
|
||||
if follow_links:
|
||||
links = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
||||
for link in links:
|
||||
full_link = urljoin(current_url, link)
|
||||
parsed_link = urlparse(full_link)
|
||||
|
||||
if parsed_link.netloc == base_domain:
|
||||
if full_link not in visited_pages:
|
||||
if link_pattern:
|
||||
if re.search(link_pattern, full_link):
|
||||
pages_queue.append(full_link)
|
||||
else:
|
||||
pages_queue.append(full_link)
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
results["errors"].append({"url": current_url, "error": str(e)})
|
||||
|
||||
results["resources_found"] = len(resources_to_download)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {
|
||||
executor.submit(download_resource, url, destination_dir, default_headers): url
|
||||
for url in resources_to_download
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result["status"] == "success":
|
||||
results["downloaded"].append(result)
|
||||
elif result["status"] == "skipped":
|
||||
results["skipped"].append(result)
|
||||
else:
|
||||
results["errors"].append(result)
|
||||
|
||||
if log_file and (results["downloaded"] or results["metadata"]):
|
||||
log_path = os.path.expanduser(log_file)
|
||||
os.makedirs(os.path.dirname(log_path) if os.path.dirname(log_path) else '.', exist_ok=True)
|
||||
with open(log_path, 'w', newline='') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=['filename', 'url', 'size', 'source_page'])
|
||||
writer.writeheader()
|
||||
for item in results["downloaded"]:
|
||||
writer.writerow({
|
||||
'filename': item.get('filename', ''),
|
||||
'url': item['url'],
|
||||
'size': item.get('size', 0),
|
||||
'source_page': start_url
|
||||
})
|
||||
|
||||
results["total_downloaded"] = len(results["downloaded"])
|
||||
results["total_skipped"] = len(results["skipped"])
|
||||
results["total_errors"] = len(results["errors"])
|
||||
|
||||
results["downloaded"] = results["downloaded"][:20]
|
||||
results["skipped"] = results["skipped"][:10]
|
||||
results["errors"] = results["errors"][:10]
|
||||
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def bulk_download_urls(
|
||||
urls: str,
|
||||
destination_dir: str,
|
||||
max_workers: int = 5,
|
||||
min_size_kb: int = 0,
|
||||
max_size_kb: int = 0,
|
||||
log_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Download multiple URLs concurrently.
|
||||
|
||||
Args:
|
||||
urls: Newline-separated list of URLs or path to file containing URLs.
|
||||
destination_dir: Directory to save downloaded files.
|
||||
max_workers: Number of concurrent download threads.
|
||||
min_size_kb: Minimum file size in KB to keep.
|
||||
max_size_kb: Maximum file size in KB to keep (0 for unlimited).
|
||||
log_file: Path to CSV log file for results.
|
||||
|
||||
Returns:
|
||||
Dict with status, downloaded files, and any errors.
|
||||
"""
|
||||
import os
|
||||
import csv
|
||||
from urllib.parse import urlparse, unquote
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
min_size_bytes = min_size_kb * 1024
|
||||
max_size_bytes = max_size_kb * 1024 if max_size_kb > 0 else float('inf')
|
||||
|
||||
results = {
|
||||
"status": "success",
|
||||
"destination_dir": destination_dir,
|
||||
"downloaded": [],
|
||||
"skipped": [],
|
||||
"errors": []
|
||||
}
|
||||
|
||||
url_list = []
|
||||
if os.path.isfile(os.path.expanduser(urls)):
|
||||
with open(os.path.expanduser(urls), 'r') as f:
|
||||
url_list = [line.strip() for line in f if line.strip() and line.strip().startswith('http')]
|
||||
else:
|
||||
url_list = [u.strip() for u in urls.split('\n') if u.strip() and u.strip().startswith('http')]
|
||||
|
||||
def extract_filename(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
path = unquote(parsed.path)
|
||||
filename = os.path.basename(path)
|
||||
if not filename or filename == '/':
|
||||
filename = f"download_{hash(url) % 100000}"
|
||||
return filename
|
||||
|
||||
def download_url(url: str, dest_dir: str, headers: dict) -> dict:
|
||||
filename = extract_filename(url)
|
||||
dest_path = os.path.join(dest_dir, filename)
|
||||
|
||||
if os.path.exists(dest_path):
|
||||
base, ext = os.path.splitext(filename)
|
||||
counter = 1
|
||||
while os.path.exists(dest_path):
|
||||
dest_path = os.path.join(dest_dir, f"{base}_{counter}{ext}")
|
||||
counter += 1
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, stream=True, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
with open(dest_path, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
file_size = os.path.getsize(dest_path)
|
||||
|
||||
if file_size < min_size_bytes:
|
||||
os.remove(dest_path)
|
||||
return {"status": "skipped", "url": url, "reason": f"File too small ({file_size} bytes)"}
|
||||
|
||||
if file_size > max_size_bytes:
|
||||
os.remove(dest_path)
|
||||
return {"status": "skipped", "url": url, "reason": f"File too large ({file_size} bytes)"}
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"url": url,
|
||||
"path": dest_path,
|
||||
"filename": os.path.basename(dest_path),
|
||||
"size": file_size
|
||||
}
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {"status": "error", "url": url, "error": str(e)}
|
||||
|
||||
try:
|
||||
os.makedirs(destination_dir, exist_ok=True)
|
||||
default_headers = get_default_headers()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {
|
||||
executor.submit(download_url, url, destination_dir, default_headers): url
|
||||
for url in url_list
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result["status"] == "success":
|
||||
results["downloaded"].append(result)
|
||||
elif result["status"] == "skipped":
|
||||
results["skipped"].append(result)
|
||||
else:
|
||||
results["errors"].append(result)
|
||||
|
||||
if log_file and results["downloaded"]:
|
||||
log_path = os.path.expanduser(log_file)
|
||||
os.makedirs(os.path.dirname(log_path) if os.path.dirname(log_path) else '.', exist_ok=True)
|
||||
with open(log_path, 'w', newline='') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=['filename', 'url', 'size'])
|
||||
writer.writeheader()
|
||||
for item in results["downloaded"]:
|
||||
writer.writerow({
|
||||
'filename': item['filename'],
|
||||
'url': item['url'],
|
||||
'size': item['size']
|
||||
})
|
||||
|
||||
results["total_urls"] = len(url_list)
|
||||
results["total_downloaded"] = len(results["downloaded"])
|
||||
results["total_skipped"] = len(results["skipped"])
|
||||
results["total_errors"] = len(results["errors"])
|
||||
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": str(e)}
|
||||
|
||||
return results
|
||||
|
||||
Reference in New Issue
Block a user