905 lines
35 KiB
Python
905 lines
35 KiB
Python
import base64
|
|
import imghdr
|
|
import logging
|
|
import random
|
|
import time
|
|
import requests
|
|
from typing import Optional, Dict, Any
|
|
|
|
from rp.core.operations import Validator, ValidationError
|
|
|
|
logger = logging.getLogger("rp")
|
|
|
|
NETWORK_TRANSIENT_ERRORS = (
|
|
requests.exceptions.ConnectionError,
|
|
requests.exceptions.Timeout,
|
|
requests.exceptions.ChunkedEncodingError,
|
|
)
|
|
|
|
MAX_RETRIES = 3
|
|
BASE_DELAY = 1.0
|
|
|
|
# Realistic User-Agents
|
|
USER_AGENTS = [
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
|
|
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59",
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1",
|
|
"Mozilla/5.0 (iPad; CPU OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1",
|
|
"Mozilla/5.0 (Android 11; Mobile; rv:68.0) Gecko/68.0 Firefox/88.0",
|
|
]
|
|
|
|
|
|
def get_default_headers():
|
|
"""Get default realistic headers with variations."""
|
|
accept_languages = [
|
|
"en-US,en;q=0.5",
|
|
"en-US,en;q=0.9",
|
|
"en-GB,en;q=0.5",
|
|
"en-US,en;q=0.5;fr;q=0.3",
|
|
]
|
|
headers = {
|
|
"User-Agent": random.choice(USER_AGENTS),
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Accept-Language": random.choice(accept_languages),
|
|
"Accept-Encoding": "gzip, deflate, br",
|
|
"DNT": "1",
|
|
"Connection": "keep-alive",
|
|
"Upgrade-Insecure-Requests": "1",
|
|
}
|
|
# Sometimes add Cache-Control
|
|
if random.random() < 0.3:
|
|
headers["Cache-Control"] = "no-cache"
|
|
# Sometimes add Referer
|
|
if random.random() < 0.2:
|
|
headers["Referer"] = "https://www.google.com/"
|
|
return headers
|
|
|
|
|
|
def http_fetch(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
|
|
"""Fetch content from an HTTP URL with automatic retry.
|
|
|
|
Args:
|
|
url: The URL to fetch.
|
|
headers: Optional HTTP headers.
|
|
|
|
Returns:
|
|
Dict with status and content.
|
|
"""
|
|
try:
|
|
url = Validator.string(url, "url", min_length=1, max_length=8192)
|
|
except ValidationError as e:
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
last_error = None
|
|
|
|
for attempt in range(MAX_RETRIES):
|
|
try:
|
|
default_headers = get_default_headers()
|
|
if headers:
|
|
default_headers.update(headers)
|
|
|
|
response = requests.get(url, headers=default_headers, timeout=30)
|
|
response.raise_for_status()
|
|
|
|
content_type = response.headers.get("Content-Type", "").lower()
|
|
if "text" in content_type or "json" in content_type or "xml" in content_type:
|
|
content = response.text
|
|
return {"status": "success", "content": content[:10000]}
|
|
else:
|
|
content = response.content
|
|
content_length = len(content)
|
|
if content_length > 10000:
|
|
return {
|
|
"status": "success",
|
|
"content_type": content_type,
|
|
"size_bytes": content_length,
|
|
"message": f"Binary content ({content_length} bytes). Use download_to_file to save it.",
|
|
}
|
|
else:
|
|
return {
|
|
"status": "success",
|
|
"content_type": content_type,
|
|
"size_bytes": content_length,
|
|
"content_base64": base64.b64encode(content).decode("utf-8"),
|
|
}
|
|
|
|
except NETWORK_TRANSIENT_ERRORS as e:
|
|
last_error = e
|
|
if attempt < MAX_RETRIES - 1:
|
|
delay = BASE_DELAY * (2 ** attempt)
|
|
logger.warning(f"http_fetch attempt {attempt + 1} failed: {e}. Retrying in {delay:.1f}s")
|
|
time.sleep(delay)
|
|
continue
|
|
|
|
except requests.exceptions.HTTPError as e:
|
|
return {"status": "error", "error": f"HTTP error: {e}"}
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
return {"status": "error", "error": f"Failed after {MAX_RETRIES} retries: {last_error}"}
|
|
|
|
|
|
def download_to_file(
|
|
source_url: str, destination_path: str, headers: Optional[Dict[str, str]] = None
|
|
) -> Dict[str, Any]:
|
|
"""Download content from an HTTP URL to a file with retry and safe write.
|
|
|
|
Args:
|
|
source_url: The URL to download from.
|
|
destination_path: The path to save the downloaded content.
|
|
headers: Optional HTTP headers.
|
|
|
|
Returns:
|
|
Dict with status, downloaded_from, and downloaded_to on success, or status and error on failure.
|
|
"""
|
|
import os
|
|
|
|
try:
|
|
source_url = Validator.string(source_url, "source_url", min_length=1, max_length=8192)
|
|
destination_path = Validator.string(destination_path, "destination_path", min_length=1, max_length=4096)
|
|
except ValidationError as e:
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
temp_path = destination_path + ".download"
|
|
last_error = None
|
|
|
|
for attempt in range(MAX_RETRIES):
|
|
try:
|
|
default_headers = get_default_headers()
|
|
if headers:
|
|
default_headers.update(headers)
|
|
|
|
response = requests.get(source_url, headers=default_headers, stream=True, timeout=60)
|
|
response.raise_for_status()
|
|
|
|
with open(temp_path, "wb") as file:
|
|
for chunk in response.iter_content(chunk_size=8192):
|
|
file.write(chunk)
|
|
|
|
os.replace(temp_path, destination_path)
|
|
|
|
content_type = response.headers.get("Content-Type", "").lower()
|
|
if content_type.startswith("image/"):
|
|
img_type = imghdr.what(destination_path)
|
|
if img_type is None:
|
|
return {
|
|
"status": "success",
|
|
"downloaded_from": source_url,
|
|
"downloaded_to": destination_path,
|
|
"is_valid_image": False,
|
|
"warning": "Downloaded content is not a valid image, consider finding a different source.",
|
|
}
|
|
else:
|
|
return {
|
|
"status": "success",
|
|
"downloaded_from": source_url,
|
|
"downloaded_to": destination_path,
|
|
"is_valid_image": True,
|
|
}
|
|
else:
|
|
return {
|
|
"status": "success",
|
|
"downloaded_from": source_url,
|
|
"downloaded_to": destination_path,
|
|
}
|
|
|
|
except NETWORK_TRANSIENT_ERRORS as e:
|
|
last_error = e
|
|
if attempt < MAX_RETRIES - 1:
|
|
delay = BASE_DELAY * (2 ** attempt)
|
|
logger.warning(f"download_to_file attempt {attempt + 1} failed: {e}. Retrying in {delay:.1f}s")
|
|
time.sleep(delay)
|
|
continue
|
|
|
|
except requests.exceptions.HTTPError as e:
|
|
if os.path.exists(temp_path):
|
|
os.remove(temp_path)
|
|
return {"status": "error", "error": f"HTTP error: {e}"}
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
if os.path.exists(temp_path):
|
|
os.remove(temp_path)
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
except Exception as e:
|
|
if os.path.exists(temp_path):
|
|
os.remove(temp_path)
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
if os.path.exists(temp_path):
|
|
try:
|
|
os.remove(temp_path)
|
|
except OSError:
|
|
pass
|
|
|
|
return {"status": "error", "error": f"Failed after {MAX_RETRIES} retries: {last_error}"}
|
|
|
|
|
|
def _perform_search(
|
|
base_url: str, query: str, params: Optional[Dict[str, str]] = None
|
|
) -> Dict[str, Any]:
|
|
try:
|
|
default_headers = get_default_headers()
|
|
search_params = {"query": query}
|
|
if params:
|
|
search_params.update(params)
|
|
|
|
response = requests.get(base_url, headers=default_headers, params=search_params, timeout=30)
|
|
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
|
|
|
|
try:
|
|
data = response.json()
|
|
except ValueError:
|
|
data = response.text
|
|
return {"status": "success", "content": data}
|
|
except requests.exceptions.RequestException as e:
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
|
|
def web_search(query: str) -> Dict[str, Any]:
|
|
"""Perform a web search.
|
|
|
|
Args:
|
|
query: Search query.
|
|
|
|
Returns:
|
|
Dict with status and search results.
|
|
"""
|
|
base_url = "https://static.molodetz.nl/search.cgi"
|
|
return _perform_search(base_url, query)
|
|
|
|
|
|
def web_search_news(query: str) -> Dict[str, Any]:
|
|
"""Perform a web search for news.
|
|
|
|
Args:
|
|
query: Search query for news.
|
|
|
|
Returns:
|
|
Dict with status and news search results.
|
|
"""
|
|
base_url = "https://static.molodetz.nl/search.cgi"
|
|
return _perform_search(base_url, query)
|
|
|
|
|
|
def scrape_images(
|
|
url: str,
|
|
destination_dir: str,
|
|
full_size: bool = True,
|
|
extensions: Optional[str] = None,
|
|
min_size_kb: int = 0,
|
|
max_size_kb: int = 0,
|
|
max_workers: int = 5,
|
|
extract_captions: bool = False,
|
|
log_file: Optional[str] = None
|
|
) -> Dict[str, Any]:
|
|
"""Scrape and download all images from a webpage with filtering and concurrent downloads.
|
|
|
|
Args:
|
|
url: The webpage URL to scrape for images.
|
|
destination_dir: Directory to save downloaded images.
|
|
full_size: If True, attempt to find full-size image URLs instead of thumbnails.
|
|
extensions: Comma-separated list of extensions to filter (e.g., "jpg,png,gif"). Default: all images.
|
|
min_size_kb: Minimum file size in KB to keep (0 for no minimum).
|
|
max_size_kb: Maximum file size in KB to keep (0 for no maximum).
|
|
max_workers: Number of concurrent download threads.
|
|
extract_captions: If True, extract alt text and captions.
|
|
log_file: Path to CSV log file for metadata.
|
|
|
|
Returns:
|
|
Dict with status, downloaded files list, and any errors.
|
|
"""
|
|
import os
|
|
import re
|
|
import csv
|
|
from urllib.parse import urljoin, urlparse, unquote
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
allowed_extensions = None
|
|
if extensions:
|
|
allowed_extensions = [ext.strip().lower().lstrip('.') for ext in extensions.split(',')]
|
|
|
|
min_size_bytes = min_size_kb * 1024
|
|
max_size_bytes = max_size_kb * 1024 if max_size_kb > 0 else float('inf')
|
|
|
|
def normalize_url(base_url: str, img_url: str) -> str:
|
|
if img_url.startswith(('http://', 'https://')):
|
|
return img_url
|
|
return urljoin(base_url, img_url)
|
|
|
|
def extract_filename(img_url: str) -> str:
|
|
parsed = urlparse(img_url)
|
|
path = unquote(parsed.path)
|
|
filename = os.path.basename(path)
|
|
filename = re.sub(r'\?.*$', '', filename)
|
|
filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
|
|
return filename if filename else f"image_{hash(img_url) % 10000}.jpg"
|
|
|
|
def is_valid_image_url(img_url: str) -> bool:
|
|
lower_url = img_url.lower()
|
|
img_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.svg', '.ico')
|
|
if any(ext in lower_url for ext in img_extensions):
|
|
return True
|
|
if '_media' in lower_url or '/images/' in lower_url or '/img/' in lower_url:
|
|
return True
|
|
return False
|
|
|
|
def get_full_size_url(img_url: str, base_url: str) -> str:
|
|
clean_url = re.sub(r'\?w=\d+.*$', '', img_url)
|
|
clean_url = re.sub(r'\?.*tok=[a-f0-9]+.*$', '', clean_url)
|
|
clean_url = re.sub(r'&w=\d+&h=\d+', '', clean_url)
|
|
clean_url = re.sub(r'[?&]cache=.*$', '', clean_url)
|
|
if '_media' in clean_url:
|
|
clean_url = clean_url.replace('%3A', ':').replace('%2F', '/')
|
|
return clean_url
|
|
|
|
def extract_dokuwiki_images(html: str, base_url: str) -> list:
|
|
images = []
|
|
media_patterns = [
|
|
r'href=["\']([^"\']*/_media/[^"\']+)["\']',
|
|
r'href=["\']([^"\']*/_detail/[^"\']+)["\']',
|
|
r'src=["\']([^"\']*/_media/[^"\']+)["\']',
|
|
]
|
|
for pattern in media_patterns:
|
|
matches = re.findall(pattern, html, re.IGNORECASE)
|
|
for match in matches:
|
|
full_url = normalize_url(base_url, match)
|
|
if '_detail' in full_url:
|
|
full_url = full_url.replace('/_detail/', '/_media/')
|
|
full_url = re.sub(r'\?id=[^&]*', '', full_url)
|
|
full_url = re.sub(r'&.*$', '', full_url)
|
|
images.append({"url": full_url, "caption": ""})
|
|
return images
|
|
|
|
def extract_standard_images(html: str, base_url: str) -> list:
|
|
images = []
|
|
img_pattern = r'<img[^>]+src=["\']([^"\']+)["\'][^>]*(?:alt=["\']([^"\']*)["\'])?[^>]*>'
|
|
alt_pattern = r'<img[^>]*alt=["\']([^"\']*)["\'][^>]*src=["\']([^"\']+)["\'][^>]*>'
|
|
|
|
for match in re.finditer(img_pattern, html, re.IGNORECASE):
|
|
src = match.group(1)
|
|
alt = match.group(2) or ""
|
|
if is_valid_image_url(src):
|
|
images.append({"url": normalize_url(base_url, src), "caption": alt})
|
|
|
|
for match in re.finditer(alt_pattern, html, re.IGNORECASE):
|
|
alt = match.group(1) or ""
|
|
src = match.group(2)
|
|
if is_valid_image_url(src):
|
|
existing = [i for i in images if i["url"] == normalize_url(base_url, src)]
|
|
if not existing:
|
|
images.append({"url": normalize_url(base_url, src), "caption": alt})
|
|
|
|
link_patterns = [
|
|
r'<a[^>]+href=["\']([^"\']+\.(?:jpg|jpeg|png|gif|webp))["\']',
|
|
r'srcset=["\']([^"\',\s]+)',
|
|
r'data-src=["\']([^"\']+)["\']',
|
|
]
|
|
for pattern in link_patterns:
|
|
matches = re.findall(pattern, html, re.IGNORECASE)
|
|
for match in matches:
|
|
if is_valid_image_url(match):
|
|
url = normalize_url(base_url, match)
|
|
existing = [i for i in images if i["url"] == url]
|
|
if not existing:
|
|
images.append({"url": url, "caption": ""})
|
|
return images
|
|
|
|
def download_image(img_data: dict, dest_dir: str, headers: dict) -> dict:
|
|
img_url = img_data["url"]
|
|
caption = img_data.get("caption", "")
|
|
filename = extract_filename(img_url)
|
|
if not filename:
|
|
return {"status": "error", "url": img_url, "error": "Could not extract filename"}
|
|
|
|
dest_path = os.path.join(dest_dir, filename)
|
|
|
|
if os.path.exists(dest_path):
|
|
return {"status": "skipped", "url": img_url, "path": dest_path, "reason": "already exists"}
|
|
|
|
try:
|
|
head_response = requests.head(img_url, headers=headers, timeout=10, allow_redirects=True)
|
|
content_length = int(head_response.headers.get('Content-Length', 0))
|
|
|
|
if content_length > 0:
|
|
if content_length < min_size_bytes:
|
|
return {"status": "skipped", "url": img_url, "reason": f"Too small ({content_length} bytes)"}
|
|
if content_length > max_size_bytes:
|
|
return {"status": "skipped", "url": img_url, "reason": f"Too large ({content_length} bytes)"}
|
|
|
|
img_response = requests.get(img_url, headers=headers, stream=True, timeout=30)
|
|
img_response.raise_for_status()
|
|
|
|
content_type = img_response.headers.get('Content-Type', '').lower()
|
|
if 'text/html' in content_type:
|
|
return {"status": "error", "url": img_url, "error": "URL returned HTML instead of image"}
|
|
|
|
with open(dest_path, 'wb') as f:
|
|
for chunk in img_response.iter_content(chunk_size=8192):
|
|
f.write(chunk)
|
|
|
|
file_size = os.path.getsize(dest_path)
|
|
|
|
if file_size < min_size_bytes:
|
|
os.remove(dest_path)
|
|
return {"status": "skipped", "url": img_url, "reason": f"File too small ({file_size} bytes)"}
|
|
|
|
if file_size > max_size_bytes:
|
|
os.remove(dest_path)
|
|
return {"status": "skipped", "url": img_url, "reason": f"File too large ({file_size} bytes)"}
|
|
|
|
if file_size < 100:
|
|
os.remove(dest_path)
|
|
return {"status": "error", "url": img_url, "error": f"File too small ({file_size} bytes)"}
|
|
|
|
return {
|
|
"status": "success",
|
|
"url": img_url,
|
|
"path": dest_path,
|
|
"size": file_size,
|
|
"caption": caption
|
|
}
|
|
except requests.exceptions.RequestException as e:
|
|
return {"status": "error", "url": img_url, "error": str(e)}
|
|
|
|
try:
|
|
os.makedirs(destination_dir, exist_ok=True)
|
|
|
|
default_headers = get_default_headers()
|
|
response = requests.get(url, headers=default_headers, timeout=30)
|
|
response.raise_for_status()
|
|
html = response.text
|
|
|
|
image_data = []
|
|
dokuwiki_images = extract_dokuwiki_images(html, url)
|
|
image_data.extend(dokuwiki_images)
|
|
standard_images = extract_standard_images(html, url)
|
|
|
|
existing_urls = {i["url"] for i in image_data}
|
|
for img in standard_images:
|
|
if img["url"] not in existing_urls:
|
|
image_data.append(img)
|
|
existing_urls.add(img["url"])
|
|
|
|
if full_size:
|
|
for img in image_data:
|
|
img["url"] = get_full_size_url(img["url"], url)
|
|
|
|
if allowed_extensions:
|
|
filtered = []
|
|
for img in image_data:
|
|
lower_url = img["url"].lower()
|
|
if any(lower_url.endswith('.' + ext) or ('.' + ext + '?') in lower_url or ('.' + ext + '&') in lower_url for ext in allowed_extensions):
|
|
filtered.append(img)
|
|
image_data = filtered
|
|
|
|
downloaded = []
|
|
errors = []
|
|
skipped = []
|
|
|
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = {
|
|
executor.submit(download_image, img, destination_dir, default_headers): img
|
|
for img in image_data
|
|
}
|
|
|
|
for future in as_completed(futures):
|
|
result = future.result()
|
|
if result["status"] == "success":
|
|
downloaded.append(result)
|
|
elif result["status"] == "skipped":
|
|
skipped.append(result)
|
|
else:
|
|
errors.append(result)
|
|
|
|
if log_file and downloaded:
|
|
log_path = os.path.expanduser(log_file)
|
|
os.makedirs(os.path.dirname(log_path) if os.path.dirname(log_path) else '.', exist_ok=True)
|
|
with open(log_path, 'w', newline='') as f:
|
|
writer = csv.DictWriter(f, fieldnames=['filename', 'url', 'size', 'caption'])
|
|
writer.writeheader()
|
|
for item in downloaded:
|
|
writer.writerow({
|
|
'filename': os.path.basename(item['path']),
|
|
'url': item['url'],
|
|
'size': item['size'],
|
|
'caption': item.get('caption', '')
|
|
})
|
|
|
|
captions_text = ""
|
|
if extract_captions and downloaded:
|
|
caption_lines = []
|
|
for item in downloaded:
|
|
if item.get('caption'):
|
|
caption_lines.append(f"{os.path.basename(item['path'])}: {item['caption']}")
|
|
captions_text = "\n".join(caption_lines)
|
|
|
|
return {
|
|
"status": "success",
|
|
"source_url": url,
|
|
"destination_dir": destination_dir,
|
|
"total_found": len(image_data),
|
|
"downloaded": len(downloaded),
|
|
"skipped": len(skipped),
|
|
"errors": len(errors),
|
|
"files": downloaded[:20],
|
|
"skipped_files": skipped[:5] if skipped else [],
|
|
"error_details": errors[:5] if errors else [],
|
|
"captions": captions_text if extract_captions else None,
|
|
"log_file": log_file if log_file else None
|
|
}
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
return {"status": "error", "error": f"Failed to fetch page: {str(e)}"}
|
|
except Exception as e:
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
|
|
def crawl_and_download(
|
|
start_url: str,
|
|
destination_dir: str,
|
|
resource_pattern: str = r'\.(jpg|jpeg|png|gif|svg|pdf|mp3|mp4)$',
|
|
max_pages: int = 10,
|
|
follow_links: bool = True,
|
|
link_pattern: Optional[str] = None,
|
|
min_size_kb: int = 0,
|
|
max_size_kb: int = 0,
|
|
max_workers: int = 5,
|
|
log_file: Optional[str] = None,
|
|
extract_metadata: bool = False
|
|
) -> Dict[str, Any]:
|
|
"""Crawl website pages and download matching resources with metadata extraction.
|
|
|
|
Args:
|
|
start_url: Starting URL to crawl.
|
|
destination_dir: Directory to save downloaded resources.
|
|
resource_pattern: Regex pattern for resource URLs to download.
|
|
max_pages: Maximum number of pages to crawl.
|
|
follow_links: If True, follow links to other pages on same domain.
|
|
link_pattern: Regex pattern for links to follow (e.g., "page=\\d+" for pagination).
|
|
min_size_kb: Minimum file size in KB to download.
|
|
max_size_kb: Maximum file size in KB to download (0 for unlimited).
|
|
max_workers: Number of concurrent download threads.
|
|
log_file: Path to CSV log file for metadata.
|
|
extract_metadata: If True, extract additional metadata (title, description, etc.).
|
|
|
|
Returns:
|
|
Dict with status, pages crawled, resources downloaded, and metadata.
|
|
"""
|
|
import os
|
|
import re
|
|
import csv
|
|
from urllib.parse import urljoin, urlparse, unquote
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from collections import deque
|
|
|
|
min_size_bytes = min_size_kb * 1024
|
|
max_size_bytes = max_size_kb * 1024 if max_size_kb > 0 else float('inf')
|
|
|
|
results = {
|
|
"status": "success",
|
|
"start_url": start_url,
|
|
"destination_dir": destination_dir,
|
|
"pages_crawled": 0,
|
|
"resources_found": 0,
|
|
"downloaded": [],
|
|
"skipped": [],
|
|
"errors": [],
|
|
"metadata": []
|
|
}
|
|
|
|
visited_pages = set()
|
|
visited_resources = set()
|
|
pages_queue = deque([start_url])
|
|
resources_to_download = []
|
|
|
|
parsed_start = urlparse(start_url)
|
|
base_domain = parsed_start.netloc
|
|
|
|
def extract_filename(resource_url: str) -> str:
|
|
parsed = urlparse(resource_url)
|
|
path = unquote(parsed.path)
|
|
filename = os.path.basename(path)
|
|
filename = re.sub(r'\?.*$', '', filename)
|
|
filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
|
|
return filename if filename else f"resource_{hash(resource_url) % 100000}"
|
|
|
|
def extract_page_metadata(html: str, page_url: str) -> dict:
|
|
metadata = {"url": page_url, "title": "", "description": "", "creator": ""}
|
|
|
|
title_match = re.search(r'<title[^>]*>([^<]+)</title>', html, re.IGNORECASE)
|
|
if title_match:
|
|
metadata["title"] = title_match.group(1).strip()
|
|
|
|
desc_match = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
|
if not desc_match:
|
|
desc_match = re.search(r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+name=["\']description["\']', html, re.IGNORECASE)
|
|
if desc_match:
|
|
metadata["description"] = desc_match.group(1).strip()
|
|
|
|
author_match = re.search(r'<meta[^>]+name=["\']author["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
|
if author_match:
|
|
metadata["creator"] = author_match.group(1).strip()
|
|
|
|
return metadata
|
|
|
|
def download_resource(resource_url: str, dest_dir: str, headers: dict) -> dict:
|
|
filename = extract_filename(resource_url)
|
|
dest_path = os.path.join(dest_dir, filename)
|
|
|
|
if os.path.exists(dest_path):
|
|
return {"status": "skipped", "url": resource_url, "path": dest_path, "reason": "already exists"}
|
|
|
|
try:
|
|
head_response = requests.head(resource_url, headers=headers, timeout=10, allow_redirects=True)
|
|
content_length = int(head_response.headers.get('Content-Length', 0))
|
|
|
|
if content_length > 0:
|
|
if content_length < min_size_bytes:
|
|
return {"status": "skipped", "url": resource_url, "reason": f"Too small ({content_length} bytes)"}
|
|
if content_length > max_size_bytes:
|
|
return {"status": "skipped", "url": resource_url, "reason": f"Too large ({content_length} bytes)"}
|
|
|
|
response = requests.get(resource_url, headers=headers, stream=True, timeout=60)
|
|
response.raise_for_status()
|
|
|
|
content_type = response.headers.get('Content-Type', '').lower()
|
|
if 'text/html' in content_type:
|
|
return {"status": "error", "url": resource_url, "error": "URL returned HTML"}
|
|
|
|
with open(dest_path, 'wb') as f:
|
|
for chunk in response.iter_content(chunk_size=8192):
|
|
f.write(chunk)
|
|
|
|
file_size = os.path.getsize(dest_path)
|
|
|
|
if file_size < min_size_bytes:
|
|
os.remove(dest_path)
|
|
return {"status": "skipped", "url": resource_url, "reason": f"File too small ({file_size} bytes)"}
|
|
|
|
if file_size > max_size_bytes:
|
|
os.remove(dest_path)
|
|
return {"status": "skipped", "url": resource_url, "reason": f"File too large ({file_size} bytes)"}
|
|
|
|
return {
|
|
"status": "success",
|
|
"url": resource_url,
|
|
"path": dest_path,
|
|
"filename": filename,
|
|
"size": file_size
|
|
}
|
|
except requests.exceptions.RequestException as e:
|
|
return {"status": "error", "url": resource_url, "error": str(e)}
|
|
|
|
try:
|
|
os.makedirs(destination_dir, exist_ok=True)
|
|
default_headers = get_default_headers()
|
|
|
|
while pages_queue and len(visited_pages) < max_pages:
|
|
current_url = pages_queue.popleft()
|
|
|
|
if current_url in visited_pages:
|
|
continue
|
|
|
|
visited_pages.add(current_url)
|
|
|
|
try:
|
|
response = requests.get(current_url, headers=default_headers, timeout=30)
|
|
response.raise_for_status()
|
|
html = response.text
|
|
results["pages_crawled"] += 1
|
|
|
|
if extract_metadata:
|
|
page_meta = extract_page_metadata(html, current_url)
|
|
results["metadata"].append(page_meta)
|
|
|
|
resource_urls = re.findall(r'(?:href|src)=["\']([^"\']+)["\']', html)
|
|
for res_url in resource_urls:
|
|
full_url = urljoin(current_url, res_url)
|
|
if re.search(resource_pattern, full_url, re.IGNORECASE):
|
|
if full_url not in visited_resources:
|
|
visited_resources.add(full_url)
|
|
resources_to_download.append(full_url)
|
|
|
|
if follow_links:
|
|
links = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
|
for link in links:
|
|
full_link = urljoin(current_url, link)
|
|
parsed_link = urlparse(full_link)
|
|
|
|
if parsed_link.netloc == base_domain:
|
|
if full_link not in visited_pages:
|
|
if link_pattern:
|
|
if re.search(link_pattern, full_link):
|
|
pages_queue.append(full_link)
|
|
else:
|
|
pages_queue.append(full_link)
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
results["errors"].append({"url": current_url, "error": str(e)})
|
|
|
|
results["resources_found"] = len(resources_to_download)
|
|
|
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = {
|
|
executor.submit(download_resource, url, destination_dir, default_headers): url
|
|
for url in resources_to_download
|
|
}
|
|
|
|
for future in as_completed(futures):
|
|
result = future.result()
|
|
if result["status"] == "success":
|
|
results["downloaded"].append(result)
|
|
elif result["status"] == "skipped":
|
|
results["skipped"].append(result)
|
|
else:
|
|
results["errors"].append(result)
|
|
|
|
if log_file and (results["downloaded"] or results["metadata"]):
|
|
log_path = os.path.expanduser(log_file)
|
|
os.makedirs(os.path.dirname(log_path) if os.path.dirname(log_path) else '.', exist_ok=True)
|
|
with open(log_path, 'w', newline='') as f:
|
|
writer = csv.DictWriter(f, fieldnames=['filename', 'url', 'size', 'source_page'])
|
|
writer.writeheader()
|
|
for item in results["downloaded"]:
|
|
writer.writerow({
|
|
'filename': item.get('filename', ''),
|
|
'url': item['url'],
|
|
'size': item.get('size', 0),
|
|
'source_page': start_url
|
|
})
|
|
|
|
results["total_downloaded"] = len(results["downloaded"])
|
|
results["total_skipped"] = len(results["skipped"])
|
|
results["total_errors"] = len(results["errors"])
|
|
|
|
results["downloaded"] = results["downloaded"][:20]
|
|
results["skipped"] = results["skipped"][:10]
|
|
results["errors"] = results["errors"][:10]
|
|
|
|
except Exception as e:
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
return results
|
|
|
|
|
|
def bulk_download_urls(
|
|
urls: str,
|
|
destination_dir: str,
|
|
max_workers: int = 5,
|
|
min_size_kb: int = 0,
|
|
max_size_kb: int = 0,
|
|
log_file: Optional[str] = None
|
|
) -> Dict[str, Any]:
|
|
"""Download multiple URLs concurrently.
|
|
|
|
Args:
|
|
urls: Newline-separated list of URLs or path to file containing URLs.
|
|
destination_dir: Directory to save downloaded files.
|
|
max_workers: Number of concurrent download threads.
|
|
min_size_kb: Minimum file size in KB to keep.
|
|
max_size_kb: Maximum file size in KB to keep (0 for unlimited).
|
|
log_file: Path to CSV log file for results.
|
|
|
|
Returns:
|
|
Dict with status, downloaded files, and any errors.
|
|
"""
|
|
import os
|
|
import csv
|
|
from urllib.parse import urlparse, unquote
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
min_size_bytes = min_size_kb * 1024
|
|
max_size_bytes = max_size_kb * 1024 if max_size_kb > 0 else float('inf')
|
|
|
|
results = {
|
|
"status": "success",
|
|
"destination_dir": destination_dir,
|
|
"downloaded": [],
|
|
"skipped": [],
|
|
"errors": []
|
|
}
|
|
|
|
url_list = []
|
|
if os.path.isfile(os.path.expanduser(urls)):
|
|
with open(os.path.expanduser(urls), 'r') as f:
|
|
url_list = [line.strip() for line in f if line.strip() and line.strip().startswith('http')]
|
|
else:
|
|
url_list = [u.strip() for u in urls.split('\n') if u.strip() and u.strip().startswith('http')]
|
|
|
|
def extract_filename(url: str) -> str:
|
|
parsed = urlparse(url)
|
|
path = unquote(parsed.path)
|
|
filename = os.path.basename(path)
|
|
if not filename or filename == '/':
|
|
filename = f"download_{hash(url) % 100000}"
|
|
return filename
|
|
|
|
def download_url(url: str, dest_dir: str, headers: dict) -> dict:
|
|
filename = extract_filename(url)
|
|
dest_path = os.path.join(dest_dir, filename)
|
|
|
|
if os.path.exists(dest_path):
|
|
base, ext = os.path.splitext(filename)
|
|
counter = 1
|
|
while os.path.exists(dest_path):
|
|
dest_path = os.path.join(dest_dir, f"{base}_{counter}{ext}")
|
|
counter += 1
|
|
|
|
try:
|
|
response = requests.get(url, headers=headers, stream=True, timeout=60)
|
|
response.raise_for_status()
|
|
|
|
with open(dest_path, 'wb') as f:
|
|
for chunk in response.iter_content(chunk_size=8192):
|
|
f.write(chunk)
|
|
|
|
file_size = os.path.getsize(dest_path)
|
|
|
|
if file_size < min_size_bytes:
|
|
os.remove(dest_path)
|
|
return {"status": "skipped", "url": url, "reason": f"File too small ({file_size} bytes)"}
|
|
|
|
if file_size > max_size_bytes:
|
|
os.remove(dest_path)
|
|
return {"status": "skipped", "url": url, "reason": f"File too large ({file_size} bytes)"}
|
|
|
|
return {
|
|
"status": "success",
|
|
"url": url,
|
|
"path": dest_path,
|
|
"filename": os.path.basename(dest_path),
|
|
"size": file_size
|
|
}
|
|
except requests.exceptions.RequestException as e:
|
|
return {"status": "error", "url": url, "error": str(e)}
|
|
|
|
try:
|
|
os.makedirs(destination_dir, exist_ok=True)
|
|
default_headers = get_default_headers()
|
|
|
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = {
|
|
executor.submit(download_url, url, destination_dir, default_headers): url
|
|
for url in url_list
|
|
}
|
|
|
|
for future in as_completed(futures):
|
|
result = future.result()
|
|
if result["status"] == "success":
|
|
results["downloaded"].append(result)
|
|
elif result["status"] == "skipped":
|
|
results["skipped"].append(result)
|
|
else:
|
|
results["errors"].append(result)
|
|
|
|
if log_file and results["downloaded"]:
|
|
log_path = os.path.expanduser(log_file)
|
|
os.makedirs(os.path.dirname(log_path) if os.path.dirname(log_path) else '.', exist_ok=True)
|
|
with open(log_path, 'w', newline='') as f:
|
|
writer = csv.DictWriter(f, fieldnames=['filename', 'url', 'size'])
|
|
writer.writeheader()
|
|
for item in results["downloaded"]:
|
|
writer.writerow({
|
|
'filename': item['filename'],
|
|
'url': item['url'],
|
|
'size': item['size']
|
|
})
|
|
|
|
results["total_urls"] = len(url_list)
|
|
results["total_downloaded"] = len(results["downloaded"])
|
|
results["total_skipped"] = len(results["skipped"])
|
|
results["total_errors"] = len(results["errors"])
|
|
|
|
except Exception as e:
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
return results
|