commit 018b4e431a49afd7339569053ef75de99a52fc55 Author: retoor Date: Thu Oct 2 21:17:36 2025 +0200 Initial commit. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1543560 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.db-shm +*.db +*.db-wal +__pycache__ diff --git a/README.md b/README.md new file mode 100644 index 0000000..5e86abb --- /dev/null +++ b/README.md @@ -0,0 +1,105 @@ +# RSS Feed Manager + +A FastAPI application for managing RSS feeds with SQLite database. + +## Project Structure + +``` +. +├── app.py # Main FastAPI application +├── routers.py # API routes +├── requirements.txt # Python dependencies +├── feeds.db # SQLite database (auto-created) +└── templates/ # Jinja2 templates + ├── base.html # Base template + ├── index.html # Main splash screen + ├── manage_list.html # Feed list view + ├── manage_create.html # Create form + ├── manage_upload.html # Upload form + ├── manage_sync.html # Sync page with live updates + ├── manage_edit.html # Edit form + ├── newspapers_list.html # Newspapers list + ├── newspaper_view.html # Pure newspaper layout + └── sync_logs_list.html # Detailed sync history +``` + +## Installation + +1. Install dependencies: +```bash +pip install -r requirements.txt +``` + +2. Run the application: +```bash +python app.py +``` + +3. Open browser at `http://localhost:8000` + +## Features + +- **Upload JSON**: Upload RSS feed configuration files +- **Sync by URL**: Automatically upsert feeds based on URL (unique field) +- **CRUD Operations**: Create, Read, Update, Delete feeds +- **Google Search Theme**: Clean, familiar interface +- **Backend Rendered**: All pages rendered server-side with Jinja2 + +## Database + +Uses SQLite with `dataset` library (1.6.2) for simple ORM operations. + +### Feeds Table +- id (auto-generated) +- name +- url (unique identifier for upsert) +- type +- category +- structure (JSON field) +- last_synced (timestamp) + +### Articles Table +- id (auto-generated) +- feed_name +- feed_url +- title +- link +- description +- content (clean text extracted by trafilatura) +- published +- author +- guid (unique identifier for upsert) +- created_at (timestamp) +- last_synchronized (timestamp) + +### Sync Logs Table +- id (auto-generated) +- sync_time (timestamp) +- total_feeds (integer) +- completed_feeds (integer) +- failed_feeds (integer) +- total_articles_processed (integer) +- new_articles (integer) +- elapsed_seconds (float) +- avg_req_per_sec (float) +- timed_out (boolean) + +### Newspapers Table +- id (auto-generated) +- created_at (timestamp) +- article_count (integer) +- articles_json (JSON string) + +## Routes + +- `/` - Main splash screen +- `/manage` - List all feeds +- `/manage/create` - Create new feed +- `/manage/upload` - Upload JSON file +- `/manage/sync` - Synchronize all RSS feeds +- `/manage/edit/{id}` - Edit feed +- `/manage/delete/{id}` - Delete feed +- `/newspapers` - List all newspapers +- `/newspaper/{id}` - View newspaper (pure newspaper layout) +- `/sync-logs` - View complete synchronization history with detailed statistics +- `/ws/sync` - WebSocket endpoint for live sync updates diff --git a/app.py b/app.py new file mode 100644 index 0000000..ef81a4e --- /dev/null +++ b/app.py @@ -0,0 +1,38 @@ +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +import dataset +import asyncio +from datetime import datetime + +app = FastAPI(title="RSS Feed Manager") + +# Database setup +db = dataset.connect('sqlite:///feeds.db') + +# Templates setup +templates = Jinja2Templates(directory="templates") + +# Import routers +from routers import router as manage_router, run_sync_task + +app.include_router(manage_router) + +@app.on_event("startup") +async def startup_event(): + # Ensure feeds table exists + feeds_table = db['feeds'] + # Start background sync task + asyncio.create_task(hourly_sync_task()) + +async def hourly_sync_task(): + while True: + await asyncio.sleep(3600) # Wait 1 hour + try: + await run_sync_task() + except Exception as e: + print(f"Error in hourly sync: {e}") + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8592) diff --git a/feeds.json b/feeds.json new file mode 100644 index 0000000..9a68657 --- /dev/null +++ b/feeds.json @@ -0,0 +1,1749 @@ +[ + { + "name": "TechCrunch", + "url": "https://techcrunch.com/feed/", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "The Verge", + "url": "https://www.theverge.com/rss/index.xml", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Ars Technica", + "url": "http://feeds.arstechnica.com/arstechnica/index/", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Engadget", + "url": "https://www.engadget.com/rss.xml", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "WIRED", + "url": "https://www.wired.com/feed/rss", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "CNET", + "url": "https://www.cnet.com/rss/news/", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "VentureBeat", + "url": "https://venturebeat.com/feed/", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Mashable", + "url": "http://feeds.mashable.com/Mashable", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Gizmodo", + "url": "https://gizmodo.com/rss", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "TechRepublic", + "url": "https://www.techrepublic.com/rssfeeds/articles/", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "ExtremeTech", + "url": "https://www.extremetech.com/feed", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "TechSpot", + "url": "https://www.techspot.com/backend.xml", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Stack Overflow Blog", + "url": "https://stackoverflow.blog/feed/", + "type": "rss", + "category": "programming", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "DEV Community", + "url": "https://dev.to/feed", + "type": "rss", + "category": "programming", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Hacker News", + "url": "https://news.ycombinator.com/rss", + "type": "rss", + "category": "programming", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "CSS-Tricks", + "url": "https://css-tricks.com/feed/", + "type": "rss", + "category": "frontend", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Smashing Magazine", + "url": "https://www.smashingmagazine.com/feed/", + "type": "rss", + "category": "frontend", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "SitePoint", + "url": "https://www.sitepoint.com/feed/", + "type": "rss", + "category": "frontend", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Google Developers Blog", + "url": "https://developers.googleblog.com/feeds/posts/default", + "type": "rss", + "category": "programming", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "Atom 1.0", + "encoding": "UTF-8" + } + }, + { + "name": "GitHub Blog", + "url": "https://github.blog/feed/", + "type": "rss", + "category": "programming", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "OpenAI Blog", + "url": "https://openai.com/blog/rss/", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Google AI Blog", + "url": "https://ai.googleblog.com/feeds/posts/default", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "Atom 1.0", + "encoding": "UTF-8" + } + }, + { + "name": "Machine Learning Mastery", + "url": "https://machinelearningmastery.com/blog/feed/", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Towards Data Science", + "url": "https://towardsdatascience.com/feed", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "MIT News - AI", + "url": "https://news.mit.edu/rss/topic/artificial-intelligence2", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "DeepMind", + "url": "https://deepmind.com/blog/feed/basic/", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "The Hacker News", + "url": "http://feeds.feedburner.com/TheHackersNews", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Dark Reading", + "url": "https://www.darkreading.com/rss.xml", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Krebs on Security", + "url": "https://krebsonsecurity.com/feed/", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Graham Cluley", + "url": "https://grahamcluley.com/feed/", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Troy Hunt", + "url": "https://www.troyhunt.com/rss/", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Schneier on Security", + "url": "https://www.schneier.com/blog/atom.xml", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "Atom 1.0", + "encoding": "UTF-8" + } + }, + { + "name": "DevOps.com", + "url": "https://devops.com/feed/", + "type": "rss", + "category": "devops", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "The New Stack", + "url": "https://thenewstack.io/feed/", + "type": "rss", + "category": "devops", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "AWS News Blog", + "url": "https://aws.amazon.com/blogs/aws/feed/", + "type": "rss", + "category": "cloud", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Azure Blog", + "url": "https://azure.microsoft.com/en-us/blog/feed/", + "type": "rss", + "category": "cloud", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Kubernetes Blog", + "url": "https://kubernetes.io/feed.xml", + "type": "rss", + "category": "devops", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Android Developers Blog", + "url": "https://android-developers.googleblog.com/feeds/posts/default", + "type": "rss", + "category": "mobile", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "Atom 1.0", + "encoding": "UTF-8" + } + }, + { + "name": "iOS Developer News", + "url": "https://iosdevweekly.com/issues.rss", + "type": "rss", + "category": "mobile", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Ray Wenderlich", + "url": "https://www.raywenderlich.com/feed", + "type": "rss", + "category": "mobile", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "KDnuggets", + "url": "https://www.kdnuggets.com/feed", + "type": "rss", + "category": "data_science", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Data Science Central", + "url": "https://www.datasciencecentral.com/feed/", + "type": "rss", + "category": "data_science", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "CoinTelegraph", + "url": "https://cointelegraph.com/rss", + "type": "rss", + "category": "blockchain", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "CoinDesk", + "url": "https://www.coindesk.com/arc/outboundfeeds/rss/", + "type": "rss", + "category": "blockchain", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Blockchain News", + "url": "https://blockchain.news/feed", + "type": "rss", + "category": "blockchain", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "UX Collective", + "url": "https://uxdesign.cc/feed", + "type": "rss", + "category": "design", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "UX Planet", + "url": "https://uxplanet.org/feed", + "type": "rss", + "category": "design", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Nielsen Norman Group", + "url": "https://www.nngroup.com/feed/rss/", + "type": "rss", + "category": "design", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Netflix TechBlog", + "url": "https://netflixtechblog.com/feed", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Uber Engineering", + "url": "https://eng.uber.com/feed/", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Spotify Engineering", + "url": "https://engineering.atspotify.com/feed/", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Airbnb Engineering", + "url": "https://medium.com/feed/airbnb-engineering", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Facebook Engineering", + "url": "https://engineering.fb.com/feed/", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Dropbox Tech", + "url": "https://dropbox.tech/feed", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Slack Engineering", + "url": "https://slack.engineering/feed/", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Cloudflare Blog", + "url": "https://blog.cloudflare.com/rss/", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "GitHub Releases - Example: Kubernetes", + "url": "https://github.com/kubernetes/kubernetes/releases.atom", + "type": "atom", + "category": "releases", + "structure": { + "items": [ + "title", + "link", + "id", + "updated", + "content", + "author" + ], + "format": "Atom 1.0", + "encoding": "UTF-8", + "special_notes": "Replace 'kubernetes/kubernetes' with any owner/repo" + } + }, + { + "name": "GitHub Commits - Example: React", + "url": "https://github.com/facebook/react/commits/main.atom", + "type": "atom", + "category": "commits", + "structure": { + "items": [ + "title", + "link", + "id", + "updated", + "content", + "author" + ], + "format": "Atom 1.0", + "encoding": "UTF-8", + "special_notes": "Replace 'facebook/react' with any owner/repo and 'main' with branch" + } + }, + { + "name": "BBC Technology", + "url": "http://feeds.bbci.co.uk/news/technology/rss.xml", + "type": "rss", + "category": "news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Reuters Technology", + "url": "https://www.reutersagency.com/feed/?best-topics=tech&post_type=best", + "type": "rss", + "category": "news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Guardian Technology", + "url": "https://www.theguardian.com/technology/rss", + "type": "rss", + "category": "news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Slashdot", + "url": "http://rss.slashdot.org/Slashdot/slashdotMain", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "TechRadar", + "url": "https://www.techradar.com/rss", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "MIT Technology Review", + "url": "https://www.technologyreview.com/feed/", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Fast Company Technology", + "url": "https://www.fastcompany.com/technology/rss", + "type": "rss", + "category": "tech_news", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Codrops", + "url": "https://tympanus.net/codrops/feed/", + "type": "rss", + "category": "frontend", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "A List Apart", + "url": "https://alistapart.com/main/feed/", + "type": "rss", + "category": "frontend", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "web.dev", + "url": "https://web.dev/feed.xml", + "type": "rss", + "category": "frontend", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Mozilla Hacks", + "url": "https://hacks.mozilla.org/feed/", + "type": "rss", + "category": "frontend", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Chrome Developers", + "url": "https://developer.chrome.com/feeds/blog.xml", + "type": "rss", + "category": "frontend", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "SANS Internet Storm Center", + "url": "https://isc.sans.edu/rssfeed.xml", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "WeLiveSecurity", + "url": "https://www.welivesecurity.com/en/rss/feed/", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Securelist by Kaspersky", + "url": "https://securelist.com/feed/", + "type": "rss", + "category": "security", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "The Gradient", + "url": "https://thegradient.pub/rss/", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Distill", + "url": "https://distill.pub/rss.xml", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Berkeley AI Research", + "url": "https://bair.berkeley.edu/blog/feed.xml", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Unite.AI", + "url": "https://www.unite.ai/feed/", + "type": "rss", + "category": "ai_ml", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Google Cloud Blog", + "url": "https://cloud.google.com/blog/rss/", + "type": "rss", + "category": "cloud", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "category" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Docker Blog", + "url": "https://www.docker.com/blog/feed/", + "type": "rss", + "category": "devops", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "HashiCorp Blog", + "url": "https://www.hashicorp.com/blog/rss.xml", + "type": "rss", + "category": "devops", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "React Blog", + "url": "https://reactjs.org/feed.xml", + "type": "rss", + "category": "javascript", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Vue.js News", + "url": "https://news.vuejs.org/feed.rss", + "type": "rss", + "category": "javascript", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Angular Blog", + "url": "https://blog.angular.io/feed", + "type": "rss", + "category": "javascript", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Node.js Blog", + "url": "https://nodejs.org/en/feed/blog.xml", + "type": "rss", + "category": "javascript", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Ethereum Blog", + "url": "https://blog.ethereum.org/feed.xml", + "type": "rss", + "category": "blockchain", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Bitcoin Magazine", + "url": "https://bitcoinmagazine.com/feed", + "type": "rss", + "category": "blockchain", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Reddit - r/technology", + "url": "https://www.reddit.com/r/technology/.rss", + "type": "rss", + "category": "social_tech", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Reddit - r/programming", + "url": "https://www.reddit.com/r/programming/.rss", + "type": "rss", + "category": "social_tech", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Reddit - r/MachineLearning", + "url": "https://www.reddit.com/r/MachineLearning/.rss", + "type": "rss", + "category": "social_tech", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Product Hunt", + "url": "https://www.producthunt.com/feed", + "type": "rss", + "category": "product", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "First Round Review", + "url": "http://firstround.com/review/feed.xml", + "type": "rss", + "category": "startup", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Tom's Hardware", + "url": "https://www.tomshardware.com/feeds/all", + "type": "rss", + "category": "hardware", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "AnandTech", + "url": "https://www.anandtech.com/rss/", + "type": "rss", + "category": "hardware", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Twitter Engineering", + "url": "https://blog.twitter.com/engineering/en_us/blog.rss", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Instagram Engineering", + "url": "https://instagram-engineering.com/feed", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid", + "author" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "LinkedIn Engineering", + "url": "https://engineering.linkedin.com/blog.rss", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Stripe Blog", + "url": "https://stripe.com/blog/feed.rss", + "type": "rss", + "category": "engineering", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "RSS 2.0", + "encoding": "UTF-8" + } + }, + { + "name": "Stack Overflow - JavaScript", + "url": "https://stackoverflow.com/feeds/tag?tagnames=javascript&sort=newest", + "type": "rss", + "category": "qa", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "Atom 1.0", + "encoding": "UTF-8" + } + }, + { + "name": "Stack Overflow - Python", + "url": "https://stackoverflow.com/feeds/tag?tagnames=python&sort=newest", + "type": "rss", + "category": "qa", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "Atom 1.0", + "encoding": "UTF-8" + } + }, + { + "name": "Stack Overflow - React", + "url": "https://stackoverflow.com/feeds/tag?tagnames=reactjs&sort=newest", + "type": "rss", + "category": "qa", + "structure": { + "items": [ + "title", + "link", + "description", + "pubDate", + "guid" + ], + "format": "Atom 1.0", + "encoding": "UTF-8" + } + } +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6770bb5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.104.1 +uvicorn==0.24.0 +jinja2==3.1.2 +python-multipart==0.0.6 +dataset==1.6.2 +aiohttp==3.9.1 +feedparser==6.0.10 +websockets==12.0 +trafilatura==1.6.2 diff --git a/routers.py b/routers.py new file mode 100644 index 0000000..8c5b817 --- /dev/null +++ b/routers.py @@ -0,0 +1,457 @@ +from fastapi import APIRouter, Request, UploadFile, File, Form, WebSocket, WebSocketDisconnect +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates +import dataset +import json +import aiohttp +import feedparser +import asyncio +from datetime import datetime +import time +import trafilatura + +router = APIRouter() +templates = Jinja2Templates(directory="templates") +db = dataset.connect('sqlite:///feeds.db') + +# Browser-like headers +HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1' +} + +async def fetch_article_content(session, url): + try: + async with session.get(url, headers=HEADERS, timeout=aiohttp.ClientTimeout(total=10)) as response: + html = await response.text() + clean_text = trafilatura.extract(html) + return clean_text if clean_text else "" + except: + return "" + +async def perform_sync(): + feeds_table = db['feeds'] + articles_table = db['articles'] + feeds = list(feeds_table.all()) + + total_feeds = len(feeds) + completed = 0 + total_articles_added = 0 + total_articles_processed = 0 + failed_feeds = 0 + new_articles = [] + start_time = time.time() + timeout = 300 # 5 minutes + + # Create newspaper immediately at start + newspapers_table = db['newspapers'] + sync_time = datetime.now().isoformat() + newspaper_id = newspapers_table.insert({ + 'created_at': sync_time, + 'article_count': 0, + 'articles_json': json.dumps([]) + }) + + connector = aiohttp.TCPConnector(limit=10) + async with aiohttp.ClientSession(connector=connector) as session: + tasks = [] + for feed in feeds: + tasks.append(fetch_feed(session, feed['url'], feed['name'])) + + for coro in asyncio.as_completed(tasks): + elapsed = time.time() - start_time + if elapsed >= timeout: + break + + feed_url, feed_name, content, error = await coro + completed += 1 + + if error: + failed_feeds += 1 + continue + + parsed = feedparser.parse(content) + + for entry in parsed.entries: + total_articles_processed += 1 + article_link = entry.get('link', '') + + # Extract clean text content + clean_text = await fetch_article_content(session, article_link) if article_link else "" + + article_data = { + 'feed_name': feed_name, + 'feed_url': feed_url, + 'title': entry.get('title', ''), + 'link': article_link, + 'description': entry.get('description', '') or entry.get('summary', ''), + 'content': clean_text, + 'published': entry.get('published', '') or entry.get('updated', ''), + 'author': entry.get('author', ''), + 'guid': entry.get('id', '') or entry.get('link', ''), + 'created_at': datetime.now().isoformat(), + 'last_synchronized': datetime.now().isoformat() + } + + existing = articles_table.find_one(guid=article_data['guid']) + if not existing: + new_articles.append(article_data) + total_articles_added += 1 + + articles_table.upsert(article_data, ['guid']) + + feeds_table.update({ + 'id': [f for f in feeds if f['url'] == feed_url][0]['id'], + 'last_synced': datetime.now().isoformat() + }, ['id']) + + # Update newspaper immediately after each feed + newspapers_table.update({ + 'id': newspaper_id, + 'article_count': len(new_articles), + 'articles_json': json.dumps(new_articles) + }, ['id']) + + elapsed_total = time.time() - start_time + avg_req_per_sec = completed / elapsed_total if elapsed_total > 0 else 0 + + # Record sync log with detailed statistics + sync_logs_table = db['sync_logs'] + sync_logs_table.insert({ + 'sync_time': sync_time, + 'total_feeds': total_feeds, + 'completed_feeds': completed, + 'failed_feeds': failed_feeds, + 'total_articles_processed': total_articles_processed, + 'new_articles': total_articles_added, + 'elapsed_seconds': round(elapsed_total, 2), + 'avg_req_per_sec': round(avg_req_per_sec, 2), + 'timed_out': elapsed_total >= timeout + }) + + return { + 'total_feeds': total_feeds, + 'total_articles': total_articles_added, + 'elapsed': elapsed_total, + 'new_articles': new_articles + } + +async def run_sync_task(): + return await perform_sync() + +@router.get("/config", response_class=HTMLResponse) +async def index(request: Request): + return templates.TemplateResponse("index.html", {"request": request}) + +@router.get("/manage", response_class=HTMLResponse) +async def manage_list(request: Request): + feeds_table = db['feeds'] + feeds = list(feeds_table.all()) + return templates.TemplateResponse("manage_list.html", { + "request": request, + "feeds": feeds, + "total": len(feeds) + }) + +@router.get("/manage/create", response_class=HTMLResponse) +async def manage_create_page(request: Request): + return templates.TemplateResponse("manage_create.html", {"request": request}) + +@router.post("/manage/create") +async def manage_create( + name: str = Form(...), + url: str = Form(...), + type: str = Form(...), + category: str = Form(...) +): + feeds_table = db['feeds'] + feeds_table.insert({ + 'name': name, + 'url': url, + 'type': type, + 'category': category + }) + + return RedirectResponse(url="/manage", status_code=303) + +@router.get("/manage/upload", response_class=HTMLResponse) +async def manage_upload_page(request: Request): + return templates.TemplateResponse("manage_upload.html", {"request": request}) + +@router.post("/manage/upload") +async def manage_upload(file: UploadFile = File(...)): + content = await file.read() + feeds_data = json.loads(content) + + feeds_table = db['feeds'] + + for feed in feeds_data: + feeds_table.upsert(feed, ['url']) + + return RedirectResponse(url="/manage", status_code=303) + +@router.get("/manage/edit/{feed_id}", response_class=HTMLResponse) +async def manage_edit_page(request: Request, feed_id: int): + feeds_table = db['feeds'] + feed = feeds_table.find_one(id=feed_id) + return templates.TemplateResponse("manage_edit.html", { + "request": request, + "feed": feed + }) + +@router.post("/manage/edit/{feed_id}") +async def manage_edit( + feed_id: int, + name: str = Form(...), + url: str = Form(...), + type: str = Form(...), + category: str = Form(...) +): + feeds_table = db['feeds'] + feeds_table.update({ + 'id': feed_id, + 'name': name, + 'url': url, + 'type': type, + 'category': category + }, ['id']) + + return RedirectResponse(url="/manage", status_code=303) + +@router.post("/manage/delete/{feed_id}") +async def manage_delete(feed_id: int): + feeds_table = db['feeds'] + feeds_table.delete(id=feed_id) + + return RedirectResponse(url="/manage", status_code=303) + +@router.get("/manage/sync", response_class=HTMLResponse) +async def manage_sync_page(request: Request): + feeds_table = db['feeds'] + total_feeds = len(list(feeds_table.all())) + articles_table = db['articles'] + total_articles = len(list(articles_table.all())) + + return templates.TemplateResponse("manage_sync.html", { + "request": request, + "total_feeds": total_feeds, + "total_articles": total_articles + }) + +async def fetch_feed(session, feed_url, feed_name): + try: + async with session.get(feed_url, headers=HEADERS, timeout=aiohttp.ClientTimeout(total=30)) as response: + content = await response.text() + return feed_url, feed_name, content, None + except Exception as e: + return feed_url, feed_name, None, str(e) + +@router.websocket("/ws/sync") +async def websocket_sync(websocket: WebSocket): + await websocket.accept() + + try: + feeds_table = db['feeds'] + articles_table = db['articles'] + feeds = list(feeds_table.all()) + + total_feeds = len(feeds) + completed = 0 + total_articles_added = 0 + new_articles = [] + start_time = time.time() + timeout = 300 # 5 minutes + + await websocket.send_json({ + "type": "start", + "total": total_feeds, + "message": f"Starting synchronization of {total_feeds} feeds (5 minute timeout)..." + }) + + # Create newspaper immediately at start + newspapers_table = db['newspapers'] + sync_time = datetime.now().isoformat() + newspaper_id = newspapers_table.insert({ + 'created_at': sync_time, + 'article_count': 0, + 'articles_json': json.dumps([]) + }) + + connector = aiohttp.TCPConnector(limit=10) + async with aiohttp.ClientSession(connector=connector) as session: + tasks = [] + for feed in feeds: + tasks.append(fetch_feed(session, feed['url'], feed['name'])) + + for coro in asyncio.as_completed(tasks): + elapsed = time.time() - start_time + if elapsed >= timeout: + await websocket.send_json({ + "type": "timeout", + "message": "5 minute timeout reached. Newspaper already contains all articles found so far...", + "completed": completed, + "total": total_feeds + }) + break + + feed_url, feed_name, content, error = await coro + completed += 1 + req_per_sec = completed / elapsed if elapsed > 0 else 0 + + if error: + await websocket.send_json({ + "type": "error", + "feed": feed_name, + "url": feed_url, + "error": error, + "completed": completed, + "total": total_feeds, + "req_per_sec": round(req_per_sec, 2) + }) + continue + + await websocket.send_json({ + "type": "fetching", + "feed": feed_name, + "url": feed_url, + "completed": completed, + "total": total_feeds, + "req_per_sec": round(req_per_sec, 2) + }) + + parsed = feedparser.parse(content) + articles_count = 0 + + for entry in parsed.entries: + article_data = { + 'feed_name': feed_name, + 'feed_url': feed_url, + 'title': entry.get('title', ''), + 'link': entry.get('link', ''), + 'description': entry.get('description', '') or entry.get('summary', ''), + 'published': entry.get('published', '') or entry.get('updated', ''), + 'author': entry.get('author', ''), + 'guid': entry.get('id', '') or entry.get('link', ''), + 'created_at': datetime.now().isoformat(), + 'last_synchronized': datetime.now().isoformat() + } + + existing = articles_table.find_one(guid=article_data['guid']) + if not existing: + new_articles.append(article_data) + articles_count += 1 + + articles_table.upsert(article_data, ['guid']) + + total_articles_added += articles_count + + feeds_table.update({ + 'id': [f for f in feeds if f['url'] == feed_url][0]['id'], + 'last_synced': datetime.now().isoformat() + }, ['id']) + + # Update newspaper immediately after each feed + newspapers_table.update({ + 'id': newspaper_id, + 'article_count': len(new_articles), + 'articles_json': json.dumps(new_articles) + }, ['id']) + + await websocket.send_json({ + "type": "parsed", + "feed": feed_name, + "articles": articles_count, + "completed": completed, + "total": total_feeds, + "total_articles": total_articles_added, + "req_per_sec": round(req_per_sec, 2) + }) + + elapsed_total = time.time() - start_time + + # Record sync log + sync_logs_table = db['sync_logs'] + sync_logs_table.insert({ + 'sync_time': sync_time, + 'total_feeds': total_feeds, + 'new_articles': total_articles_added, + 'elapsed_seconds': round(elapsed_total, 2) + }) + + await websocket.send_json({ + "type": "complete", + "total_feeds": total_feeds, + "total_articles": total_articles_added, + "elapsed": round(elapsed_total, 2), + "avg_req_per_sec": round(total_feeds / elapsed_total if elapsed_total > 0 else 0, 2) + }) + + except WebSocketDisconnect: + pass + except Exception as e: + await websocket.send_json({ + "type": "error", + "message": str(e) + }) + +@router.get("/newspapers", response_class=HTMLResponse) +async def newspapers_list(request: Request): + newspapers_table = db['newspapers'] + newspapers = list(newspapers_table.all()) + newspapers.reverse() + + return templates.TemplateResponse("newspapers_list.html", { + "request": request, + "newspapers": newspapers + }) + +@router.get("/sync-logs", response_class=HTMLResponse) +async def sync_logs_list(request: Request): + sync_logs_table = db['sync_logs'] + sync_logs = list(sync_logs_table.all()) + sync_logs.reverse() + + return templates.TemplateResponse("sync_logs_list.html", { + "request": request, + "sync_logs": sync_logs + }) + +@router.get("/newspaper/{newspaper_id}", response_class=HTMLResponse) +async def newspaper_view(request: Request, newspaper_id: int): + newspapers_table = db['newspapers'] + newspaper = newspapers_table.find_one(id=newspaper_id) + + if not newspaper: + return RedirectResponse(url="/newspapers") + + articles = json.loads(newspaper['articles_json']) + + return templates.TemplateResponse("newspaper_view.html", { + "request": request, + "newspaper": newspaper, + "articles": articles + }) + +@router.get("/", response_class=HTMLResponse) +async def newspaper_latest(request: Request): + newspapers_table = db['newspapers'] + newspaper = None + try: + newspaper = list(db.query("select * from newspapers order by id desc limit 1"))[0] + except IndexError: + pass + + if not newspaper: + return RedirectResponse(url="/newspapers") + + articles = json.loads(newspaper['articles_json']) + + return templates.TemplateResponse("newspaper_view.html", { + "request": request, + "newspaper": newspaper, + "articles": articles + }) diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..378face --- /dev/null +++ b/templates/base.html @@ -0,0 +1,162 @@ + + + + + + {% block title %}RSS Feed Manager{% endblock %} + + {% block extra_css %}{% endblock %} + + +
+ RSS Feed Manager +
+ +
+ {% block content %}{% endblock %} +
+ + {% block extra_js %}{% endblock %} + + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..811888b --- /dev/null +++ b/templates/index.html @@ -0,0 +1,45 @@ +{% extends "base.html" %} + +{% block title %}RSS Feed Manager{% endblock %} + +{% block extra_css %} + +{% endblock %} + +{% block content %} +
+

RSS Feed Manager

+

+ Manage your RSS feeds efficiently +

+
+ Manage Feeds + View Newspapers + Sync Logs +
+
+{% endblock %} diff --git a/templates/manage_create.html b/templates/manage_create.html new file mode 100644 index 0000000..3b03911 --- /dev/null +++ b/templates/manage_create.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} + +{% block title %}Create Feed - RSS Feed Manager{% endblock %} + +{% block content %} +

Create New Feed

+ +
+
+
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + Cancel +
+
+
+{% endblock %} diff --git a/templates/manage_edit.html b/templates/manage_edit.html new file mode 100644 index 0000000..6154ca1 --- /dev/null +++ b/templates/manage_edit.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} + +{% block title %}Edit Feed - RSS Feed Manager{% endblock %} + +{% block content %} +

Edit Feed

+ +
+
+
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + Cancel +
+
+
+{% endblock %} diff --git a/templates/manage_list.html b/templates/manage_list.html new file mode 100644 index 0000000..25699fc --- /dev/null +++ b/templates/manage_list.html @@ -0,0 +1,64 @@ +{% extends "base.html" %} + +{% block title %}Manage Feeds - RSS Feed Manager{% endblock %} + +{% block content %} +

Manage Feeds

+ +
+ Total Feeds: {{ total }} +
+ +
+ Create New Feed + Upload Feeds + Sync All Feeds + View Newspapers + Sync Logs + Back to Home +
+ +{% if feeds %} + + + + + + + + + + + + + {% for feed in feeds %} + + + + + + + + + {% endfor %} + +
NameURLTypeCategoryLast SyncedActions
{{ feed.name }}{{ feed.url }}{{ feed.type }}{{ feed.category }} + {% if feed.last_synced %} + {{ feed.last_synced[:19] }} + {% else %} + Never + {% endif %} + +
+ Edit +
+ +
+
+
+{% else %} +

+ No feeds found. Upload a JSON file to get started. +

+{% endif %} +{% endblock %} diff --git a/templates/manage_sync.html b/templates/manage_sync.html new file mode 100644 index 0000000..863e5c7 --- /dev/null +++ b/templates/manage_sync.html @@ -0,0 +1,240 @@ +{% extends "base.html" %} + +{% block title %}Sync Feeds - RSS Feed Manager{% endblock %} + +{% block extra_css %} + +{% endblock %} + +{% block content %} +
+

Synchronize RSS Feeds

+ +
+ Total Feeds: {{ total_feeds }} | Total Articles in DB: {{ total_articles }} +
+ +
+ + Back to Manage +
+ + + + + + +
+{% endblock %} + +{% block extra_js %} + +{% endblock %} diff --git a/templates/manage_upload.html b/templates/manage_upload.html new file mode 100644 index 0000000..fb8c79d --- /dev/null +++ b/templates/manage_upload.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} + +{% block title %}Upload Feeds - RSS Feed Manager{% endblock %} + +{% block content %} +

Upload Feeds

+ +
+
+
+ + +
+ +
+ + Cancel +
+
+ +
+ Note: The upload will synchronize feeds based on their URL. + Existing feeds with the same URL will be updated, and new feeds will be added. +
+
+{% endblock %} diff --git a/templates/newspaper_view.html b/templates/newspaper_view.html new file mode 100644 index 0000000..485aa32 --- /dev/null +++ b/templates/newspaper_view.html @@ -0,0 +1,198 @@ + + + + + + Tech News - {{ newspaper.created_at[:10] }} + + + +
+
+
TECH NEWS
+
{{ newspaper.created_at[:10] }} - {{ newspaper.created_at[11:19] }}
+
+ +
+

+ {{ newspaper.article_count }} New Articles from Latest Synchronization +

+
+ +
+ {% for article in articles %} +
+

+ {{ article.title }} +

+
+ {{ article.feed_name }} + {% if article.author %} + | By {{ article.author }} + {% endif %} + {% if article.published %} + | {{ article.published[:10] }} + {% endif %} +
+
+ {% set full_content = article.content if article.content else article.description %} + {% set clean_text = full_content|striptags %} + {% set display_text = clean_text[:500] if article.content else clean_text[:300] %} + {{ display_text }}{% if clean_text|length > (500 if article.content else 300) %}...{% endif %} +
+ + {% set words = full_content.split() %} + {% for word in words %} + {% if 'https' in word and ('.jpg' in word or '.jpeg' in word or '.png' in word or '.gif' in word or '.webp' in word) %} + Article image + {% endif %} + {% endfor %} +
+ {% endfor %} +
+ +
+ Generated from RSS Feed Manager | Synchronized at {{ newspaper.created_at[:19].replace('T', ' ') }} +
+
+ + diff --git a/templates/newspapers_list.html b/templates/newspapers_list.html new file mode 100644 index 0000000..3f61324 --- /dev/null +++ b/templates/newspapers_list.html @@ -0,0 +1,39 @@ +{% extends "base.html" %} + +{% block title %}Newspapers - RSS Feed Manager{% endblock %} + +{% block content %} +

Newspapers

+ +
+ View Sync Logs + Back to Manage +
+ +{% if newspapers %} + + + + + + + + + + {% for newspaper in newspapers %} + + + + + + {% endfor %} + +
Date & TimeArticlesActions
{{ newspaper.created_at[:19].replace('T', ' ') }}{{ newspaper.article_count }} + Read Newspaper +
+{% else %} +

+ No newspapers generated yet. Run a synchronization to generate one. +

+{% endif %} +{% endblock %} diff --git a/templates/sync_logs_list.html b/templates/sync_logs_list.html new file mode 100644 index 0000000..dd7c471 --- /dev/null +++ b/templates/sync_logs_list.html @@ -0,0 +1,73 @@ +{% extends "base.html" %} + +{% block title %}Synchronization Logs - RSS Feed Manager{% endblock %} + +{% block content %} +

Synchronization Logs

+ +
+ Back to Newspapers + Back to Manage +
+ +{% if sync_logs %} + + + + + + + + + + + + + + + + {% for log in sync_logs %} + + + + + + + + + + + + {% endfor %} + +
Sync TimeTotal FeedsCompletedFailedTotal ArticlesNew ArticlesDuration (s)Req/sStatus
{{ log.sync_time[:19].replace('T', ' ') }}{{ log.total_feeds }}{{ log.completed_feeds }} + {{ log.failed_feeds }} + {{ log.total_articles_processed }} + {{ log.new_articles }} + {{ log.elapsed_seconds }}{{ log.avg_req_per_sec }} + {% if log.timed_out %} + Timeout + {% else %} + Complete + {% endif %} +
+ +
+

Statistics Legend

+ +
+{% else %} +

+ No synchronization logs available. Run a sync to see logs here. +

+{% endif %} +{% endblock %}