philandClaude
25086fc01b
- Modular architecture with separate modules for scraping, parsing, security, validation, and caching - Comprehensive security measures including HTML sanitization, rate limiting, and input validation - Robust error handling with custom exceptions and retry logic - HTTP caching with ETags and Last-Modified headers for efficiency - Pre-compiled regex patterns for improved performance - Comprehensive test suite with 66 tests covering all major functionality - Docker support for containerized deployment - Configuration management with environment variable support - Working parser that successfully extracts 32 articles from Warhammer Community 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
111 lines
4.2 KiB
Python
111 lines
4.2 KiB
Python
"""HTML parsing and article extraction functionality."""
|
|
|
|
import logging
|
|
from datetime import datetime
|
|
from typing import List, Dict, Any, Optional
|
|
import pytz
|
|
from bs4 import BeautifulSoup
|
|
|
|
from .config import Config
|
|
from .validation import validate_link
|
|
from .exceptions import ParseError
|
|
from .security import sanitize_text_content, sanitize_html_content
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def sanitize_text(text: Optional[str]) -> str:
|
|
"""Sanitize text content to prevent injection attacks"""
|
|
return sanitize_text_content(text)
|
|
|
|
|
|
def extract_articles_from_html(html: str, base_url: str) -> List[Dict[str, Any]]:
|
|
"""Extract articles from HTML content."""
|
|
logger.info("Parsing HTML content with BeautifulSoup")
|
|
|
|
# Sanitize HTML content first for security
|
|
sanitized_html = sanitize_html_content(html)
|
|
|
|
try:
|
|
soup = BeautifulSoup(sanitized_html, 'html.parser')
|
|
except Exception as e:
|
|
raise ParseError(f"Failed to parse HTML content: {e}")
|
|
|
|
# Define a timezone (UTC in this case)
|
|
timezone = pytz.UTC
|
|
|
|
# Find all articles in the page - look for article elements with shared- classes (all article types)
|
|
all_articles = soup.find_all('article')
|
|
article_elements = []
|
|
for article in all_articles:
|
|
classes = article.get('class', [])
|
|
if classes and any('shared-' in cls for cls in classes):
|
|
article_elements.append(article)
|
|
logger.info(f"Found {len(article_elements)} article elements on page")
|
|
|
|
articles: List[Dict[str, Any]] = []
|
|
seen_urls: set = set() # Set to track seen URLs and avoid duplicates
|
|
|
|
for article in article_elements:
|
|
# Extract and sanitize the title
|
|
title_tag = None
|
|
for selector in Config.TITLE_SELECTORS:
|
|
class_name = selector.split('.')[1] if '.' in selector else selector
|
|
title_tag = article.find('h3', class_=class_name)
|
|
if title_tag:
|
|
break
|
|
|
|
raw_title = title_tag.text.strip() if title_tag else 'No title'
|
|
title = sanitize_text(raw_title)
|
|
|
|
# Extract and validate the link - look for btn-cover class first, then any anchor
|
|
link_tag = article.find('a', class_='btn-cover', href=True) or article.find('a', href=True)
|
|
raw_link = link_tag['href'] if link_tag else None
|
|
link = validate_link(raw_link, base_url)
|
|
|
|
# Skip this entry if the link is None or the URL has already been seen
|
|
if not link or link in seen_urls:
|
|
logger.debug(f"Skipping duplicate or invalid article: {title}")
|
|
continue # Skip duplicates or invalid entries
|
|
|
|
seen_urls.add(link) # Add the URL to the set of seen URLs
|
|
logger.debug(f"Processing article: {title[:50]}...")
|
|
|
|
# Extract the publication date and ignore reading time
|
|
date = None
|
|
for time_tag in article.find_all('time'):
|
|
raw_date = time_tag.text.strip()
|
|
|
|
# Ignore "min" time blocks (reading time)
|
|
if "min" not in raw_date.lower():
|
|
try:
|
|
# Parse the actual date (e.g., "05 Jun 25")
|
|
date = datetime.strptime(raw_date, '%d %b %y')
|
|
date = timezone.localize(date) # Localize with UTC
|
|
break # Stop after finding the correct date
|
|
except ValueError:
|
|
# Try alternative date formats if the first one fails
|
|
try:
|
|
# Try format like "Jun 05, 2025"
|
|
date = datetime.strptime(raw_date, '%b %d, %Y')
|
|
date = timezone.localize(date)
|
|
break
|
|
except ValueError:
|
|
continue
|
|
|
|
# If no valid date is found, use the current date as a fallback
|
|
if not date:
|
|
date = datetime.now(timezone)
|
|
|
|
# Add the article to the list with its publication date
|
|
articles.append({
|
|
'title': title,
|
|
'link': link,
|
|
'date': date
|
|
})
|
|
|
|
# Sort the articles by publication date (newest first)
|
|
articles.sort(key=lambda x: x['date'], reverse=True)
|
|
logger.info(f"Successfully extracted {len(articles)} unique articles")
|
|
|
|
return articles |