philandClaude
25086fc01b
- Modular architecture with separate modules for scraping, parsing, security, validation, and caching - Comprehensive security measures including HTML sanitization, rate limiting, and input validation - Robust error handling with custom exceptions and retry logic - HTTP caching with ETags and Last-Modified headers for efficiency - Pre-compiled regex patterns for improved performance - Comprehensive test suite with 66 tests covering all major functionality - Docker support for containerized deployment - Configuration management with environment variable support - Working parser that successfully extracts 32 articles from Warhammer Community 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
77 lines
3.0 KiB
Python
77 lines
3.0 KiB
Python
"""Configuration management for RSS Warhammer scraper."""
|
|
|
|
import os
|
|
from typing import List, Optional
|
|
|
|
|
|
class Config:
|
|
"""Configuration class for RSS scraper settings."""
|
|
|
|
# Security settings
|
|
ALLOWED_DOMAINS: List[str] = [
|
|
'warhammer-community.com',
|
|
'www.warhammer-community.com'
|
|
]
|
|
|
|
# Scraping limits
|
|
MAX_SCROLL_ITERATIONS: int = int(os.getenv('MAX_SCROLL_ITERATIONS', '5'))
|
|
MAX_CONTENT_SIZE: int = int(os.getenv('MAX_CONTENT_SIZE', str(10 * 1024 * 1024))) # 10MB
|
|
MAX_TITLE_LENGTH: int = int(os.getenv('MAX_TITLE_LENGTH', '500'))
|
|
|
|
# Timing settings
|
|
SCROLL_DELAY_SECONDS: float = float(os.getenv('SCROLL_DELAY_SECONDS', '2.0'))
|
|
PAGE_TIMEOUT_MS: int = int(os.getenv('PAGE_TIMEOUT_MS', '120000'))
|
|
|
|
# Default URLs and paths
|
|
DEFAULT_URL: str = os.getenv('DEFAULT_URL', 'https://www.warhammer-community.com/en-gb/')
|
|
DEFAULT_OUTPUT_DIR: str = os.getenv('DEFAULT_OUTPUT_DIR', '.')
|
|
|
|
# File names
|
|
RSS_FILENAME: str = os.getenv('RSS_FILENAME', 'warhammer_rss_feed.xml')
|
|
DEBUG_HTML_FILENAME: str = os.getenv('DEBUG_HTML_FILENAME', 'page.html')
|
|
|
|
# Feed metadata
|
|
FEED_TITLE: str = os.getenv('FEED_TITLE', 'Warhammer Community RSS Feed')
|
|
FEED_DESCRIPTION: str = os.getenv('FEED_DESCRIPTION', 'Latest Warhammer Community Articles')
|
|
|
|
# Security patterns to remove from content
|
|
DANGEROUS_PATTERNS: List[str] = [
|
|
'<script', '</script', 'javascript:', 'data:', 'vbscript:'
|
|
]
|
|
|
|
# CSS selectors for article parsing
|
|
TITLE_SELECTORS: List[str] = [
|
|
'h3.newsCard-title-sm',
|
|
'h3.newsCard-title-lg'
|
|
]
|
|
|
|
@classmethod
|
|
def get_output_dir(cls, override: Optional[str] = None) -> str:
|
|
"""Get output directory with optional override."""
|
|
return override or cls.DEFAULT_OUTPUT_DIR
|
|
|
|
@classmethod
|
|
def get_allowed_domains(cls) -> List[str]:
|
|
"""Get list of allowed domains for scraping."""
|
|
env_domains = os.getenv('ALLOWED_DOMAINS')
|
|
if env_domains:
|
|
return [domain.strip() for domain in env_domains.split(',')]
|
|
return cls.ALLOWED_DOMAINS
|
|
|
|
@classmethod
|
|
def validate_config(cls) -> None:
|
|
"""Validate configuration values."""
|
|
if cls.MAX_SCROLL_ITERATIONS < 0:
|
|
raise ValueError("MAX_SCROLL_ITERATIONS must be non-negative")
|
|
if cls.MAX_CONTENT_SIZE <= 0:
|
|
raise ValueError("MAX_CONTENT_SIZE must be positive")
|
|
if cls.MAX_TITLE_LENGTH <= 0:
|
|
raise ValueError("MAX_TITLE_LENGTH must be positive")
|
|
if cls.SCROLL_DELAY_SECONDS < 0:
|
|
raise ValueError("SCROLL_DELAY_SECONDS must be non-negative")
|
|
if cls.PAGE_TIMEOUT_MS <= 0:
|
|
raise ValueError("PAGE_TIMEOUT_MS must be positive")
|
|
if not cls.DEFAULT_URL.startswith(('http://', 'https://')):
|
|
raise ValueError("DEFAULT_URL must be a valid HTTP/HTTPS URL")
|
|
if not cls.get_allowed_domains():
|
|
raise ValueError("ALLOWED_DOMAINS cannot be empty") |