rss_warhammer-community/src/rss_scraper/cache.py

"""Caching utilities for avoiding redundant scraping."""

import os
import json
import hashlib
import logging
from datetime import datetime, timedelta
from typing import Optional, Dict, Any, List
import requests

from .config import Config
from .exceptions import FileOperationError

logger = logging.getLogger(__name__)


class ContentCache:
    """Cache for storing and retrieving scraped content."""
    
    def __init__(self, cache_dir: str = "cache"):
        self.cache_dir = cache_dir
        self.cache_file = os.path.join(cache_dir, "content_cache.json")
        self.etag_file = os.path.join(cache_dir, "etags.json")
        self.max_cache_age_hours = 24  # Cache expires after 24 hours
        
        # Ensure cache directory exists
        os.makedirs(cache_dir, exist_ok=True)
    
    def _get_cache_key(self, url: str) -> str:
        """Generate cache key from URL."""
        return hashlib.sha256(url.encode()).hexdigest()
    
    def _load_cache(self) -> Dict[str, Any]:
        """Load cache from file."""
        try:
            if os.path.exists(self.cache_file):
                with open(self.cache_file, 'r') as f:
                    return json.load(f)
        except Exception as e:
            logger.warning(f"Failed to load cache: {e}")
        return {}
    
    def _save_cache(self, cache_data: Dict[str, Any]) -> None:
        """Save cache to file."""
        try:
            with open(self.cache_file, 'w') as f:
                json.dump(cache_data, f, indent=2, default=str)
        except Exception as e:
            logger.error(f"Failed to save cache: {e}")
            raise FileOperationError(f"Failed to save cache: {e}")
    
    def _load_etags(self) -> Dict[str, str]:
        """Load ETags from file."""
        try:
            if os.path.exists(self.etag_file):
                with open(self.etag_file, 'r') as f:
                    return json.load(f)
        except Exception as e:
            logger.warning(f"Failed to load ETags: {e}")
        return {}
    
    def _save_etags(self, etag_data: Dict[str, str]) -> None:
        """Save ETags to file."""
        try:
            with open(self.etag_file, 'w') as f:
                json.dump(etag_data, f, indent=2)
        except Exception as e:
            logger.warning(f"Failed to save ETags: {e}")
    
    def _is_cache_valid(self, cached_entry: Dict[str, Any]) -> bool:
        """Check if cached entry is still valid."""
        try:
            cached_time = datetime.fromisoformat(cached_entry['timestamp'])
            expiry_time = cached_time + timedelta(hours=self.max_cache_age_hours)
            return datetime.now() < expiry_time
        except (KeyError, ValueError):
            return False
    
    def check_if_content_changed(self, url: str) -> Optional[Dict[str, str]]:
        """Check if content has changed using conditional requests."""
        etags = self._load_etags()
        cache_key = self._get_cache_key(url)
        
        headers = {}
        if cache_key in etags:
            headers['If-None-Match'] = etags[cache_key]
        
        try:
            logger.debug(f"Checking if content changed for {url}")
            response = requests.head(url, headers=headers, timeout=10)
            
            # 304 means not modified
            if response.status_code == 304:
                logger.info(f"Content not modified for {url}")
                return {'status': 'not_modified'}
            
            # Update ETag if available
            if 'etag' in response.headers:
                etags[cache_key] = response.headers['etag']
                self._save_etags(etags)
                logger.debug(f"Updated ETag for {url}")
            
            return {'status': 'modified', 'etag': response.headers.get('etag')}
            
        except requests.RequestException as e:
            logger.warning(f"Failed to check content modification for {url}: {e}")
            # If we can't check, assume it's modified
            return {'status': 'modified'}
    
    def get_cached_content(self, url: str) -> Optional[str]:
        """Get cached HTML content if available and valid."""
        cache_data = self._load_cache()
        cache_key = self._get_cache_key(url)
        
        if cache_key not in cache_data:
            logger.debug(f"No cached content for {url}")
            return None
        
        cached_entry = cache_data[cache_key]
        
        if not self._is_cache_valid(cached_entry):
            logger.debug(f"Cached content for {url} has expired")
            # Remove expired entry
            del cache_data[cache_key]
            self._save_cache(cache_data)
            return None
        
        logger.info(f"Using cached content for {url}")
        return cached_entry['content']
    
    def cache_content(self, url: str, content: str) -> None:
        """Cache HTML content with timestamp."""
        cache_data = self._load_cache()
        cache_key = self._get_cache_key(url)
        
        cache_data[cache_key] = {
            'url': url,
            'content': content,
            'timestamp': datetime.now().isoformat(),
            'size': len(content)
        }
        
        self._save_cache(cache_data)
        logger.info(f"Cached content for {url} ({len(content)} bytes)")
    
    def get_cached_articles(self, url: str) -> Optional[List[Dict[str, Any]]]:
        """Get cached articles if available and valid."""
        cache_data = self._load_cache()
        cache_key = self._get_cache_key(url) + "_articles"
        
        if cache_key not in cache_data:
            return None
        
        cached_entry = cache_data[cache_key]
        
        if not self._is_cache_valid(cached_entry):
            # Remove expired entry
            del cache_data[cache_key]
            self._save_cache(cache_data)
            return None
        
        logger.info(f"Using cached articles for {url}")
        return cached_entry['articles']
    
    def cache_articles(self, url: str, articles: List[Dict[str, Any]]) -> None:
        """Cache extracted articles."""
        cache_data = self._load_cache()
        cache_key = self._get_cache_key(url) + "_articles"
        
        # Convert datetime objects to strings for JSON serialization
        serializable_articles = []
        for article in articles:
            serializable_article = article.copy()
            if 'date' in serializable_article and hasattr(serializable_article['date'], 'isoformat'):
                serializable_article['date'] = serializable_article['date'].isoformat()
            serializable_articles.append(serializable_article)
        
        cache_data[cache_key] = {
            'url': url,
            'articles': serializable_articles,
            'timestamp': datetime.now().isoformat(),
            'count': len(articles)
        }
        
        self._save_cache(cache_data)
        logger.info(f"Cached {len(articles)} articles for {url}")
    
    def clear_cache(self) -> None:
        """Clear all cached content."""
        try:
            if os.path.exists(self.cache_file):
                os.remove(self.cache_file)
            if os.path.exists(self.etag_file):
                os.remove(self.etag_file)
            logger.info("Cache cleared successfully")
        except Exception as e:
            logger.error(f"Failed to clear cache: {e}")
            raise FileOperationError(f"Failed to clear cache: {e}")
    
    def get_cache_info(self) -> Dict[str, Any]:
        """Get information about cached content."""
        cache_data = self._load_cache()
        etags = self._load_etags()
        
        info = {
            'cache_file': self.cache_file,
            'etag_file': self.etag_file,
            'cache_entries': len(cache_data),
            'etag_entries': len(etags),
            'cache_size_bytes': 0
        }
        
        if os.path.exists(self.cache_file):
            info['cache_size_bytes'] = os.path.getsize(self.cache_file)
        
        return info