Trixy3/trixy_core/scheduler/persistence.py

"""
Persistence System for Trixy Scheduler

This module provides comprehensive persistence capabilities for the scheduler system,
including schedule storage, loading, backup, and recovery functionality. Supports
multiple storage formats and ensures data integrity.

Key Features:
- JSON-based schedule storage with human-readable format
- Automatic backup and recovery mechanisms
- Data validation and integrity checking
- Atomic operations to prevent data corruption
- Version management and migration support
- Compression and encryption support (optional)
- Thread-safe file operations

Storage Format:
- Primary format: JSON with complete schedule definitions
- Backup format: Timestamped copies with rotation
- Migration support: Version-aware loading and conversion
- Validation: Schema validation and data integrity checks

Usage:
    from trixy_core.scheduler.persistence import SchedulePersistence
    
    # Create persistence manager
    persistence = SchedulePersistence("/path/to/schedules.json")
    
    # Save schedules
    schedules = [schedule1, schedule2, schedule3]
    persistence.save_schedules(schedules)
    
    # Load schedules
    loaded_schedules = persistence.load_schedules()
    
    # Create backup
    backup_path = persistence.create_backup()
    
    # Restore from backup
    persistence.restore_from_backup(backup_path)
"""

import os
import json
import time
import shutil
import gzip
import threading
import tempfile
from datetime import datetime, timedelta
from pathlib import Path
from typing import List, Dict, Any, Optional, Union, Tuple
from dataclasses import dataclass, field
from enum import Enum
import hashlib
import uuid


def pprint(message: str) -> None:
    """
    Persistence logging function that adapts based on mode.
    Uses the same pattern as specified in CLAUDE.md.
    """
    print(f"[SCHEDULER.PERSISTENCE] {message}")


class PersistenceError(Exception):
    """Base exception for persistence-related errors."""
    pass


class ValidationError(PersistenceError):
    """Raised when data validation fails."""
    pass


class CorruptionError(PersistenceError):
    """Raised when data corruption is detected."""
    pass


class StorageFormat(Enum):
    """Supported storage formats."""
    JSON = "json"
    JSON_COMPRESSED = "json.gz"


class BackupPolicy(Enum):
    """Backup retention policies."""
    NONE = "none"
    DAILY = "daily"
    WEEKLY = "weekly"
    MONTHLY = "monthly"
    CUSTOM = "custom"


@dataclass
class PersistenceConfig:
    """Configuration for persistence system."""
    storage_path: str
    backup_directory: Optional[str] = None
    backup_policy: BackupPolicy = BackupPolicy.DAILY
    max_backups: int = 30
    compress_backups: bool = True
    validate_on_load: bool = True
    atomic_writes: bool = True
    file_permissions: int = 0o644
    create_directories: bool = True
    
    def __post_init__(self):
        """Post-initialization validation."""
        if self.max_backups < 0:
            raise ValueError("Max backups cannot be negative")
        
        if not self.storage_path:
            raise ValueError("Storage path is required")


@dataclass
class BackupInfo:
    """Information about a backup file."""
    path: str
    timestamp: float
    size_bytes: int
    checksum: str
    schedule_count: int
    version: str = "1.0.0"
    
    @property
    def age_hours(self) -> float:
        """Get backup age in hours."""
        return (time.time() - self.timestamp) / 3600
    
    @property
    def creation_date(self) -> datetime:
        """Get backup creation date."""
        return datetime.fromtimestamp(self.timestamp)
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary."""
        return {
            'path': self.path,
            'timestamp': self.timestamp,
            'size_bytes': self.size_bytes,
            'checksum': self.checksum,
            'schedule_count': self.schedule_count,
            'version': self.version,
            'creation_date': self.creation_date.isoformat(),
            'age_hours': self.age_hours
        }


class ScheduleStorage:
    """
    Low-level storage operations for schedule data.
    
    Handles the actual file I/O operations with integrity checking
    and atomic operations.
    """
    
    def __init__(self, config: PersistenceConfig):
        """
        Initialize storage.
        
        Args:
            config: Persistence configuration
        """
        self.config = config
        self._lock = threading.RLock()
        
        # Ensure storage directory exists
        if config.create_directories:
            storage_dir = os.path.dirname(config.storage_path)
            if storage_dir:
                os.makedirs(storage_dir, exist_ok=True)
            
            if config.backup_directory:
                os.makedirs(config.backup_directory, exist_ok=True)
    
    def read_data(self, file_path: str) -> Dict[str, Any]:
        """
        Read data from file with integrity checking.
        
        Args:
            file_path: Path to file to read
            
        Returns:
            Dict[str, Any]: Loaded data
            
        Raises:
            PersistenceError: If file cannot be read or is corrupted
        """
        if not os.path.exists(file_path):
            raise PersistenceError(f"File does not exist: {file_path}")
        
        try:
            # Determine format from extension
            if file_path.endswith('.gz'):
                with gzip.open(file_path, 'rt', encoding='utf-8') as f:
                    data = json.load(f)
            else:
                with open(file_path, 'r', encoding='utf-8') as f:
                    data = json.load(f)
            
            # Validate data structure
            if self.config.validate_on_load:
                self._validate_data_structure(data)
            
            return data
            
        except json.JSONDecodeError as e:
            raise CorruptionError(f"Invalid JSON in file {file_path}: {e}")
        except Exception as e:
            raise PersistenceError(f"Failed to read file {file_path}: {e}")
    
    def write_data(self, file_path: str, data: Dict[str, Any], compressed: bool = False) -> None:
        """
        Write data to file with atomic operations.
        
        Args:
            file_path: Path to file to write
            data: Data to write
            compressed: Whether to compress the file
            
        Raises:
            PersistenceError: If file cannot be written
        """
        with self._lock:
            # Validate data before writing
            self._validate_data_structure(data)
            
            if self.config.atomic_writes:
                self._write_atomic(file_path, data, compressed)
            else:
                self._write_direct(file_path, data, compressed)
    
    def _write_atomic(self, file_path: str, data: Dict[str, Any], compressed: bool) -> None:
        """Write data atomically using temporary file."""
        temp_path = f"{file_path}.tmp.{uuid.uuid4().hex[:8]}"
        
        try:
            # Write to temporary file
            self._write_direct(temp_path, data, compressed)
            
            # Atomic move to final location
            shutil.move(temp_path, file_path)
            
            # Set file permissions
            os.chmod(file_path, self.config.file_permissions)
            
        except Exception as e:
            # Clean up temporary file on error
            if os.path.exists(temp_path):
                try:
                    os.remove(temp_path)
                except:
                    pass
            raise PersistenceError(f"Failed to write file {file_path}: {e}")
    
    def _write_direct(self, file_path: str, data: Dict[str, Any], compressed: bool) -> None:
        """Write data directly to file."""
        try:
            if compressed:
                with gzip.open(file_path, 'wt', encoding='utf-8') as f:
                    json.dump(data, f, indent=2, ensure_ascii=False, sort_keys=True)
            else:
                with open(file_path, 'w', encoding='utf-8') as f:
                    json.dump(data, f, indent=2, ensure_ascii=False, sort_keys=True)
                    
        except Exception as e:
            raise PersistenceError(f"Failed to write data to {file_path}: {e}")
    
    def _validate_data_structure(self, data: Dict[str, Any]) -> None:
        """
        Validate data structure.
        
        Args:
            data: Data to validate
            
        Raises:
            ValidationError: If data structure is invalid
        """
        required_fields = ['version', 'timestamp', 'schedules']
        
        for field in required_fields:
            if field not in data:
                raise ValidationError(f"Missing required field: {field}")
        
        if not isinstance(data['schedules'], list):
            raise ValidationError("Schedules field must be a list")
        
        # Validate version format
        version = data['version']
        if not isinstance(version, str) or not version.count('.') >= 1:
            raise ValidationError(f"Invalid version format: {version}")
    
    def calculate_checksum(self, file_path: str) -> str:
        """
        Calculate file checksum.
        
        Args:
            file_path: Path to file
            
        Returns:
            str: MD5 checksum
        """
        hash_md5 = hashlib.md5()
        
        try:
            with open(file_path, 'rb') as f:
                for chunk in iter(lambda: f.read(4096), b""):
                    hash_md5.update(chunk)
            return hash_md5.hexdigest()
        except Exception as e:
            raise PersistenceError(f"Failed to calculate checksum for {file_path}: {e}")
    
    def get_file_info(self, file_path: str) -> Dict[str, Any]:
        """
        Get file information.
        
        Args:
            file_path: Path to file
            
        Returns:
            Dict[str, Any]: File information
        """
        try:
            stat = os.stat(file_path)
            return {
                'path': file_path,
                'size_bytes': stat.st_size,
                'modified_time': stat.st_mtime,
                'created_time': stat.st_ctime,
                'permissions': oct(stat.st_mode)[-3:],
                'exists': True
            }
        except FileNotFoundError:
            return {
                'path': file_path,
                'exists': False
            }
        except Exception as e:
            raise PersistenceError(f"Failed to get file info for {file_path}: {e}")


class SchedulePersistence:
    """
    High-level persistence manager for schedule data.
    
    Provides methods for saving, loading, backup, and recovery
    of schedule configurations.
    """
    
    def __init__(
        self,
        storage_path: str,
        backup_directory: Optional[str] = None,
        config: Optional[PersistenceConfig] = None
    ):
        """
        Initialize persistence manager.
        
        Args:
            storage_path: Path to main storage file
            backup_directory: Directory for backups (optional)
            config: Persistence configuration (optional)
        """
        if config is None:
            config = PersistenceConfig(
                storage_path=storage_path,
                backup_directory=backup_directory
            )
        
        self.config = config
        self.storage = ScheduleStorage(config)
        self._lock = threading.RLock()
        
        # Default backup directory
        if not self.config.backup_directory:
            self.config.backup_directory = os.path.join(
                os.path.dirname(storage_path), 'backups'
            )
        
        pprint(f"SchedulePersistence initialized: {storage_path}")
    
    def save_schedules(self, schedules: List['ScheduleEntry']) -> None:
        """
        Save schedules to storage.
        
        Args:
            schedules: List of schedule entries to save
            
        Raises:
            PersistenceError: If save operation fails
        """
        with self._lock:
            pprint(f"Saving {len(schedules)} schedules to {self.config.storage_path}")
            
            # Create backup before saving if file exists
            if os.path.exists(self.config.storage_path):
                try:
                    self._create_backup_if_needed()
                except Exception as e:
                    pprint(f"Warning: Failed to create backup before save: {e}")
            
            # Convert schedules to dictionary format
            data = self._schedules_to_dict(schedules)
            
            # Write to storage
            compressed = self.config.storage_path.endswith('.gz')
            self.storage.write_data(self.config.storage_path, data, compressed)
            
            pprint(f"Successfully saved {len(schedules)} schedules")
    
    def load_schedules(self) -> List['ScheduleEntry']:
        """
        Load schedules from storage.
        
        Returns:
            List[ScheduleEntry]: List of loaded schedule entries
            
        Raises:
            PersistenceError: If load operation fails
        """
        if not os.path.exists(self.config.storage_path):
            pprint(f"Storage file does not exist: {self.config.storage_path}")
            return []
        
        with self._lock:
            pprint(f"Loading schedules from {self.config.storage_path}")
            
            try:
                data = self.storage.read_data(self.config.storage_path)
                schedules = self._dict_to_schedules(data)
                
                pprint(f"Successfully loaded {len(schedules)} schedules")
                return schedules
                
            except Exception as e:
                pprint(f"Failed to load schedules: {e}")
                
                # Try to recover from backup
                return self._recover_from_backup()
    
    def _schedules_to_dict(self, schedules: List['ScheduleEntry']) -> Dict[str, Any]:
        """
        Convert schedules to dictionary format.
        
        Args:
            schedules: List of schedule entries
            
        Returns:
            Dict[str, Any]: Dictionary representation
        """
        return {
            'version': '1.0.0',
            'timestamp': time.time(),
            'created_date': datetime.now().isoformat(),
            'schedule_count': len(schedules),
            'schedules': [schedule.to_dict() for schedule in schedules],
            'metadata': {
                'created_by': 'Trixy Scheduler System',
                'format': 'json',
                'encoding': 'utf-8'
            }
        }
    
    def _dict_to_schedules(self, data: Dict[str, Any]) -> List['ScheduleEntry']:
        """
        Convert dictionary format to schedules.
        
        Args:
            data: Dictionary data
            
        Returns:
            List[ScheduleEntry]: List of schedule entries
        """
        # Import here to avoid circular imports
        from .schedule_entry import ScheduleEntry
        from .triggers import TriggerFactory
        from .actions import ActionFactory
        
        schedules = []
        
        for schedule_data in data.get('schedules', []):
            try:
                # Create schedule entry
                schedule = ScheduleEntry(
                    name=schedule_data['name'],
                    description=schedule_data.get('description', ''),
                    enabled=schedule_data.get('enabled', True)
                )
                
                # Restore triggers
                for trigger_data in schedule_data.get('triggers', []):
                    trigger = TriggerFactory.from_dict(trigger_data)
                    schedule.add_trigger(trigger)
                
                # Restore actions
                for action_data in schedule_data.get('actions', []):
                    action = ActionFactory.from_dict(action_data)
                    schedule.add_action(action)
                
                # Restore metadata
                schedule.created_at = schedule_data.get('created_at', schedule.created_at)
                schedule.last_modified = schedule_data.get('last_modified', schedule.last_modified)
                schedule.last_executed = schedule_data.get('last_executed')
                schedule.execution_count = schedule_data.get('execution_count', 0)
                schedule.error_count = schedule_data.get('error_count', 0)
                
                schedules.append(schedule)
                
            except Exception as e:
                pprint(f"Failed to load schedule '{schedule_data.get('name', 'unknown')}': {e}")
                # Continue loading other schedules
        
        return schedules
    
    def create_backup(self, backup_name: Optional[str] = None) -> str:
        """
        Create a backup of the current schedules.
        
        Args:
            backup_name: Optional custom backup name
            
        Returns:
            str: Path to created backup
            
        Raises:
            PersistenceError: If backup creation fails
        """
        if not os.path.exists(self.config.storage_path):
            raise PersistenceError("Cannot create backup: storage file does not exist")
        
        with self._lock:
            # Generate backup filename
            if backup_name is None:
                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
                backup_name = f"schedules_backup_{timestamp}"
            
            # Add appropriate extension
            if self.config.compress_backups:
                backup_path = os.path.join(self.config.backup_directory, f"{backup_name}.json.gz")
            else:
                backup_path = os.path.join(self.config.backup_directory, f"{backup_name}.json")
            
            # Ensure backup directory exists
            os.makedirs(os.path.dirname(backup_path), exist_ok=True)
            
            try:
                # Read current data
                data = self.storage.read_data(self.config.storage_path)
                
                # Add backup metadata
                data['backup_info'] = {
                    'original_path': self.config.storage_path,
                    'backup_timestamp': time.time(),
                    'backup_date': datetime.now().isoformat()
                }
                
                # Write backup
                self.storage.write_data(backup_path, data, self.config.compress_backups)
                
                pprint(f"Created backup: {backup_path}")
                
                # Clean up old backups
                self._cleanup_old_backups()
                
                return backup_path
                
            except Exception as e:
                raise PersistenceError(f"Failed to create backup: {e}")
    
    def restore_from_backup(self, backup_path: str) -> None:
        """
        Restore schedules from a backup.
        
        Args:
            backup_path: Path to backup file
            
        Raises:
            PersistenceError: If restore operation fails
        """
        if not os.path.exists(backup_path):
            raise PersistenceError(f"Backup file does not exist: {backup_path}")
        
        with self._lock:
            pprint(f"Restoring schedules from backup: {backup_path}")
            
            try:
                # Create backup of current state first
                if os.path.exists(self.config.storage_path):
                    emergency_backup = self.create_backup("emergency_pre_restore")
                    pprint(f"Created emergency backup: {emergency_backup}")
                
                # Read backup data
                backup_data = self.storage.read_data(backup_path)
                
                # Remove backup metadata before restoring
                if 'backup_info' in backup_data:
                    del backup_data['backup_info']
                
                # Write to main storage
                compressed = self.config.storage_path.endswith('.gz')
                self.storage.write_data(self.config.storage_path, backup_data, compressed)
                
                pprint(f"Successfully restored schedules from backup")
                
            except Exception as e:
                raise PersistenceError(f"Failed to restore from backup: {e}")
    
    def _recover_from_backup(self) -> List['ScheduleEntry']:
        """
        Attempt to recover schedules from most recent backup.
        
        Returns:
            List[ScheduleEntry]: Recovered schedules or empty list
        """
        pprint("Attempting to recover from backup...")
        
        backups = self.list_backups()
        if not backups:
            pprint("No backups available for recovery")
            return []
        
        # Try most recent backup first
        for backup_info in backups:
            try:
                pprint(f"Trying to recover from backup: {backup_info.path}")
                data = self.storage.read_data(backup_info.path)
                schedules = self._dict_to_schedules(data)
                
                pprint(f"Successfully recovered {len(schedules)} schedules from backup")
                return schedules
                
            except Exception as e:
                pprint(f"Failed to recover from backup {backup_info.path}: {e}")
                continue
        
        pprint("All backup recovery attempts failed")
        return []
    
    def _create_backup_if_needed(self) -> None:
        """Create backup if needed based on backup policy."""
        if self.config.backup_policy == BackupPolicy.NONE:
            return
        
        # Check if backup is needed
        backups = self.list_backups()
        
        if not backups:
            # No backups exist, create one
            self.create_backup()
            return
        
        # Check if we need a new backup based on policy
        latest_backup = backups[0]  # list_backups returns sorted by timestamp desc
        
        hours_since_backup = latest_backup.age_hours
        
        backup_needed = False
        
        if self.config.backup_policy == BackupPolicy.DAILY and hours_since_backup >= 24:
            backup_needed = True
        elif self.config.backup_policy == BackupPolicy.WEEKLY and hours_since_backup >= 168:  # 7 days
            backup_needed = True
        elif self.config.backup_policy == BackupPolicy.MONTHLY and hours_since_backup >= 720:  # 30 days
            backup_needed = True
        
        if backup_needed:
            self.create_backup()
    
    def _cleanup_old_backups(self) -> None:
        """Clean up old backups based on retention policy."""
        if self.config.max_backups <= 0:
            return
        
        backups = self.list_backups()
        
        if len(backups) > self.config.max_backups:
            # Remove oldest backups
            backups_to_remove = backups[self.config.max_backups:]
            
            for backup_info in backups_to_remove:
                try:
                    os.remove(backup_info.path)
                    pprint(f"Removed old backup: {backup_info.path}")
                except Exception as e:
                    pprint(f"Failed to remove old backup {backup_info.path}: {e}")
    
    def list_backups(self) -> List[BackupInfo]:
        """
        List available backups.
        
        Returns:
            List[BackupInfo]: List of backup information, sorted by timestamp (newest first)
        """
        if not os.path.exists(self.config.backup_directory):
            return []
        
        backups = []
        
        try:
            for filename in os.listdir(self.config.backup_directory):
                if not (filename.endswith('.json') or filename.endswith('.json.gz')):
                    continue
                
                backup_path = os.path.join(self.config.backup_directory, filename)
                
                try:
                    # Get file info
                    file_info = self.storage.get_file_info(backup_path)
                    
                    if not file_info.get('exists', False):
                        continue
                    
                    # Try to get schedule count from file
                    schedule_count = 0
                    try:
                        data = self.storage.read_data(backup_path)
                        schedule_count = data.get('schedule_count', len(data.get('schedules', [])))
                    except:
                        pass  # Count will remain 0
                    
                    # Calculate checksum
                    checksum = self.storage.calculate_checksum(backup_path)
                    
                    backup_info = BackupInfo(
                        path=backup_path,
                        timestamp=file_info['modified_time'],
                        size_bytes=file_info['size_bytes'],
                        checksum=checksum,
                        schedule_count=schedule_count
                    )
                    
                    backups.append(backup_info)
                    
                except Exception as e:
                    pprint(f"Failed to get backup info for {backup_path}: {e}")
        
        except Exception as e:
            pprint(f"Failed to list backups: {e}")
        
        # Sort by timestamp (newest first)
        backups.sort(key=lambda b: b.timestamp, reverse=True)
        
        return backups
    
    def get_storage_info(self) -> Dict[str, Any]:
        """
        Get information about storage.
        
        Returns:
            Dict[str, Any]: Storage information
        """
        info = {
            'storage_path': self.config.storage_path,
            'backup_directory': self.config.backup_directory,
            'backup_policy': self.config.backup_policy.value,
            'max_backups': self.config.max_backups,
            'storage_exists': os.path.exists(self.config.storage_path),
            'backup_count': len(self.list_backups())
        }
        
        # Add file info if storage exists
        if info['storage_exists']:
            file_info = self.storage.get_file_info(self.config.storage_path)
            info.update(file_info)
            
            # Add schedule count
            try:
                data = self.storage.read_data(self.config.storage_path)
                info['schedule_count'] = data.get('schedule_count', len(data.get('schedules', [])))
            except:
                info['schedule_count'] = 0
        
        return info
    
    def validate_storage(self) -> List[str]:
        """
        Validate storage integrity.
        
        Returns:
            List[str]: List of validation errors (empty if valid)
        """
        errors = []
        
        # Check if storage file exists
        if not os.path.exists(self.config.storage_path):
            errors.append(f"Storage file does not exist: {self.config.storage_path}")
            return errors
        
        try:
            # Try to load and validate data
            data = self.storage.read_data(self.config.storage_path)
            self.storage._validate_data_structure(data)
            
            # Try to convert to schedules (validates structure)
            schedules = self._dict_to_schedules(data)
            
            # Validate each schedule
            for i, schedule in enumerate(schedules):
                schedule_errors = schedule.validate()
                for error in schedule_errors:
                    errors.append(f"Schedule {i} ({schedule.name}): {error}")
            
        except Exception as e:
            errors.append(f"Storage validation failed: {e}")
        
        return errors


# Convenience functions

def load_schedules_from_file(file_path: str) -> List['ScheduleEntry']:
    """
    Load schedules from a file.
    
    Args:
        file_path: Path to schedule file
        
    Returns:
        List[ScheduleEntry]: List of loaded schedules
    """
    persistence = SchedulePersistence(file_path)
    return persistence.load_schedules()


def save_schedules_to_file(schedules: List['ScheduleEntry'], file_path: str) -> None:
    """
    Save schedules to a file.
    
    Args:
        schedules: List of schedules to save
        file_path: Path to save file
    """
    persistence = SchedulePersistence(file_path)
    persistence.save_schedules(schedules)


def create_backup(
    source_file: str,
    backup_directory: Optional[str] = None,
    backup_name: Optional[str] = None
) -> str:
    """
    Create a backup of a schedule file.
    
    Args:
        source_file: Source schedule file
        backup_directory: Directory for backup (optional)
        backup_name: Custom backup name (optional)
        
    Returns:
        str: Path to created backup
    """
    persistence = SchedulePersistence(source_file, backup_directory)
    return persistence.create_backup(backup_name)


def restore_from_backup(target_file: str, backup_file: str) -> None:
    """
    Restore schedules from a backup file.
    
    Args:
        target_file: Target schedule file
        backup_file: Backup file to restore from
    """
    persistence = SchedulePersistence(target_file)
    persistence.restore_from_backup(backup_file)


# Module exports
__all__ = [
    'SchedulePersistence',
    'ScheduleStorage',
    'PersistenceError',
    'ValidationError',
    'CorruptionError',
    'PersistenceConfig',
    'BackupInfo',
    'StorageFormat',
    'BackupPolicy',
    'load_schedules_from_file',
    'save_schedules_to_file',
    'create_backup',
    'restore_from_backup',
    'pprint'
]