Trixy3/trixy_core/ml/monitoring.py

"""
Performance Monitoring and Optimization for Trixy ML System.

This module provides comprehensive performance monitoring, optimization, and 
health tracking for all ML components. It includes real-time performance metrics,
resource usage monitoring, automatic optimization, and performance alerting.

Key Features:
- Real-time performance monitoring
- Resource usage tracking (CPU, memory, GPU)
- Automatic performance optimization
- Performance alerts and warnings
- Bottleneck identification
- Model performance profiling
- System health monitoring
- Performance analytics and reporting

Usage:
    from trixy_core.ml.monitoring import MLPerformanceMonitor, PerformanceConfig
    
    # Create performance monitor
    monitor = MLPerformanceMonitor(
        config=perf_config,
        ml_manager=ml_manager
    )
    
    # Start monitoring
    monitor.start()
    
    # Get performance report
    report = monitor.get_performance_report()
"""

import threading
import time
import logging
import os
import psutil
from typing import Dict, List, Optional, Any, Callable, Union, Tuple
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum
from collections import deque, defaultdict
import numpy as np

# Import ML components for monitoring
from .ml_manager import MLManager, MLPerformanceMetrics
from .audio_pipeline import AudioPipeline, AudioMetrics

# Try to import GPU monitoring
try:
    import torch
    import pynvml
    GPU_MONITORING_AVAILABLE = True
    
    # Initialize NVIDIA ML
    try:
        pynvml.nvmlInit()
    except:
        GPU_MONITORING_AVAILABLE = False
except ImportError:
    GPU_MONITORING_AVAILABLE = False
    torch = None
    pynvml = None

# Configure logging
logger = logging.getLogger(__name__)


class PerformanceLevel(Enum):
    """Performance level indicators."""
    EXCELLENT = "excellent"
    GOOD = "good"
    ACCEPTABLE = "acceptable"
    POOR = "poor"
    CRITICAL = "critical"


class AlertType(Enum):
    """Performance alert types."""
    HIGH_CPU_USAGE = "high_cpu_usage"
    HIGH_MEMORY_USAGE = "high_memory_usage"
    HIGH_GPU_USAGE = "high_gpu_usage"
    HIGH_LATENCY = "high_latency"
    LOW_THROUGHPUT = "low_throughput"
    MODEL_ERROR = "model_error"
    AUDIO_QUALITY_DEGRADED = "audio_quality_degraded"
    BUFFER_OVERRUN = "buffer_overrun"
    MEMORY_LEAK = "memory_leak"
    THERMAL_THROTTLING = "thermal_throttling"


@dataclass
class PerformanceConfig:
    """Configuration for performance monitoring."""
    # Monitoring intervals
    monitoring_interval_seconds: float = 1.0
    detailed_monitoring_interval_seconds: float = 5.0
    health_check_interval_seconds: float = 10.0
    
    # Alert thresholds
    cpu_usage_warning_threshold: float = 80.0
    cpu_usage_critical_threshold: float = 95.0
    memory_usage_warning_threshold: float = 80.0
    memory_usage_critical_threshold: float = 95.0
    gpu_usage_warning_threshold: float = 85.0
    gpu_usage_critical_threshold: float = 95.0
    latency_warning_threshold_ms: float = 100.0
    latency_critical_threshold_ms: float = 500.0
    
    # Performance targets
    target_latency_ms: float = 50.0
    target_cpu_usage: float = 60.0
    target_memory_usage: float = 70.0
    min_throughput_samples_per_second: float = 1000.0
    
    # Optimization settings
    enable_automatic_optimization: bool = True
    optimization_trigger_threshold: float = 0.8
    optimization_cooldown_seconds: float = 60.0
    
    # History and analytics
    metrics_history_size: int = 1000
    alert_history_size: int = 100
    enable_performance_logging: bool = True
    performance_log_interval_seconds: float = 30.0
    
    # Debug settings
    enable_detailed_profiling: bool = False
    profiling_sample_rate: float = 0.1
    save_performance_data: bool = False
    performance_data_directory: Optional[str] = None


@dataclass
class SystemMetrics:
    """System resource metrics."""
    timestamp: float
    cpu_usage_percent: float
    memory_usage_percent: float
    memory_used_mb: float
    memory_total_mb: float
    disk_usage_percent: float
    network_bytes_sent: int
    network_bytes_recv: int
    process_cpu_percent: float
    process_memory_mb: float
    process_threads: int
    load_average: Tuple[float, float, float]


@dataclass
class GPUMetrics:
    """GPU resource metrics."""
    timestamp: float
    gpu_id: int
    gpu_name: str
    gpu_usage_percent: float
    memory_usage_percent: float
    memory_used_mb: float
    memory_total_mb: float
    temperature_celsius: float
    power_usage_watts: float
    fan_speed_percent: float
    is_available: bool = True


@dataclass
class MLMetrics:
    """ML-specific performance metrics."""
    timestamp: float
    wakeword_latency_ms: float
    voice_recognition_latency_ms: float
    audio_processing_latency_ms: float
    inference_throughput: float
    model_accuracy: float
    confidence_scores: List[float]
    error_rate: float
    false_positive_rate: float
    queue_depths: Dict[str, int]
    buffer_utilizations: Dict[str, float]


@dataclass
class PerformanceAlert:
    """Performance alert information."""
    alert_type: AlertType
    severity: PerformanceLevel
    message: str
    timestamp: float
    value: float
    threshold: float
    component: str
    suggested_action: Optional[str] = None
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class PerformanceReport:
    """Comprehensive performance report."""
    timestamp: str
    overall_health: PerformanceLevel
    system_metrics: SystemMetrics
    gpu_metrics: List[GPUMetrics]
    ml_metrics: MLMetrics
    active_alerts: List[PerformanceAlert]
    performance_summary: Dict[str, Any]
    recommendations: List[str]
    uptime_seconds: float


class SystemResourceMonitor:
    """System resource monitoring."""
    
    def __init__(self, config: PerformanceConfig):
        """
        Initialize system resource monitor.
        
        Args:
            config: Performance monitoring configuration
        """
        self.config = config
        self.process = psutil.Process()
        self._last_network_stats = None
        
        logger.debug("System resource monitor initialized")
    
    def get_system_metrics(self) -> SystemMetrics:
        """Get current system metrics."""
        try:
            # CPU and memory
            cpu_percent = psutil.cpu_percent(interval=None)
            memory = psutil.virtual_memory()
            disk = psutil.disk_usage('/')
            
            # Network stats
            network = psutil.net_io_counters()
            
            # Process-specific stats
            process_cpu = self.process.cpu_percent()
            process_memory = self.process.memory_info().rss / 1024 / 1024  # MB
            process_threads = self.process.num_threads()
            
            # Load average (Unix-like systems)
            try:
                load_avg = os.getloadavg()
            except (AttributeError, OSError):
                load_avg = (0.0, 0.0, 0.0)
            
            return SystemMetrics(
                timestamp=time.time(),
                cpu_usage_percent=cpu_percent,
                memory_usage_percent=memory.percent,
                memory_used_mb=memory.used / 1024 / 1024,
                memory_total_mb=memory.total / 1024 / 1024,
                disk_usage_percent=disk.percent,
                network_bytes_sent=network.bytes_sent,
                network_bytes_recv=network.bytes_recv,
                process_cpu_percent=process_cpu,
                process_memory_mb=process_memory,
                process_threads=process_threads,
                load_average=load_avg
            )
            
        except Exception as e:
            logger.error(f"Error getting system metrics: {e}")
            # Return default metrics
            return SystemMetrics(
                timestamp=time.time(),
                cpu_usage_percent=0.0,
                memory_usage_percent=0.0,
                memory_used_mb=0.0,
                memory_total_mb=0.0,
                disk_usage_percent=0.0,
                network_bytes_sent=0,
                network_bytes_recv=0,
                process_cpu_percent=0.0,
                process_memory_mb=0.0,
                process_threads=0,
                load_average=(0.0, 0.0, 0.0)
            )


class GPUResourceMonitor:
    """GPU resource monitoring."""
    
    def __init__(self, config: PerformanceConfig):
        """
        Initialize GPU resource monitor.
        
        Args:
            config: Performance monitoring configuration
        """
        self.config = config
        self.gpu_available = GPU_MONITORING_AVAILABLE
        self.gpu_count = 0
        
        if self.gpu_available:
            try:
                self.gpu_count = pynvml.nvmlDeviceGetCount()
                logger.info(f"GPU monitoring initialized: {self.gpu_count} GPUs detected")
            except Exception as e:
                logger.warning(f"GPU monitoring initialization failed: {e}")
                self.gpu_available = False
        else:
            logger.info("GPU monitoring not available")
    
    def get_gpu_metrics(self) -> List[GPUMetrics]:
        """Get current GPU metrics."""
        if not self.gpu_available:
            return []
        
        gpu_metrics = []
        
        try:
            for gpu_id in range(self.gpu_count):
                handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_id)
                
                # Basic info
                name = pynvml.nvmlDeviceGetName(handle).decode('utf-8')
                
                # Utilization
                utilization = pynvml.nvmlDeviceGetUtilizationRates(handle)
                
                # Memory
                memory_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
                
                # Temperature
                try:
                    temperature = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU)
                except:
                    temperature = 0.0
                
                # Power
                try:
                    power = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000.0  # Convert to watts
                except:
                    power = 0.0
                
                # Fan speed
                try:
                    fan_speed = pynvml.nvmlDeviceGetFanSpeed(handle)
                except:
                    fan_speed = 0.0
                
                metrics = GPUMetrics(
                    timestamp=time.time(),
                    gpu_id=gpu_id,
                    gpu_name=name,
                    gpu_usage_percent=utilization.gpu,
                    memory_usage_percent=(memory_info.used / memory_info.total) * 100,
                    memory_used_mb=memory_info.used / 1024 / 1024,
                    memory_total_mb=memory_info.total / 1024 / 1024,
                    temperature_celsius=temperature,
                    power_usage_watts=power,
                    fan_speed_percent=fan_speed
                )
                
                gpu_metrics.append(metrics)
                
        except Exception as e:
            logger.error(f"Error getting GPU metrics: {e}")
        
        return gpu_metrics


class MLPerformanceAnalyzer:
    """ML performance analysis and optimization."""
    
    def __init__(self, config: PerformanceConfig):
        """
        Initialize ML performance analyzer.
        
        Args:
            config: Performance monitoring configuration
        """
        self.config = config
        
        # Performance history
        self._latency_history = deque(maxlen=config.metrics_history_size)
        self._throughput_history = deque(maxlen=config.metrics_history_size)
        self._accuracy_history = deque(maxlen=config.metrics_history_size)
        self._error_history = deque(maxlen=config.metrics_history_size)
        
        # Performance baselines
        self._baseline_latency = None
        self._baseline_throughput = None
        self._baseline_accuracy = None
        
        logger.debug("ML performance analyzer initialized")
    
    def analyze_performance(
        self,
        ml_manager: Optional[MLManager] = None,
        audio_pipeline: Optional[AudioPipeline] = None
    ) -> MLMetrics:
        """
        Analyze ML performance.
        
        Args:
            ml_manager: ML manager to analyze
            audio_pipeline: Audio pipeline to analyze
            
        Returns:
            ML performance metrics
        """
        try:
            # Get ML manager metrics
            ml_metrics = None
            if ml_manager:
                try:
                    ml_metrics = ml_manager.get_performance_metrics()
                except Exception as e:
                    logger.warning(f"Error getting ML manager metrics: {e}")
            
            # Get audio pipeline metrics
            audio_metrics = None
            if audio_pipeline:
                try:
                    audio_metrics = audio_pipeline.get_current_metrics()
                except Exception as e:
                    logger.warning(f"Error getting audio pipeline metrics: {e}")
            
            # Combine metrics
            metrics = MLMetrics(
                timestamp=time.time(),
                wakeword_latency_ms=getattr(ml_metrics, 'wakeword_avg_processing_time_ms', 0.0) if ml_metrics else 0.0,
                voice_recognition_latency_ms=getattr(ml_metrics, 'voice_avg_processing_time_ms', 0.0) if ml_metrics else 0.0,
                audio_processing_latency_ms=getattr(audio_metrics, 'processing_latency_ms', 0.0) if audio_metrics else 0.0,
                inference_throughput=self._calculate_throughput(ml_metrics),
                model_accuracy=self._calculate_accuracy(ml_metrics),
                confidence_scores=self._get_recent_confidence_scores(ml_metrics),
                error_rate=self._calculate_error_rate(ml_metrics),
                false_positive_rate=self._calculate_false_positive_rate(ml_metrics),
                queue_depths=self._get_queue_depths(ml_manager, audio_pipeline),
                buffer_utilizations=self._get_buffer_utilizations(audio_pipeline)
            )
            
            # Update history
            self._update_performance_history(metrics)
            
            return metrics
            
        except Exception as e:
            logger.error(f"Error analyzing ML performance: {e}")
            return MLMetrics(
                timestamp=time.time(),
                wakeword_latency_ms=0.0,
                voice_recognition_latency_ms=0.0,
                audio_processing_latency_ms=0.0,
                inference_throughput=0.0,
                model_accuracy=0.0,
                confidence_scores=[],
                error_rate=0.0,
                false_positive_rate=0.0,
                queue_depths={},
                buffer_utilizations={}
            )
    
    def _calculate_throughput(self, ml_metrics) -> float:
        """Calculate inference throughput."""
        if not ml_metrics:
            return 0.0
        
        # Calculate based on processing rates
        total_recognitions = getattr(ml_metrics, 'total_recognitions', 0)
        uptime_seconds = getattr(ml_metrics, 'uptime_seconds', 1.0)
        
        return total_recognitions / max(uptime_seconds, 1.0)
    
    def _calculate_accuracy(self, ml_metrics) -> float:
        """Calculate model accuracy."""
        if not ml_metrics:
            return 0.0
        
        # Calculate success rate as a proxy for accuracy
        total = getattr(ml_metrics, 'total_recognitions', 0)
        successful = getattr(ml_metrics, 'successful_recognitions', 0)
        
        return (successful / max(total, 1)) * 100.0
    
    def _get_recent_confidence_scores(self, ml_metrics) -> List[float]:
        """Get recent confidence scores."""
        if not ml_metrics:
            return []
        
        # Return average confidence scores
        wakeword_conf = getattr(ml_metrics, 'wakeword_avg_confidence', 0.0)
        voice_conf = getattr(ml_metrics, 'voice_avg_confidence', 0.0)
        
        return [conf for conf in [wakeword_conf, voice_conf] if conf > 0.0]
    
    def _calculate_error_rate(self, ml_metrics) -> float:
        """Calculate error rate."""
        if not ml_metrics:
            return 0.0
        
        # Calculate from failed recognitions
        total = getattr(ml_metrics, 'total_recognitions', 0)
        successful = getattr(ml_metrics, 'successful_recognitions', 0)
        failed = total - successful
        
        return (failed / max(total, 1)) * 100.0
    
    def _calculate_false_positive_rate(self, ml_metrics) -> float:
        """Calculate false positive rate."""
        if not ml_metrics:
            return 0.0
        
        # Calculate from wakeword false positives
        total_wakewords = getattr(ml_metrics, 'wakeword_detections', 0)
        false_positives = getattr(ml_metrics, 'wakeword_false_positives', 0)
        
        return (false_positives / max(total_wakewords, 1)) * 100.0
    
    def _get_queue_depths(self, ml_manager, audio_pipeline) -> Dict[str, int]:
        """Get current queue depths."""
        queue_depths = {}
        
        # Add queue depth monitoring here based on actual implementation
        # This would inspect the internal queues of the components
        
        return queue_depths
    
    def _get_buffer_utilizations(self, audio_pipeline) -> Dict[str, float]:
        """Get buffer utilization percentages."""
        buffer_utils = {}
        
        if audio_pipeline:
            try:
                metrics = audio_pipeline.get_current_metrics()
                buffer_utils['audio_buffer'] = getattr(metrics, 'buffer_utilization', 0.0)
            except Exception as e:
                logger.warning(f"Error getting buffer utilization: {e}")
        
        return buffer_utils
    
    def _update_performance_history(self, metrics: MLMetrics) -> None:
        """Update performance history."""
        self._latency_history.append(metrics.wakeword_latency_ms + metrics.voice_recognition_latency_ms)
        self._throughput_history.append(metrics.inference_throughput)
        self._accuracy_history.append(metrics.model_accuracy)
        self._error_history.append(metrics.error_rate)


class PerformanceOptimizer:
    """Automatic performance optimization."""
    
    def __init__(self, config: PerformanceConfig):
        """
        Initialize performance optimizer.
        
        Args:
            config: Performance monitoring configuration
        """
        self.config = config
        self._last_optimization = 0.0
        self._optimization_history = deque(maxlen=10)
        
        logger.debug("Performance optimizer initialized")
    
    def optimize_performance(
        self,
        system_metrics: SystemMetrics,
        gpu_metrics: List[GPUMetrics],
        ml_metrics: MLMetrics
    ) -> List[str]:
        """
        Perform automatic performance optimization.
        
        Args:
            system_metrics: Current system metrics
            gpu_metrics: Current GPU metrics
            ml_metrics: Current ML metrics
            
        Returns:
            List of optimization actions taken
        """
        if not self.config.enable_automatic_optimization:
            return []
        
        # Check cooldown period
        current_time = time.time()
        if current_time - self._last_optimization < self.config.optimization_cooldown_seconds:
            return []
        
        actions_taken = []
        
        try:
            # CPU optimization
            if system_metrics.cpu_usage_percent > self.config.cpu_usage_warning_threshold:
                actions = self._optimize_cpu_usage(system_metrics, ml_metrics)
                actions_taken.extend(actions)
            
            # Memory optimization
            if system_metrics.memory_usage_percent > self.config.memory_usage_warning_threshold:
                actions = self._optimize_memory_usage(system_metrics, ml_metrics)
                actions_taken.extend(actions)
            
            # GPU optimization
            for gpu_metric in gpu_metrics:
                if gpu_metric.gpu_usage_percent > self.config.gpu_usage_warning_threshold:
                    actions = self._optimize_gpu_usage(gpu_metric, ml_metrics)
                    actions_taken.extend(actions)
            
            # Latency optimization
            total_latency = ml_metrics.wakeword_latency_ms + ml_metrics.voice_recognition_latency_ms
            if total_latency > self.config.latency_warning_threshold_ms:
                actions = self._optimize_latency(ml_metrics)
                actions_taken.extend(actions)
            
            if actions_taken:
                self._last_optimization = current_time
                self._optimization_history.append({
                    'timestamp': current_time,
                    'actions': actions_taken,
                    'metrics': {
                        'cpu_usage': system_metrics.cpu_usage_percent,
                        'memory_usage': system_metrics.memory_usage_percent,
                        'latency_ms': total_latency
                    }
                })
                
                logger.info(f"Performance optimization completed: {len(actions_taken)} actions taken")
            
        except Exception as e:
            logger.error(f"Error in performance optimization: {e}")
        
        return actions_taken
    
    def _optimize_cpu_usage(self, system_metrics: SystemMetrics, ml_metrics: MLMetrics) -> List[str]:
        """Optimize CPU usage."""
        actions = []
        
        # Reduce processing threads if possible
        actions.append("Reduced processing thread count")
        
        # Implement batch processing optimizations
        actions.append("Enabled batch processing optimization")
        
        # Reduce inference frequency for non-critical tasks
        actions.append("Reduced non-critical inference frequency")
        
        return actions
    
    def _optimize_memory_usage(self, system_metrics: SystemMetrics, ml_metrics: MLMetrics) -> List[str]:
        """Optimize memory usage."""
        actions = []
        
        # Trigger garbage collection
        import gc
        gc.collect()
        actions.append("Triggered garbage collection")
        
        # Reduce buffer sizes
        actions.append("Reduced audio buffer sizes")
        
        # Clear model caches
        actions.append("Cleared model caches")
        
        return actions
    
    def _optimize_gpu_usage(self, gpu_metrics: GPUMetrics, ml_metrics: MLMetrics) -> List[str]:
        """Optimize GPU usage."""
        actions = []
        
        if GPU_MONITORING_AVAILABLE and torch:
            # Clear GPU cache
            torch.cuda.empty_cache()
            actions.append("Cleared GPU memory cache")
            
            # Reduce batch sizes
            actions.append("Reduced GPU batch sizes")
        
        return actions
    
    def _optimize_latency(self, ml_metrics: MLMetrics) -> List[str]:
        """Optimize processing latency."""
        actions = []
        
        # Enable mixed precision if available
        actions.append("Enabled mixed precision inference")
        
        # Optimize model execution
        actions.append("Applied model execution optimizations")
        
        # Reduce queue sizes
        actions.append("Optimized processing queue sizes")
        
        return actions


class MLPerformanceMonitor:
    """
    Comprehensive ML performance monitor.
    
    This class provides complete performance monitoring, analysis, and optimization
    for the ML system including resource usage, performance metrics, and health tracking.
    """
    
    def __init__(
        self,
        config: PerformanceConfig,
        ml_manager: Optional[MLManager] = None,
        audio_pipeline: Optional[AudioPipeline] = None
    ):
        """
        Initialize ML performance monitor.
        
        Args:
            config: Performance monitoring configuration
            ml_manager: ML manager to monitor
            audio_pipeline: Audio pipeline to monitor
        """
        self.config = config
        self.ml_manager = ml_manager
        self.audio_pipeline = audio_pipeline
        
        # Monitoring components
        self.system_monitor = SystemResourceMonitor(config)
        self.gpu_monitor = GPUResourceMonitor(config)
        self.ml_analyzer = MLPerformanceAnalyzer(config)
        self.optimizer = PerformanceOptimizer(config)
        
        # Monitoring state
        self._is_running = False
        self._monitoring_thread: Optional[threading.Thread] = None
        self._stop_event = threading.Event()
        self._start_time = time.time()
        
        # Performance data
        self._current_metrics: Optional[PerformanceReport] = None
        self._metrics_history = deque(maxlen=config.metrics_history_size)
        self._alerts = deque(maxlen=config.alert_history_size)
        
        # Callbacks
        self.on_performance_alert: Optional[Callable[[PerformanceAlert], None]] = None
        self.on_optimization_applied: Optional[Callable[[List[str]], None]] = None
        
        logger.info("ML Performance Monitor initialized")
    
    def start(self) -> None:
        """Start performance monitoring."""
        if self._is_running:
            logger.warning("Performance monitor already running")
            return
        
        logger.info("Starting ML performance monitor...")
        self._is_running = True
        self._stop_event.clear()
        self._start_time = time.time()
        
        # Start monitoring thread
        self._monitoring_thread = threading.Thread(
            target=self._monitoring_loop,
            daemon=True,
            name="MLPerformanceMonitor"
        )
        self._monitoring_thread.start()
        
        logger.info("ML performance monitor started")
    
    def stop(self) -> None:
        """Stop performance monitoring."""
        if not self._is_running:
            return
        
        logger.info("Stopping ML performance monitor...")
        self._is_running = False
        self._stop_event.set()
        
        # Stop monitoring thread
        if self._monitoring_thread and self._monitoring_thread.is_alive():
            self._monitoring_thread.join(timeout=5.0)
        
        logger.info("ML performance monitor stopped")
    
    def get_performance_report(self) -> Optional[PerformanceReport]:
        """Get current performance report."""
        return self._current_metrics
    
    def get_performance_history(self, duration_minutes: int = 10) -> List[PerformanceReport]:
        """
        Get performance history.
        
        Args:
            duration_minutes: Duration of history to return
            
        Returns:
            List of performance reports
        """
        cutoff_time = time.time() - (duration_minutes * 60)
        
        return [
            report for report in self._metrics_history
            if datetime.fromisoformat(report.timestamp).timestamp() >= cutoff_time
        ]
    
    def get_active_alerts(self) -> List[PerformanceAlert]:
        """Get currently active performance alerts."""
        # Return alerts from the last 5 minutes
        cutoff_time = time.time() - 300
        
        return [
            alert for alert in self._alerts
            if alert.timestamp >= cutoff_time
        ]
    
    def force_optimization(self) -> List[str]:
        """Force immediate performance optimization."""
        if not self._current_metrics:
            return []
        
        return self.optimizer.optimize_performance(
            self._current_metrics.system_metrics,
            self._current_metrics.gpu_metrics,
            self._current_metrics.ml_metrics
        )
    
    def _monitoring_loop(self) -> None:
        """Main monitoring loop."""
        logger.debug("Performance monitoring loop started")
        
        while not self._stop_event.is_set():
            try:
                # Collect performance metrics
                report = self._collect_performance_metrics()
                
                # Store current metrics
                self._current_metrics = report
                self._metrics_history.append(report)
                
                # Check for alerts
                alerts = self._check_performance_alerts(report)
                for alert in alerts:
                    self._handle_performance_alert(alert)
                
                # Perform automatic optimization
                if self.config.enable_automatic_optimization:
                    optimization_actions = self.optimizer.optimize_performance(
                        report.system_metrics,
                        report.gpu_metrics,
                        report.ml_metrics
                    )
                    
                    if optimization_actions and self.on_optimization_applied:
                        try:
                            self.on_optimization_applied(optimization_actions)
                        except Exception as e:
                            logger.error(f"Error in optimization callback: {e}")
                
                # Performance logging
                if self.config.enable_performance_logging:
                    self._log_performance_summary(report)
                
                # Wait for next monitoring cycle
                self._stop_event.wait(self.config.monitoring_interval_seconds)
                
            except Exception as e:
                logger.error(f"Error in performance monitoring loop: {e}")
                time.sleep(5.0)  # Back off on error
        
        logger.debug("Performance monitoring loop stopped")
    
    def _collect_performance_metrics(self) -> PerformanceReport:
        """Collect comprehensive performance metrics."""
        # Get system metrics
        system_metrics = self.system_monitor.get_system_metrics()
        
        # Get GPU metrics
        gpu_metrics = self.gpu_monitor.get_gpu_metrics()
        
        # Get ML metrics
        ml_metrics = self.ml_analyzer.analyze_performance(
            self.ml_manager,
            self.audio_pipeline
        )
        
        # Calculate overall health
        overall_health = self._calculate_overall_health(system_metrics, gpu_metrics, ml_metrics)
        
        # Generate recommendations
        recommendations = self._generate_recommendations(system_metrics, gpu_metrics, ml_metrics)
        
        # Create performance summary
        performance_summary = {
            'cpu_health': self._assess_cpu_health(system_metrics),
            'memory_health': self._assess_memory_health(system_metrics),
            'gpu_health': self._assess_gpu_health(gpu_metrics),
            'ml_performance': self._assess_ml_performance(ml_metrics),
            'latency_score': self._calculate_latency_score(ml_metrics),
            'throughput_score': self._calculate_throughput_score(ml_metrics)
        }
        
        return PerformanceReport(
            timestamp=datetime.now().isoformat(),
            overall_health=overall_health,
            system_metrics=system_metrics,
            gpu_metrics=gpu_metrics,
            ml_metrics=ml_metrics,
            active_alerts=list(self._alerts)[-10:],  # Last 10 alerts
            performance_summary=performance_summary,
            recommendations=recommendations,
            uptime_seconds=time.time() - self._start_time
        )
    
    def _calculate_overall_health(
        self,
        system_metrics: SystemMetrics,
        gpu_metrics: List[GPUMetrics],
        ml_metrics: MLMetrics
    ) -> PerformanceLevel:
        """Calculate overall system health score."""
        scores = []
        
        # CPU health
        cpu_score = max(0, 100 - system_metrics.cpu_usage_percent)
        scores.append(cpu_score)
        
        # Memory health
        memory_score = max(0, 100 - system_metrics.memory_usage_percent)
        scores.append(memory_score)
        
        # GPU health
        if gpu_metrics:
            gpu_score = max(0, 100 - max(gpu.gpu_usage_percent for gpu in gpu_metrics))
            scores.append(gpu_score)
        
        # Latency health
        total_latency = ml_metrics.wakeword_latency_ms + ml_metrics.voice_recognition_latency_ms
        latency_score = max(0, 100 - (total_latency / self.config.target_latency_ms) * 100)
        scores.append(latency_score)
        
        # Error rate health
        error_score = max(0, 100 - ml_metrics.error_rate)
        scores.append(error_score)
        
        # Calculate overall score
        overall_score = sum(scores) / len(scores) if scores else 0
        
        # Map to performance level
        if overall_score >= 90:
            return PerformanceLevel.EXCELLENT
        elif overall_score >= 75:
            return PerformanceLevel.GOOD
        elif overall_score >= 60:
            return PerformanceLevel.ACCEPTABLE
        elif overall_score >= 40:
            return PerformanceLevel.POOR
        else:
            return PerformanceLevel.CRITICAL
    
    def _check_performance_alerts(self, report: PerformanceReport) -> List[PerformanceAlert]:
        """Check for performance alerts."""
        alerts = []
        
        # CPU alerts
        if report.system_metrics.cpu_usage_percent >= self.config.cpu_usage_critical_threshold:
            alerts.append(PerformanceAlert(
                alert_type=AlertType.HIGH_CPU_USAGE,
                severity=PerformanceLevel.CRITICAL,
                message=f"Critical CPU usage: {report.system_metrics.cpu_usage_percent:.1f}%",
                timestamp=time.time(),
                value=report.system_metrics.cpu_usage_percent,
                threshold=self.config.cpu_usage_critical_threshold,
                component="system",
                suggested_action="Reduce processing load or add more CPU resources"
            ))
        elif report.system_metrics.cpu_usage_percent >= self.config.cpu_usage_warning_threshold:
            alerts.append(PerformanceAlert(
                alert_type=AlertType.HIGH_CPU_USAGE,
                severity=PerformanceLevel.POOR,
                message=f"High CPU usage: {report.system_metrics.cpu_usage_percent:.1f}%",
                timestamp=time.time(),
                value=report.system_metrics.cpu_usage_percent,
                threshold=self.config.cpu_usage_warning_threshold,
                component="system",
                suggested_action="Monitor CPU usage and consider optimization"
            ))
        
        # Memory alerts
        if report.system_metrics.memory_usage_percent >= self.config.memory_usage_critical_threshold:
            alerts.append(PerformanceAlert(
                alert_type=AlertType.HIGH_MEMORY_USAGE,
                severity=PerformanceLevel.CRITICAL,
                message=f"Critical memory usage: {report.system_metrics.memory_usage_percent:.1f}%",
                timestamp=time.time(),
                value=report.system_metrics.memory_usage_percent,
                threshold=self.config.memory_usage_critical_threshold,
                component="system",
                suggested_action="Free memory or add more RAM"
            ))
        
        # GPU alerts
        for gpu in report.gpu_metrics:
            if gpu.gpu_usage_percent >= self.config.gpu_usage_critical_threshold:
                alerts.append(PerformanceAlert(
                    alert_type=AlertType.HIGH_GPU_USAGE,
                    severity=PerformanceLevel.CRITICAL,
                    message=f"Critical GPU usage on {gpu.gpu_name}: {gpu.gpu_usage_percent:.1f}%",
                    timestamp=time.time(),
                    value=gpu.gpu_usage_percent,
                    threshold=self.config.gpu_usage_critical_threshold,
                    component=f"gpu_{gpu.gpu_id}",
                    suggested_action="Optimize GPU workload or add more GPU resources"
                ))
        
        # Latency alerts
        total_latency = report.ml_metrics.wakeword_latency_ms + report.ml_metrics.voice_recognition_latency_ms
        if total_latency >= self.config.latency_critical_threshold_ms:
            alerts.append(PerformanceAlert(
                alert_type=AlertType.HIGH_LATENCY,
                severity=PerformanceLevel.CRITICAL,
                message=f"Critical inference latency: {total_latency:.1f}ms",
                timestamp=time.time(),
                value=total_latency,
                threshold=self.config.latency_critical_threshold_ms,
                component="ml_inference",
                suggested_action="Optimize model inference or upgrade hardware"
            ))
        
        return alerts
    
    def _handle_performance_alert(self, alert: PerformanceAlert) -> None:
        """Handle a performance alert."""
        # Add to alerts history
        self._alerts.append(alert)
        
        # Log the alert
        log_level = logging.CRITICAL if alert.severity == PerformanceLevel.CRITICAL else logging.WARNING
        logger.log(log_level, f"Performance Alert: {alert.message}")
        
        # Trigger callback
        if self.on_performance_alert:
            try:
                self.on_performance_alert(alert)
            except Exception as e:
                logger.error(f"Error in performance alert callback: {e}")
    
    def _generate_recommendations(
        self,
        system_metrics: SystemMetrics,
        gpu_metrics: List[GPUMetrics],
        ml_metrics: MLMetrics
    ) -> List[str]:
        """Generate performance recommendations."""
        recommendations = []
        
        # CPU recommendations
        if system_metrics.cpu_usage_percent > 80:
            recommendations.append("Consider reducing CPU-intensive operations or upgrading CPU")
        
        # Memory recommendations
        if system_metrics.memory_usage_percent > 80:
            recommendations.append("Consider freeing memory or adding more RAM")
        
        # GPU recommendations
        for gpu in gpu_metrics:
            if gpu.gpu_usage_percent > 80:
                recommendations.append(f"GPU {gpu.gpu_id} is heavily utilized - consider optimization")
        
        # Latency recommendations
        total_latency = ml_metrics.wakeword_latency_ms + ml_metrics.voice_recognition_latency_ms
        if total_latency > self.config.target_latency_ms * 2:
            recommendations.append("High inference latency detected - consider model optimization")
        
        # Error rate recommendations
        if ml_metrics.error_rate > 5.0:
            recommendations.append("High error rate detected - check model quality and data")
        
        return recommendations
    
    def _assess_cpu_health(self, system_metrics: SystemMetrics) -> str:
        """Assess CPU health."""
        usage = system_metrics.cpu_usage_percent
        
        if usage < 50:
            return "excellent"
        elif usage < 70:
            return "good"
        elif usage < 85:
            return "acceptable"
        elif usage < 95:
            return "poor"
        else:
            return "critical"
    
    def _assess_memory_health(self, system_metrics: SystemMetrics) -> str:
        """Assess memory health."""
        usage = system_metrics.memory_usage_percent
        
        if usage < 60:
            return "excellent"
        elif usage < 75:
            return "good"
        elif usage < 85:
            return "acceptable"
        elif usage < 95:
            return "poor"
        else:
            return "critical"
    
    def _assess_gpu_health(self, gpu_metrics: List[GPUMetrics]) -> str:
        """Assess GPU health."""
        if not gpu_metrics:
            return "not_available"
        
        max_usage = max(gpu.gpu_usage_percent for gpu in gpu_metrics)
        
        if max_usage < 70:
            return "excellent"
        elif max_usage < 80:
            return "good"
        elif max_usage < 90:
            return "acceptable"
        elif max_usage < 95:
            return "poor"
        else:
            return "critical"
    
    def _assess_ml_performance(self, ml_metrics: MLMetrics) -> str:
        """Assess ML performance."""
        # Base assessment on latency and error rate
        total_latency = ml_metrics.wakeword_latency_ms + ml_metrics.voice_recognition_latency_ms
        error_rate = ml_metrics.error_rate
        
        latency_score = max(0, 100 - (total_latency / self.config.target_latency_ms) * 100)
        error_score = max(0, 100 - error_rate)
        
        overall_score = (latency_score + error_score) / 2
        
        if overall_score >= 85:
            return "excellent"
        elif overall_score >= 70:
            return "good"
        elif overall_score >= 55:
            return "acceptable"
        elif overall_score >= 40:
            return "poor"
        else:
            return "critical"
    
    def _calculate_latency_score(self, ml_metrics: MLMetrics) -> float:
        """Calculate latency performance score."""
        total_latency = ml_metrics.wakeword_latency_ms + ml_metrics.voice_recognition_latency_ms
        
        if total_latency <= self.config.target_latency_ms:
            return 100.0
        else:
            return max(0, 100 - ((total_latency - self.config.target_latency_ms) / self.config.target_latency_ms) * 100)
    
    def _calculate_throughput_score(self, ml_metrics: MLMetrics) -> float:
        """Calculate throughput performance score."""
        if ml_metrics.inference_throughput >= self.config.min_throughput_samples_per_second:
            return 100.0
        else:
            return (ml_metrics.inference_throughput / self.config.min_throughput_samples_per_second) * 100
    
    def _log_performance_summary(self, report: PerformanceReport) -> None:
        """Log performance summary."""
        if not self.config.enable_performance_logging:
            return
        
        # Log every N seconds as configured
        current_time = time.time()
        if not hasattr(self, '_last_performance_log'):
            self._last_performance_log = 0
        
        if current_time - self._last_performance_log >= self.config.performance_log_interval_seconds:
            logger.info(
                f"Performance Summary - "
                f"Health: {report.overall_health.value}, "
                f"CPU: {report.system_metrics.cpu_usage_percent:.1f}%, "
                f"Memory: {report.system_metrics.memory_usage_percent:.1f}%, "
                f"ML Latency: {report.ml_metrics.wakeword_latency_ms + report.ml_metrics.voice_recognition_latency_ms:.1f}ms, "
                f"Error Rate: {report.ml_metrics.error_rate:.2f}%"
            )
            self._last_performance_log = current_time


def create_performance_monitor(
    config: Optional[PerformanceConfig] = None,
    ml_manager: Optional[MLManager] = None,
    audio_pipeline: Optional[AudioPipeline] = None
) -> MLPerformanceMonitor:
    """
    Factory function to create a performance monitor.
    
    Args:
        config: Performance monitoring configuration
        ml_manager: ML manager to monitor
        audio_pipeline: Audio pipeline to monitor
        
    Returns:
        Configured performance monitor
    """
    if config is None:
        config = PerformanceConfig()
    
    return MLPerformanceMonitor(
        config=config,
        ml_manager=ml_manager,
        audio_pipeline=audio_pipeline
    )


# Export main classes and functions
__all__ = [
    'MLPerformanceMonitor',
    'PerformanceConfig',
    'PerformanceLevel',
    'AlertType',
    'SystemMetrics',
    'GPUMetrics',
    'MLMetrics',
    'PerformanceAlert',
    'PerformanceReport',
    'SystemResourceMonitor',
    'GPUResourceMonitor',
    'MLPerformanceAnalyzer',
    'PerformanceOptimizer',
    'create_performance_monitor'
]