integration.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908
  1. """
  2. Application Container Integration for Trixy ML System.
  3. This module provides comprehensive integration between the ML components and the
  4. Trixy application container system. It handles component lifecycle management,
  5. dependency injection, and seamless integration with the event system, configuration
  6. management, and other Trixy core components.
  7. Key Features:
  8. - ML component factory for application container
  9. - Lifecycle management (initialization, startup, shutdown)
  10. - Configuration integration with config manager
  11. - Event system integration
  12. - Health monitoring and status reporting
  13. - Network manager integration for client mode
  14. - Performance monitoring integration
  15. - Error handling and recovery
  16. Usage:
  17. from trixy_core.ml.integration import MLContainerIntegration
  18. # Create ML integration for application container
  19. ml_integration = MLContainerIntegration(application_container)
  20. # Initialize ML components
  21. ml_integration.initialize()
  22. # Start ML system
  23. ml_integration.start()
  24. """
  25. import logging
  26. import threading
  27. import time
  28. from typing import Dict, List, Optional, Any, Callable, Union
  29. from dataclasses import dataclass, field
  30. from datetime import datetime
  31. from enum import Enum
  32. import traceback
  33. # Import ML components
  34. from .ml_manager import MLManager, MLConfig, MLSystemMode as MLMode, create_ml_manager, create_default_ml_config
  35. from .audio_pipeline import AudioPipeline, AudioPipelineConfig, create_audio_pipeline
  36. from .workflow import MLWorkflowManager, WorkflowConfig, create_workflow_manager
  37. from .events import MLEventManager, create_ml_event_manager
  38. from .monitoring import MLPerformanceMonitor, PerformanceConfig, create_performance_monitor
  39. from .config import MLConfigManager, MLSystemConfig, create_ml_config
  40. # Import application container components
  41. try:
  42. from ..application import ApplicationContainer, ComponentStatus, ComponentHealth
  43. from ..events import EventHandler
  44. from ..config import ConfigManager, DeploymentMode
  45. from ..network import NetworkManager
  46. APPLICATION_CONTAINER_AVAILABLE = True
  47. except ImportError:
  48. APPLICATION_CONTAINER_AVAILABLE = False
  49. ApplicationContainer = object
  50. ComponentStatus = object
  51. ComponentHealth = object
  52. EventHandler = object
  53. ConfigManager = object
  54. DeploymentMode = object
  55. NetworkManager = object
  56. # Configure logging
  57. logger = logging.getLogger(__name__)
  58. class MLIntegrationState(Enum):
  59. """ML integration states."""
  60. NOT_INITIALIZED = "not_initialized"
  61. INITIALIZING = "initializing"
  62. INITIALIZED = "initialized"
  63. STARTING = "starting"
  64. RUNNING = "running"
  65. STOPPING = "stopping"
  66. STOPPED = "stopped"
  67. ERROR = "error"
  68. @dataclass
  69. class MLComponentStatus:
  70. """Status of individual ML components."""
  71. name: str
  72. status: MLIntegrationState
  73. health: ComponentStatus
  74. last_check: float
  75. errors: List[str] = field(default_factory=list)
  76. warnings: List[str] = field(default_factory=list)
  77. info: Dict[str, Any] = field(default_factory=dict)
  78. def is_healthy(self) -> bool:
  79. """Check if component is healthy."""
  80. return (
  81. self.status in [MLIntegrationState.RUNNING, MLIntegrationState.INITIALIZED] and
  82. len(self.errors) == 0
  83. )
  84. @dataclass
  85. class MLIntegrationConfig:
  86. """Configuration for ML integration."""
  87. # Component enablement
  88. enable_ml_manager: bool = True
  89. enable_audio_pipeline: bool = True
  90. enable_workflow_manager: bool = True
  91. enable_event_integration: bool = True
  92. enable_performance_monitoring: bool = True
  93. # Integration settings
  94. auto_start_components: bool = True
  95. health_check_interval: float = 10.0
  96. component_timeout_seconds: float = 30.0
  97. enable_component_recovery: bool = True
  98. max_recovery_attempts: int = 3
  99. # Performance settings
  100. startup_delay_seconds: float = 1.0
  101. shutdown_timeout_seconds: float = 10.0
  102. # Debug settings
  103. enable_integration_logging: bool = True
  104. log_component_status: bool = False
  105. class MLContainerIntegrationError(Exception):
  106. """ML container integration errors."""
  107. pass
  108. class MLContainerIntegration:
  109. """
  110. ML system integration with Trixy application container.
  111. This class provides seamless integration between the ML components and the
  112. Trixy application container, handling lifecycle management, dependency
  113. injection, and component coordination.
  114. """
  115. def __init__(
  116. self,
  117. application_container: Any,
  118. integration_config: Optional[MLIntegrationConfig] = None
  119. ):
  120. """
  121. Initialize ML container integration.
  122. Args:
  123. application_container: Trixy application container
  124. integration_config: Integration configuration
  125. """
  126. self.application_container = application_container
  127. self.integration_config = integration_config or MLIntegrationConfig()
  128. # Integration state
  129. self._state = MLIntegrationState.NOT_INITIALIZED
  130. self._state_lock = threading.RLock()
  131. # ML components
  132. self._ml_manager: Optional[MLManager] = None
  133. self._audio_pipeline: Optional[AudioPipeline] = None
  134. self._workflow_manager: Optional[MLWorkflowManager] = None
  135. self._event_manager: Optional[MLEventManager] = None
  136. self._performance_monitor: Optional[MLPerformanceMonitor] = None
  137. self._ml_config_manager: Optional[MLConfigManager] = None
  138. # Component status tracking
  139. self._component_status: Dict[str, MLComponentStatus] = {}
  140. self._health_check_thread: Optional[threading.Thread] = None
  141. self._stop_health_checks = threading.Event()
  142. # Configuration
  143. self._ml_config: Optional[MLSystemConfig] = None
  144. # Container dependencies
  145. self._event_handler: Optional[EventHandler] = None
  146. self._config_manager: Optional[ConfigManager] = None
  147. self._network_manager: Optional[NetworkManager] = None
  148. # Recovery tracking
  149. self._recovery_attempts: Dict[str, int] = {}
  150. logger.info("ML Container Integration initialized")
  151. def initialize(self) -> None:
  152. """Initialize the ML integration system."""
  153. with self._state_lock:
  154. if self._state != MLIntegrationState.NOT_INITIALIZED:
  155. logger.warning("ML integration already initialized")
  156. return
  157. logger.info("Initializing ML container integration...")
  158. self._state = MLIntegrationState.INITIALIZING
  159. try:
  160. # Get container dependencies
  161. self._get_container_dependencies()
  162. # Load ML configuration
  163. self._load_ml_configuration()
  164. # Initialize ML components
  165. self._initialize_ml_components()
  166. # Start health monitoring
  167. if self.integration_config.health_check_interval > 0:
  168. self._start_health_monitoring()
  169. self._state = MLIntegrationState.INITIALIZED
  170. logger.info("ML container integration initialized successfully")
  171. except Exception as e:
  172. self._state = MLIntegrationState.ERROR
  173. logger.error(f"Failed to initialize ML integration: {e}")
  174. raise MLContainerIntegrationError(f"Initialization failed: {e}") from e
  175. def start(self) -> None:
  176. """Start the ML integration system."""
  177. with self._state_lock:
  178. if self._state not in [MLIntegrationState.INITIALIZED, MLIntegrationState.STOPPED]:
  179. logger.warning(f"Cannot start ML integration from state: {self._state.value}")
  180. return
  181. logger.info("Starting ML container integration...")
  182. self._state = MLIntegrationState.STARTING
  183. try:
  184. # Start components in dependency order
  185. if self.integration_config.auto_start_components:
  186. self._start_ml_components()
  187. self._state = MLIntegrationState.RUNNING
  188. logger.info("ML container integration started successfully")
  189. except Exception as e:
  190. self._state = MLIntegrationState.ERROR
  191. logger.error(f"Failed to start ML integration: {e}")
  192. raise MLContainerIntegrationError(f"Start failed: {e}") from e
  193. def stop(self) -> None:
  194. """Stop the ML integration system."""
  195. with self._state_lock:
  196. if self._state in [MLIntegrationState.STOPPED, MLIntegrationState.NOT_INITIALIZED]:
  197. return
  198. logger.info("Stopping ML container integration...")
  199. old_state = self._state
  200. self._state = MLIntegrationState.STOPPING
  201. try:
  202. # Stop health monitoring
  203. self._stop_health_monitoring()
  204. # Stop ML components
  205. self._stop_ml_components()
  206. self._state = MLIntegrationState.STOPPED
  207. logger.info("ML container integration stopped successfully")
  208. except Exception as e:
  209. self._state = MLIntegrationState.ERROR
  210. logger.error(f"Error stopping ML integration: {e}")
  211. def get_state(self) -> MLIntegrationState:
  212. """Get current integration state."""
  213. with self._state_lock:
  214. return self._state
  215. def get_ml_manager(self) -> Optional[MLManager]:
  216. """Get ML manager instance."""
  217. return self._ml_manager
  218. def get_audio_pipeline(self) -> Optional[AudioPipeline]:
  219. """Get audio pipeline instance."""
  220. return self._audio_pipeline
  221. def get_workflow_manager(self) -> Optional[MLWorkflowManager]:
  222. """Get workflow manager instance."""
  223. return self._workflow_manager
  224. def get_event_manager(self) -> Optional[MLEventManager]:
  225. """Get ML event manager instance."""
  226. return self._event_manager
  227. def get_performance_monitor(self) -> Optional[MLPerformanceMonitor]:
  228. """Get performance monitor instance."""
  229. return self._performance_monitor
  230. def get_component_health(self) -> Dict[str, MLComponentStatus]:
  231. """Get health status of all ML components."""
  232. return self._component_status.copy()
  233. def is_healthy(self) -> bool:
  234. """Check if all ML components are healthy."""
  235. if self._state != MLIntegrationState.RUNNING:
  236. return False
  237. return all(status.is_healthy() for status in self._component_status.values())
  238. def restart_component(self, component_name: str) -> bool:
  239. """
  240. Restart a specific ML component.
  241. Args:
  242. component_name: Name of component to restart
  243. Returns:
  244. True if restart was successful
  245. """
  246. logger.info(f"Restarting ML component: {component_name}")
  247. try:
  248. # Stop component
  249. if component_name == "ml_manager" and self._ml_manager:
  250. self._ml_manager.stop()
  251. elif component_name == "audio_pipeline" and self._audio_pipeline:
  252. self._audio_pipeline.stop()
  253. elif component_name == "workflow_manager" and self._workflow_manager:
  254. self._workflow_manager.stop()
  255. elif component_name == "event_manager" and self._event_manager:
  256. self._event_manager.stop()
  257. elif component_name == "performance_monitor" and self._performance_monitor:
  258. self._performance_monitor.stop()
  259. # Wait a moment
  260. time.sleep(1.0)
  261. # Start component
  262. if component_name == "ml_manager" and self._ml_manager:
  263. self._ml_manager.start()
  264. elif component_name == "audio_pipeline" and self._audio_pipeline:
  265. self._audio_pipeline.start()
  266. elif component_name == "workflow_manager" and self._workflow_manager:
  267. self._workflow_manager.start()
  268. elif component_name == "event_manager" and self._event_manager:
  269. self._event_manager.start()
  270. elif component_name == "performance_monitor" and self._performance_monitor:
  271. self._performance_monitor.start()
  272. logger.info(f"Component {component_name} restarted successfully")
  273. return True
  274. except Exception as e:
  275. logger.error(f"Failed to restart component {component_name}: {e}")
  276. return False
  277. def _get_container_dependencies(self) -> None:
  278. """Get dependencies from application container."""
  279. if not APPLICATION_CONTAINER_AVAILABLE:
  280. logger.warning("Application container not available")
  281. return
  282. try:
  283. # Get event handler
  284. if hasattr(self.application_container, 'get_event_handler'):
  285. self._event_handler = self.application_container.get_event_handler()
  286. logger.debug("Event handler obtained from container")
  287. # Get config manager
  288. if hasattr(self.application_container, 'get_config_manager'):
  289. self._config_manager = self.application_container.get_config_manager()
  290. logger.debug("Config manager obtained from container")
  291. # Get network manager (for client mode)
  292. if hasattr(self.application_container, 'get_network_manager'):
  293. try:
  294. self._network_manager = self.application_container.get_network_manager()
  295. logger.debug("Network manager obtained from container")
  296. except Exception as e:
  297. logger.warning(f"Network manager not available: {e}")
  298. except Exception as e:
  299. logger.error(f"Error getting container dependencies: {e}")
  300. raise
  301. def _load_ml_configuration(self) -> None:
  302. """Load ML configuration."""
  303. try:
  304. # Create ML config manager
  305. self._ml_config_manager = MLConfigManager()
  306. # Determine deployment mode
  307. mode = MLMode.CLIENT # Default
  308. if self.application_container and hasattr(self.application_container, 'get_mode'):
  309. container_mode = self.application_container.get_mode()
  310. if hasattr(container_mode, 'value'):
  311. mode_str = container_mode.value
  312. if mode_str == "server":
  313. mode = MLMode.SERVER
  314. elif mode_str == "standalone":
  315. mode = MLMode.STANDALONE
  316. elif mode_str == "client":
  317. mode = MLMode.CLIENT
  318. # Load configuration
  319. self._ml_config = self._ml_config_manager.load_config(mode)
  320. logger.info(f"ML configuration loaded for {mode.value} mode")
  321. except Exception as e:
  322. logger.error(f"Error loading ML configuration: {e}")
  323. # Create default configuration
  324. self._ml_config = create_ml_config()
  325. def _initialize_ml_components(self) -> None:
  326. """Initialize all ML components."""
  327. logger.info("Initializing ML components...")
  328. # Initialize ML Event Manager first (others may depend on it)
  329. if self.integration_config.enable_event_integration:
  330. self._initialize_event_manager()
  331. # Initialize Audio Pipeline
  332. if self.integration_config.enable_audio_pipeline:
  333. self._initialize_audio_pipeline()
  334. # Initialize ML Manager
  335. if self.integration_config.enable_ml_manager:
  336. self._initialize_ml_manager()
  337. # Initialize Workflow Manager
  338. if self.integration_config.enable_workflow_manager:
  339. self._initialize_workflow_manager()
  340. # Initialize Performance Monitor
  341. if self.integration_config.enable_performance_monitoring:
  342. self._initialize_performance_monitor()
  343. logger.info("ML components initialized")
  344. def _initialize_event_manager(self) -> None:
  345. """Initialize ML event manager."""
  346. try:
  347. logger.debug("Initializing ML event manager...")
  348. self._event_manager = create_ml_event_manager(
  349. main_event_handler=self._event_handler,
  350. enable_correlation=True,
  351. enable_persistence=False,
  352. max_event_history=1000
  353. )
  354. self._update_component_status(
  355. "event_manager",
  356. MLIntegrationState.INITIALIZED,
  357. ComponentStatus.INITIALIZED
  358. )
  359. logger.debug("ML event manager initialized")
  360. except Exception as e:
  361. logger.error(f"Error initializing ML event manager: {e}")
  362. self._update_component_status(
  363. "event_manager",
  364. MLIntegrationState.ERROR,
  365. ComponentStatus.ERROR,
  366. errors=[str(e)]
  367. )
  368. raise
  369. def _initialize_audio_pipeline(self) -> None:
  370. """Initialize audio pipeline."""
  371. try:
  372. logger.debug("Initializing audio pipeline...")
  373. # Create audio pipeline configuration
  374. if self._ml_config and self._ml_config.audio_pipeline_config:
  375. audio_config = self._ml_config.audio_pipeline_config
  376. else:
  377. audio_config = AudioPipelineConfig(
  378. sample_rate=self._ml_config.sample_rate if self._ml_config else 16000,
  379. channels=self._ml_config.channels if self._ml_config else 1,
  380. chunk_size=self._ml_config.chunk_size if self._ml_config else 1024
  381. )
  382. self._audio_pipeline = create_audio_pipeline(
  383. config=audio_config,
  384. ml_manager=None # Will be set later
  385. )
  386. self._update_component_status(
  387. "audio_pipeline",
  388. MLIntegrationState.INITIALIZED,
  389. ComponentStatus.INITIALIZED
  390. )
  391. logger.debug("Audio pipeline initialized")
  392. except Exception as e:
  393. logger.error(f"Error initializing audio pipeline: {e}")
  394. self._update_component_status(
  395. "audio_pipeline",
  396. MLIntegrationState.ERROR,
  397. ComponentStatus.ERROR,
  398. errors=[str(e)]
  399. )
  400. raise
  401. def _initialize_ml_manager(self) -> None:
  402. """Initialize ML manager."""
  403. try:
  404. logger.debug("Initializing ML manager...")
  405. # Create ML configuration
  406. if self._ml_config:
  407. ml_config = MLConfig(
  408. mode=self._ml_config.deployment_mode,
  409. enable_wakeword=self._ml_config.enable_wakeword,
  410. enable_voice_recognition=self._ml_config.enable_voice_recognition,
  411. enable_continuous_recognition=self._ml_config.enable_continuous_recognition,
  412. wakeword_config=self._ml_config.wakeword_config,
  413. voice_recognition_config=self._ml_config.voice_recognition_config,
  414. sample_rate=self._ml_config.sample_rate,
  415. chunk_size=self._ml_config.chunk_size
  416. )
  417. else:
  418. ml_config = create_default_ml_config()
  419. self._ml_manager = create_ml_manager(
  420. config=ml_config,
  421. application=self.application_container,
  422. event_handler=self._event_handler,
  423. network_manager=self._network_manager
  424. )
  425. # Link with audio pipeline
  426. if self._audio_pipeline:
  427. self._audio_pipeline.ml_manager = self._ml_manager
  428. self._update_component_status(
  429. "ml_manager",
  430. MLIntegrationState.INITIALIZED,
  431. ComponentStatus.INITIALIZED
  432. )
  433. logger.debug("ML manager initialized")
  434. except Exception as e:
  435. logger.error(f"Error initializing ML manager: {e}")
  436. self._update_component_status(
  437. "ml_manager",
  438. MLIntegrationState.ERROR,
  439. ComponentStatus.ERROR,
  440. errors=[str(e)]
  441. )
  442. raise
  443. def _initialize_workflow_manager(self) -> None:
  444. """Initialize workflow manager."""
  445. try:
  446. logger.debug("Initializing workflow manager...")
  447. if not self._ml_manager:
  448. logger.warning("ML manager not available for workflow manager")
  449. return
  450. # Create workflow configuration
  451. workflow_config = WorkflowConfig(
  452. enable_workflow_logging=self.integration_config.enable_integration_logging
  453. )
  454. self._workflow_manager = create_workflow_manager(
  455. ml_manager=self._ml_manager,
  456. network_manager=self._network_manager,
  457. event_handler=self._event_handler,
  458. config=workflow_config
  459. )
  460. self._update_component_status(
  461. "workflow_manager",
  462. MLIntegrationState.INITIALIZED,
  463. ComponentStatus.INITIALIZED
  464. )
  465. logger.debug("Workflow manager initialized")
  466. except Exception as e:
  467. logger.error(f"Error initializing workflow manager: {e}")
  468. self._update_component_status(
  469. "workflow_manager",
  470. MLIntegrationState.ERROR,
  471. ComponentStatus.ERROR,
  472. errors=[str(e)]
  473. )
  474. raise
  475. def _initialize_performance_monitor(self) -> None:
  476. """Initialize performance monitor."""
  477. try:
  478. logger.debug("Initializing performance monitor...")
  479. # Create performance configuration
  480. performance_config = PerformanceConfig(
  481. enable_performance_logging=self.integration_config.enable_integration_logging,
  482. monitoring_interval_seconds=1.0,
  483. enable_automatic_optimization=True
  484. )
  485. self._performance_monitor = create_performance_monitor(
  486. config=performance_config,
  487. ml_manager=self._ml_manager,
  488. audio_pipeline=self._audio_pipeline
  489. )
  490. self._update_component_status(
  491. "performance_monitor",
  492. MLIntegrationState.INITIALIZED,
  493. ComponentStatus.INITIALIZED
  494. )
  495. logger.debug("Performance monitor initialized")
  496. except Exception as e:
  497. logger.error(f"Error initializing performance monitor: {e}")
  498. self._update_component_status(
  499. "performance_monitor",
  500. MLIntegrationState.ERROR,
  501. ComponentStatus.ERROR,
  502. errors=[str(e)]
  503. )
  504. # Performance monitor is not critical - don't raise
  505. def _start_ml_components(self) -> None:
  506. """Start all ML components."""
  507. logger.info("Starting ML components...")
  508. # Start components in dependency order
  509. startup_order = [
  510. ("event_manager", self._event_manager),
  511. ("audio_pipeline", self._audio_pipeline),
  512. ("ml_manager", self._ml_manager),
  513. ("workflow_manager", self._workflow_manager),
  514. ("performance_monitor", self._performance_monitor)
  515. ]
  516. for component_name, component in startup_order:
  517. if component:
  518. try:
  519. logger.debug(f"Starting {component_name}...")
  520. # Add startup delay if configured
  521. if self.integration_config.startup_delay_seconds > 0:
  522. time.sleep(self.integration_config.startup_delay_seconds)
  523. component.start()
  524. self._update_component_status(
  525. component_name,
  526. MLIntegrationState.RUNNING,
  527. ComponentStatus.RUNNING
  528. )
  529. logger.debug(f"{component_name} started")
  530. except Exception as e:
  531. logger.error(f"Error starting {component_name}: {e}")
  532. self._update_component_status(
  533. component_name,
  534. MLIntegrationState.ERROR,
  535. ComponentStatus.ERROR,
  536. errors=[str(e)]
  537. )
  538. # Some components are critical for operation
  539. if component_name in ["ml_manager", "workflow_manager"]:
  540. raise
  541. logger.info("ML components started")
  542. def _stop_ml_components(self) -> None:
  543. """Stop all ML components."""
  544. logger.info("Stopping ML components...")
  545. # Stop components in reverse order
  546. shutdown_order = [
  547. ("performance_monitor", self._performance_monitor),
  548. ("workflow_manager", self._workflow_manager),
  549. ("ml_manager", self._ml_manager),
  550. ("audio_pipeline", self._audio_pipeline),
  551. ("event_manager", self._event_manager)
  552. ]
  553. for component_name, component in shutdown_order:
  554. if component:
  555. try:
  556. logger.debug(f"Stopping {component_name}...")
  557. component.stop()
  558. self._update_component_status(
  559. component_name,
  560. MLIntegrationState.STOPPED,
  561. ComponentStatus.STOPPED
  562. )
  563. logger.debug(f"{component_name} stopped")
  564. except Exception as e:
  565. logger.error(f"Error stopping {component_name}: {e}")
  566. self._update_component_status(
  567. component_name,
  568. MLIntegrationState.ERROR,
  569. ComponentStatus.ERROR,
  570. errors=[str(e)]
  571. )
  572. logger.info("ML components stopped")
  573. def _start_health_monitoring(self) -> None:
  574. """Start health monitoring thread."""
  575. if self._health_check_thread and self._health_check_thread.is_alive():
  576. return
  577. logger.debug("Starting ML health monitoring...")
  578. self._stop_health_checks.clear()
  579. self._health_check_thread = threading.Thread(
  580. target=self._health_monitoring_loop,
  581. daemon=True,
  582. name="MLHealthMonitor"
  583. )
  584. self._health_check_thread.start()
  585. logger.debug("ML health monitoring started")
  586. def _stop_health_monitoring(self) -> None:
  587. """Stop health monitoring thread."""
  588. if not self._health_check_thread or not self._health_check_thread.is_alive():
  589. return
  590. logger.debug("Stopping ML health monitoring...")
  591. self._stop_health_checks.set()
  592. self._health_check_thread.join(timeout=5.0)
  593. logger.debug("ML health monitoring stopped")
  594. def _health_monitoring_loop(self) -> None:
  595. """Health monitoring loop."""
  596. logger.debug("ML health monitoring loop started")
  597. while not self._stop_health_checks.is_set():
  598. try:
  599. self._check_component_health()
  600. # Log component status if enabled
  601. if self.integration_config.log_component_status:
  602. self._log_component_status()
  603. # Wait for next check
  604. self._stop_health_checks.wait(self.integration_config.health_check_interval)
  605. except Exception as e:
  606. logger.error(f"Error in health monitoring loop: {e}")
  607. time.sleep(5.0)
  608. logger.debug("ML health monitoring loop stopped")
  609. def _check_component_health(self) -> None:
  610. """Check health of all ML components."""
  611. components = [
  612. ("event_manager", self._event_manager),
  613. ("audio_pipeline", self._audio_pipeline),
  614. ("ml_manager", self._ml_manager),
  615. ("workflow_manager", self._workflow_manager),
  616. ("performance_monitor", self._performance_monitor)
  617. ]
  618. for component_name, component in components:
  619. if component:
  620. try:
  621. # Check if component is responsive
  622. is_healthy = self._check_component_responsiveness(component_name, component)
  623. if is_healthy:
  624. self._update_component_status(
  625. component_name,
  626. MLIntegrationState.RUNNING,
  627. ComponentStatus.RUNNING
  628. )
  629. else:
  630. self._handle_component_failure(component_name, "Health check failed")
  631. except Exception as e:
  632. logger.error(f"Error checking health of {component_name}: {e}")
  633. self._handle_component_failure(component_name, str(e))
  634. def _check_component_responsiveness(self, component_name: str, component: Any) -> bool:
  635. """Check if component is responsive."""
  636. try:
  637. # Basic responsiveness checks
  638. if hasattr(component, 'is_running'):
  639. return component.is_running()
  640. elif hasattr(component, 'get_current_state'):
  641. state = component.get_current_state()
  642. return state is not None
  643. elif hasattr(component, '_is_running'):
  644. return getattr(component, '_is_running', False)
  645. else:
  646. # Assume healthy if no specific check available
  647. return True
  648. except Exception as e:
  649. logger.warning(f"Error checking responsiveness of {component_name}: {e}")
  650. return False
  651. def _handle_component_failure(self, component_name: str, error_message: str) -> None:
  652. """Handle component failure."""
  653. logger.warning(f"Component failure detected: {component_name} - {error_message}")
  654. self._update_component_status(
  655. component_name,
  656. MLIntegrationState.ERROR,
  657. ComponentStatus.ERROR,
  658. errors=[error_message]
  659. )
  660. # Attempt recovery if enabled
  661. if self.integration_config.enable_component_recovery:
  662. self._attempt_component_recovery(component_name, error_message)
  663. def _attempt_component_recovery(self, component_name: str, error_message: str) -> None:
  664. """Attempt to recover a failed component."""
  665. recovery_count = self._recovery_attempts.get(component_name, 0)
  666. if recovery_count >= self.integration_config.max_recovery_attempts:
  667. logger.error(f"Max recovery attempts reached for {component_name}")
  668. return
  669. logger.info(f"Attempting recovery for {component_name} (attempt {recovery_count + 1})")
  670. try:
  671. success = self.restart_component(component_name)
  672. if success:
  673. logger.info(f"Recovery successful for {component_name}")
  674. self._recovery_attempts[component_name] = 0 # Reset counter
  675. else:
  676. self._recovery_attempts[component_name] = recovery_count + 1
  677. except Exception as e:
  678. logger.error(f"Recovery failed for {component_name}: {e}")
  679. self._recovery_attempts[component_name] = recovery_count + 1
  680. def _update_component_status(
  681. self,
  682. component_name: str,
  683. state: MLIntegrationState,
  684. health: ComponentStatus,
  685. errors: Optional[List[str]] = None,
  686. warnings: Optional[List[str]] = None,
  687. info: Optional[Dict[str, Any]] = None
  688. ) -> None:
  689. """Update component status."""
  690. self._component_status[component_name] = MLComponentStatus(
  691. name=component_name,
  692. status=state,
  693. health=health,
  694. last_check=time.time(),
  695. errors=errors or [],
  696. warnings=warnings or [],
  697. info=info or {}
  698. )
  699. def _log_component_status(self) -> None:
  700. """Log current component status."""
  701. if not self.integration_config.log_component_status:
  702. return
  703. healthy_count = sum(1 for status in self._component_status.values() if status.is_healthy())
  704. total_count = len(self._component_status)
  705. logger.info(f"ML Component Health: {healthy_count}/{total_count} healthy")
  706. for name, status in self._component_status.items():
  707. if not status.is_healthy():
  708. logger.warning(f"Component {name}: {status.status.value} - {status.errors}")
  709. def create_ml_integration(
  710. application_container: Any,
  711. integration_config: Optional[MLIntegrationConfig] = None
  712. ) -> MLContainerIntegration:
  713. """
  714. Factory function to create ML container integration.
  715. Args:
  716. application_container: Trixy application container
  717. integration_config: Integration configuration
  718. Returns:
  719. Configured ML container integration
  720. """
  721. return MLContainerIntegration(
  722. application_container=application_container,
  723. integration_config=integration_config
  724. )
  725. # Export main classes and functions
  726. __all__ = [
  727. 'MLContainerIntegration',
  728. 'MLIntegrationConfig',
  729. 'MLIntegrationState',
  730. 'MLComponentStatus',
  731. 'MLContainerIntegrationError',
  732. 'create_ml_integration'
  733. ]