config.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802
  1. """
  2. Configuration management and argument parsing for ML trainers.
  3. This module provides comprehensive configuration management including
  4. file-based configuration, command-line argument parsing, and validation
  5. with support for inheritance and environment variable override.
  6. """
  7. import os
  8. import sys
  9. import json
  10. import yaml
  11. import argparse
  12. import logging
  13. from pathlib import Path
  14. from typing import Any, Dict, List, Optional, Union, Type
  15. from dataclasses import dataclass, field, fields, asdict
  16. from abc import ABC, abstractmethod
  17. import torch
  18. from .base import TrainerConfig, ModelFormat, TrainingState
  19. class ConfigError(Exception):
  20. """Exception raised for configuration-related errors."""
  21. pass
  22. class ConfigValidator(ABC):
  23. """Abstract base class for configuration validators."""
  24. @abstractmethod
  25. def validate(self, config: Dict[str, Any]) -> List[str]:
  26. """Validate configuration and return list of error messages."""
  27. pass
  28. class TrainerConfigValidator(ConfigValidator):
  29. """Validator for trainer configuration."""
  30. def validate(self, config: Dict[str, Any]) -> List[str]:
  31. """Validate trainer configuration."""
  32. errors = []
  33. # Required fields
  34. required_fields = ['trainer_name', 'model_name', 'data_dir']
  35. for field in required_fields:
  36. if field not in config or not config[field]:
  37. errors.append(f"Required field '{field}' is missing or empty")
  38. # Numeric validations
  39. numeric_validations = {
  40. 'batch_size': (1, 1024),
  41. 'learning_rate': (1e-8, 1.0),
  42. 'num_epochs': (1, 10000),
  43. 'min_epochs': (0, 10000),
  44. 'max_epochs': (1, 50000),
  45. 'early_stopping_patience': (1, 1000),
  46. 'validation_split': (0.0, 1.0),
  47. 'test_split': (0.0, 1.0),
  48. 'sample_rate': (8000, 48000),
  49. 'n_mels': (10, 128),
  50. 'audio_length': (0.1, 30.0)
  51. }
  52. for field, (min_val, max_val) in numeric_validations.items():
  53. if field in config:
  54. value = config[field]
  55. if not isinstance(value, (int, float)) or not (min_val <= value <= max_val):
  56. errors.append(f"Field '{field}' must be between {min_val} and {max_val}")
  57. # Validation split + test split check
  58. if 'validation_split' in config and 'test_split' in config:
  59. if config['validation_split'] + config['test_split'] >= 1.0:
  60. errors.append("validation_split + test_split must be less than 1.0")
  61. # Cross-field epoch validations
  62. if 'min_epochs' in config and 'num_epochs' in config:
  63. if config['min_epochs'] > config['num_epochs']:
  64. errors.append("min_epochs cannot be greater than num_epochs")
  65. if 'max_epochs' in config and 'num_epochs' in config:
  66. if config['max_epochs'] < config['num_epochs']:
  67. errors.append("max_epochs cannot be less than num_epochs")
  68. if 'min_epochs' in config and 'max_epochs' in config:
  69. if config['min_epochs'] > config['max_epochs']:
  70. errors.append("min_epochs cannot be greater than max_epochs")
  71. # Directory existence checks
  72. directory_fields = ['data_dir', 'output_dir', 'config_dir']
  73. for field in directory_fields:
  74. if field in config and config[field]:
  75. path = Path(config[field])
  76. if field == 'data_dir' and not path.exists():
  77. errors.append(f"Data directory does not exist: {config[field]}")
  78. # Model format validation
  79. if 'model_format' in config:
  80. format_value = config['model_format']
  81. if isinstance(format_value, str):
  82. valid_formats = [fmt.value for fmt in ModelFormat]
  83. if format_value not in valid_formats:
  84. errors.append(f"Invalid model format: {format_value}. Valid options: {valid_formats}")
  85. # Device validation
  86. if 'device' in config:
  87. device = config['device']
  88. if device not in ['auto', 'cpu', 'cuda', 'mps']:
  89. errors.append(f"Invalid device: {device}. Valid options: auto, cpu, cuda, mps")
  90. return errors
  91. class WakewordConfigValidator(ConfigValidator):
  92. """Validator for wakeword trainer configuration."""
  93. def validate(self, config: Dict[str, Any]) -> List[str]:
  94. """Validate wakeword-specific configuration."""
  95. errors = []
  96. # Wakeword-specific validations
  97. if 'wakeword_classes' in config:
  98. classes = config['wakeword_classes']
  99. if not isinstance(classes, dict):
  100. errors.append("wakeword_classes must be a dictionary")
  101. elif len(classes) < 2:
  102. errors.append("At least 2 wakeword classes required (including negative)")
  103. # Audio processing for wakewords
  104. if 'audio_length' in config:
  105. if config['audio_length'] < 0.5 or config['audio_length'] > 5.0:
  106. errors.append("Wakeword audio_length should be between 0.5 and 5.0 seconds")
  107. return errors
  108. class VoiceRecognitionConfigValidator(ConfigValidator):
  109. """Validator for voice recognition trainer configuration."""
  110. def validate(self, config: Dict[str, Any]) -> List[str]:
  111. """Validate voice recognition specific configuration."""
  112. errors = []
  113. # Speaker-specific validations
  114. if 'num_speakers' in config:
  115. num_speakers = config['num_speakers']
  116. if not isinstance(num_speakers, int) or num_speakers < 2:
  117. errors.append("num_speakers must be at least 2")
  118. elif num_speakers > 1000:
  119. errors.append("num_speakers should not exceed 1000 for practical purposes")
  120. if 'embedding_dimension' in config:
  121. embedding_dim = config['embedding_dimension']
  122. if not isinstance(embedding_dim, int) or embedding_dim < 64:
  123. errors.append("embedding_dimension must be at least 64")
  124. elif embedding_dim > 2048:
  125. errors.append("embedding_dimension should not exceed 2048")
  126. # Audio processing for voice recognition
  127. if 'audio_length' in config:
  128. if config['audio_length'] < 1.0 or config['audio_length'] > 10.0:
  129. errors.append("Voice recognition audio_length should be between 1.0 and 10.0 seconds")
  130. return errors
  131. class ConfigManager:
  132. """
  133. Comprehensive configuration manager for ML trainers.
  134. Supports loading from files, command-line arguments, environment variables,
  135. and provides validation and inheritance capabilities.
  136. """
  137. def __init__(self, config_dir: str = "./config", logger: Optional[logging.Logger] = None):
  138. """
  139. Initialize configuration manager.
  140. Args:
  141. config_dir: Directory containing configuration files
  142. logger: Optional logger instance
  143. """
  144. self.config_dir = Path(config_dir)
  145. self.logger = logger or logging.getLogger(__name__)
  146. self.validators: Dict[str, ConfigValidator] = {
  147. 'base': TrainerConfigValidator(),
  148. 'wakeword': WakewordConfigValidator(),
  149. 'voice_recognition': VoiceRecognitionConfigValidator()
  150. }
  151. self.loaded_configs: Dict[str, Dict[str, Any]] = {}
  152. def load_config_file(self, filepath: Union[str, Path]) -> Dict[str, Any]:
  153. """
  154. Load configuration from file (JSON or YAML).
  155. Args:
  156. filepath: Path to configuration file
  157. Returns:
  158. Configuration dictionary
  159. """
  160. filepath = Path(filepath)
  161. if not filepath.exists():
  162. raise ConfigError(f"Configuration file not found: {filepath}")
  163. try:
  164. with open(filepath, 'r', encoding='utf-8') as f:
  165. if filepath.suffix.lower() in ['.yml', '.yaml']:
  166. try:
  167. config = yaml.safe_load(f)
  168. except ImportError:
  169. raise ConfigError("PyYAML not available for YAML configuration files")
  170. elif filepath.suffix.lower() == '.json':
  171. config = json.load(f)
  172. else:
  173. raise ConfigError(f"Unsupported configuration file format: {filepath.suffix}")
  174. self.logger.info(f"Loaded configuration from: {filepath}")
  175. return config or {}
  176. except (json.JSONDecodeError, yaml.YAMLError) as e:
  177. raise ConfigError(f"Error parsing configuration file {filepath}: {str(e)}")
  178. except Exception as e:
  179. raise ConfigError(f"Error loading configuration file {filepath}: {str(e)}")
  180. def save_config_file(self, config: Dict[str, Any], filepath: Union[str, Path]):
  181. """
  182. Save configuration to file.
  183. Args:
  184. config: Configuration dictionary
  185. filepath: Path to save configuration
  186. """
  187. filepath = Path(filepath)
  188. filepath.parent.mkdir(parents=True, exist_ok=True)
  189. try:
  190. with open(filepath, 'w', encoding='utf-8') as f:
  191. if filepath.suffix.lower() in ['.yml', '.yaml']:
  192. try:
  193. yaml.safe_dump(config, f, default_flow_style=False, allow_unicode=True)
  194. except ImportError:
  195. raise ConfigError("PyYAML not available for YAML configuration files")
  196. elif filepath.suffix.lower() == '.json':
  197. json.dump(config, f, indent=2, ensure_ascii=False)
  198. else:
  199. raise ConfigError(f"Unsupported configuration file format: {filepath.suffix}")
  200. self.logger.info(f"Saved configuration to: {filepath}")
  201. except Exception as e:
  202. raise ConfigError(f"Error saving configuration file {filepath}: {str(e)}")
  203. def load_trainer_config(self, trainer_name: str, config_name: str = None) -> TrainerConfig:
  204. """
  205. Load trainer configuration with inheritance and validation.
  206. Args:
  207. trainer_name: Name of the trainer (wakeword, voice_recognition, etc.)
  208. config_name: Specific configuration name (defaults to trainer_name)
  209. Returns:
  210. TrainerConfig instance
  211. """
  212. if config_name is None:
  213. config_name = trainer_name
  214. # Load base configuration
  215. base_config = self._load_base_config()
  216. # Load trainer-specific configuration
  217. trainer_config = self._load_trainer_specific_config(trainer_name, config_name)
  218. # Merge configurations (trainer-specific overrides base)
  219. merged_config = self._merge_configs(base_config, trainer_config)
  220. # Apply environment variable overrides
  221. merged_config = self._apply_env_overrides(merged_config)
  222. # Validate configuration
  223. self._validate_config(merged_config, trainer_name)
  224. # Convert to TrainerConfig object
  225. return self._dict_to_trainer_config(merged_config)
  226. def _load_base_config(self) -> Dict[str, Any]:
  227. """Load base trainer configuration."""
  228. base_config_path = self.config_dir / "trainer_base.json"
  229. if base_config_path.exists():
  230. return self.load_config_file(base_config_path)
  231. else:
  232. # Return default base configuration
  233. self.logger.info("Using default base configuration")
  234. return {
  235. "batch_size": 32,
  236. "learning_rate": 0.001,
  237. "num_epochs": 100,
  238. "min_epochs": 10,
  239. "early_stopping_patience": 10,
  240. "validation_split": 0.2,
  241. "test_split": 0.1,
  242. "sample_rate": 16000,
  243. "audio_length": 1.5,
  244. "n_mels": 40,
  245. "use_augmentation": True,
  246. "log_level": "INFO",
  247. "device": "auto"
  248. }
  249. def _load_trainer_specific_config(self, trainer_name: str, config_name: str) -> Dict[str, Any]:
  250. """Load trainer-specific configuration."""
  251. config_paths = [
  252. self.config_dir / f"{config_name}.json",
  253. self.config_dir / f"{config_name}.yml",
  254. self.config_dir / f"{config_name}.yaml",
  255. self.config_dir / f"{trainer_name}.json",
  256. self.config_dir / f"{trainer_name}.yml",
  257. self.config_dir / f"{trainer_name}.yaml"
  258. ]
  259. for config_path in config_paths:
  260. if config_path.exists():
  261. return self.load_config_file(config_path)
  262. self.logger.warning(f"No specific configuration found for {trainer_name}/{config_name}")
  263. return {}
  264. def _merge_configs(self, base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
  265. """Merge configuration dictionaries (override takes precedence)."""
  266. merged = base.copy()
  267. for key, value in override.items():
  268. if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
  269. # Recursively merge nested dictionaries
  270. merged[key] = self._merge_configs(merged[key], value)
  271. else:
  272. merged[key] = value
  273. return merged
  274. def _apply_env_overrides(self, config: Dict[str, Any]) -> Dict[str, Any]:
  275. """Apply environment variable overrides."""
  276. env_prefix = "TRIXY_TRAINER_"
  277. for key, value in os.environ.items():
  278. if key.startswith(env_prefix):
  279. config_key = key[len(env_prefix):].lower()
  280. # Convert environment variable value to appropriate type
  281. try:
  282. # Try to parse as JSON first (for complex types)
  283. parsed_value = json.loads(value)
  284. except json.JSONDecodeError:
  285. # Fall back to string/numeric parsing
  286. if value.lower() in ['true', 'false']:
  287. parsed_value = value.lower() == 'true'
  288. elif value.isdigit():
  289. parsed_value = int(value)
  290. else:
  291. try:
  292. parsed_value = float(value)
  293. except ValueError:
  294. parsed_value = value
  295. config[config_key] = parsed_value
  296. self.logger.info(f"Applied environment override: {config_key} = {parsed_value}")
  297. return config
  298. def _validate_config(self, config: Dict[str, Any], trainer_name: str):
  299. """Validate configuration using appropriate validators."""
  300. all_errors = []
  301. # Always validate with base validator
  302. base_errors = self.validators['base'].validate(config)
  303. all_errors.extend(base_errors)
  304. # Validate with trainer-specific validator if available
  305. if trainer_name in self.validators:
  306. trainer_errors = self.validators[trainer_name].validate(config)
  307. all_errors.extend(trainer_errors)
  308. if all_errors:
  309. error_msg = f"Configuration validation failed:\n" + "\n".join(f"- {error}" for error in all_errors)
  310. raise ConfigError(error_msg)
  311. def _dict_to_trainer_config(self, config_dict: Dict[str, Any]) -> TrainerConfig:
  312. """Convert configuration dictionary to TrainerConfig object."""
  313. # Handle enum conversions
  314. if 'model_format' in config_dict:
  315. if isinstance(config_dict['model_format'], str):
  316. config_dict['model_format'] = ModelFormat(config_dict['model_format'])
  317. # Filter only valid TrainerConfig fields
  318. trainer_config_fields = {f.name for f in fields(TrainerConfig)}
  319. filtered_config = {k: v for k, v in config_dict.items() if k in trainer_config_fields}
  320. # Store additional config in custom_params
  321. custom_params = {k: v for k, v in config_dict.items() if k not in trainer_config_fields}
  322. if custom_params:
  323. # Merge with existing custom_params if any
  324. existing_custom_params = filtered_config.get('custom_params', {})
  325. existing_custom_params.update(custom_params)
  326. filtered_config['custom_params'] = existing_custom_params
  327. try:
  328. return TrainerConfig(**filtered_config)
  329. except Exception as e:
  330. raise ConfigError(f"Error creating TrainerConfig: {str(e)}")
  331. def create_argument_parser(self, trainer_name: str = "trainer") -> argparse.ArgumentParser:
  332. """
  333. Create command-line argument parser for trainer.
  334. Args:
  335. trainer_name: Name of the trainer for help text
  336. Returns:
  337. Configured ArgumentParser
  338. """
  339. parser = argparse.ArgumentParser(
  340. description=f"Trixy {trainer_name.title()} Trainer",
  341. formatter_class=argparse.ArgumentDefaultsHelpFormatter
  342. )
  343. # General arguments
  344. parser.add_argument(
  345. '--config', '-c', type=str,
  346. help='Configuration file path'
  347. )
  348. parser.add_argument(
  349. '--trainer-name', type=str, default=trainer_name,
  350. help='Name of the trainer'
  351. )
  352. parser.add_argument(
  353. '--model-name', type=str,
  354. help='Name of the model'
  355. )
  356. parser.add_argument(
  357. '--output-dir', type=str, default='./models',
  358. help='Output directory for models'
  359. )
  360. parser.add_argument(
  361. '--data-dir', type=str, default='./trainer/data',
  362. help='Data directory'
  363. )
  364. # Training parameters
  365. training_group = parser.add_argument_group('Training Parameters')
  366. training_group.add_argument(
  367. '--batch-size', type=int, default=32,
  368. help='Batch size for training'
  369. )
  370. training_group.add_argument(
  371. '--learning-rate', '--lr', type=float, default=0.001,
  372. help='Learning rate'
  373. )
  374. training_group.add_argument(
  375. '--num-epochs', '--epochs', type=int, default=100,
  376. help='Number of training epochs'
  377. )
  378. training_group.add_argument(
  379. '--min-epochs', type=int, default=10,
  380. help='Minimum number of epochs'
  381. )
  382. training_group.add_argument(
  383. '--max-epochs', type=int, default=1000,
  384. help='Maximum number of epochs'
  385. )
  386. training_group.add_argument(
  387. '--early-stopping-patience', type=int, default=10,
  388. help='Early stopping patience'
  389. )
  390. # Model parameters
  391. model_group = parser.add_argument_group('Model Parameters')
  392. model_group.add_argument(
  393. '--model-format', choices=['pth', 'pt', 'onnx'], default='pth',
  394. help='Model file format'
  395. )
  396. model_group.add_argument(
  397. '--use-mixed-precision', action='store_true', default=True,
  398. help='Use mixed precision training'
  399. )
  400. model_group.add_argument(
  401. '--no-mixed-precision', dest='use_mixed_precision', action='store_false',
  402. help='Disable mixed precision training'
  403. )
  404. # Data parameters
  405. data_group = parser.add_argument_group('Data Parameters')
  406. data_group.add_argument(
  407. '--validation-split', type=float, default=0.2,
  408. help='Validation split ratio'
  409. )
  410. data_group.add_argument(
  411. '--test-split', type=float, default=0.1,
  412. help='Test split ratio'
  413. )
  414. data_group.add_argument(
  415. '--sample-rate', type=int, default=16000,
  416. help='Audio sample rate'
  417. )
  418. data_group.add_argument(
  419. '--audio-length', type=float, default=1.5,
  420. help='Audio length in seconds'
  421. )
  422. data_group.add_argument(
  423. '--n-mels', type=int, default=40,
  424. help='Number of mel filterbanks'
  425. )
  426. # Augmentation
  427. aug_group = parser.add_argument_group('Data Augmentation')
  428. aug_group.add_argument(
  429. '--use-augmentation', action='store_true', default=True,
  430. help='Use data augmentation'
  431. )
  432. aug_group.add_argument(
  433. '--no-augmentation', dest='use_augmentation', action='store_false',
  434. help='Disable data augmentation'
  435. )
  436. aug_group.add_argument(
  437. '--noise-factor', type=float, default=0.1,
  438. help='Noise augmentation factor'
  439. )
  440. aug_group.add_argument(
  441. '--speed-factor', type=float, default=0.1,
  442. help='Speed augmentation factor'
  443. )
  444. # Security
  445. security_group = parser.add_argument_group('Security')
  446. security_group.add_argument(
  447. '--password', type=str,
  448. help='Password for model encryption'
  449. )
  450. security_group.add_argument(
  451. '--no-password-protection', dest='use_password_protection',
  452. action='store_false', default=True,
  453. help='Disable password protection'
  454. )
  455. # Hardware
  456. hardware_group = parser.add_argument_group('Hardware')
  457. hardware_group.add_argument(
  458. '--device', choices=['auto', 'cpu', 'cuda', 'mps'], default='auto',
  459. help='Training device'
  460. )
  461. hardware_group.add_argument(
  462. '--num-workers', type=int, default=4,
  463. help='Number of data loader workers'
  464. )
  465. # Logging and checkpoints
  466. logging_group = parser.add_argument_group('Logging and Checkpoints')
  467. logging_group.add_argument(
  468. '--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], default='INFO',
  469. help='Logging level'
  470. )
  471. logging_group.add_argument(
  472. '--save-checkpoints', action='store_true', default=True,
  473. help='Save training checkpoints'
  474. )
  475. logging_group.add_argument(
  476. '--no-checkpoints', dest='save_checkpoints', action='store_false',
  477. help='Disable checkpoint saving'
  478. )
  479. logging_group.add_argument(
  480. '--checkpoint-interval', type=int, default=10,
  481. help='Checkpoint save interval (epochs)'
  482. )
  483. # Resume training
  484. resume_group = parser.add_argument_group('Resume Training')
  485. resume_group.add_argument(
  486. '--resume', '--resume-from-checkpoint', type=str,
  487. help='Resume training from checkpoint'
  488. )
  489. # Actions
  490. action_group = parser.add_argument_group('Actions')
  491. action_group.add_argument(
  492. '--train', action='store_true', default=True,
  493. help='Train the model'
  494. )
  495. action_group.add_argument(
  496. '--test-only', action='store_true',
  497. help='Only test the model (requires trained model)'
  498. )
  499. action_group.add_argument(
  500. '--prepare-data-only', action='store_true',
  501. help='Only prepare data without training'
  502. )
  503. action_group.add_argument(
  504. '--validate-config-only', action='store_true',
  505. help='Only validate configuration'
  506. )
  507. return parser
  508. def parse_args_to_config(self, args: argparse.Namespace, trainer_name: str) -> TrainerConfig:
  509. """
  510. Convert command-line arguments to TrainerConfig.
  511. Args:
  512. args: Parsed command-line arguments
  513. trainer_name: Name of the trainer
  514. Returns:
  515. TrainerConfig instance
  516. """
  517. # Start with configuration file if provided
  518. if hasattr(args, 'config') and args.config:
  519. config = self.load_config_file(args.config)
  520. else:
  521. config = self.load_trainer_config(trainer_name).to_dict()
  522. # Override with command-line arguments
  523. arg_dict = vars(args)
  524. for key, value in arg_dict.items():
  525. if value is not None:
  526. # Convert hyphens to underscores for config keys
  527. config_key = key.replace('-', '_')
  528. config[config_key] = value
  529. # Handle special cases
  530. if hasattr(args, 'resume') and args.resume:
  531. config['resume_from_checkpoint'] = True
  532. config['checkpoint_path'] = args.resume
  533. if hasattr(args, 'model_format') and args.model_format:
  534. config['model_format'] = f".{args.model_format}"
  535. # Validate and convert to TrainerConfig
  536. self._validate_config(config, trainer_name)
  537. return self._dict_to_trainer_config(config)
  538. def generate_config_template(self, trainer_name: str, filepath: str = None) -> str:
  539. """
  540. Generate configuration file template.
  541. Args:
  542. trainer_name: Name of the trainer
  543. filepath: Optional file path to save template
  544. Returns:
  545. Configuration template as string
  546. """
  547. template = {
  548. "# Configuration for Trixy ML Trainer": None,
  549. "trainer_name": trainer_name,
  550. "model_name": f"{trainer_name}_model",
  551. "description": f"Configuration for {trainer_name} trainer",
  552. "# Data settings": None,
  553. "data_dir": f"./trainer/data/{trainer_name}",
  554. "output_dir": "./models",
  555. "validation_split": 0.2,
  556. "test_split": 0.1,
  557. "# Training parameters": None,
  558. "batch_size": 32,
  559. "learning_rate": 0.001,
  560. "num_epochs": 100,
  561. "min_epochs": 10,
  562. "early_stopping_patience": 10,
  563. "# Model settings": None,
  564. "model_format": ".pth",
  565. "use_mixed_precision": True,
  566. "use_password_protection": True,
  567. "# Audio processing": None,
  568. "sample_rate": 16000,
  569. "audio_length": 1.5,
  570. "n_mels": 40,
  571. "n_fft": 512,
  572. "hop_length": 160,
  573. "# Data augmentation": None,
  574. "use_augmentation": True,
  575. "noise_factor": 0.1,
  576. "speed_factor": 0.1,
  577. "pitch_factor": 0.1,
  578. "volume_factor": 0.2,
  579. "# Hardware": None,
  580. "device": "auto",
  581. "num_workers": 4,
  582. "# Logging": None,
  583. "log_level": "INFO",
  584. "save_checkpoints": True,
  585. "checkpoint_interval": 10
  586. }
  587. # Add trainer-specific settings
  588. if trainer_name == "wakeword":
  589. template.update({
  590. "# Wakeword specific": None,
  591. "wakeword_classes": {
  592. 0: "custom",
  593. 1: "system_command",
  594. 2: "negative"
  595. },
  596. "detection_threshold": 0.5
  597. })
  598. elif trainer_name == "voice_recognition":
  599. template.update({
  600. "# Voice recognition specific": None,
  601. "num_speakers": 2,
  602. "embedding_dimension": 256,
  603. "architecture": "ecapa_tdnn"
  604. })
  605. # Convert to JSON string
  606. template_str = "{\n"
  607. for key, value in template.items():
  608. if value is None: # Comment line
  609. template_str += f" \"{key}\",\n"
  610. elif isinstance(value, str):
  611. template_str += f" \"{key}\": \"{value}\",\n"
  612. elif isinstance(value, dict):
  613. template_str += f" \"{key}\": {json.dumps(value, indent=4)},\n"
  614. else:
  615. template_str += f" \"{key}\": {json.dumps(value)},\n"
  616. template_str = template_str.rstrip(',\n') + "\n}"
  617. # Save to file if requested
  618. if filepath:
  619. with open(filepath, 'w', encoding='utf-8') as f:
  620. f.write(template_str)
  621. self.logger.info(f"Configuration template saved to: {filepath}")
  622. return template_str
  623. def list_available_configs(self) -> List[str]:
  624. """List all available configuration files."""
  625. configs = []
  626. if self.config_dir.exists():
  627. for pattern in ['*.json', '*.yml', '*.yaml']:
  628. configs.extend([f.stem for f in self.config_dir.glob(pattern)])
  629. return sorted(list(set(configs)))
  630. def get_config_info(self, config_name: str) -> Dict[str, Any]:
  631. """Get information about a configuration file."""
  632. config_paths = [
  633. self.config_dir / f"{config_name}.json",
  634. self.config_dir / f"{config_name}.yml",
  635. self.config_dir / f"{config_name}.yaml"
  636. ]
  637. for config_path in config_paths:
  638. if config_path.exists():
  639. config = self.load_config_file(config_path)
  640. return {
  641. 'name': config_name,
  642. 'path': str(config_path),
  643. 'format': config_path.suffix,
  644. 'size': config_path.stat().st_size,
  645. 'trainer_name': config.get('trainer_name', 'unknown'),
  646. 'model_name': config.get('model_name', 'unknown'),
  647. 'description': config.get('description', ''),
  648. 'keys': list(config.keys())
  649. }
  650. raise ConfigError(f"Configuration not found: {config_name}")
  651. def create_trainer_cli(trainer_name: str) -> argparse.ArgumentParser:
  652. """
  653. Create command-line interface for a trainer.
  654. Args:
  655. trainer_name: Name of the trainer
  656. Returns:
  657. Configured ArgumentParser
  658. """
  659. config_manager = ConfigManager()
  660. return config_manager.create_argument_parser(trainer_name)
  661. def load_config_from_args(args: argparse.Namespace, trainer_name: str) -> TrainerConfig:
  662. """
  663. Load configuration from command-line arguments.
  664. Args:
  665. args: Parsed command-line arguments
  666. trainer_name: Name of the trainer
  667. Returns:
  668. TrainerConfig instance
  669. """
  670. config_manager = ConfigManager()
  671. return config_manager.parse_args_to_config(args, trainer_name)