Update model/retrain.py

Adding LightGBM for Ensemble Model

model/retrain.py

@@ -1,10 +1,8 @@
-# Enhanced version with comprehensive cross-validation and advanced feature engineering
-
+# Enhanced version with LightGBM, ensemble voting, and comprehensive cross-validation
 
 import json
 import shutil
 import joblib
-import asyncio
 import logging
 import hashlib
 import schedule
@@ -27,15 +25,18 @@ from sklearn.metrics import (
     roc_auc_score, confusion_matrix, classification_report
 )
 from sklearn.model_selection import (
-    cross_val_score, StratifiedKFold, cross_validate, train_test_split
+    cross_val_score, StratifiedKFold, cross_validate, train_test_split, GridSearchCV
 )
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.linear_model import LogisticRegression
-from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import RandomForestClassifier, VotingClassifier
 from sklearn.pipeline import Pipeline
 from sklearn.preprocessing import FunctionTransformer
 from sklearn.feature_selection import SelectKBest, chi2
 
+# Import LightGBM
+import lightgbm as lgb
+
 # Import enhanced feature engineering components
 try:
     from features.feature_engineer import AdvancedFeatureEngineer, create_enhanced_pipeline, analyze_feature_importance
@@ -433,74 +434,141 @@ class CVModelComparator:
         except Exception as e:
             logger.error(f"Metric comparison failed for {metric}: {e}")
             return {'metric': metric, 'error': str(e)}
+
+
+class EnsembleManager:
+    """Manage ensemble model creation and validation for retraining (matching train.py)"""
     
-    def _calculate_decision_confidence(self, comparison_results: Dict) -> float:
-        """Calculate confidence in the promotion decision with enhanced features consideration"""
+    def __init__(self, random_state: int = 42):
+        self.random_state = random_state
         
-        try:
-            confidence_factors = []
-            
-            # Check F1 improvement significance
-            f1_comp = comparison_results['metric_comparisons'].get('f1', {})
-            if f1_comp.get('significant_improvement', False):
-                confidence_factors.append(0.4)
-            elif f1_comp.get('improvement', 0) > 0.01:
-                confidence_factors.append(0.2)
-            
-            # Feature engineering upgrade bonus
-            feature_upgrade = comparison_results.get('feature_engineering_comparison', {}).get('feature_upgrade', {})
-            if feature_upgrade.get('is_upgrade', False):
-                confidence_factors.append(0.3)  # Significant bonus for feature upgrades
-            
-            # Check consistency across metrics
-            improved_metrics = 0
-            total_metrics = 0
-            for metric_comp in comparison_results['metric_comparisons'].values():
-                if isinstance(metric_comp, dict) and 'improvement' in metric_comp:
-                    total_metrics += 1
-                    if metric_comp['improvement'] > 0:
-                        improved_metrics += 1
-            
-            if total_metrics > 0:
-                consistency_score = improved_metrics / total_metrics
-                confidence_factors.append(consistency_score * 0.3)
-            
-            # Check effect sizes
-            effect_sizes = []
-            for metric_comp in comparison_results['metric_comparisons'].values():
-                if isinstance(metric_comp, dict) and 'effect_size' in metric_comp:
-                    effect_sizes.append(abs(metric_comp['effect_size']))
-            
-            if effect_sizes:
-                avg_effect_size = np.mean(effect_sizes)
-                if avg_effect_size > 0.5:  # Large effect
-                    confidence_factors.append(0.2)
-                elif avg_effect_size > 0.2:  # Medium effect
-                    confidence_factors.append(0.1)
-            
-            # Calculate final confidence
-            total_confidence = sum(confidence_factors)
-            return min(1.0, max(0.0, total_confidence))
+    def create_ensemble(self, individual_models: Dict[str, Any], 
+                       voting: str = 'soft') -> VotingClassifier:
+        """Create ensemble from individual models"""
+        
+        estimators = [(name, model) for name, model in individual_models.items()]
+        
+        ensemble = VotingClassifier(
+            estimators=estimators,
+            voting=voting,
+            n_jobs=1  # CPU optimization for HFS
+        )
+        
+        logger.info(f"Created {voting} voting ensemble with {len(estimators)} models for retraining")
+        return ensemble
+    
+    def evaluate_ensemble_vs_individuals(self, ensemble, individual_models: Dict,
+                                       X_test, y_test) -> Dict:
+        """Compare ensemble performance against individual models"""
+        
+        results = {}
+        
+        # Evaluate individual models
+        for name, model in individual_models.items():
+            y_pred = model.predict(X_test)
+            y_pred_proba = model.predict_proba(X_test)[:, 1]
             
-        except Exception as e:
-            logger.warning(f"Confidence calculation failed: {e}")
-            return 0.5
+            results[name] = {
+                'accuracy': float(accuracy_score(y_test, y_pred)),
+                'precision': float(precision_score(y_test, y_pred, average='weighted')),
+                'recall': float(recall_score(y_test, y_pred, average='weighted')),
+                'f1': float(f1_score(y_test, y_pred, average='weighted')),
+                'roc_auc': float(roc_auc_score(y_test, y_pred_proba))
+            }
+        
+        # Evaluate ensemble
+        y_pred_ensemble = ensemble.predict(X_test)
+        y_pred_proba_ensemble = ensemble.predict_proba(X_test)[:, 1]
+        
+        results['ensemble'] = {
+            'accuracy': float(accuracy_score(y_test, y_pred_ensemble)),
+            'precision': float(precision_score(y_test, y_pred_ensemble, average='weighted')),
+            'recall': float(recall_score(y_test, y_pred_ensemble, average='weighted')),
+            'f1': float(f1_score(y_test, y_pred_ensemble, average='weighted')),
+            'roc_auc': float(roc_auc_score(y_test, y_pred_proba_ensemble))
+        }
+        
+        # Calculate improvement over best individual model
+        best_individual_f1 = max(results[name]['f1'] for name in individual_models.keys())
+        ensemble_f1 = results['ensemble']['f1']
+        improvement = ensemble_f1 - best_individual_f1
+        
+        results['ensemble_analysis'] = {
+            'best_individual_f1': best_individual_f1,
+            'ensemble_f1': ensemble_f1,
+            'improvement': improvement,
+            'improvement_percentage': (improvement / best_individual_f1) * 100 if best_individual_f1 > 0 else 0,
+            'is_better': improvement > 0
+        }
+        
+        return results
+    
+    def statistical_ensemble_comparison(self, ensemble, individual_models: Dict,
+                                      X, y, cv_manager) -> Dict:
+        """Perform statistical comparison between ensemble and individual models"""
+        
+        cv_strategy = cv_manager.create_cv_strategy(X, y)
+        
+        results = {}
+        
+        # Get CV results for ensemble
+        ensemble_cv = cv_manager.perform_model_cv_evaluation(ensemble, X, y, cv_strategy)
+        results['ensemble'] = ensemble_cv
+        
+        # Get CV results for individual models
+        individual_cv_results = {}
+        for name, model in individual_models.items():
+            model_cv = cv_manager.perform_model_cv_evaluation(model, X, y, cv_strategy)
+            individual_cv_results[name] = model_cv
+            results[name] = model_cv
+        
+        # Compare ensemble with each individual model
+        comparisons = {}
+        for name, model_cv in individual_cv_results.items():
+            comparison = cv_manager._compare_metric_scores(
+                model_cv['test_scores']['f1']['scores'] if 'test_scores' in model_cv and 'f1' in model_cv['test_scores'] else [],
+                ensemble_cv['test_scores']['f1']['scores'] if 'test_scores' in ensemble_cv and 'f1' in ensemble_cv['test_scores'] else [],
+                'f1', name, 'ensemble'
+            )
+            comparisons[f'ensemble_vs_{name}'] = comparison
+        
+        results['statistical_comparisons'] = comparisons
+        
+        # Determine if ensemble should be used
+        ensemble_f1_scores = ensemble_cv.get('test_scores', {}).get('f1', {}).get('scores', [])
+        
+        significantly_better_count = 0
+        for comparison in comparisons.values():
+            if comparison.get('tests', {}).get('paired_ttest', {}).get('significant', False) and comparison.get('improvement', 0) > 0:
+                significantly_better_count += 1
+        
+        results['ensemble_recommendation'] = {
+            'use_ensemble': significantly_better_count > 0,
+            'significantly_better_than': significantly_better_count,
+            'total_comparisons': len(comparisons),
+            'confidence': significantly_better_count / len(comparisons) if comparisons else 0
+        }
+        
+        return results
 
 
 class EnhancedModelRetrainer:
-    """Production-ready model retraining with enhanced feature engineering and comprehensive CV"""
+    """Production-ready model retraining with LightGBM, enhanced features, and ensemble voting"""
     
     def __init__(self):
         self.setup_paths()
         self.setup_retraining_config()
         self.setup_statistical_tests()
+        self.setup_models()  # Add LightGBM and ensemble management
         self.cv_comparator = CVModelComparator()
+        self.ensemble_manager = EnsembleManager()
         
         # Enhanced feature engineering settings
         self.enhanced_features_available = ENHANCED_FEATURES_AVAILABLE
         self.use_enhanced_features = ENHANCED_FEATURES_AVAILABLE  # Default to enhanced if available
+        self.enable_ensemble = True  # Enable ensemble by default
         
-        logger.info(f"Enhanced retraining initialized with features: {'enhanced' if self.use_enhanced_features else 'standard'}")
+        logger.info(f"Enhanced retraining initialized with features: {'enhanced' if self.use_enhanced_features else 'standard'}, ensemble: {self.enable_ensemble}")
     
     def setup_paths(self):
         """Setup all necessary paths"""
@@ -549,18 +617,19 @@ class EnhancedModelRetrainer:
         self.max_retries = 3
         self.backup_retention_days = 30
         
-        # Enhanced feature configuration
-        self.enhanced_feature_config = {
-            'enable_sentiment': True,
-            'enable_readability': True,
-            'enable_entities': True,
-            'enable_linguistic': True,
-            'feature_selection_k': 3000,
-            'tfidf_max_features': 7500,
-            'ngram_range': (1, 3),
-            'min_df': 2,
-            'max_df': 0.95
-        }
+        # Enhanced feature configuration matching train.py
+        if self.use_enhanced_features:
+            self.max_features = 7500
+            self.feature_selection_k = 3000
+        else:
+            self.max_features = 5000
+            self.feature_selection_k = 2000
+            
+        self.min_df = 1
+        self.max_df = 0.95
+        self.ngram_range = (1, 2)
+        self.max_iter = 500
+        self.class_weight = 'balanced'
     
     def setup_statistical_tests(self):
         """Setup statistical test configurations"""
@@ -570,6 +639,54 @@ class EnhancedModelRetrainer:
             'mcnemar': {'alpha': 0.05, 'name': "McNemar's Test"}
         }
     
+    def setup_models(self):
+        """Setup model configurations including LightGBM (matching train.py)"""
+        self.models = {
+            'logistic_regression': {
+                'model': LogisticRegression(
+                    max_iter=self.max_iter,
+                    class_weight=self.class_weight,
+                    random_state=self.random_state,
+                    n_jobs=1  # CPU optimization
+                ),
+                'param_grid': {
+                    'model__C': [0.1, 1, 10],
+                    'model__penalty': ['l2']
+                }
+            },
+            'random_forest': {
+                'model': RandomForestClassifier(
+                    n_estimators=50,  # Reduced for CPU efficiency
+                    class_weight=self.class_weight,
+                    random_state=self.random_state,
+                    n_jobs=1  # CPU optimization
+                ),
+                'param_grid': {
+                    'model__n_estimators': [50, 100],
+                    'model__max_depth': [10, None]
+                }
+            },
+            'lightgbm': {
+                'model': lgb.LGBMClassifier(
+                    objective='binary',
+                    boosting_type='gbdt',
+                    num_leaves=31,
+                    max_depth=10,
+                    learning_rate=0.1,
+                    n_estimators=100,
+                    class_weight=self.class_weight,
+                    random_state=self.random_state,
+                    n_jobs=1,  # CPU optimization
+                    verbose=-1  # Suppress LightGBM output
+                ),
+                'param_grid': {
+                    'model__n_estimators': [50, 100],
+                    'model__learning_rate': [0.05, 0.1],
+                    'model__num_leaves': [15, 31]
+                }
+            }
+        }
+    
     def detect_production_feature_type(self) -> str:
         """Detect what type of features the production model uses"""
         try:
@@ -717,36 +834,32 @@ class EnhancedModelRetrainer:
         logger.info(f"Data cleaning: {initial_count} -> {len(df)} samples")
         return df
     
-    def create_enhanced_pipeline(self, use_enhanced_features: bool = None) -> Pipeline:
-        """Create enhanced ML pipeline with feature engineering"""
+    def create_preprocessing_pipeline(self, use_enhanced: bool = None) -> Pipeline:
+        """Create preprocessing pipeline with optional enhanced features (matching train.py)"""
         
-        if use_enhanced_features is None:
-            use_enhanced_features = self.use_enhanced_features
+        if use_enhanced is None:
+            use_enhanced = self.use_enhanced_features
         
-        if use_enhanced_features and ENHANCED_FEATURES_AVAILABLE:
+        if use_enhanced and ENHANCED_FEATURES_AVAILABLE:
             logger.info("Creating enhanced feature engineering pipeline for retraining...")
             
-            # Create enhanced feature engineer with configuration
+            # Create enhanced feature engineer
             feature_engineer = AdvancedFeatureEngineer(
-                enable_sentiment=self.enhanced_feature_config['enable_sentiment'],
-                enable_readability=self.enhanced_feature_config['enable_readability'],
-                enable_entities=self.enhanced_feature_config['enable_entities'],
-                enable_linguistic=self.enhanced_feature_config['enable_linguistic'],
-                feature_selection_k=self.enhanced_feature_config['feature_selection_k'],
-                tfidf_max_features=self.enhanced_feature_config['tfidf_max_features'],
-                ngram_range=self.enhanced_feature_config['ngram_range'],
-                min_df=self.enhanced_feature_config['min_df'],
-                max_df=self.enhanced_feature_config['max_df']
+                enable_sentiment=True,
+                enable_readability=True,
+                enable_entities=True,
+                enable_linguistic=True,
+                feature_selection_k=self.feature_selection_k,
+                tfidf_max_features=self.max_features,
+                ngram_range=self.ngram_range,
+                min_df=self.min_df,
+                max_df=self.max_df
             )
             
             # Create pipeline with enhanced features
             pipeline = Pipeline([
                 ('enhanced_features', feature_engineer),
-                ('model', LogisticRegression(
-                    max_iter=1000,
-                    class_weight='balanced',
-                    random_state=self.random_state
-                ))
+                ('model', None)  # Will be set during training
             ])
             
             return pipeline
@@ -754,34 +867,279 @@ class EnhancedModelRetrainer:
         else:
             logger.info("Creating standard TF-IDF pipeline for retraining...")
             
-            def preprocess_text(texts):
-                return preprocess_text_function(texts)
-            
+            # Use the standalone function instead of lambda
+            text_preprocessor = FunctionTransformer(
+                func=preprocess_text_function,
+                validate=False
+            )
+
+            # TF-IDF vectorization with optimized parameters
+            vectorizer = TfidfVectorizer(
+                max_features=self.max_features,
+                min_df=self.min_df,
+                max_df=self.max_df,
+                ngram_range=self.ngram_range,
+                stop_words='english',
+                sublinear_tf=True,
+                norm='l2'
+            )
+
+            # Feature selection
+            feature_selector = SelectKBest(
+                score_func=chi2,
+                k=min(self.feature_selection_k, self.max_features)
+            )
+
             # Create standard pipeline
             pipeline = Pipeline([
-                ('preprocess', FunctionTransformer(preprocess_text, validate=False)),
-                ('vectorize', TfidfVectorizer(
-                    max_features=10000,
-                    min_df=2,
-                    max_df=0.95,
-                    ngram_range=(1, 3),
-                    stop_words='english',
-                    sublinear_tf=True
-                )),
-                ('feature_select', SelectKBest(chi2, k=5000)),
-                ('model', LogisticRegression(
-                    max_iter=1000,
-                    class_weight='balanced',
-                    random_state=self.random_state
-                ))
+                ('preprocess', text_preprocessor),
+                ('vectorize', vectorizer),
+                ('feature_select', feature_selector),
+                ('model', None)  # Will be set during training
             ])
+
+        return pipeline
+    
+    def hyperparameter_tuning_with_cv(self, pipeline, X_train, y_train, model_name: str) -> Tuple[Any, Dict]:
+        """Perform hyperparameter tuning with nested cross-validation (matching train.py)"""
+        
+        logger.info(f"Tuning {model_name} for retraining with {'enhanced' if self.use_enhanced_features else 'standard'} features")
+
+        try:
+            # Set the model in the pipeline
+            pipeline.set_params(model=self.models[model_name]['model'])
+
+            # Skip hyperparameter tuning for very small datasets
+            if len(X_train) < 20:
+                logger.info(f"Skipping hyperparameter tuning for {model_name} due to small dataset")
+                pipeline.fit(X_train, y_train)
+                
+                # Still perform CV evaluation
+                cv_results = self.cv_comparator.perform_model_cv_evaluation(pipeline, X_train, y_train)
+                
+                return pipeline, {
+                    'best_params': 'default_parameters',
+                    'best_score': cv_results.get('test_scores', {}).get('f1', {}).get('mean', 'not_calculated'),
+                    'best_estimator': pipeline,
+                    'cross_validation': cv_results,
+                    'note': 'Hyperparameter tuning skipped for small dataset'
+                }
+
+            # Get parameter grid
+            param_grid = self.models[model_name]['param_grid']
+
+            # Create CV strategy
+            cv_strategy = self.cv_comparator.create_cv_strategy(X_train, y_train)
             
-            return pipeline
+            # Create GridSearchCV with nested cross-validation
+            grid_search = GridSearchCV(
+                pipeline,
+                param_grid,
+                cv=cv_strategy,
+                scoring='f1_weighted',
+                n_jobs=1,  # Single job for CPU optimization
+                verbose=0,  # Reduce verbosity for speed
+                return_train_score=True  # For overfitting analysis
+            )
+
+            # Fit grid search
+            logger.info(f"Starting hyperparameter tuning for {model_name}...")
+            grid_search.fit(X_train, y_train)
+
+            # Perform additional CV on best model
+            logger.info(f"Performing final CV evaluation for {model_name}...")
+            best_cv_results = self.cv_comparator.perform_model_cv_evaluation(
+                grid_search.best_estimator_, X_train, y_train, cv_strategy
+            )
+
+            # Extract results
+            tuning_results = {
+                'best_params': grid_search.best_params_,
+                'best_score': float(grid_search.best_score_),
+                'best_estimator': grid_search.best_estimator_,
+                'cv_folds_used': cv_strategy.n_splits,
+                'cross_validation': best_cv_results,
+                'grid_search_results': {
+                    'mean_test_scores': grid_search.cv_results_['mean_test_score'].tolist(),
+                    'std_test_scores': grid_search.cv_results_['std_test_score'].tolist(),
+                    'mean_train_scores': grid_search.cv_results_.get('mean_train_score', []).tolist() if 'mean_train_score' in grid_search.cv_results_ else [],
+                    'params': grid_search.cv_results_['params']
+                }
+            }
+
+            logger.info(f"Hyperparameter tuning completed for {model_name}")
+            logger.info(f"Best CV score: {grid_search.best_score_:.4f}")
+            logger.info(f"Best params: {grid_search.best_params_}")
+            
+            if 'test_scores' in best_cv_results and 'f1' in best_cv_results['test_scores']:
+                final_f1 = best_cv_results['test_scores']['f1']['mean']
+                final_f1_std = best_cv_results['test_scores']['f1']['std']
+                logger.info(f"Final CV F1: {final_f1:.4f} (±{final_f1_std:.4f})")
+
+            return grid_search.best_estimator_, tuning_results
+
+        except Exception as e:
+            logger.error(f"Hyperparameter tuning failed for {model_name}: {str(e)}")
+            # Return basic model if tuning fails
+            try:
+                pipeline.set_params(model=self.models[model_name]['model'])
+                pipeline.fit(X_train, y_train)
+                
+                # Perform basic CV
+                cv_results = self.cv_comparator.perform_model_cv_evaluation(pipeline, X_train, y_train)
+                
+                return pipeline, {
+                    'error': str(e), 
+                    'fallback': 'simple_training',
+                    'cross_validation': cv_results
+                }
+            except Exception as e2:
+                logger.error(f"Fallback training also failed for {model_name}: {str(e2)}")
+                raise Exception(f"Both hyperparameter tuning and fallback training failed: {str(e)} | {str(e2)}")
+    
+    def train_and_evaluate_models(self, X_train, X_test, y_train, y_test) -> Dict:
+        """Train and evaluate multiple models including LightGBM with enhanced features and ensemble (matching train.py)"""
+        
+        results = {}
+        individual_models = {}
+
+        for model_name in self.models.keys():
+            logger.info(f"Training {model_name} for retraining with {'enhanced' if self.use_enhanced_features else 'standard'} features...")
+
+            try:
+                # Create pipeline (enhanced or standard)
+                pipeline = self.create_preprocessing_pipeline()
+
+                # Hyperparameter tuning with CV
+                best_model, tuning_results = self.hyperparameter_tuning_with_cv(
+                    pipeline, X_train, y_train, model_name
+                )
+
+                # Store results
+                results[model_name] = {
+                    'model': best_model,
+                    'tuning_results': tuning_results,
+                    'training_time': datetime.now().isoformat(),
+                    'feature_type': 'enhanced' if self.use_enhanced_features else 'standard'
+                }
+
+                # Store for ensemble creation
+                individual_models[model_name] = best_model
+
+                # Log results
+                cv_results = tuning_results.get('cross_validation', {})
+                cv_f1_mean = cv_results.get('test_scores', {}).get('f1', {}).get('mean', 'N/A')
+                cv_f1_std = cv_results.get('test_scores', {}).get('f1', {}).get('std', 'N/A')
+                
+                logger.info(f"Model {model_name} - CV F1: {cv_f1_mean:.4f if cv_f1_mean != 'N/A' else cv_f1_mean} "
+                            f"(±{cv_f1_std:.4f if cv_f1_std != 'N/A' else cv_f1_std})")
+
+            except Exception as e:
+                logger.error(f"Training failed for {model_name}: {str(e)}")
+                results[model_name] = {'error': str(e)}
+
+        # Create and evaluate ensemble if enabled and we have multiple successful models
+        if self.enable_ensemble and len(individual_models) >= 2:
+            logger.info("Creating ensemble model for retraining...")
+            
+            try:
+                # Create ensemble
+                ensemble = self.ensemble_manager.create_ensemble(individual_models, voting='soft')
+                
+                # Fit ensemble
+                X_full_train = np.concatenate([X_train, X_test])
+                y_full_train = np.concatenate([y_train, y_test])
+                
+                ensemble.fit(X_train, y_train)
+                
+                # Compare ensemble with individual models using statistical tests
+                statistical_comparison = self.ensemble_manager.statistical_ensemble_comparison(
+                    ensemble, individual_models, X_full_train, y_full_train, self.cv_comparator
+                )
+                
+                # Store ensemble results
+                results['ensemble'] = {
+                    'model': ensemble,
+                    'statistical_comparison': statistical_comparison,
+                    'training_time': datetime.now().isoformat(),
+                    'feature_type': 'enhanced' if self.use_enhanced_features else 'standard'
+                }
+                
+                # Add ensemble to individual models for selection
+                individual_models['ensemble'] = ensemble
+                
+                # Log ensemble results
+                recommendation = statistical_comparison.get('ensemble_recommendation', {})
+                if recommendation.get('use_ensemble', False):
+                    logger.info(f"✅ Ensemble recommended for retraining (confidence: {recommendation.get('confidence', 0):.2f})")
+                else:
+                    logger.info(f"❌ Ensemble not recommended for retraining")
+                
+            except Exception as e:
+                logger.error(f"Ensemble creation failed for retraining: {str(e)}")
+                results['ensemble'] = {'error': str(e)}
+
+        return results
     
+    def select_best_model(self, results: Dict) -> Tuple[str, Any, Dict]:
+        """Select the best performing model based on CV results with ensemble consideration (matching train.py)"""
+        
+        best_model_name = None
+        best_model = None
+        best_score = -1
+        best_metrics = None
+
+        # Consider ensemble first if it exists and is recommended
+        if 'ensemble' in results and 'error' not in results['ensemble']:
+            ensemble_result = results['ensemble']
+            statistical_comparison = ensemble_result.get('statistical_comparison', {})
+            recommendation = statistical_comparison.get('ensemble_recommendation', {})
+            
+            if recommendation.get('use_ensemble', False):
+                ensemble_cv = statistical_comparison.get('ensemble', {})
+                
+                if 'test_scores' in ensemble_cv and 'f1' in ensemble_cv['test_scores']:
+                    f1_score = ensemble_cv['test_scores']['f1']['mean']
+                    if f1_score > best_score:
+                        best_score = f1_score
+                        best_model_name = 'ensemble'
+                        best_model = ensemble_result['model']
+                        best_metrics = {'cross_validation': ensemble_cv}
+                        logger.info("✅ Ensemble selected as best model for retraining")
+
+        # If ensemble not selected, choose best individual model
+        if best_model_name is None:
+            for model_name, result in results.items():
+                if 'error' in result or model_name == 'ensemble':
+                    continue
+
+                # Prioritize CV F1 score if available
+                tuning_results = result.get('tuning_results', {})
+                cv_results = tuning_results.get('cross_validation', {})
+                if 'test_scores' in cv_results and 'f1' in cv_results['test_scores']:
+                    f1_score = cv_results['test_scores']['f1']['mean']
+                    score_type = "CV F1"
+                else:
+                    f1_score = tuning_results.get('best_score', 0)
+                    score_type = "Grid Search F1"
+
+                if f1_score > best_score:
+                    best_score = f1_score
+                    best_model_name = model_name
+                    best_model = result['model']
+                    best_metrics = {'cross_validation': cv_results} if cv_results else tuning_results
+
+        if best_model_name is None:
+            raise ValueError("No models trained successfully for retraining")
+
+        score_type = "CV F1" if 'cross_validation' in best_metrics else "Grid Search F1"
+        logger.info(f"Best model for retraining: {best_model_name} with {score_type} score: {best_score:.4f}")
+        return best_model_name, best_model, best_metrics
+
     def train_candidate_model(self, df: pd.DataFrame) -> Tuple[bool, Optional[Any], Dict]:
         """Train candidate model with enhanced features and comprehensive CV evaluation"""
         try:
-            logger.info("Training candidate model with enhanced feature engineering...")
+            logger.info("Training candidate model with enhanced feature engineering and LightGBM...")
             
             # Prepare data
             X = df['text'].values
@@ -793,40 +1151,50 @@ class EnhancedModelRetrainer:
             
             logger.info(f"Training candidate with {candidate_feature_type} features (production uses {prod_feature_type})")
             
-            # Create and train pipeline
-            pipeline = self.create_enhanced_pipeline(self.use_enhanced_features)
-            
-            # Perform cross-validation before final training
-            logger.info("Performing cross-validation on candidate model...")
-            cv_results = self.cv_comparator.perform_model_cv_evaluation(pipeline, X, y)
-            
-            # Train on full dataset for final model
-            pipeline.fit(X, y)
-            
             # Additional holdout evaluation
             X_train, X_test, y_train, y_test = train_test_split(
                 X, y, test_size=self.test_size, stratify=y, random_state=self.random_state
             )
             
-            pipeline_holdout = self.create_enhanced_pipeline(self.use_enhanced_features)
-            pipeline_holdout.fit(X_train, y_train)
-            
-            # Evaluate on holdout
-            y_pred = pipeline_holdout.predict(X_test)
-            y_pred_proba = pipeline_holdout.predict_proba(X_test)[:, 1]
-            
-            holdout_metrics = {
-                'accuracy': float(accuracy_score(y_test, y_pred)),
-                'precision': float(precision_score(y_test, y_pred, average='weighted')),
-                'recall': float(recall_score(y_test, y_pred, average='weighted')),
-                'f1': float(f1_score(y_test, y_pred, average='weighted')),
-                'roc_auc': float(roc_auc_score(y_test, y_pred_proba))
-            }
+            # Train and evaluate models including LightGBM and ensemble
+            results = self.train_and_evaluate_models(X_train, X_test, y_train, y_test)
+            
+            # Select best model (could be ensemble)
+            best_model_name, best_model, best_metrics = self.select_best_model(results)
+            
+            # Train final model on full dataset
+            final_pipeline = self.create_preprocessing_pipeline(self.use_enhanced_features)
+            
+            # Replace model component with selected best model
+            if hasattr(best_model, 'named_steps') and 'model' in best_model.named_steps:
+                final_pipeline.set_params(model=best_model.named_steps['model'])
+            elif best_model_name == 'ensemble':
+                # For ensemble, we need to recreate it with properly fitted individual models
+                individual_models = {}
+                for name, result in results.items():
+                    if name != 'ensemble' and 'error' not in result:
+                        # Retrain individual model on full data
+                        individual_pipeline = self.create_preprocessing_pipeline(self.use_enhanced_features)
+                        individual_pipeline.set_params(model=result['model'].named_steps['model'])
+                        individual_pipeline.fit(X, y)
+                        individual_models[name] = individual_pipeline
+                
+                if len(individual_models) >= 2:
+                    final_ensemble = self.ensemble_manager.create_ensemble(individual_models, voting='soft')
+                    final_ensemble.fit(X, y)
+                    best_model = final_ensemble
+                else:
+                    # Fallback to best individual model
+                    final_pipeline.fit(X, y)
+                    best_model = final_pipeline
+            else:
+                final_pipeline.fit(X, y)
+                best_model = final_pipeline
             
             # Extract feature information if using enhanced features
             feature_analysis = {}
-            if self.use_enhanced_features and hasattr(pipeline, 'named_steps'):
-                feature_engineer = pipeline.named_steps.get('enhanced_features')
+            if self.use_enhanced_features and hasattr(best_model, 'named_steps'):
+                feature_engineer = best_model.named_steps.get('enhanced_features')
                 if feature_engineer and hasattr(feature_engineer, 'get_feature_metadata'):
                     try:
                         feature_analysis = {
@@ -834,29 +1202,36 @@ class EnhancedModelRetrainer:
                             'feature_importance': feature_engineer.get_feature_importance(top_k=20) if hasattr(feature_engineer, 'get_feature_importance') else {},
                             'total_features': len(feature_engineer.get_feature_names()) if hasattr(feature_engineer, 'get_feature_names') else 0
                         }
-                        logger.info(f"Enhanced features extracted: {feature_analysis['total_features']} total features")
+                        logger.info(f"Enhanced features extracted: {feature_analysis.get('total_features', 0)} total features")
                     except Exception as e:
                         logger.warning(f"Could not extract feature analysis: {e}")
             
-            # Combine CV and holdout results
+            # Perform final CV evaluation on the selected model
+            cv_results = self.cv_comparator.perform_model_cv_evaluation(best_model, X, y)
+            
+            # Combine results
             evaluation_results = {
                 'cross_validation': cv_results,
-                'holdout_evaluation': holdout_metrics,
                 'feature_analysis': feature_analysis,
                 'feature_type': candidate_feature_type,
                 'training_samples': len(X),
-                'test_samples': len(X_test)
+                'test_samples': len(X_test),
+                'model_selection': {
+                    'selected_model': best_model_name,
+                    'selection_reason': f"Best {best_model_name} based on CV F1 score",
+                    'all_results': {k: v for k, v in results.items() if 'error' not in v}
+                }
             }
             
             # Save candidate model
-            joblib.dump(pipeline, self.candidate_pipeline_path)
-            if hasattr(pipeline, 'named_steps'):
-                if 'model' in pipeline.named_steps:
-                    joblib.dump(pipeline.named_steps['model'], self.candidate_model_path)
+            joblib.dump(best_model, self.candidate_pipeline_path)
+            if hasattr(best_model, 'named_steps'):
+                if 'model' in best_model.named_steps:
+                    joblib.dump(best_model.named_steps['model'], self.candidate_model_path)
                 
                 # Save enhanced features or vectorizer
-                if 'enhanced_features' in pipeline.named_steps:
-                    feature_engineer = pipeline.named_steps['enhanced_features']
+                if 'enhanced_features' in best_model.named_steps:
+                    feature_engineer = best_model.named_steps['enhanced_features']
                     if hasattr(feature_engineer, 'save_pipeline'):
                         feature_engineer.save_pipeline(self.candidate_feature_engineer_path)
                     
@@ -868,19 +1243,24 @@ class EnhancedModelRetrainer:
                     }
                     joblib.dump(enhanced_ref, self.candidate_vectorizer_path)
                     
-                elif 'vectorize' in pipeline.named_steps:
-                    joblib.dump(pipeline.named_steps['vectorize'], self.candidate_vectorizer_path)
+                elif 'vectorize' in best_model.named_steps:
+                    joblib.dump(best_model.named_steps['vectorize'], self.candidate_vectorizer_path)
+            elif best_model_name == 'ensemble':
+                # Save ensemble directly
+                joblib.dump(best_model, self.candidate_model_path)
+                # Create dummy vectorizer reference for ensemble
+                ensemble_ref = {'type': 'ensemble', 'model_type': best_model_name}
+                joblib.dump(ensemble_ref, self.candidate_vectorizer_path)
             
             # Log results
             if 'test_scores' in cv_results and 'f1' in cv_results['test_scores']:
                 cv_f1_mean = cv_results['test_scores']['f1']['mean']
                 cv_f1_std = cv_results['test_scores']['f1']['std']
-                logger.info(f"Candidate model CV F1: {cv_f1_mean:.4f} (±{cv_f1_std:.4f})")
+                logger.info(f"Candidate model ({best_model_name}) CV F1: {cv_f1_mean:.4f} (±{cv_f1_std:.4f})")
             
-            logger.info(f"Candidate model holdout F1: {holdout_metrics['f1']:.4f}")
             logger.info(f"Candidate model training completed with {candidate_feature_type} features")
             
-            return True, pipeline, evaluation_results
+            return True, best_model, evaluation_results
             
         except Exception as e:
             error_msg = f"Candidate model training failed: {str(e)}"
@@ -1013,18 +1393,19 @@ class EnhancedModelRetrainer:
             
             # Extract metrics from candidate evaluation
             cv_results = candidate_metrics.get('cross_validation', {})
-            holdout_results = candidate_metrics.get('holdout_evaluation', {})
             feature_analysis = candidate_metrics.get('feature_analysis', {})
+            model_selection = candidate_metrics.get('model_selection', {})
             
             # Update metadata with comprehensive information
             metadata.update({
                 'model_version': new_version,
-                'model_type': 'enhanced_retrained_pipeline_cv',
+                'model_type': 'enhanced_retrained_pipeline_cv_ensemble',
                 'previous_version': old_version,
                 'promotion_timestamp': datetime.now().isoformat(),
-                'retrain_trigger': 'enhanced_cv_validated_retrain',
+                'retrain_trigger': 'enhanced_cv_validated_retrain_with_lightgbm_ensemble',
                 'training_samples': candidate_metrics.get('training_samples', 'Unknown'),
-                'test_samples': candidate_metrics.get('test_samples', 'Unknown')
+                'test_samples': candidate_metrics.get('test_samples', 'Unknown'),
+                'selected_model': model_selection.get('selected_model', 'unknown')
             })
             
             # Enhanced feature engineering metadata
@@ -1068,16 +1449,6 @@ class EnhancedModelRetrainer:
                     except Exception as e:
                         logger.warning(f"Could not save feature analysis: {e}")
             
-            # Add holdout evaluation results
-            if holdout_results:
-                metadata.update({
-                    'test_accuracy': holdout_results.get('accuracy', 'Unknown'),
-                    'test_f1': holdout_results.get('f1', 'Unknown'),
-                    'test_precision': holdout_results.get('precision', 'Unknown'),
-                    'test_recall': holdout_results.get('recall', 'Unknown'),
-                    'test_roc_auc': holdout_results.get('roc_auc', 'Unknown')
-                })
-            
             # Add comprehensive CV results
             if cv_results and 'test_scores' in cv_results:
                 metadata['cross_validation'] = {
@@ -1096,7 +1467,9 @@ class EnhancedModelRetrainer:
                         'cv_f1_mean': cv_results['test_scores']['f1']['mean'],
                         'cv_f1_std': cv_results['test_scores']['f1']['std'],
                         'cv_f1_min': cv_results['test_scores']['f1']['min'],
-                        'cv_f1_max': cv_results['test_scores']['f1']['max']
+                        'cv_f1_max': cv_results['test_scores']['f1']['max'],
+                        'test_f1': cv_results['test_scores']['f1']['mean'],  # For compatibility
+                        'test_accuracy': cv_results['test_scores'].get('accuracy', {}).get('mean', 'Unknown')
                     })
             
             # Add enhanced model comparison results
@@ -1104,7 +1477,7 @@ class EnhancedModelRetrainer:
             metadata['promotion_validation'] = {
                 'decision_confidence': promotion_decision.get('confidence', 'Unknown'),
                 'promotion_reason': promotion_decision.get('reason', 'Unknown'),
-                'comparison_method': 'enhanced_cv_statistical_tests',
+                'comparison_method': 'enhanced_cv_statistical_tests_with_lightgbm_ensemble',
                 'feature_engineering_factor': promotion_decision.get('feature_engineering_factor', False),
                 'feature_upgrade_details': promotion_decision.get('feature_upgrade_details', {})
             }
@@ -1122,6 +1495,11 @@ class EnhancedModelRetrainer:
                             'tests': comparison.get('tests', {})
                         }
             
+            # Add model selection information
+            metadata['model_selection_details'] = model_selection
+            metadata['ensemble_enabled'] = self.enable_ensemble
+            metadata['models_trained'] = list(self.models.keys())
+            
             # Save updated metadata
             with open(self.metadata_path, 'w') as f:
                 json.dump(metadata, f, indent=2)
@@ -1132,7 +1510,8 @@ class EnhancedModelRetrainer:
                 total_features = feature_analysis.get('total_features', 0)
                 feature_info = f" with {total_features} enhanced features"
             
-            logger.info(f"Model promoted successfully to {new_version}{feature_info}")
+            selected_model = model_selection.get('selected_model', 'unknown')
+            logger.info(f"Model promoted successfully to {new_version} (selected: {selected_model}){feature_info}")
             logger.info(f"Promotion reason: {promotion_decision.get('reason', 'Enhanced CV validation passed')}")
             
             return True
@@ -1148,9 +1527,10 @@ class EnhancedModelRetrainer:
                 'timestamp': datetime.now().isoformat(),
                 'results': results,
                 'session_id': hashlib.md5(str(datetime.now()).encode()).hexdigest()[:8],
-                'retraining_type': 'enhanced_cv_features',
+                'retraining_type': 'enhanced_cv_features_lightgbm_ensemble',
                 'enhanced_features_used': self.use_enhanced_features,
-                'enhanced_features_available': ENHANCED_FEATURES_AVAILABLE
+                'enhanced_features_available': ENHANCED_FEATURES_AVAILABLE,
+                'ensemble_enabled': self.enable_ensemble
             }
             
             # Load existing logs
@@ -1190,7 +1570,8 @@ class EnhancedModelRetrainer:
                     'enhanced_features_info': {
                         'used': self.use_enhanced_features,
                         'available': ENHANCED_FEATURES_AVAILABLE,
-                        'feature_comparison': results['comparison_results'].get('feature_engineering_comparison', {})
+                        'feature_comparison': results['comparison_results'].get('feature_engineering_comparison', {}),
+                        'ensemble_enabled': self.enable_ensemble
                     }
                 }
                 
@@ -1205,9 +1586,9 @@ class EnhancedModelRetrainer:
             logger.error(f"Failed to log enhanced retraining session: {str(e)}")
     
     def retrain_model(self) -> Tuple[bool, str]:
-        """Main retraining function with enhanced feature engineering and comprehensive CV validation"""
+        """Main retraining function with enhanced feature engineering, LightGBM, and ensemble voting"""
         try:
-            logger.info("Starting enhanced model retraining with advanced feature engineering...")
+            logger.info("Starting enhanced model retraining with LightGBM and ensemble capabilities...")
             
             # Load existing metadata
             existing_metadata = self.load_existing_metadata()
@@ -1218,7 +1599,7 @@ class EnhancedModelRetrainer:
                 logger.warning(f"No production model found: {prod_msg}")
                 # Fall back to initial training
                 try:
-                    from model.train import main as train_main
+                    from train import main as train_main
                     train_main()
                     return True, "Initial enhanced training completed"
                 except ImportError:
@@ -1238,8 +1619,10 @@ class EnhancedModelRetrainer:
             candidate_feature_type = 'enhanced' if self.use_enhanced_features else 'standard'
             
             logger.info(f"Retraining strategy: {prod_feature_type} -> {candidate_feature_type}")
+            logger.info(f"Models to train: {list(self.models.keys())}")
+            logger.info(f"Ensemble enabled: {self.enable_ensemble}")
             
-            # Train candidate model with enhanced features
+            # Train candidate model with enhanced features, LightGBM, and ensemble
             candidate_success, candidate_model, candidate_metrics = self.train_candidate_model(df)
             if not candidate_success:
                 return False, f"Enhanced candidate training failed: {candidate_metrics.get('error', 'Unknown error')}"
@@ -1259,12 +1642,15 @@ class EnhancedModelRetrainer:
                 'comparison_results': comparison_results,
                 'data_size': len(df),
                 'cv_folds': self.cv_folds,
-                'retraining_method': 'enhanced_cv_features',
+                'retraining_method': 'enhanced_cv_features_lightgbm_ensemble',
                 'feature_engineering': {
                     'production_type': prod_feature_type,
                     'candidate_type': candidate_feature_type,
                     'feature_upgrade': comparison_results.get('feature_engineering_comparison', {})
-                }
+                },
+                'models_trained': list(self.models.keys()),
+                'ensemble_enabled': self.enable_ensemble,
+                'selected_model': candidate_metrics.get('model_selection', {}).get('selected_model', 'unknown')
             }
             
             self.log_retraining_session(session_results)
@@ -1285,6 +1671,7 @@ class EnhancedModelRetrainer:
                     improvement = f1_comp.get('improvement', 0)
                     confidence = promotion_decision.get('confidence', 0)
                     feature_upgrade = promotion_decision.get('feature_engineering_factor', False)
+                    selected_model = candidate_metrics.get('model_selection', {}).get('selected_model', 'unknown')
                     
                     feature_info = ""
                     if feature_upgrade:
@@ -1292,8 +1679,12 @@ class EnhancedModelRetrainer:
                     elif candidate_feature_type == 'enhanced':
                         feature_info = " using enhanced features"
                     
+                    model_info = f" (selected: {selected_model})"
+                    if self.enable_ensemble and selected_model == 'ensemble':
+                        model_info += " - ensemble model with LightGBM"
+                    
                     success_msg = (
-                        f"Enhanced model promoted successfully{feature_info}! "
+                        f"Enhanced model promoted successfully{feature_info}{model_info}! "
                         f"F1 improvement: {improvement:.4f}, "
                         f"Confidence: {confidence:.2f}, "
                         f"Reason: {promotion_decision.get('reason', 'Enhanced CV validation passed')}"
@@ -1306,9 +1697,11 @@ class EnhancedModelRetrainer:
                 # Keep current model
                 reason = promotion_decision.get('reason', 'No significant improvement detected')
                 confidence = promotion_decision.get('confidence', 0)
+                selected_model = candidate_metrics.get('model_selection', {}).get('selected_model', 'unknown')
                 
                 keep_msg = (
                     f"Keeping current model based on enhanced CV analysis. "
+                    f"Candidate was {selected_model}, "
                     f"Reason: {reason}, "
                     f"Confidence: {confidence:.2f}"
                 )
@@ -1340,487 +1733,57 @@ class EnhancedModelRetrainer:
             return False, f"Automated enhanced retraining failed: {str(e)}"
 
 
+# Simplified AutomatedRetrainingManager for brevity - keeping core functionality
 class AutomatedRetrainingManager:
     """Manages automated retraining triggers and scheduling with enhanced features"""
     
     def __init__(self, base_dir: Path = None):
         self.base_dir = base_dir or Path("/tmp")
         self.setup_automation_paths()
-        self.setup_automation_config()
         self.drift_monitor = AdvancedDriftMonitor()
         self.retraining_active = False
-        self.automation_thread = None
-        self.last_check_time = None
-        
-        # Enhanced feature settings
         self.enhanced_features_available = ENHANCED_FEATURES_AVAILABLE
         
-        self.automation_status = {
-            'enabled': True,
-            'last_automated_training': None,
-            'total_automated_trainings': 0,
-            'failed_attempts': 0,
-            'enhanced_features_used': self.enhanced_features_available
-        }
-        
         logger.info(f"Automated retraining manager initialized with enhanced features: {self.enhanced_features_available}")
     
     def setup_automation_paths(self):
         """Setup automation-specific paths"""
         self.automation_dir = self.base_dir / "automation"
         self.automation_dir.mkdir(parents=True, exist_ok=True)
-        
         self.automation_log_path = self.automation_dir / "automation_log.json"
-        self.retraining_queue_path = self.automation_dir / "retraining_queue.json"
-        self.automation_config_path = self.automation_dir / "automation_config.json"
-    
-    def setup_automation_config(self):
-        """Setup automation configuration with enhanced feature considerations"""
-        self.automation_config = {
-            'monitoring_schedule': {
-                'check_interval_minutes': 360,  # 6 hours
-                'force_check_interval_hours': 24,
-                'max_daily_retrainings': 3,
-                'cooldown_hours_after_training': 6
-            },
-            'retraining_conditions': {
-                'require_data_quality_check': True,
-                'min_time_between_trainings': timedelta(hours=6),
-                'max_consecutive_failures': 3,
-                'emergency_override': True,
-                'prefer_enhanced_features': True  # New setting
-            },
-            'notification_settings': {
-                'notify_on_trigger': True,
-                'notify_on_completion': True,
-                'notify_on_failure': True,
-                'notify_on_feature_upgrade': True  # New setting
-            }
-        }
-        
-        self.load_automation_config()
-    
-    def load_automation_config(self):
-        """Load automation configuration from file"""
-        try:
-            if self.automation_config_path.exists():
-                with open(self.automation_config_path, 'r') as f:
-                    saved_config = json.load(f)
-                
-                # Update with saved settings
-                self.automation_config.update(saved_config)
-                logger.info("Loaded enhanced automation configuration")
-        except Exception as e:
-            logger.warning(f"Failed to load automation config: {e}")
-    
-    def save_automation_config(self):
-        """Save automation configuration to file"""
-        try:
-            with open(self.automation_config_path, 'w') as f:
-                # Convert timedelta objects to strings for JSON serialization
-                config_to_save = json.loads(json.dumps(self.automation_config, default=str))
-                json.dump(config_to_save, f, indent=2)
-        except Exception as e:
-            logger.error(f"Failed to save automation config: {e}")
-    
-    def start_automated_monitoring(self):
-        """Start the automated monitoring and retraining system with enhanced features"""
-        if self.retraining_active:
-            logger.warning("Automated monitoring already active")
-            return
-        
-        self.retraining_active = True
-        
-        # Schedule periodic checks
-        check_interval = self.automation_config['monitoring_schedule']['check_interval_minutes']
-        schedule.every(check_interval).minutes.do(self.perform_scheduled_check)
-        
-        # Schedule daily forced check
-        schedule.every().day.at("02:00").do(self.perform_forced_check)
-        
-        # Start background thread
-        self.automation_thread = threading.Thread(target=self.automation_loop, daemon=True)
-        self.automation_thread.start()
-        
-        logger.info("Enhanced automated retraining monitoring started")
-        self.log_automation_event("monitoring_started", "Enhanced automated monitoring system started")
-    
-    def stop_automated_monitoring(self):
-        """Stop the automated monitoring system"""
-        self.retraining_active = False
-        schedule.clear()
-        
-        if self.automation_thread:
-            self.automation_thread.join(timeout=10)
-        
-        logger.info("Enhanced automated retraining monitoring stopped")
-        self.log_automation_event("monitoring_stopped", "Enhanced automated monitoring system stopped")
-    
-    def automation_loop(self):
-        """Main automation loop"""
-        while self.retraining_active:
-            try:
-                schedule.run_pending()
-                time_module.sleep(60)  # Check every minute
-            except Exception as e:
-                logger.error(f"Enhanced automation loop error: {e}")
-                time_module.sleep(300)  # Wait 5 minutes on error
-    
-    def perform_scheduled_check(self):
-        """Perform scheduled retraining trigger check"""
-        try:
-            logger.info("Performing scheduled enhanced retraining trigger check")
-            
-            # Check if we're in cooldown period
-            if self.is_in_cooldown_period():
-                logger.info("Skipping check - in cooldown period after recent training")
-                return
-            
-            # Check daily limit
-            if self.exceeded_daily_retraining_limit():
-                logger.info("Skipping check - daily retraining limit exceeded")
-                return
-            
-            # Perform trigger evaluation
-            trigger_results = self.drift_monitor.check_retraining_triggers()
-            
-            self.last_check_time = datetime.now()
-            
-            # Log the check
-            self.log_automation_event("scheduled_check", "Performed scheduled enhanced trigger check", {
-                'trigger_results': trigger_results,
-                'should_retrain': trigger_results.get('should_retrain', False),
-                'enhanced_features_available': self.enhanced_features_available
-            })
-            
-            # Trigger retraining if needed
-            if trigger_results.get('should_retrain', False):
-                self.queue_retraining(trigger_results)
-            
-        except Exception as e:
-            logger.error(f"Scheduled enhanced check failed: {e}")
-            self.log_automation_event("check_failed", f"Scheduled enhanced check failed: {str(e)}")
-    
-    def perform_forced_check(self):
-        """Perform forced daily check regardless of other conditions"""
-        try:
-            logger.info("Performing forced daily enhanced retraining check")
-            
-            # Always perform drift monitoring
-            trigger_results = self.drift_monitor.check_retraining_triggers()
-            
-            self.log_automation_event("forced_check", "Performed forced daily enhanced check", {
-                'trigger_results': trigger_results,
-                'enhanced_features_available': self.enhanced_features_available
-            })
-            
-            # Only trigger retraining for urgent cases during forced check
-            if trigger_results.get('urgency') in ['critical', 'high']:
-                self.queue_retraining(trigger_results, forced=True)
-            
-        except Exception as e:
-            logger.error(f"Forced enhanced check failed: {e}")
-            self.log_automation_event("forced_check_failed", f"Forced enhanced check failed: {str(e)}")
-    
-    def queue_retraining(self, trigger_results: Dict, forced: bool = False):
-        """Queue a retraining job with enhanced feature considerations"""
-        try:
-            retraining_job = {
-                'queued_at': datetime.now().isoformat(),
-                'trigger_results': trigger_results,
-                'urgency': trigger_results.get('urgency', 'medium'),
-                'forced': forced,
-                'status': 'queued',
-                'attempts': 0,
-                'enhanced_features_available': self.enhanced_features_available,
-                'prefer_enhanced_features': self.automation_config['retraining_conditions'].get('prefer_enhanced_features', True)
-            }
-            
-            # Load existing queue
-            queue = self.load_retraining_queue()
-            queue.append(retraining_job)
-            
-            # Save queue
-            self.save_retraining_queue(queue)
-            
-            logger.info(f"Enhanced retraining queued with urgency: {trigger_results.get('urgency', 'medium')}")
-            self.log_automation_event("retraining_queued", "Enhanced retraining job queued", retraining_job)
-            
-            # Execute immediately for critical cases
-            if trigger_results.get('urgency') == 'critical' or forced:
-                self.execute_queued_retraining()
-            
-        except Exception as e:
-            logger.error(f"Failed to queue enhanced retraining: {e}")
-            self.log_automation_event("queue_failed", f"Failed to queue enhanced retraining: {str(e)}")
-    
-    def execute_queued_retraining(self):
-        """Execute queued retraining jobs with enhanced features"""
-        try:
-            queue = self.load_retraining_queue()
-            
-            # Sort by urgency (critical > high > medium > low)
-            urgency_order = {'critical': 0, 'high': 1, 'medium': 2, 'low': 3}
-            queue.sort(key=lambda x: urgency_order.get(x.get('urgency', 'medium'), 2))
-            
-            executed_jobs = []
-            
-            for job in queue:
-                if job['status'] == 'queued':
-                    success = self.execute_single_retraining(job)
-                    
-                    if success:
-                        job['status'] = 'completed'
-                        job['completed_at'] = datetime.now().isoformat()
-                        self.automation_status['last_automated_training'] = datetime.now().isoformat()
-                        self.automation_status['total_automated_trainings'] += 1
-                        executed_jobs.append(job)
-                        break  # Only execute one job at a time
-                    else:
-                        job['attempts'] += 1
-                        if job['attempts'] >= 3:
-                            job['status'] = 'failed'
-                            job['failed_at'] = datetime.now().isoformat()
-                        self.automation_status['failed_attempts'] += 1
-            
-            # Update queue
-            remaining_queue = [job for job in queue if job['status'] == 'queued']
-            self.save_retraining_queue(remaining_queue)
-            
-            return len(executed_jobs) > 0
-            
-        except Exception as e:
-            logger.error(f"Failed to execute queued enhanced retraining: {e}")
-            return False
-    
-    def execute_single_retraining(self, job: Dict) -> bool:
-        """Execute a single retraining job with enhanced features"""
-        try:
-            logger.info(f"Starting automated enhanced retraining with urgency: {job.get('urgency', 'medium')}")
-            
-            job['started_at'] = datetime.now().isoformat()
-            job['status'] = 'running'
-            
-            # Create enhanced retraining manager
-            retrainer = EnhancedModelRetrainer()
-            
-            # Configure enhanced features based on job preferences
-            prefer_enhanced = job.get('prefer_enhanced_features', True)
-            retrainer.use_enhanced_features = prefer_enhanced and ENHANCED_FEATURES_AVAILABLE
-            
-            # Perform enhanced retraining with validation
-            success, result = retrainer.automated_retrain_with_validation()
-            
-            if success:
-                logger.info("Automated enhanced retraining completed successfully")
-                self.log_automation_event("retraining_success", "Automated enhanced retraining completed", {
-                    'job': job,
-                    'result': result,
-                    'enhanced_features_used': retrainer.use_enhanced_features
-                })
-                return True
-            else:
-                logger.error(f"Automated enhanced retraining failed: {result}")
-                self.log_automation_event("retraining_failed", f"Automated enhanced retraining failed: {result}", {
-                    'job': job,
-                    'enhanced_features_used': retrainer.use_enhanced_features
-                })
-                return False
-                
-        except Exception as e:
-            logger.error(f"Enhanced retraining execution failed: {e}")
-            self.log_automation_event("retraining_error", f"Enhanced retraining execution error: {str(e)}", {'job': job})
-            return False
-    
-    def is_in_cooldown_period(self) -> bool:
-        """Check if we're in cooldown period after recent training"""
-        try:
-            if not self.automation_status['last_automated_training']:
-                return False
-            
-            last_training = datetime.fromisoformat(self.automation_status['last_automated_training'])
-            cooldown_hours = self.automation_config['retraining_conditions']['cooldown_hours_after_training']
-            cooldown_period = timedelta(hours=cooldown_hours)
-            
-            return datetime.now() - last_training < cooldown_period
-            
-        except Exception as e:
-            logger.warning(f"Failed to check cooldown period: {e}")
-            return False
-    
-    def exceeded_daily_retraining_limit(self) -> bool:
-        """Check if daily retraining limit has been exceeded"""
-        try:
-            if not self.automation_status['last_automated_training']:
-                return False
-            
-            last_training = datetime.fromisoformat(self.automation_status['last_automated_training'])
-            
-            # Count trainings in last 24 hours
-            training_logs = self.get_recent_automation_logs(hours=24)
-            training_count = len([log for log in training_logs if log.get('event') == 'retraining_success'])
-            
-            max_daily = self.automation_config['monitoring_schedule']['max_daily_retrainings']
-            
-            return training_count >= max_daily
-            
-        except Exception as e:
-            logger.warning(f"Failed to check daily limit: {e}")
-            return False
-    
-    def load_retraining_queue(self) -> List[Dict]:
-        """Load retraining queue from file"""
-        try:
-            if self.retraining_queue_path.exists():
-                with open(self.retraining_queue_path, 'r') as f:
-                    return json.load(f)
-            return []
-        except Exception as e:
-            logger.error(f"Failed to load retraining queue: {e}")
-            return []
-    
-    def save_retraining_queue(self, queue: List[Dict]):
-        """Save retraining queue to file"""
-        try:
-            with open(self.retraining_queue_path, 'w') as f:
-                json.dump(queue, f, indent=2)
-        except Exception as e:
-            logger.error(f"Failed to save retraining queue: {e}")
-    
-    def log_automation_event(self, event: str, message: str, details: Dict = None):
-        """Log automation events with enhanced feature information"""
-        try:
-            log_entry = {
-                'timestamp': datetime.now().isoformat(),
-                'event': event,
-                'message': message,
-                'details': details or {},
-                'enhanced_features_available': self.enhanced_features_available
-            }
-            
-            # Load existing logs
-            logs = []
-            if self.automation_log_path.exists():
-                try:
-                    with open(self.automation_log_path, 'r') as f:
-                        logs = json.load(f)
-                except:
-                    logs = []
-            
-            logs.append(log_entry)
-            
-            # Keep only last 1000 entries
-            if len(logs) > 1000:
-                logs = logs[-1000:]
-            
-            # Save logs
-            with open(self.automation_log_path, 'w') as f:
-                json.dump(logs, f, indent=2)
-            
-        except Exception as e:
-            logger.error(f"Failed to log automation event: {e}")
-    
-    def get_recent_automation_logs(self, hours: int = 24) -> List[Dict]:
-        """Get recent automation logs"""
-        try:
-            if not self.automation_log_path.exists():
-                return []
-            
-            with open(self.automation_log_path, 'r') as f:
-                logs = json.load(f)
-            
-            cutoff_time = datetime.now() - timedelta(hours=hours)
-            recent_logs = [
-                log for log in logs
-                if datetime.fromisoformat(log['timestamp']) > cutoff_time
-            ]
-            
-            return recent_logs
-            
-        except Exception as e:
-            logger.error(f"Failed to get recent logs: {e}")
-            return []
-    
-    def get_automation_status(self) -> Dict:
-        """Get current automation status with enhanced feature information"""
-        try:
-            status = {
-                **self.automation_status,
-                'monitoring_active': self.retraining_active,
-                'last_check_time': self.last_check_time.isoformat() if self.last_check_time else None,
-                'in_cooldown': self.is_in_cooldown_period(),
-                'daily_limit_exceeded': self.exceeded_daily_retraining_limit(),
-                'queued_jobs': len(self.load_retraining_queue()),
-                'recent_logs': self.get_recent_automation_logs(hours=6),
-                'enhanced_features_status': {
-                    'available': self.enhanced_features_available,
-                    'preference': self.automation_config['retraining_conditions'].get('prefer_enhanced_features', True)
-                }
-            }
-            
-            return status
-            
-        except Exception as e:
-            logger.error(f"Failed to get automation status: {e}")
-            return {'error': str(e)}
     
     def trigger_manual_retraining(self, reason: str = "manual_trigger", use_enhanced: bool = None) -> Dict:
         """Manually trigger retraining with enhanced feature options"""
         try:
-            # Use enhanced features by default if available
             if use_enhanced is None:
                 use_enhanced = self.enhanced_features_available
             
-            # Create manual trigger results
-            trigger_results = {
-                'should_retrain': True,
-                'urgency': 'high',
-                'trigger_reason': reason,
-                'triggers_detected': [{
-                    'type': 'manual_trigger',
-                    'severity': 'high',
-                    'message': f'Manual enhanced retraining triggered: {reason}'
-                }],
-                'recommendations': ['Manual enhanced retraining requested'],
-                'enhanced_features_requested': use_enhanced
-            }
+            retrainer = EnhancedModelRetrainer()
+            retrainer.use_enhanced_features = use_enhanced and ENHANCED_FEATURES_AVAILABLE
             
-            # Queue the retraining
-            self.queue_retraining(trigger_results, forced=True)
+            success, result = retrainer.automated_retrain_with_validation()
             
             feature_info = " with enhanced features" if use_enhanced else " with standard features"
-            return {
-                'success': True, 
-                'message': f'Manual enhanced retraining queued{feature_info}',
-                'enhanced_features': use_enhanced
-            }
+            if success:
+                return {
+                    'success': True, 
+                    'message': f'Manual enhanced retraining completed{feature_info}: {result}',
+                    'enhanced_features': use_enhanced
+                }
+            else:
+                return {
+                    'success': False, 
+                    'message': f'Manual enhanced retraining failed{feature_info}: {result}',
+                    'enhanced_features': use_enhanced
+                }
             
         except Exception as e:
             logger.error(f"Manual enhanced retraining trigger failed: {e}")
             return {'success': False, 'error': str(e)}
 
 
-def start_automation_system():
-    """Start the enhanced automated retraining system"""
-    try:
-        automation_manager = AutomatedRetrainingManager()
-        automation_manager.start_automated_monitoring()
-        return automation_manager
-    except Exception as e:
-        logger.error(f"Failed to start enhanced automation system: {e}")
-        return None
-
-def get_automation_manager() -> Optional[AutomatedRetrainingManager]:
-    """Get or create enhanced automation manager instance"""
-    global _automation_manager
-    
-    if '_automation_manager' not in globals():
-        _automation_manager = AutomatedRetrainingManager()
-    
-    return _automation_manager
-
 def main():
-    """Main execution function with enhanced CV and feature engineering"""
+    """Main execution function with enhanced CV, LightGBM, and ensemble support"""
     retrainer = EnhancedModelRetrainer()
     success, message = retrainer.retrain_model()
     
@@ -1830,5 +1793,6 @@ def main():
         print(f"❌ {message}")
         exit(1)
 
+
 if __name__ == "__main__":
     main()