Update app/fastapi_server.py

app/fastapi_server.py

@@ -678,6 +678,57 @@ async def predict(
                 detail="Model is not available. Please try again later."
             )
 
+        # NEW: Data validation before prediction
+        try:
+            from data.data_validator import DataValidator
+            from data.validation_schemas import ValidationLevel
+            
+            validator = DataValidator()
+            validation_result = validator.validate_text(request.text)
+            
+            # Log validation result
+            validation_entry = {
+                'timestamp': datetime.now().isoformat(),
+                'text_length': len(request.text),
+                'validation_level': validation_result.validation_level.value,
+                'quality_score': validation_result.quality_score,
+                'issues': [issue.dict() for issue in validation_result.issues],
+                'passed_validation': validation_result.validation_level != ValidationLevel.INVALID,
+                'client_ip': client_ip,
+                'user_agent': user_agent
+            }
+            
+            # Save validation results
+            try:
+                validation_log_path = path_manager.get_logs_path("validation_log.json")
+                if validation_log_path.exists():
+                    with open(validation_log_path, 'r') as f:
+                        validation_data = json.load(f)
+                else:
+                    validation_data = []
+                
+                validation_data.append(validation_entry)
+                
+                # Keep only last 1000 entries
+                if len(validation_data) > 1000:
+                    validation_data = validation_data[-1000:]
+                
+                with open(validation_log_path, 'w') as f:
+                    json.dump(validation_data, f, indent=2)
+            except Exception as e:
+                logger.warning(f"Could not save validation log: {e}")
+            
+            # Block invalid inputs
+            if validation_result.validation_level == ValidationLevel.INVALID:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Input validation failed: {validation_result.issues[0].message if validation_result.issues else 'Invalid input'}"
+                )
+                
+        except ImportError:
+            logger.warning("Data validation components not available, proceeding without validation")
+            validation_result = None
+
         # Prepare request data for routing
         request_data = {
             'client_id': client_ip,
@@ -738,48 +789,15 @@ async def predict(
             processing_time=processing_time
         )
 
-        # Validation logging - NEW ADDITION
-        validation_entry = {
-            'timestamp': datetime.now().isoformat(),
-            'text_length': len(request.text),
-            'prediction': label,
-            'confidence': confidence,
-            'validation_passed': confidence > 0.6,  # Define validation threshold
-            'quality_score': confidence,
-            'model_version': model_manager.model_metadata.get('model_version', 'unknown'),
-            'processing_time': processing_time,
-            'client_ip': client_ip,
-            'environment': environment
-        }
-        
-        # Save to validation log
-        try:
-            validation_log_path = path_manager.get_logs_path("validation_log.json")
-            if validation_log_path.exists():
-                with open(validation_log_path, 'r') as f:
-                    validation_data = json.load(f)
-            else:
-                validation_data = []
-            
-            validation_data.append(validation_entry)
-            
-            # Keep only last 1000 entries to prevent file from growing too large
-            if len(validation_data) > 1000:
-                validation_data = validation_data[-1000:]
-            
-            with open(validation_log_path, 'w') as f:
-                json.dump(validation_data, f, indent=2)
-        except Exception as e:
-            logger.warning(f"Could not save validation log: {e}")
-
-        # Log prediction (background task)
+        # Log prediction (background task) - ENHANCED with validation info
         background_tasks.add_task(
-            log_prediction,
+            log_prediction_with_validation,
             request.text,
             label,
             confidence,
             client_ip,
-            processing_time
+            processing_time,
+            validation_result
         )
 
         return response
@@ -825,6 +843,50 @@ async def predict(
         )
 
 
+async def log_prediction_with_validation(text: str, prediction: str, confidence: float, 
+                                       client_ip: str, processing_time: float, 
+                                       validation_result=None):
+    """Enhanced logging function that includes validation data"""
+    try:
+        prediction_entry = {
+            'timestamp': datetime.now().isoformat(),
+            'prediction': prediction,
+            'confidence': confidence,
+            'processing_time': processing_time,
+            'client_ip': client_ip,
+            'text_length': len(text),
+            'text_preview': text[:100] + "..." if len(text) > 100 else text
+        }
+        
+        # Add validation information if available
+        if validation_result:
+            prediction_entry.update({
+                'validation_level': validation_result.validation_level.value,
+                'quality_score': validation_result.quality_score,
+                'validation_issues_count': len(validation_result.issues)
+            })
+        
+        prediction_log_path = path_manager.get_logs_path("prediction_log.json")
+        
+        if prediction_log_path.exists():
+            with open(prediction_log_path, 'r') as f:
+                prediction_data = json.load(f)
+        else:
+            prediction_data = []
+        
+        prediction_data.append(prediction_entry)
+        
+        # Keep only last 1000 entries
+        if len(prediction_data) > 1000:
+            prediction_data = prediction_data[-1000:]
+        
+        with open(prediction_log_path, 'w') as f:
+            json.dump(prediction_data, f, indent=2)
+            
+    except Exception as e:
+        logger.error(f"Failed to log prediction: {e}")
+
+
 @app.post("/predict/batch", response_model=BatchPredictionResponse)
 async def predict_batch(
     request: BatchPredictionRequest,
@@ -1338,79 +1400,119 @@ async def get_metrics():
         )
 
 def get_validation_stats():
-   """Get validation statistics from various sources"""
-   try:
-       stats = {
-           'last_updated': datetime.now().isoformat(),
-           'total_validations': 0,
-           'total_articles': 0,
-           'total_valid_articles': 0,
-           'average_quality_score': 0.0,
-           'source_statistics': {},
-           'validation_history': [],
-           'quality_trends': []
-       }
-       
-       # Try to load validation data from logs
-       validation_log_path = path_manager.get_logs_path("validation_log.json")
-       if validation_log_path.exists():
-           with open(validation_log_path, 'r') as f:
-               validation_data = json.load(f)
-               if validation_data:
-                   stats['total_validations'] = len(validation_data)
-                   stats['validation_history'] = validation_data[-10:]  # Last 10 entries
-       
-       # Try to load prediction data for article count
-       prediction_log_path = path_manager.get_logs_path("prediction_log.json") 
-       if prediction_log_path.exists():
-           with open(prediction_log_path, 'r') as f:
-               prediction_data = json.load(f)
-               if prediction_data:
-                   stats['total_articles'] = len(prediction_data)
-                   
-                   # Calculate success rate (predictions with high confidence)
-                   high_confidence_predictions = [
-                       p for p in prediction_data 
-                       if p.get('confidence', 0) > 0.7
-                   ]
-                   stats['total_valid_articles'] = len(high_confidence_predictions)
-                   
-                   # Calculate average confidence as quality score
-                   if prediction_data:
-                       avg_confidence = sum(p.get('confidence', 0) for p in prediction_data) / len(prediction_data)
-                       stats['average_quality_score'] = avg_confidence
-       
-       # Load activity log for additional metrics
-       activity_log_path = path_manager.get_activity_log_path()
-       if activity_log_path.exists():
-           with open(activity_log_path, 'r') as f:
-               activity_data = json.load(f)
-               if activity_data:
-                   stats['last_updated'] = activity_data[-1].get('timestamp', datetime.now().isoformat())
-       
-       # Try to load monitoring data for additional validation metrics
-       monitoring_log_path = path_manager.get_logs_path("monitoring_log.json")
-       if monitoring_log_path.exists():
-           with open(monitoring_log_path, 'r') as f:
-               monitoring_data = json.load(f)
-               if monitoring_data:
-                   # Extract quality trends from monitoring data
-                   quality_entries = [
-                       {
-                           'timestamp': entry.get('timestamp'),
-                           'quality_score': entry.get('quality_score', 0)
-                       }
-                       for entry in monitoring_data
-                       if entry.get('quality_score') is not None
-                   ]
-                   stats['quality_trends'] = quality_entries[-10:]
-       
-       return stats if any(stats[k] for k in ['total_validations', 'total_articles']) else None
-       
-   except Exception as e:
-       logger.warning(f"Could not load validation stats: {e}")
-       return None
+    """Get validation statistics from actual validation logs"""
+    try:
+        stats = {
+            'last_updated': datetime.now().isoformat(),
+            'total_validations': 0,
+            'total_articles': 0,
+            'total_valid_articles': 0,
+            'average_quality_score': 0.0,
+            'validation_breakdown': {},
+            'source_statistics': {},
+            'validation_history': [],
+            'quality_trends': []
+        }
+        
+        # Load actual validation data
+        validation_log_path = path_manager.get_logs_path("validation_log.json")
+        if validation_log_path.exists():
+            with open(validation_log_path, 'r') as f:
+                validation_data = json.load(f)
+                
+            if validation_data:
+                stats['total_validations'] = len(validation_data)
+                stats['total_articles'] = len(validation_data)
+                
+                # Analyze validation levels
+                level_counts = {}
+                quality_scores = []
+                
+                for entry in validation_data:
+                    level = entry.get('validation_level', 'unknown')
+                    level_counts[level] = level_counts.get(level, 0) + 1
+                    
+                    if entry.get('quality_score'):
+                        quality_scores.append(entry['quality_score'])
+                    
+                    if entry.get('passed_validation', False):
+                        stats['total_valid_articles'] += 1
+                
+                stats['validation_breakdown'] = level_counts
+                stats['average_quality_score'] = sum(quality_scores) / len(quality_scores) if quality_scores else 0.0
+                stats['validation_history'] = validation_data[-10:]  # Last 10
+                
+                # Quality trends over time
+                for entry in validation_data[-20:]:  # Last 20 for trends
+                    if entry.get('quality_score') is not None:
+                        stats['quality_trends'].append({
+                            'timestamp': entry.get('timestamp'),
+                            'quality_score': entry.get('quality_score')
+                        })
+        
+        return stats if stats['total_validations'] > 0 else None
+        
+    except Exception as e:
+        logger.warning(f"Could not load validation stats: {e}")
+        return None
+
+
+# Data Quality Report Endpoint
+@app.get("/validation/quality-report")
+async def get_validation_quality_report():
+    """Get detailed validation quality report"""
+    try:
+        stats = get_validation_stats()
+        
+        if not stats:
+            return {
+                'error': 'No validation data available',
+                'message': 'No validation statistics available yet'
+            }
+        
+        # Generate quality assessment
+        avg_quality = stats.get('average_quality_score', 0)
+        validation_breakdown = stats.get('validation_breakdown', {})
+        
+        quality_level = 'poor'
+        if avg_quality > 0.8:
+            quality_level = 'excellent'
+        elif avg_quality > 0.6:
+            quality_level = 'good'
+        elif avg_quality > 0.4:
+            quality_level = 'fair'
+        
+        # Generate recommendations
+        recommendations = []
+        invalid_count = validation_breakdown.get('INVALID', 0)
+        total = stats.get('total_validations', 1)
+        
+        if invalid_count / total > 0.1:
+            recommendations.append("High rate of invalid inputs detected - consider input preprocessing")
+        
+        if avg_quality < 0.5:
+            recommendations.append("Low average quality scores - review data sources")
+        
+        return {
+            'overall_statistics': {
+                'total_articles': stats.get('total_articles', 0),
+                'overall_success_rate': stats.get('total_valid_articles', 0) / max(stats.get('total_articles', 1), 1)
+            },
+            'quality_assessment': {
+                'quality_level': quality_level,
+                'average_quality_score': avg_quality
+            },
+            'validation_breakdown': validation_breakdown,
+            'recommendations': recommendations,
+            'timestamp': datetime.now().isoformat()
+        }
+        
+    except Exception as e:
+        logger.error(f"Quality report generation failed: {e}")
+        raise HTTPException(status_code=500, detail="Failed to generate quality report")
+
 
+# Statistics Validation Endpoint
 @app.get("/validation/statistics")
 async def get_validation_statistics():
     """Get comprehensive validation statistics"""
@@ -1454,18 +1556,56 @@ async def get_validation_statistics():
 # Adding fallback to build quality report from metadata if generate_quality_report fails; improved error handling, logging, and richer report structure
 @app.get("/validation/quality-report")
 async def get_quality_report():
-    """Get comprehensive data quality report"""
+    """Get comprehensive data quality report with real validation data"""
     try:
-        # First try the existing generate_quality_report function
-        try:
-            report = generate_quality_report()
+        # Try to get real validation statistics
+        validation_stats = get_validation_stats()
+        
+        if validation_stats and validation_stats.get('total_validations', 0) > 0:
+            # Generate report from real validation data
+            avg_quality = validation_stats.get('average_quality_score', 0.0)
+            breakdown = validation_stats.get('validation_breakdown', {})
+            total_validations = validation_stats.get('total_validations', 0)
             
-            if report and 'error' not in report:
-                return report
-        except Exception as e:
-            logger.warning(f"generate_quality_report failed: {e}, falling back to metadata")
+            # Assess quality level
+            if avg_quality > 0.8:
+                quality_level = "excellent"
+            elif avg_quality > 0.6:
+                quality_level = "good" 
+            elif avg_quality > 0.4:
+                quality_level = "fair"
+            else:
+                quality_level = "poor"
+            
+            # Generate recommendations
+            recommendations = []
+            invalid_rate = breakdown.get('INVALID', 0) / max(total_validations, 1)
+            
+            if invalid_rate > 0.1:
+                recommendations.append("High rate of invalid inputs - consider input preprocessing")
+            if avg_quality < 0.5:
+                recommendations.append("Low average quality scores - review data sources")
+            if breakdown.get('LOW', 0) / max(total_validations, 1) > 0.2:
+                recommendations.append("Many low-quality inputs detected - implement content filtering")
+            
+            return {
+                "report_timestamp": datetime.now().isoformat(),
+                "data_source": "real_validation_logs",
+                "overall_statistics": {
+                    "total_articles": validation_stats.get('total_articles', 0),
+                    "total_validations": total_validations,
+                    "overall_success_rate": validation_stats.get('total_valid_articles', 0) / max(validation_stats.get('total_articles', 1), 1)
+                },
+                "quality_assessment": {
+                    "quality_level": quality_level,
+                    "average_quality_score": avg_quality
+                },
+                "validation_breakdown": breakdown,
+                "recommendations": recommendations,
+                "quality_trends": validation_stats.get('quality_trends', [])
+            }
         
-        # Fallback: Generate report from model metadata
+        # Fallback to existing metadata-based approach
         metadata_path = path_manager.get_metadata_path()
         
         if not metadata_path.exists():
@@ -1477,9 +1617,10 @@ async def get_quality_report():
         with open(metadata_path, 'r') as f:
             metadata = json.load(f)
         
-        # Create quality report from metadata
+        # Create quality report from metadata (existing code)
         quality_report = {
             "report_timestamp": datetime.now().isoformat(),
+            "data_source": "model_metadata",
             "overall_statistics": {
                 "total_articles": (metadata.get('train_size', 0) + metadata.get('test_size', 0)),
                 "overall_success_rate": 0.85 if metadata.get('test_f1', 0) > 0.7 else 0.65
@@ -1509,16 +1650,6 @@ async def get_quality_report():
         
     except HTTPException:
         raise
-    except FileNotFoundError:
-        raise HTTPException(
-            status_code=404,
-            detail="No validation statistics available"
-        )
-    except json.JSONDecodeError:
-        raise HTTPException(
-            status_code=500,
-            detail="Invalid metadata format"
-        )
     except Exception as e:
         logger.error(f"Failed to generate quality report: {e}")
         raise HTTPException(