Check point 4

app.py

@@ -10,13 +10,12 @@ import torchaudio
 from scipy.spatial.distance import cosine
 from RealtimeSTT import AudioToTextRecorder
 from fastapi import FastAPI, APIRouter
-from fastrtc import Stream, AsyncStreamHandler
+from fastrtc import Stream, AsyncStreamHandler, WebRTC
 import json
 import asyncio
 import uvicorn
 from queue import Queue
 import logging
-from fastrtc import WebRTC
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -420,7 +419,7 @@ class RealtimeSpeakerDiarization:
             # Setup recorder configuration
             recorder_config = {
                 'spinner': False,
-                'use_microphone': False,  # Change to False for Hugging Face Spaces
+                'use_microphone': False,  # Must be False since we're using FastRTC
                 'model': FINAL_TRANSCRIPTION_MODEL,
                 'language': TRANSCRIPTION_LANGUAGE,
                 'silero_sensitivity': SILERO_SENSITIVITY,
@@ -430,7 +429,7 @@ class RealtimeSpeakerDiarization:
                 'pre_recording_buffer_duration': PRE_RECORDING_BUFFER_DURATION,
                 'min_gap_between_recordings': 0,
                 'enable_realtime_transcription': True,
-                'realtime_processing_pause': 0.1,
+                'realtime_processing_pause': 0.05,  # Faster updates for live transcription
                 'realtime_model_type': REALTIME_TRANSCRIPTION_MODEL,
                 'on_realtime_transcription_update': self.live_text_detected,
                 'beam_size': FINAL_BEAM_SIZE,
@@ -448,7 +447,8 @@ class RealtimeSpeakerDiarization:
             self.transcription_thread = threading.Thread(target=self.run_transcription, daemon=True)
             self.transcription_thread.start()
             
-            return "Recording started successfully!"
+            logger.info("Recording started with FastRTC integration")
+            return "Recording started successfully! Speak now..."
             
         except Exception as e:
             logger.error(f"Error starting recording: {e}")
@@ -587,11 +587,17 @@ class DiarizationHandler(AsyncStreamHandler):
                 return
                 
             # Extract audio data
-            audio_data = getattr(frame, 'data', frame)
+            if hasattr(frame, 'data'):
+                audio_data = frame.data
+            else:
+                audio_data = frame
             
             # Convert to numpy array
             if isinstance(audio_data, bytes):
                 audio_array = np.frombuffer(audio_data, dtype=np.int16).astype(np.float32) / 32768.0
+            elif isinstance(audio_data, tuple) and len(audio_data) >= 2:
+                sample_rate, data = audio_data
+                audio_array = np.array(data, dtype=np.float32)
             elif isinstance(audio_data, (list, tuple)):
                 audio_array = np.array(audio_data, dtype=np.float32)
             else:
@@ -609,8 +615,16 @@ class DiarizationHandler(AsyncStreamHandler):
                 chunk = np.array(self.audio_buffer[:self.buffer_size])
                 self.audio_buffer = self.audio_buffer[self.buffer_size:]
                 
-                # Process asynchronously
+                # Process both for speaker detection and feed to the recorder for transcription
                 await self.process_audio_async(chunk)
+
+                # If recorder exists, feed audio for transcription
+                if self.diarization_system.recorder:
+                    # Convert to bytes for the recorder's audio buffer
+                    audio_bytes = (chunk * 32768.0).astype(np.int16).tobytes()
+                    if hasattr(self.diarization_system.recorder, '_handle_audio'):
+                        # Send audio to the recorder's audio buffer
+                        self.diarization_system.recorder._handle_audio(audio_bytes)
                 
         except Exception as e:
             logger.error(f"Error in FastRTC receive: {e}")
@@ -627,6 +641,14 @@ class DiarizationHandler(AsyncStreamHandler):
             )
         except Exception as e:
             logger.error(f"Error in async audio processing: {e}")
+            
+    async def start_up(self):
+        """Called when stream starts"""
+        logger.info("FastRTC stream handler started")
+        
+    async def shutdown(self):
+        """Called when stream ends"""
+        logger.info("FastRTC stream handler shutdown")
 
 
 # Global instances
@@ -639,9 +661,14 @@ def initialize_system():
     try:
         success = diarization_system.initialize_models()
         if success:
-            # Update the Stream's handler to use our DiarizationHandler
-            stream.handler = DiarizationHandler(diarization_system)
-            return "✅ System initialized successfully!"
+            # Create a fresh handler that uses our diarization system
+            handler = DiarizationHandler(diarization_system)
+            
+            # Update the Stream's handler
+            stream.handler = handler
+            
+            logger.info("FastRTC handler initialized successfully")
+            return "✅ System initialized successfully! Click 'Start' to begin recording."
         else:
             return "❌ Failed to initialize system. Check logs for details."
     except Exception as e:
@@ -658,7 +685,8 @@ def start_recording():
 
 def on_start():
     result = start_recording()
-    return result, gr.update(interactive=False), gr.update(interactive=True)
+    # When starting recording, update UI and return WebRTC component with autostart=True
+    return result, gr.update(interactive=False), gr.update(interactive=True), gr.update(autostart=True)
 
 def stop_recording():
     """Stop recording and transcription"""
@@ -698,6 +726,15 @@ def get_status():
     except Exception as e:
         return f"Error getting status: {str(e)}"
 
+def refresh_conversation():
+    """Get the current conversation and update live transcription status"""
+    has_live = diarization_system.last_transcription != ""
+    status = "🟢 **Live Transcription Status:** Active" if has_live else "🟠 **Live Transcription Status:** Ready (No speech detected)" 
+    if not diarization_system.is_running:
+        status = "🔴 **Live Transcription Status:** Not running"
+    
+    return get_conversation(), status
+
 # Create Gradio interface
 def create_interface():
     with gr.Blocks(title="Real-time Speaker Diarization", theme=gr.themes.Soft()) as interface:
@@ -706,11 +743,17 @@ def create_interface():
         
         with gr.Row():
             with gr.Column(scale=2):
-                # Replace WebRTC with standard Gradio audio component
-                audio_component = gr.Audio(
-                    label="Audio Input",
-                    sources=["microphone"],
-                    streaming=True
+                # Replace standard Gradio audio with FastRTC WebRTC component
+                audio_component = WebRTC(
+                    stream=stream,
+                    label="Audio Input (FastRTC)",
+                    show_audio_waveform=True,
+                    autostart=False,
+                )
+                
+                # Add live transcription status indicator
+                live_transcription_status = gr.Markdown(
+                    "🔴 **Live Transcription Status:** Waiting to initialize...",
                 )
                 
                 # Conversation display
@@ -786,7 +829,8 @@ def create_interface():
         
         def on_start():
             result = start_recording()
-            return result, gr.update(interactive=False), gr.update(interactive=True)
+            # When starting recording, update UI and return WebRTC component with autostart=True
+            return result, gr.update(interactive=False), gr.update(interactive=True), gr.update(autostart=True)
         
         def on_stop():
             result = stop_recording()
@@ -814,7 +858,7 @@ def create_interface():
         
         start_btn.click(
             fn=on_start,
-            outputs=[status_output, start_btn, stop_btn]
+            outputs=[status_output, start_btn, stop_btn, audio_component]
         )
         
         stop_btn.click(
@@ -835,27 +879,12 @@ def create_interface():
         
         # Auto-refresh conversation display every 1 second
         conversation_timer = gr.Timer(1)
-        conversation_timer.tick(refresh_conversation, outputs=[conversation_output])
+        conversation_timer.tick(refresh_conversation, outputs=[conversation_output, live_transcription_status])
         
         # Auto-refresh status every 2 seconds  
         status_timer = gr.Timer(2)
         status_timer.tick(refresh_status, outputs=[status_output])
 
-        # Process audio from Gradio component
-        def process_audio_input(audio_data):
-            if audio_data is not None and diarization_system.is_running:
-                # Extract audio data
-                if isinstance(audio_data, tuple) and len(audio_data) >= 2:
-                    sample_rate, audio_array = audio_data[0], audio_data[1]
-                    diarization_system.process_audio_chunk(audio_array, sample_rate)
-            return get_conversation()
-        
-        # Connect audio component to processing function
-        audio_component.stream(
-            fn=process_audio_input,
-            outputs=[conversation_output]
-        )
-
     return interface