Update app.py

app.py

@@ -1,23 +1,26 @@
+import io
 import re
+import wave
+
 import gradio as gr
 import numpy as np
-import os
-import io
-import wave
-import threading
-import subprocess
-import sys
-import time
 
-from huggingface_hub import snapshot_download
-from tools.fish_e2e import FishE2EAgent, FishE2EEventType
-from tools.schema import ServeMessage, ServeTextPart, ServeVQPart
+from .fish_e2e import FishE2EAgent, FishE2EEventType
+from .schema import ServeMessage, ServeTextPart, ServeVQPart
+
+
+def wav_chunk_header(sample_rate=44100, bit_depth=16, channels=1):
+    buffer = io.BytesIO()
+
+    with wave.open(buffer, "wb") as wav_file:
+        wav_file.setnchannels(channels)
+        wav_file.setsampwidth(bit_depth // 8)
+        wav_file.setframerate(sample_rate)
+
+    wav_header_bytes = buffer.getvalue()
+    buffer.close()
+    return wav_header_bytes
 
-# Download Weights
-os.makedirs("checkpoints", exist_ok=True)
-snapshot_download(repo_id="fishaudio/fish-speech-1.4", local_dir="./checkpoints/fish-speech-1.4")
-snapshot_download(repo_id="fishaudio/fish-agent-v0.1-3b", local_dir="./checkpoints/fish-agent-v0.1-3b")
-SYSTEM_PROMPT = 'You are a voice assistant created by Fish Audio, offering end-to-end voice interaction for a seamless user experience. You are required to first transcribe the user\'s speech, then answer it in the following format: "Question: [USER_SPEECH]\n\nResponse: [YOUR_RESPONSE]\n"。You are required to use the following voice in this conversation.'
 
 class ChatState:
     def __init__(self):
@@ -54,17 +57,6 @@ class ChatState:
 def clear_fn():
     return [], ChatState(), None, None, None
 
-def wav_chunk_header(sample_rate=44100, bit_depth=16, channels=1):
-    buffer = io.BytesIO()
-
-    with wave.open(buffer, "wb") as wav_file:
-        wav_file.setnchannels(channels)
-        wav_file.setsampwidth(bit_depth // 8)
-        wav_file.setframerate(sample_rate)
-
-    wav_header_bytes = buffer.getvalue()
-    buffer.close()
-    return wav_header_bytes
 
 async def process_audio_input(
     sys_audio_input, sys_text_input, audio_input, state: ChatState, text_input: str
@@ -117,16 +109,16 @@ async def process_audio_input(
     ):
         if event.type == FishE2EEventType.USER_CODES:
             append_to_chat_ctx(ServeVQPart(codes=event.vq_codes), role="user")
-            
         elif event.type == FishE2EEventType.SPEECH_SEGMENT:
+            np_audio = np.frombuffer(event.frame.data, dtype=np.int16)
+            result_audio += np_audio
             append_to_chat_ctx(ServeVQPart(codes=event.vq_codes))
-            yield state.get_history(), wav_chunk_header() + event.frame.data, None, None
-            
+            yield state.get_history(), (44100, result_audio), None, None
         elif event.type == FishE2EEventType.TEXT_SEGMENT:
             append_to_chat_ctx(ServeTextPart(text=event.text))
-            yield state.get_history(), None, None, None
+            yield state.get_history(), (44100, result_audio), None, None
 
-    yield state.get_history(), None, None, None
+    yield state.get_history(), (44100, result_audio), None, None
 
 
 async def process_text_input(
@@ -153,16 +145,28 @@ def create_demo():
                     type="messages",
                 )
 
+                # notes = gr.Markdown(
+                #     """
+                # # Fish Agent
+                # 1. 此Demo为Fish Audio自研端到端语言模型Fish Agent 3B版本.
+                # 2. 你可以在我们的官方仓库找到代码以及权重，但是相关内容全部基于 CC BY-NC-SA 4.0 许可证发布.
+                # 3. Demo为早期灰度测试版本，推理速度尚待优化.
+                # # 特色
+                # 1. 该模型自动集成ASR与TTS部分，不需要外挂其它模型，即真正的端到端，而非三段式(ASR+LLM+TTS).
+                # 2. 模型可以使用reference audio控制说话音色.
+                # 3. 可以生成具有较强情感与韵律的音频.
+                # """
+                # )
                 notes = gr.Markdown(
                     """
-                # Fish Agent
-                1. This demo is the Fish Audio self-developed end-to-end language model Fish Agent 3B version.
-                2. You can find the code and weights in our official repository, but all related content is released under the CC BY-NC-SA 4.0 license.
-                3. The demo is an early beta version, and inference speed is yet to be optimized.
-                # Features
-                1. This model automatically integrates ASR and TTS components, requiring no external models, making it truly end-to-end rather than a three-stage process (ASR+LLM+TTS).
-                2. The model can use reference audio to control speaking voice.
-                3. It can generate audio with strong emotions and prosody.
+                    # Fish Agent
+                    1. This demo is Fish Audio's self-researh end-to-end language model, Fish Agent version 3B.
+                    2. You can find the code and weights in our official repo in [gitub](https://github.com/fishaudio/fish-speech) and [hugging face](https://huggingface.co/fishaudio/fish-agent-v0.1-3b), but the content is released under a CC BY-NC-SA 4.0 licence.
+                    3. The demo is an early alpha test version, the inference speed needs to be optimised.
+                    # Features
+                    1. The model automatically integrates ASR and TTS parts, no need to plug-in other models, i.e., true end-to-end, not three-stage (ASR+LLM+TTS).
+                    2. The model can use reference audio to control the speech timbre. 
+                    3. The model can generate speech with strong emotion.
                 """
                 )
 
@@ -175,20 +179,18 @@ def create_demo():
                 )
                 sys_text_input = gr.Textbox(
                     label="What is your assistant's role?",
-                    value=SYSTEM_PROMPT,
+                    value="You are a voice assistant created by Fish Audio, offering end-to-end voice interaction for a seamless user experience. You are required to first transcribe the user's speech, then answer it in the following format: 'Question: [USER_SPEECH]\n\nAnswer: [YOUR_RESPONSE]\n'. You are required to use the following voice in this conversation.",
                     type="text",
                 )
                 audio_input = gr.Audio(
                     sources=["microphone"], type="numpy", label="Speak your message"
                 )
 
-                text_input = gr.Textbox(label="Or type your message", type="text",value="Can you give a brief introduction of yourself?")
+                text_input = gr.Textbox(label="Or type your message", type="text")
 
                 output_audio = gr.Audio(
-                    label="Assistant's Voice", 
-                    streaming=True,
-                    autoplay=True,
-                    interactive=False,
+                    label="Assistant's Voice",
+                    type="numpy",
                 )
 
                 send_button = gr.Button("Send", variant="primary")
@@ -224,18 +226,7 @@ def create_demo():
 
     return demo
 
-def run_api():
-    subprocess.run([sys.executable, "-m", "tools.api"])
 
 if __name__ == "__main__":
-    
-    # 创建并启动 API 线程
-    api_thread = threading.Thread(target=run_api, daemon=True)
-    api_thread.start()
-
-    # 给 API 一些时间启动
-    time.sleep(90)
-
-    # 创建并启动 Gradio demo
     demo = create_demo()
-    demo.launch(share=True)
+    demo.launch(server_name="127.0.0.1", server_port=7860, share=True)