From 0ec382c86b96dfedad38827efd249e66de75e2ad Mon Sep 17 00:00:00 2001 From: diudiu62 <115522593@qq.com> Date: Tue, 25 Feb 2025 09:05:24 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=9D=E8=AF=95=E9=9B=86=E6=88=90sensevoice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + astrbot/core/config/default.py | 7 ++ astrbot/core/provider/manager.py | 2 + .../sources/sensevoice_selfhosted_source.py | 83 +++++++++++++++++++ requirements.txt | 5 +- 5 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 astrbot/core/provider/sources/sensevoice_selfhosted_source.py diff --git a/.gitignore b/.gitignore index 7745d18ba..cb28c1d12 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ package.json venv/* packages/python_interpreter/workplace .venv/* + +Dockerfile_diudiu62 \ No newline at end of file diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 7a6b68b3e..f4ff839cb 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -532,6 +532,13 @@ CONFIG_METADATA_2 = { "type": "openai_whisper_selfhost", "model": "tiny", }, + "sensevoice(本地加载)": { + "whisper_hint": "(不用修改我)", + "enable": False, + "id": "sensevoice", + "type": "sensevoice_stt_selfhost", + "model": "tiny", + }, "openai_tts(API)": { "id": "openai_tts", "type": "openai_tts_api", diff --git a/astrbot/core/provider/manager.py b/astrbot/core/provider/manager.py index 2ba108e29..7187cc100 100644 --- a/astrbot/core/provider/manager.py +++ b/astrbot/core/provider/manager.py @@ -128,6 +128,8 @@ class ProviderManager(): from .sources.whisper_api_source import ProviderOpenAIWhisperAPI as ProviderOpenAIWhisperAPI case "openai_whisper_selfhost": from .sources.whisper_selfhosted_source import ProviderOpenAIWhisperSelfHost as ProviderOpenAIWhisperSelfHost + case "sensevoice_stt_selfhost": + from .sources.sensevoice_selfhosted_source import ProviderSenseVoiceSTTSelfHost as ProviderSenseVoiceSTTSelfHost case "openai_tts_api": from .sources.openai_tts_api_source import ProviderOpenAITTSAPI as ProviderOpenAITTSAPI case "fishaudio_tts_api": diff --git a/astrbot/core/provider/sources/sensevoice_selfhosted_source.py b/astrbot/core/provider/sources/sensevoice_selfhosted_source.py new file mode 100644 index 000000000..0bcb5729e --- /dev/null +++ b/astrbot/core/provider/sources/sensevoice_selfhosted_source.py @@ -0,0 +1,83 @@ +''' +Author: diudiu62 +Date: 2025-02-24 18:04:18 +LastEditTime: 2025-02-24 18:33:48 +''' +from datetime import datetime +import os +import asyncio +from funasr import AutoModel +from ..provider import STTProvider +from ..entites import ProviderType +from astrbot.core.utils.io import download_file +from ..register import register_provider_adapter +from astrbot.core import logger +from astrbot.core.utils.tencent_record_helper import tencent_silk_to_wav + +@register_provider_adapter("sensevoice_stt_selfhost", "SenseVoice 自托管语音识别 模型部署", provider_type=ProviderType.SPEECH_TO_TEXT) +class ProviderSenseVoiceSTTSelfHost(STTProvider): + def __init__( + self, + provider_config: dict, + provider_settings: dict, + ) -> None: + super().__init__(provider_config, provider_settings) + + async def initialize(self): + model_dir = "data/model/iic/SenseVoiceSmall" + loop = asyncio.get_event_loop() + logger.info("下载或者加载 SenseVoice 模型中，这可能需要一些时间 ...") + self.model = await loop.run_in_executor(None, AutoModel, + model=model_dir, + trust_remote_code=False, + # remote_code="./model.py", + vad_model="fsmn-vad", + vad_kwargs={"max_single_segment_time": 30000}, + ) + logger.info("SenseVoice 模型加载完成。") + + async def _convert_audio(self, path: str) -> str: + from pyffmpeg import FFmpeg + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") # 获取当前时间戳 + filename = timestamp + '.mp3' + ff = FFmpeg() + output_path = ff.convert(path, os.path.join('data/temp', filename)) + return output_path + + async def _is_silk_file(self, file_path): + silk_header = b"SILK" + with open(file_path, "rb") as f: + file_header = f.read(8) + + if silk_header in file_header: + return True + else: + return False + + async def get_text(self, audio_url: str) -> str: + loop = asyncio.get_event_loop() + + is_tencent = False + + if audio_url.startswith("http"): + if "multimedia.nt.qq.com.cn" in audio_url: + is_tencent = True + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") # 获取当前时间戳 + path = os.path.join("data/temp", timestamp) + await download_file(audio_url, path) + audio_url = path + + if not os.path.exists(audio_url): + raise FileNotFoundError(f"文件不存在: {audio_url}") + + if audio_url.endswith(".amr") or audio_url.endswith(".silk") or is_tencent: + is_silk = await self._is_silk_file(audio_url) + if is_silk: + logger.info("Converting silk file to wav ...") + output_path = os.path.join('data/temp', str(uuid.uuid4()) + '.wav') + await tencent_silk_to_wav(audio_url, output_path) + audio_url = output_path + + result = await loop.run_in_executor(None, self.model.transcribe, audio_url) + return result['text'] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c077496e2..aaf1562cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,7 @@ silk-python lark-oapi ormsgpack -cryptography \ No newline at end of file +cryptography + +funasr +torch~=2.6.0 \ No newline at end of file