From 541c800fb4a7415ef07c5d7d90d121e144c24897 Mon Sep 17 00:00:00 2001 From: Bifang <915779419@qq.com> Date: Tue, 13 Oct 2026 13:31:20 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=86Confucius4-R2T2=20=E6=8E=A5=E5=85=A5?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E9=A1=B5=E9=9D=A2=E7=9A=84=E5=AE=9E=E6=97=B6?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/asr/R2T2-LICENSE | 193 ++++++ app/services/asr/model_capabilities.py | 11 + app/services/asr/r2t2_model.py | 836 +++++++++++++++++++++++++ app/services/asr/r2t2_runtime.py | 132 ++++ app/services/qwen3_websocket_asr.py | 301 ++++++++- scripts/download_models_standalone.py | 10 + test_web/index.html | 35 +- 7 files changed, 1479 insertions(+), 39 deletions(-) create mode 100644 app/services/asr/R2T2-LICENSE create mode 100644 app/services/asr/r2t2_model.py create mode 100644 app/services/asr/r2t2_runtime.py diff --git a/app/services/asr/R2T2-LICENSE b/app/services/asr/R2T2-LICENSE new file mode 100644 index 0000000..6343ce6 --- /dev/null +++ b/app/services/asr/R2T2-LICENSE @@ -0,0 +1,193 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship made available under + the License, as indicated by a copyright notice that is included in + or attached to the work (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean, as submitted to the Licensor for inclusion + in the Work by the copyright owner or by an individual or Legal Entity + authorized to submit on behalf of the copyright owner. For the purposes + of this definition, "submitted" means any form of electronic, verbal, + or written communication sent to the Licensor or its representatives, + including but not limited to communication on electronic mailing lists, + source code control systems, and issue tracking systems that are managed + by, or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any Legal Entity on behalf of + whom a Contribution has been received by the Licensor and subsequently + incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a cross-claim + or counterclaim in a lawsuit) alleging that the Work or a Contribution + incorporated within the Work constitutes direct or contributory patent + infringement, then any patent licenses granted to You under this License + for that Work shall terminate as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the Work + or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You meet + the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works + a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating + that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that + You distribute, all copyright, patent, trademark, and attribution + notices from the Source form of the Work, excluding those notices + that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, You must include a readable copy of the attribution + notices contained within such NOTICE file, in at least one of the + following places: within a NOTICE text file distributed as part of + the Derivative Works; within the Source form or documentation, if + provided along with the Derivative Works; or, within a display + generated by the Derivative Works, if and wherever such third-party + notices normally appear. The contents of the NOTICE file are for + informational purposes only and do not modify the License. You may + add Your own attribution notices within Derivative Works that You + distribute, alongside or as an addendum to the NOTICE text from the + Work, provided that such additional attribution notices cannot be + construed as modifying the License. + + You may add Your own license statement for Your modifications and may + provide additional grant of rights to use, reproduce, modify, display, + perform, sublicense and distribute Your modifications and such + Derivative Works as a whole, provided Your use, reproduction, and + distribution of the Work otherwise complies with the conditions stated + in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or exemplary damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or all other + commercial damages or losses), even if such Contributor has been + advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format in use. We also recommend that + a file or directory name and a description of the purpose of the + file be included on the same "first page" as the copyright notice + for easier identification within third-party archives. + + Copyright 2026 NetEase Youdao + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. diff --git a/app/services/asr/model_capabilities.py b/app/services/asr/model_capabilities.py index 4a355e1..b068b33 100644 --- a/app/services/asr/model_capabilities.py +++ b/app/services/asr/model_capabilities.py @@ -40,6 +40,17 @@ _VAD_ASSETS = ( ) _REALTIME_ASR_ASSETS = ( + ModelAsset( + source="modelscope", + model_id="netease-youdao/Confucius4-R2T2", + description="Confucius4-R2T2 Realtime ASR", + required_patterns=("config.json",), + alternative_required_patterns=( + ("model.safetensors",), + ("model-*.safetensors",), + ), + min_total_size_bytes=1_000_000_000, + ), ModelAsset( source="modelscope", model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online", diff --git a/app/services/asr/r2t2_model.py b/app/services/asr/r2t2_model.py new file mode 100644 index 0000000..5fbd965 --- /dev/null +++ b/app/services/asr/r2t2_model.py @@ -0,0 +1,836 @@ +# coding=utf-8 +# Copyright 2026 The NetEase Youdao team. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Tuple, List, Optional +import re + +import numpy as np + +from app.services.asr.qwen3_vllm import ( + _build_chat_prompt, + _normalize_language_name, + _parse_asr_output, +) + +SAMPLE_RATE = 16000 +_ASR_TEXT_TAG = "" +_LANG_PREFIX = "language " +normalize_language_name = _normalize_language_name + + +def parse_asr_output(raw: str, user_language: Optional[str] = None) -> tuple[str, str]: + """Adapt the project's parser to the signature expected by the upstream decoder.""" + return _parse_asr_output(raw, user_language) + + +def validate_language(language: str) -> None: + """Reject empty forced-language values; aliases are normalized by Qwen utilities.""" + if not language or not str(language).strip(): + raise ValueError("language must not be empty") + + +def _build_text_prompt(context: str = "", force_language: Optional[str] = None) -> str: + """Build the same Qwen audio prompt used by the project's vLLM adapter.""" + return _build_chat_prompt(context=context, language=force_language) + +_EN2ZH_PUNCT = { + ",": "?", ".": "?", "!": "?", "?": "?", + ";": "?", ":": "?", "(": "?", ")": "?", +} +_ZH2EN_PUNCT = {v: k for k, v in _EN2ZH_PUNCT.items()} +_ALL_PUNCT_PAT = re.compile(r'[,\.!?;:()\uff0c\u3002\uff01\uff1f\uff1b\uff1a\uff08\uff09]') + + +def _normalize_punct_by_context(text: str) -> str: + """根据每个标点前字符的类型规范化标点: + - 如果前一个字符是中文,则替换为中文标点。 + - 如果前一个字符是英文字母或数字,则替换为英文标点。 + - 其他情况保留原标点。 + """ + def _replace(m): + punct = m.group() + pos = m.start() + prev_char = "" + for i in range(pos - 1, -1, -1): + if not text[i].isspace(): + prev_char = text[i] + break + if not prev_char: + return punct + if '\u4e00' <= prev_char <= '\u9fff': + return _EN2ZH_PUNCT.get(punct, punct) + elif prev_char.isascii() and (prev_char.isalnum() or prev_char in '"\''): + return _ZH2EN_PUNCT.get(punct, punct) + return punct + return _ALL_PUNCT_PAT.sub(_replace, text) + +def parse_language_output( + raw: str, + user_language: Optional[str] = None, +) -> Tuple[str, str]: + """ + 将 Qwen3-ASR 原始输出解析为 (language, text)。 + + 情况: + - 带标签:“language Chinese....” + - 带换行:“language Chinese\\n...\\n....” + - 无标签:将整个字符串视为文本。 + - “language None”:视为空音频 -> ("", "") + + 如果提供了 user_language,则强制使用该语言,并将 raw 视为纯文本 + (模型应输出不含元数据的纯转写文本)。 + + 参数: + raw:原始解码字符串。 + user_language:用户强制指定语言时使用的规范语言名称。 + + 返回: + Tuple[str, str]:(language, text) + """ + if raw is None: + return "", "" + if user_language == "English": + s = str(raw).rstrip() + else: + s = str(raw).strip() + if not s: + return "", "" + + if user_language: + # 用户显式指定了语言,因此将模型输出视为纯文本 + return user_language, s + + meta_part = s + text_part = "" + has_tag = _ASR_TEXT_TAG in s + if has_tag: + meta_part, text_part = s.split(_ASR_TEXT_TAG, 1) + else: + # 没有标签,视为纯文本 + return "", s.strip() + + meta_lower = meta_part.lower() + + # 空音频启发式判断 + if "language none" in meta_lower: + t = text_part.strip() + if not t: + return "", "" + # 如果模型仍然返回了内容,则保留内容,但语言设为未知 + return "", t + + # 从 meta 中提取“language xxx” + lang = "" + for line in meta_part.splitlines(): + line = line.strip() + if not line: + continue + low = line.lower() + if low.startswith(_LANG_PREFIX): + val = line[len(_LANG_PREFIX):].strip() + if val: + lang = normalize_language_name(val) + break + + return lang, text_part.strip() + +@dataclass +class ASRStreamingState: + """ + 单个音频流(单句)的流式 ASR 状态。 + + 属性: + unfixed_chunk_num (int):前 N 个分块不使用之前的 ASR 结果作为前缀提示(将前缀重置为 "")。 + unfixed_token_num (int):当 chunk_id >= unfixed_chunk_num 时,使用累积文本作为前缀提示前, + 回退最后 K 个 token,以减少边界抖动。 + chunk_size_sec (float):分块大小(秒)。音频将按此长度递增输入模型。 + chunk_size_samples (int):16kHz 下的分块采样点数(由 chunk_size_sec 推导)。 + chunk_id (int):当前分块索引(从 0 开始)。 + buffer (np.ndarray):尚未消费为完整分块的 PCM 采样点缓冲区。 + audio_accum (np.ndarray):从流开始到当前时刻累积的音频(不填充)。 + prompt_raw (str):由聊天模板生成的基础提示(包含生成提示),不附加前缀文本。 + context (str):上下文字符串。 + force_language (Optional[str]):如果提供,则在 prompt_raw 中追加 "language X", + 强制输出纯文本,与非流式 transcribe() 保持一致。 + language (str):最近一次解析出的语言(每个分块解码后更新)。未知/静音时为空。 + text (str):最近一次解析出的转写文本(每个分块解码后更新)。 + _raw_decoded (str):内部累积的原始解码文本(parse_asr_output 规范化之前)。 + 用于回退/token 裁剪,并作为提示前缀。 + """ + unfixed_chunk_num: int + unfixed_token_num: int + chunk_size_sec: float + chunk_size_samples: int + + chunk_id: int + buffer: np.ndarray + audio_accum: np.ndarray + + prompt_raw: str + context: str + force_language: Optional[str] + + language: str + text: str + _raw_decoded: str + + chunk_text: List + last_fixed_text: str + _first_chunk_discarded: bool + +class R2T2ASRModel: + def __init__( + self, + backend: str, + model: Any, + processor: Any, + sampling_params: Optional[Any] = None, + forced_aligner: Optional[Any] = None, + max_inference_batch_size: int = -1, + max_new_tokens: int = 512, + ): + self.backend = backend + self.model = model + self.processor = processor + self.sampling_params = sampling_params + self.forced_aligner = forced_aligner + self.max_inference_batch_size = max_inference_batch_size + self.max_new_tokens = max_new_tokens + + def _build_text_prompt(self, context: str = "", force_language: Optional[str] = None) -> str: + return _build_text_prompt(context=context, force_language=force_language) + + def init_streaming_state( + self, + context: str = "", + language: Optional[str] = None, + unfixed_chunk_num: int = 2, + unfixed_token_num: int = 5, + chunk_size_sec: float = 2.0, + ) -> ASRStreamingState: + """ + 初始化单个流的 ASR 状态。 + + 注意: + - 流式 ASR 仅支持 vLLM 后端。 + - 流式 ASR 不支持时间戳(不使用强制对齐器)。 + - 不支持批量推理。 + + 参数: + context:上下文字符串。 + language:可选的强制语言。提供时必须属于支持的语言。 + 行为与 transcribe() 相同:通过提示后缀强制输出纯文本。 + unfixed_chunk_num:前 N 个分块不使用之前的输出作为前缀提示(将前缀重置为 "")。 + unfixed_token_num:在 unfixed_chunk_num 之后将累积输出作为前缀提示时,回退最后 K 个 token。 + chunk_size_sec:分块大小(秒,音频为 16k PCM)。函数会在内部将其转换为 16kHz 下的采样点数。 + + 返回: + ASRStreamingState:可变状态对象,传递给 streaming_transcribe() 和 finish_streaming_transcribe()。 + + 异常: + ValueError: + - backend 不是 "vllm"。 + - chunk_size_sec <= 0。 + - 强制语言无效(校验规则与 transcribe() 相同)。 + """ + if self.backend != "vllm": + raise ValueError("Streaming ASR is supported only for vLLM backend (backend='vllm').") + if chunk_size_sec is None or float(chunk_size_sec) <= 0: + raise ValueError(f"chunk_size_sec must be > 0, got: {chunk_size_sec}") + + force_language = None + if language is not None and str(language).strip() != "": + ln = normalize_language_name(str(language)) + validate_language(ln) + force_language = ln + + chunk_size_samples = int(round(float(chunk_size_sec) * SAMPLE_RATE)) + chunk_size_samples = max(1, chunk_size_samples) + + prompt_raw = self._build_text_prompt(context=context, force_language=force_language) + + return ASRStreamingState( + unfixed_chunk_num=int(unfixed_chunk_num), + unfixed_token_num=int(unfixed_token_num), + chunk_size_sec=float(chunk_size_sec), + chunk_size_samples=int(chunk_size_samples), + chunk_id=0, + buffer=np.zeros((0,), dtype=np.float32), + audio_accum=np.zeros((0,), dtype=np.float32), + prompt_raw=prompt_raw, + context=context or "", + force_language=force_language, + language="", + text="", + _raw_decoded="", + chunk_text=[], + last_fixed_text="", + _first_chunk_discarded=False + ) + + def streaming_transcribe( + self, + pcm16k: np.ndarray, + state: ASRStreamingState, + max_new_tokens=None, + rollback_punctuation: bool = False, + ) -> Tuple[str, str]: + """ + 流式 ASR 解码步骤。 + + 此函数接收任意长度的 16k PCM float numpy 数组(单声道)。 + 它会缓冲输入采样点;每当累积的采样点足以组成一个完整分块(chunk_size_sec)时, + 就执行一次增量解码并更新: + + - state.language + - state.text + + 调用方只需持续向此函数传入音频,并读取 state.language/state.text。 + + 实现细节: + - 每当新分块就绪时,将其追加到 audio_accum,并将截至目前的*全部*音频重新输入模型 + (不进行填充)。 + - 按以下方式更新提示:state.prompt_raw + prefix_text + - 前缀回退策略: + * 如果 chunk_id < unfixed_chunk_num:prefix_text = "" + * 否则:从之前累积的解码文本中回退最后 unfixed_token_num 个 token。 + + 注意: + - 仅支持 vLLM 后端。 + - 不提供时间戳。 + - 仅支持单流(不支持批处理)。 + + 参数: + pcm16k:16kHz 单声道 PCM 波形(np.ndarray)。长度可以是任意非负整数。 + dtype 可以是 float32/float64/int16;将转换为 float32。 + state:init_streaming_state() 返回的流式状态。 + + 返回: + Tuple[str, str]:包含最新转写文本(``state.text``)和当前更新中已固定文本 + (``fixed_text``)的元组。状态对象会原地修改。 + + 异常: + ValueError:backend 不是 "vllm" 或状态无效。 + """ + if max_new_tokens is not None: + from vllm import SamplingParams + if self.backend != "vllm": + raise ValueError("streaming_transcribe() is supported only for vLLM backend (backend='vllm').") + if state is None: + raise ValueError("state must not be None. Call init_streaming_state() first.") + if pcm16k is None: + raise ValueError("pcm16k must not be None.") + + # 确保为一维单声道 + x = np.asarray(pcm16k) + if x.ndim != 1: + x = x.reshape(-1) + + # 如果输入为 int16,则转换为 [-1, 1] 范围内的 float32 PCM + if x.dtype == np.int16: + x = (x.astype(np.float32) / 32768.0) + else: + x = x.astype(np.float32, copy=False) + + # 追加到缓冲区 + if x.shape[0] > 0: + state.buffer = np.concatenate([state.buffer, x], axis=0) + + # 消费完整分块 + fixed_text = "" # 初始化,避免循环未执行时出现 UnboundLocalError + while state.buffer.shape[0] >= state.chunk_size_samples: + chunk = state.buffer[: state.chunk_size_samples] + state.buffer = state.buffer[state.chunk_size_samples :] + + # 累积音频(从头重新输入,不进行填充) + if state.audio_accum.shape[0] == 0: + state.audio_accum = chunk + else: + state.audio_accum = np.concatenate([state.audio_accum, chunk], axis=0) + + # 使用回退策略构建前缀 + prefix = "" + if state.chunk_id < state.unfixed_chunk_num: + prefix = "" + else: + state._raw_decoded = state._raw_decoded.split("|")[0] + cur_ids = self.processor.tokenizer.encode(state._raw_decoded) + if rollback_punctuation == True: + _punct = ",。!?、;:.!?;:" + raw_stripped = state._raw_decoded.strip() + if raw_stripped and raw_stripped[-1] in _punct: + k = 0 + else: + k = int(state.unfixed_token_num) + else: + k = int(state.unfixed_token_num) + while True: + end_idx = max(0, len(cur_ids) - k) + prefix = self.processor.tokenizer.decode(cur_ids[:end_idx]) if end_idx > 0 else "" + if '\ufffd' not in prefix: + break + else: + if end_idx == 0: + prefix = "" + break + k += 1 + + prefix = prefix.split("|")[0] + + prompt = state.prompt_raw + prefix + + # vLLM 输入:单个项目 + inp = {"prompt": prompt, "multi_modal_data": {"audio": [state.audio_accum]}} + if max_new_tokens is not None: + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=max_new_tokens, + skip_special_tokens=True, + ) + outputs = self.model.generate([inp], sampling_params=sampling_params, use_tqdm=False) + else: + outputs = self.model.generate([inp], sampling_params=self.sampling_params, use_tqdm=False) + gen_text = outputs[0].outputs[0].text + gen_text = _normalize_punct_by_context(gen_text).replace('\ufffd', '') + #print(f"prefix={prefix}") + state._raw_decoded = (prefix + gen_text) if prefix is not None else gen_text + #print(f"state._raw_decoded={state._raw_decoded}") + + lang = None + if state.force_language == None: + lang,_ = parse_language_output(state._raw_decoded, user_language=state.force_language) + + if state.force_language == "Chinese" or lang == "Chinese": + # 移除相邻中文字符之间的多余空格;在中英文混合文本中保留英文空格 + state._raw_decoded = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', state._raw_decoded) + lang, txt = parse_asr_output(state._raw_decoded, user_language=state.force_language) + + has_tag = "" in state._raw_decoded + if has_tag: + state._raw_decoded = state._raw_decoded.split("")[0] + "" + txt + else: + state._raw_decoded = txt + + state._raw_decoded = state._raw_decoded.split("|")[0] + cur_ids = self.processor.tokenizer.encode(state._raw_decoded) + if rollback_punctuation == True: + _punct = ",。!?、;:,.!?;:" + raw_stripped = state._raw_decoded.strip() + if raw_stripped and raw_stripped[-1] in _punct: + k = 0 + else: + k = int(state.unfixed_token_num) + else: + k = int(state.unfixed_token_num) + + has_tag = "" in state._raw_decoded + if has_tag and state._raw_decoded.split('')[1] == "": + k = 0 + while True: + end_idx = max(0, len(cur_ids) - k) + fixed_text = self.processor.tokenizer.decode(cur_ids[:end_idx]) if end_idx > 0 else "" + if '\ufffd' not in fixed_text: + break + else: + if end_idx == 0: + fixed_text = "" + break + k += 1 + has_tag = "" in fixed_text + if has_tag: + meta_part, fixed_text = fixed_text.split("", 1) + fixed_text = fixed_text.split("|")[0] + + has_tag = "" in state._raw_decoded + if not has_tag and state.force_language == None: + state.text = "" + fixed_text = "" + continue + + state.language = lang + state.text = txt.split("|")[0] + + state.chunk_id += 1 + + return state.text,fixed_text + + def finish_streaming_transcribe( + self, + state: ASRStreamingState, + max_new_tokens=None, + ) -> str: + """ + 完成流式 ASR。 + + 此函数会刷新 state.buffer 中剩余的缓冲音频(尾部音频)。 + 即使剩余采样点少于 chunk_size_sec,也会在不填充的情况下发送给模型。 + 然后最后一次更新 state.language/state.text。 + + 注意: + - 仅支持 vLLM 后端。 + - 不提供时间戳。 + - 仅支持单流。 + + 参数: + state:init_streaming_state() 返回的流式状态。 + max_new_tokens:最终解码时可选的最大生成 token 数。 + + 返回: + str:最新的转写文本(``state.text``)。状态对象会原地修改,且可复用于后续流式调用。 + + 异常: + ValueError:backend 不是 "vllm" 或状态无效。 + """ + if max_new_tokens is not None: + from vllm import SamplingParams + if self.backend != "vllm": + raise ValueError("finish_streaming_transcribe() is supported only for vLLM backend (backend='vllm').") + if state is None: + raise ValueError("state must not be None.") + + # 如果没有剩余缓冲区,仍按原样返回状态。 + if state.buffer is None or state.buffer.shape[0] == 0: + return state.text + + tail = state.buffer + state.buffer = np.zeros((0,), dtype=np.float32) + + # 将尾部追加到累积音频 + if state.audio_accum.shape[0] == 0: + state.audio_accum = tail + else: + state.audio_accum = np.concatenate([state.audio_accum, tail], axis=0) + + # 前缀回退策略(与逐分块处理相同) + prefix = "" + if state.chunk_id < state.unfixed_chunk_num: + prefix = "" + else: + cur_ids = self.processor.tokenizer.encode(state._raw_decoded) + end_idx = max(1, len(cur_ids) - int(state.unfixed_token_num)) + prefix = self.processor.tokenizer.decode(cur_ids[:end_idx]) + + prefix = prefix.split("|")[0] + prompt = state.prompt_raw + prefix + inp = {"prompt": prompt, "multi_modal_data": {"audio": [state.audio_accum]}} + + if max_new_tokens is not None: + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=max_new_tokens, + skip_special_tokens=True, + ) + outputs = self.model.generate([inp], sampling_params=sampling_params, use_tqdm=False) + else: + outputs = self.model.generate([inp], sampling_params=self.sampling_params, use_tqdm=False) + gen_text = outputs[0].outputs[0].text + gen_text = _normalize_punct_by_context(gen_text).replace('\ufffd', '') + + state._raw_decoded = (prefix + gen_text) if prefix is not None else gen_text + state._raw_decoded = state._raw_decoded.split("|")[0] + lang, txt = parse_asr_output(state._raw_decoded, user_language=state.force_language) + + state.language = lang + state.text = txt.split("|")[0] + state.chunk_id += 1 + return state.text + + def streaming_transcribe_no_reset( + self, + pcm16k: np.ndarray, + state: ASRStreamingState, + max_new_tokens=None, + rollback_punctuation: bool = False, + ) -> Tuple[str, str]: + """不重置累积上下文的流式 ASR 解码步骤。 + + 此变体维护滚动音频窗口,并将已固定文本累积在 ``state.last_fixed_text`` 中。 + 它会缓冲输入采样点,解码每个完整分块,并原地更新流式状态。 + + 注意: + - 仅支持 vLLM 后端。 + - 不提供时间戳。 + - 仅支持单流(不支持批处理)。 + + 参数: + pcm16k:16kHz 单声道 PCM 波形(np.ndarray)。长度可以是任意非负整数, + 函数内部会将其转换为 float32。 + state:init_streaming_state() 返回的流式状态。 + max_new_tokens:每个分块可生成的最大 token 数。 + rollback_punctuation:如果为 True,则当前输出以标点结尾时不回退 token。 + + 返回: + Tuple[str, str]:包含最新转写文本(``state.text``)和累积已固定转写文本 + (``state.last_fixed_text``)的元组。状态对象会原地修改。 + + 异常: + ValueError:backend 不是 "vllm"、状态无效或 pcm16k 为 None。 + """ + if max_new_tokens is not None: + from vllm import SamplingParams + if self.backend != "vllm": + raise ValueError("streaming_transcribe() is supported only for vLLM backend (backend='vllm').") + if state is None: + raise ValueError("state must not be None. Call init_streaming_state() first.") + if pcm16k is None: + raise ValueError("pcm16k must not be None.") + + # 确保为一维单声道 + x = np.asarray(pcm16k) + if x.ndim != 1: + x = x.reshape(-1) + + # 如果输入为 int16,则转换为 [-1, 1] 范围内的 float32 PCM + if x.dtype == np.int16: + x = (x.astype(np.float32) / 32768.0) + else: + x = x.astype(np.float32, copy=False) + + # 追加到缓冲区 + if x.shape[0] > 0: + state.buffer = np.concatenate([state.buffer, x], axis=0) + + # 消费完整分块 + fixed_text = "" # 初始化,避免循环未执行时出现 UnboundLocalError + omit_new_asr_text = "" + while state.buffer.shape[0] >= state.chunk_size_samples: + chunk = state.buffer[: state.chunk_size_samples] + state.buffer = state.buffer[state.chunk_size_samples :] + + # 累积音频(从头重新输入,不进行填充) + if state.audio_accum.shape[0] == 0: + state.audio_accum = chunk + else: + state.audio_accum = np.concatenate([state.audio_accum, chunk], axis=0) + # 当累积音频达到 16 秒时,丢弃最早的 8 秒及其对应文本 + max_samples = 16 * SAMPLE_RATE + discard_samples = 8 * SAMPLE_RATE + if state.audio_accum.shape[0] > max_samples: + keep_samples = state.audio_accum.shape[0] - discard_samples + # 计算要丢弃的准确分块数:第一个分块为 320 毫秒(5,120 个采样点),后续分块为 160 毫秒(2,560 个采样点) + normal_chunk_samples = 2560 # 160 毫秒 + if not state._first_chunk_discarded: + # 第一次丢弃:第一个分块为 320 毫秒 + first_chunk_samples = 5120 # 320 毫秒 + discard_chunks = 1 + (discard_samples - first_chunk_samples) // normal_chunk_samples # = 49 + state._first_chunk_discarded = True + else: + # 后续丢弃:所有分块均为 160 毫秒 + discard_chunks = discard_samples // normal_chunk_samples # = 50 + state.audio_accum = state.audio_accum[-keep_samples:] + state.chunk_text = state.chunk_text[discard_chunks:] + + prefix_text = "".join(state.chunk_text) + if state.force_language == None and state.language != "": + prefix = f"language {state.language}" + prefix_text + else: + prefix = prefix_text + prefix = prefix.split("|")[0] + + prompt = state.prompt_raw + prefix + + # vLLM 输入:单个项目 + inp = {"prompt": prompt, "multi_modal_data": {"audio": [state.audio_accum]}} + if max_new_tokens is not None: + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=max_new_tokens, + skip_special_tokens=True, + ) + outputs = self.model.generate([inp], sampling_params=sampling_params, use_tqdm=False) + else: + outputs = self.model.generate([inp], sampling_params=self.sampling_params, use_tqdm=False) + gen_text = outputs[0].outputs[0].text + # gen_text = gen_text.replace("#","|") + gen_text = _normalize_punct_by_context(gen_text).replace('\ufffd', '') + + state._raw_decoded = (prefix + gen_text) if prefix is not None else gen_text + + lang = None + if state.force_language == None: + lang,_ = parse_language_output(state._raw_decoded, user_language=state.force_language) + + if state.force_language == "Chinese" or lang == "Chinese": + # 移除相邻中文字符之间的多余空格;在中英文混合文本中保留英文空格 + state._raw_decoded = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', state._raw_decoded) + lang, txt = parse_asr_output(state._raw_decoded, user_language=state.force_language) + has_tag = "" in state._raw_decoded + if has_tag: + state._raw_decoded = state._raw_decoded.split("")[0] + "" + txt + else: + state._raw_decoded = txt + + state._raw_decoded = state._raw_decoded.split("|")[0] + cur_ids = self.processor.tokenizer.encode(state._raw_decoded) + if rollback_punctuation == True: + _punct = ",。!?、;:,.!?;:" + raw_stripped = state._raw_decoded.strip() + if raw_stripped and raw_stripped[-1] in _punct: + k = 0 + else: + k = int(state.unfixed_token_num) + else: + k = int(state.unfixed_token_num) + + has_tag = "" in state._raw_decoded + if has_tag and state._raw_decoded.split('')[1] == "": + k = 0 + while True: + end_idx = max(0, len(cur_ids) - k) + fixed_text = self.processor.tokenizer.decode(cur_ids[:end_idx]) if end_idx > 0 else "" + if '\ufffd' not in fixed_text: + break + else: + if end_idx == 0: + fixed_text = "" + break + k += 1 + has_tag = "" in fixed_text + if has_tag: + meta_part, fixed_text = fixed_text.split("", 1) + + has_tag = "" in state._raw_decoded + if not has_tag and state.force_language == None: + state.text = "" + fixed_text = "" + continue + + state.language = lang + state.text = txt.split("|")[0] + + state.chunk_id += 1 + prefix_stripped = prefix_text.strip() + fixed_text_stripped = fixed_text.strip() + if fixed_text_stripped.startswith(prefix_stripped): + new_asr_text = fixed_text_stripped[len(prefix_stripped):] + else: + new_asr_text = "" + new_asr_text = new_asr_text.split("|")[0] + state.chunk_text.append(new_asr_text) + state.last_fixed_text = state.last_fixed_text + new_asr_text + + + return state.text, state.last_fixed_text + + def finish_streaming_transcribe_no_reset( + self, + state: ASRStreamingState, + max_new_tokens=None, + ) -> str: + """ + 完成流式 ASR,但不重置累积上下文。 + + 此函数会刷新 state.buffer 中剩余的缓冲音频(尾部音频)。即使剩余采样点少于 + chunk_size_sec,也会在不填充的情况下发送给模型,并将新固定的文本追加到 + ``state.last_fixed_text``。 + + 注意: + - 仅支持 vLLM 后端。 + - 不提供时间戳。 + - 仅支持单流。 + + 参数: + state:init_streaming_state() 返回的流式状态。 + max_new_tokens:最终解码时可选的最大生成 token 数。 + + 返回: + str:累积的已固定转写文本(``state.last_fixed_text``)。状态对象会原地修改。 + 如果没有剩余尾部音频,则返回当前已固定文本,不再发起模型请求。 + + 异常: + ValueError:backend 不是 "vllm" 或状态无效。 + """ + if max_new_tokens is not None: + from vllm import SamplingParams + if self.backend != "vllm": + raise ValueError("finish_streaming_transcribe() is supported only for vLLM backend (backend='vllm').") + if state is None: + raise ValueError("state must not be None.") + + # 如果没有剩余缓冲区,仍按原样返回状态。 + if state.buffer is None or state.buffer.shape[0] == 0: + # print(f"22text={state.text}") + return state.last_fixed_text + + tail = state.buffer + state.buffer = np.zeros((0,), dtype=np.float32) + + # 将尾部追加到累积音频 + if state.audio_accum.shape[0] == 0: + state.audio_accum = tail + else: + state.audio_accum = np.concatenate([state.audio_accum, tail], axis=0) + + # 当累积音频达到 16 秒时,丢弃最早的 8 秒及其对应文本 + max_samples = 16 * SAMPLE_RATE + discard_samples = 8 * SAMPLE_RATE + if state.audio_accum.shape[0] > max_samples: + keep_samples = state.audio_accum.shape[0] - discard_samples + # 计算要丢弃的准确分块数:第一个分块为 320 毫秒(5,120 个采样点),后续分块为 160 毫秒(2,560 个采样点) + normal_chunk_samples = 2560 # 160 毫秒 + if not state._first_chunk_discarded: + # 第一次丢弃:第一个分块为 320 毫秒 + first_chunk_samples = 5120 # 320 毫秒 + discard_chunks = 1 + (discard_samples - first_chunk_samples) // normal_chunk_samples # = 49 + state._first_chunk_discarded = True + else: + # 后续丢弃:所有分块均为 160 毫秒 + discard_chunks = discard_samples // normal_chunk_samples # = 50 + state.audio_accum = state.audio_accum[-keep_samples:] + state.chunk_text = state.chunk_text[discard_chunks:] + + prefix_text = "".join(state.chunk_text) + if state.force_language == None and state.language != "": + prefix = f"language {state.language}" + prefix_text + else: + prefix = prefix_text + + prefix = prefix.split("|")[0] + prompt = state.prompt_raw + prefix + + inp = {"prompt": prompt, "multi_modal_data": {"audio": [state.audio_accum]}} + + if max_new_tokens is not None: + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=max_new_tokens, + skip_special_tokens=True, + ) + outputs = self.model.generate([inp], sampling_params=sampling_params, use_tqdm=False) + else: + outputs = self.model.generate([inp], sampling_params=self.sampling_params, use_tqdm=False) + gen_text = outputs[0].outputs[0].text + + gen_text = _normalize_punct_by_context(gen_text).replace('\ufffd', '') + + state._raw_decoded = (prefix + gen_text) if prefix is not None else gen_text + state._raw_decoded = state._raw_decoded.split("|")[0] + lang, txt = parse_asr_output(state._raw_decoded, user_language=state.force_language) + txt = txt.split("|")[0] + state.language = lang + state.text = txt + state.chunk_id += 1 + prefix_stripped = prefix_text.strip() + fixed_text_stripped = txt.strip() + new_asr_text = fixed_text_stripped[len(prefix_stripped):] + state.chunk_text.append(new_asr_text) + state.last_fixed_text = state.last_fixed_text + new_asr_text + return state.last_fixed_text + + + +__all__ = ["R2T2ASRModel"] diff --git a/app/services/asr/r2t2_runtime.py b/app/services/asr/r2t2_runtime.py new file mode 100644 index 0000000..ce44138 --- /dev/null +++ b/app/services/asr/r2t2_runtime.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +"""Lazy lifecycle wrapper for the R2T2 vLLM streaming model.""" + +from __future__ import annotations + +import gc +import logging +import threading +from pathlib import Path +from typing import Any, Optional + +import numpy as np + +from app.core.accelerator import get_accelerator_info +from app.core.config import settings +from app.core.device import detect_device +from app.services.asr.qwen3_engine import Qwen3ASREngine + +logger = logging.getLogger(__name__) + +R2T2_MODEL_ID = "netease-youdao/Confucius4-R2T2" +_model: Optional[Any] = None +_engine: Optional[Qwen3ASREngine] = None +_model_lock = threading.RLock() +_inference_lock = threading.Lock() + + +def get_r2t2_model_path() -> Path: + """Resolve the downloaded ModelScope checkpoint for the selected route.""" + return Path(settings.MODELSCOPE_PATH) / R2T2_MODEL_ID + + +def get_r2t2_model() -> Any: + """Load R2T2 weights with the app's vLLM backend and upstream stream decoder.""" + global _model, _engine + with _model_lock: + if _model is not None: + return _model + + accelerator = get_accelerator_info() + device = detect_device(settings.DEVICE) + if not accelerator.is_gpu or not device.startswith("cuda"): + raise RuntimeError("Confucius4-R2T2 streaming requires a CUDA GPU and vLLM") + + model_path = get_r2t2_model_path() + if not model_path.is_dir(): + raise FileNotFoundError( + f"Confucius4-R2T2 checkpoint is missing: {model_path}; " + "run the model download check before starting the service" + ) + + # Keep R2T2's vLLM-only decoder lazy so CPU-only installations do not + # import its runtime until this route is selected. + from app.services.asr.r2t2_model import R2T2ASRModel + + logger.info( + "Loading Confucius4-R2T2: model=%s device=%s", + model_path, + device, + ) + _engine = Qwen3ASREngine( + model_path=str(model_path), + device=device, + max_model_len=16384, + max_new_tokens=64, + ) + backend = _engine.model + from types import SimpleNamespace + + _model = R2T2ASRModel( + backend=_engine.backend, + model=backend._llm, + processor=SimpleNamespace(tokenizer=backend._tokenizer), + sampling_params=backend._sampling_params, + max_new_tokens=64, + ) + # Retain the engine so its batching worker and vLLM references close on switch. + _model._engine_owner = _engine + # Match the upstream server's eager chunk warmup so the first live + # utterance does not pay vLLM's initial scheduler/model setup cost. + warmup_state = _model.init_streaming_state( + context="", + language=None, + unfixed_chunk_num=0, + unfixed_token_num=1, + chunk_size_sec=0.16, + ) + with _inference_lock: + _model.streaming_transcribe_no_reset( + np.zeros((5120,), dtype=np.float32), + warmup_state, + 4, + ) + _model.finish_streaming_transcribe_no_reset(warmup_state, 4) + logger.info("Confucius4-R2T2 streaming model loaded") + return _model + + +def unload_r2t2_model() -> bool: + """Release the R2T2 vLLM instance when switching realtime model routes.""" + global _model, _engine + # Use the same lock order as get_r2t2_model (model, then inference) to + # prevent a model switch from deadlocking with a first-use warmup. + with _model_lock: + if not _inference_lock.acquire(blocking=False): + return False + try: + model = _model + if model is None: + return True + engine = _engine + if engine is not None: + engine.close() + _model = None + _engine = None + finally: + _inference_lock.release() + del model + gc.collect() + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except ImportError: + pass + return True + + +def get_r2t2_inference_lock() -> threading.Lock: + """Serialize calls because each R2T2 stream mutates shared vLLM model state.""" + return _inference_lock diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index 08d6b9b..a8015dd 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -41,6 +41,12 @@ from app.services.asr.funasr_streaming import ( get_streaming_model as get_funasr_streaming_model, unload_models as unload_funasr_models, ) +from app.services.asr.r2t2_runtime import ( + R2T2_MODEL_ID, + get_r2t2_inference_lock, + get_r2t2_model, + unload_r2t2_model, +) from app.services.asr.engines.global_models import ( get_global_punc_realtime_model, get_global_vad_model, @@ -286,6 +292,11 @@ class ConnectionContext: funasr_punc_cache: Dict[str, Any] = field(default_factory=dict) funasr_pending_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) funasr_transcript: RealtimeTranscriptAssembler = field(default_factory=RealtimeTranscriptAssembler) + # R2T2 maintains a dedicated decode state for each active speech turn. + r2t2_model: Any = None + r2t2_stream_state: Any = None + r2t2_max_new_tokens: int = 0 + r2t2_last_fixed_text: str = "" pre_roll_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) pre_roll_start_sample: int = 0 audio_samples_received: int = 0 @@ -386,7 +397,7 @@ class Qwen3ASRService: async def prepare_realtime_model(self, model_id: str) -> Dict[str, str]: """Swap and warm the selected realtime ASR route before capture begins.""" - if model_id not in {"qwen3-asr", "funasr-paraformer-streaming"}: + if model_id not in {"qwen3-asr", "funasr-paraformer-streaming", "r2t2-streaming"}: raise ValueError(f"Unsupported realtime model: {model_id}") async with self._model_switch_lock: if self._active_websocket_connections and self._current_realtime_model != model_id: @@ -415,12 +426,21 @@ class Qwen3ASRService: if settings.ASR_ENABLE_REALTIME_PUNC: await run_sync(get_global_punc_realtime_model, settings.DEVICE) await run_sync(get_global_vad_model, settings.DEVICE) + elif model_id == "r2t2-streaming": + await run_sync(get_r2t2_model) + if settings.ASR_ENABLE_REALTIME_PUNC: + await run_sync(get_global_punc_realtime_model, settings.DEVICE) + await run_sync(get_global_vad_model, settings.DEVICE) else: qwen_model_id = validate_realtime_model_id("qwen3-asr") await run_sync(runtime_router.warmup_model, qwen_model_id) elif previous_model == "funasr-paraformer-streaming": unload_funasr_models() unload_global_punc_realtime_model() + elif previous_model == "r2t2-streaming": + if not unload_r2t2_model(): + raise RuntimeError("R2T2 推理仍在运行,暂时无法释放模型;请稍后重试。") + unload_global_punc_realtime_model() else: qwen_model_id = validate_realtime_model_id("qwen3-asr") if not runtime_router.unload_model(qwen_model_id): @@ -432,6 +452,11 @@ class Qwen3ASRService: if settings.ASR_ENABLE_REALTIME_PUNC: await run_sync(get_global_punc_realtime_model, settings.DEVICE) await run_sync(get_global_vad_model, settings.DEVICE) + elif model_id == "r2t2-streaming": + await run_sync(get_r2t2_model) + if settings.ASR_ENABLE_REALTIME_PUNC: + await run_sync(get_global_punc_realtime_model, settings.DEVICE) + await run_sync(get_global_vad_model, settings.DEVICE) else: qwen_model_id = validate_realtime_model_id("qwen3-asr") await run_sync(runtime_router.warmup_model, qwen_model_id) @@ -448,11 +473,17 @@ class Qwen3ASRService: if model_id == "funasr-paraformer-streaming": unload_funasr_models() unload_global_punc_realtime_model() + elif model_id == "r2t2-streaming": + unload_r2t2_model() if previous_model == "funasr-paraformer-streaming": await run_sync(get_funasr_streaming_model) await run_sync(get_funasr_offline_model) if settings.ASR_ENABLE_REALTIME_PUNC: await run_sync(get_global_punc_realtime_model, settings.DEVICE) + elif previous_model == "r2t2-streaming": + await run_sync(get_r2t2_model) + if settings.ASR_ENABLE_REALTIME_PUNC: + await run_sync(get_global_punc_realtime_model, settings.DEVICE) else: qwen_model_id = validate_realtime_model_id("qwen3-asr") await run_sync(get_runtime_router().warmup_model, qwen_model_id) @@ -495,6 +526,8 @@ class Qwen3ASRService: ctx.engine_lease = None ctx.engine = None ctx.realtime_stream_state = None + ctx.r2t2_model = None + ctx.r2t2_stream_state = None async def _detach_session( self, @@ -1376,6 +1409,9 @@ class Qwen3ASRService: def _uses_funasr_streaming(self, ctx: ConnectionContext) -> bool: return ctx.params.get("model_id", "qwen3-asr") == "funasr-paraformer-streaming" + def _uses_r2t2_streaming(self, ctx: ConnectionContext) -> bool: + return ctx.params.get("model_id", "qwen3-asr") == "r2t2-streaming" + async def _ensure_engine(self, ctx: ConnectionContext) -> Optional[Qwen3ASREngine]: if self._uses_funasr_streaming(ctx): if ctx.funasr_model is None: @@ -1399,6 +1435,21 @@ class Qwen3ASRService: ) return None + if self._uses_r2t2_streaming(ctx): + if ctx.r2t2_model is None: + ctx.r2t2_model = await run_sync(get_r2t2_model) + logger.info( + "[%s] Using Confucius4-R2T2 streaming model: %s", + ctx.diagnostic_session_id, + R2T2_MODEL_ID, + ) + if settings.ASR_ENABLE_REALTIME_PUNC and ctx.funasr_punc_model is None: + ctx.funasr_punc_model = await run_sync( + get_global_punc_realtime_model, + settings.DEVICE, + ) + return None + if ctx.engine is not None: return ctx.engine @@ -1587,6 +1638,31 @@ class Qwen3ASRService: new_text += after return new_text + @staticmethod + def _run_r2t2_stream_chunk( + model: Any, + audio: np.ndarray, + state: Any, + max_new_tokens: int, + ) -> tuple[str, str]: + """Decode one R2T2 chunk under the shared model lock, off the event loop.""" + with get_r2t2_inference_lock(): + return model.streaming_transcribe_no_reset( + np.asarray(audio, dtype=np.float32), + state, + max_new_tokens, + ) + + @staticmethod + def _run_r2t2_stream_finish(model: Any, state: Any, max_new_tokens: int) -> str: + """Flush one R2T2 stream while keeping shared model access serialized.""" + with get_r2t2_inference_lock(): + return model.finish_streaming_transcribe_no_reset(state, max_new_tokens) + + @staticmethod + def _r2t2_last_character_is_chinese(text: str) -> bool: + return bool(text) and "\u4e00" <= text[-1] <= "\u9fff" + def _reset_realtime_speaker_change_state(self, ctx: ConnectionContext) -> None: """Discard pending online speaker evidence when an ASR turn is cut.""" task = ctx.speaker_change_probe_task @@ -1890,6 +1966,8 @@ class Qwen3ASRService: Qwen vLLM refreshes a bounded audio-window snapshot per complete model chunk, so polling for a partial more frequently cannot produce new text. """ + if self._uses_r2t2_streaming(ctx): + return int(self._stream_chunk_size_sec(ctx) * 16000) interval_sec = max( float( getattr( @@ -2115,18 +2193,21 @@ class Qwen3ASRService: return False def _stream_chunk_size_sec(self, ctx: ConnectionContext) -> float: - return max( - float( - ctx.params.get( - "chunk_size_sec", - settings.REALTIME_STREAM_CHUNK_SEC, - ) - or settings.REALTIME_STREAM_CHUNK_SEC - ), - 0.6, + configured = float( + ctx.params.get( + "chunk_size_sec", + settings.REALTIME_STREAM_CHUNK_SEC, + ) + or settings.REALTIME_STREAM_CHUNK_SEC ) + if self._uses_r2t2_streaming(ctx): + return min(max(configured, 0.08), 2.0) + return max(configured, 0.6) def _stream_unfixed_chunk_num(self, ctx: ConnectionContext) -> int: + if self._uses_r2t2_streaming(ctx): + # The upstream R2T2 WebSocket route fixes text from its first chunk. + return 0 return max( int( ctx.params.get( @@ -2139,7 +2220,8 @@ class Qwen3ASRService: ) def _stream_unfixed_token_num(self, ctx: ConnectionContext) -> int: - return max(int(ctx.params.get("unfixed_token_num", 5) or 5), 1) + default = 1 if self._uses_r2t2_streaming(ctx) else 5 + return max(int(ctx.params.get("unfixed_token_num", default) or default), 1) def _should_use_native_partial_stream( self, @@ -2147,6 +2229,8 @@ class Qwen3ASRService: engine: Qwen3ASREngine, ) -> bool: _ = engine + if self._uses_r2t2_streaming(ctx): + return True configured = ctx.params.get("enable_native_partial_stream") if configured is not None: return bool(configured) @@ -2216,6 +2300,58 @@ class Qwen3ASRService: finally: ctx.last_stream_push_ms = (time.perf_counter() - push_started) * 1000.0 return + if self._uses_r2t2_streaming(ctx): + push_started = time.perf_counter() + try: + if ctx.r2t2_stream_state is None: + if ctx.r2t2_model is None: + await self._ensure_engine(ctx) + ctx.r2t2_stream_state = await run_sync( + ctx.r2t2_model.init_streaming_state, + context=ctx.params.get("context", ""), + language=ctx.params.get("language"), + chunk_size_sec=self._stream_chunk_size_sec(ctx), + unfixed_chunk_num=self._stream_unfixed_chunk_num(ctx), + unfixed_token_num=self._stream_unfixed_token_num(ctx), + ) + initial_chunk_tokens = max( + 1, + int(self._stream_chunk_size_sec(ctx) * 2 * 16000 / 1280), + ) + ctx.r2t2_max_new_tokens = initial_chunk_tokens + chunk_tokens = max(1, int(self._stream_chunk_size_sec(ctx) * 16000 / 1280)) + max_token_floor = min(32, max(4, 2 * chunk_tokens)) + previous_fixed_text = ctx.r2t2_last_fixed_text + current_text, fixed_text = await run_sync( + self._run_r2t2_stream_chunk, + ctx.r2t2_model, + np.asarray(audio, dtype=np.float32), + ctx.r2t2_stream_state, + ctx.r2t2_max_new_tokens, + ) + ctx.r2t2_stream_state.text = str(current_text or "") + ctx.r2t2_last_fixed_text = str(fixed_text or "") + if len(ctx.r2t2_last_fixed_text) > len(previous_fixed_text): + next_max_tokens = chunk_tokens + elif self._r2t2_last_character_is_chinese(ctx.r2t2_last_fixed_text): + next_max_tokens = chunk_tokens + else: + next_max_tokens = ctx.r2t2_max_new_tokens + 0.5 + if self._r2t2_last_character_is_chinese(ctx.r2t2_last_fixed_text): + next_max_tokens *= 2 + ctx.r2t2_max_new_tokens = min(max_token_floor, int(next_max_tokens)) + except Exception as exc: + logger.warning( + "[%s] R2T2 stream push failed: segment=%d chunk_samples=%d error=%s", + ctx.diagnostic_session_id, + ctx.segment_index, + int(audio.size), + exc, + ) + ctx.r2t2_stream_state = None + finally: + ctx.last_stream_push_ms = (time.perf_counter() - push_started) * 1000.0 + return if engine is None: return if not self._should_use_native_partial_stream(ctx, engine): @@ -2286,11 +2422,77 @@ class Qwen3ASRService: finally: ctx.realtime_stream_state = None + async def _finish_r2t2_stream_text( + self, + ctx: ConnectionContext, + finalized_audio: np.ndarray, + ) -> tuple[str, str]: + """Flush R2T2's fixed-prefix decoder, replaying the turn if needed.""" + if ctx.r2t2_model is None: + ctx.r2t2_model = await run_sync(get_r2t2_model) + state = ctx.r2t2_stream_state + try: + if state is None: + state = await run_sync( + ctx.r2t2_model.init_streaming_state, + context=ctx.params.get("context", ""), + language=ctx.params.get("language"), + chunk_size_sec=self._stream_chunk_size_sec(ctx), + unfixed_chunk_num=self._stream_unfixed_chunk_num(ctx), + unfixed_token_num=self._stream_unfixed_token_num(ctx), + ) + await run_sync( + self._run_r2t2_stream_chunk, + ctx.r2t2_model, + np.asarray(finalized_audio, dtype=np.float32), + state, + max(1, int(self._stream_chunk_size_sec(ctx) * 2 * 16000 / 1280)), + ) + finish_max_tokens = max( + 1, + int(self._stream_chunk_size_sec(ctx) * 2 * 16000 / 1280), + ) + text = await run_sync( + self._run_r2t2_stream_finish, + ctx.r2t2_model, + state, + finish_max_tokens, + ) + text = self._normalize_output_text(str(text or ""), ctx, enable_itn=False) + language = self._normalize_output_language( + getattr(state, "language", ""), + ctx, + ) + return text, language + except Exception as exc: + logger.warning( + "[%s] Finalize R2T2 stream failed: segment=%d error=%s", + ctx.diagnostic_session_id, + ctx.segment_index, + exc, + ) + return "", "" + finally: + ctx.r2t2_stream_state = None + async def _decode_turn_partial_text( self, engine: Qwen3ASREngine, ctx: ConnectionContext, ) -> tuple[str, str]: + if self._uses_r2t2_streaming(ctx): + state = ctx.r2t2_stream_state + if state is None: + return "", "" + text = self._normalize_output_text( + str(getattr(state, "text", "") or ""), + ctx, + enable_itn=False, + ) + return text, self._normalize_output_language( + getattr(state, "language", ""), + ctx, + ) if self._should_use_native_partial_stream(ctx, engine) and ctx.realtime_stream_state is not None: stream_text = self._normalize_output_text( str(ctx.realtime_stream_state.last_text or ""), @@ -2409,6 +2611,9 @@ class Qwen3ASRService: ctx.best_partial_language = "" ctx.last_partial_decode_samples = 0 ctx.realtime_stream_state = None + ctx.r2t2_stream_state = None + ctx.r2t2_max_new_tokens = 0 + ctx.r2t2_last_fixed_text = "" ctx.realtime_stream_failure_logged = False ctx.funasr_cache = {} ctx.funasr_pending_audio = np.array([], dtype=np.float32) @@ -3269,6 +3474,7 @@ class Qwen3ASRService: commit_started = time.perf_counter() engine = await self._ensure_engine(ctx) funasr_streaming = self._uses_funasr_streaming(ctx) + r2t2_streaming = self._uses_r2t2_streaming(ctx) current_audio = np.asarray(ctx.segment_audio_buffer, dtype=np.float32) finalized_audio = current_audio @@ -3285,8 +3491,14 @@ class Qwen3ASRService: has_native_stream_snapshot = ( not funasr_streaming and carry_audio.size == 0 - and self._should_use_native_partial_stream(ctx, engine) - and ctx.realtime_stream_state is not None + and ( + (r2t2_streaming and ctx.r2t2_stream_state is not None) + or ( + not r2t2_streaming + and self._should_use_native_partial_stream(ctx, engine) + and ctx.realtime_stream_state is not None + ) + ) ) funasr_final_inference_ms = 0.0 if funasr_streaming: @@ -3300,6 +3512,24 @@ class Qwen3ASRService: stream_text = await self._finish_funasr_stream(ctx, finalized_audio) funasr_final_inference_ms = (time.perf_counter() - funasr_final_started) * 1000.0 stream_language = self._infer_text_language(stream_text) + elif r2t2_streaming: + if carry_audio.size > 0: + # Replay only the clipped prefix; the existing state also holds + # post-boundary samples that belong to the next turn. + ctx.r2t2_stream_state = None + stream_text, stream_language = await self._finish_r2t2_stream_text(ctx, finalized_audio) + if stream_text.strip(): + observed_text = self._replace_segment_observed_text( + ctx, + stream_text, + stream_language or self._infer_text_language(stream_text), + ) + self._update_best_partial( + ctx, + observed_text, + stream_language or self._infer_text_language(observed_text), + replace_snapshot=True, + ) elif carry_audio.size == 0: stream_text, stream_language = await self._finish_realtime_stream_text(ctx, engine) if stream_text.strip(): @@ -3319,9 +3549,10 @@ class Qwen3ASRService: ) else: ctx.realtime_stream_state = None + ctx.r2t2_stream_state = None final_transcribe_started = time.perf_counter() - if funasr_streaming: + if funasr_streaming or r2t2_streaming: # Paraformer's AutoModel has no Qwen-style ITN switch; apply the # existing project normalizer to final text when the UI enables it. segment_text = self._normalize_output_text( @@ -3371,6 +3602,7 @@ class Qwen3ASRService: ) if ( not funasr_streaming + and not r2t2_streaming and engine is not None and suspicious and finalized_audio.size > 0 @@ -3407,7 +3639,7 @@ class Qwen3ASRService: segment_text, preserve_boundary_punctuation=True, ) - if funasr_streaming: + if funasr_streaming or r2t2_streaming: # Keep CT-Transformer inference off the asyncio event loop so other # WebSocket sessions can continue sending and decoding audio. segment_text = await run_sync(self._finalize_funasr_text, ctx, segment_text) @@ -3415,7 +3647,7 @@ class Qwen3ASRService: ctx, segment_text, ) - if funasr_streaming and reason == "speaker_change": + if (funasr_streaming or r2t2_streaming) and reason == "speaker_change": # A confirmed speaker transition is a sentence boundary even when # the punctuation model did not emit a terminal mark. boundary_segment = {"text": segment_text} @@ -3681,7 +3913,7 @@ class Qwen3ASRService: ) ) model_id = str(payload.get("model_id", "qwen3-asr") or "qwen3-asr").strip() - if model_id not in {"qwen3-asr", "funasr-paraformer-streaming"}: + if model_id not in {"qwen3-asr", "funasr-paraformer-streaming", "r2t2-streaming"}: await self._send_error( websocket, f"Unsupported realtime model: {model_id}", @@ -3696,9 +3928,8 @@ class Qwen3ASRService: hotword_context = format_hotword_prompt_context( hotwords ) - if model_id == "qwen3-asr" and hotword_context: - # Keep Qwen's existing prompt-based hotword behavior while - # exposing the same hotword field for both realtime models. + if model_id in {"qwen3-asr", "r2t2-streaming"} and hotword_context: + # Both Qwen-derived decoders accept hotwords through context. context = "\n".join(part for part in (context, hotword_context) if part) ctx.params = { "model_id": model_id, @@ -3903,8 +4134,12 @@ class Qwen3ASRService: if not self._uses_funasr_streaming(ctx) and self._should_decode_turn_partial(ctx): partial_decode_started = time.perf_counter() has_native_stream_snapshot = ( - self._should_use_native_partial_stream(ctx, engine) - and ctx.realtime_stream_state is not None + (self._uses_r2t2_streaming(ctx) and ctx.r2t2_stream_state is not None) + or ( + not self._uses_r2t2_streaming(ctx) + and self._should_use_native_partial_stream(ctx, engine) + and ctx.realtime_stream_state is not None + ) ) current, current_language = await self._decode_turn_partial_text( engine, @@ -4011,11 +4246,19 @@ class Qwen3ASRService: # path, without logging transcript contents or changing events. partial_source = "window_fallback" if has_native_stream_snapshot: - partial_source = ( - "qwen_vllm_window_snapshot" - if getattr(engine, "backend", "") == "vllm" - else "qwen_native_stream" - ) + if self._uses_r2t2_streaming(ctx): + partial_source = "r2t2_streaming_snapshot" + else: + partial_source = ( + "qwen_vllm_window_snapshot" + if getattr(engine, "backend", "") == "vllm" + else "qwen_native_stream" + ) + stream_state = ( + ctx.r2t2_stream_state + if self._uses_r2t2_streaming(ctx) + else ctx.realtime_stream_state + ) logger.info( "[%s] Realtime partial result: segment=%d source=%s outcome=%s " "vad_push_ms=%.1f stream_push_ms=%.1f partial_resolve_ms=%.1f " @@ -4031,9 +4274,9 @@ class Qwen3ASRService: partial_decode_ms, len(ctx.segment_audio_buffer) / 16000.0, len(ctx.stream_window_buffer) / 16000.0, - getattr(ctx.realtime_stream_state, "chunk_count", None), + getattr(stream_state, "chunk_id", getattr(stream_state, "chunk_count", None)), getattr( - getattr(ctx.realtime_stream_state, "internal_state", None), + getattr(stream_state, "internal_state", None), "window_rolled", None, ), diff --git a/scripts/download_models_standalone.py b/scripts/download_models_standalone.py index c6423a9..0ea7b94 100644 --- a/scripts/download_models_standalone.py +++ b/scripts/download_models_standalone.py @@ -29,6 +29,16 @@ class ModelAsset: MODEL_ASSETS: tuple[ModelAsset, ...] = ( + ModelAsset( + model_id="netease-youdao/Confucius4-R2T2", + description="Confucius4-R2T2 Realtime ASR", + required_patterns=("config.json",), + alternative_required_patterns=( + ("model.safetensors",), + ("model-*.safetensors",), + ), + min_total_size_bytes=1_000_000_000, + ), ModelAsset( model_id="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404", description="FunASR Contextual Offline Final ASR", diff --git a/test_web/index.html b/test_web/index.html index 90905d2..0e9a7df 100644 --- a/test_web/index.html +++ b/test_web/index.html @@ -2183,6 +2183,7 @@ @@ -2236,8 +2237,8 @@
- - + +
@@ -3916,19 +3917,25 @@ ${transcript} function syncLiveProtocolOptions() { state.liveAsrModel = $("liveAsrModel").value || "qwen3-asr"; const useFunASR = state.liveAsrModel === "funasr-paraformer-streaming"; + const useR2T2 = state.liveAsrModel === "r2t2-streaming"; // Keep protocol-specific controls visible only when the selected backend reads them. - $("liveMinPartialSec").closest(".field").hidden = useFunASR; - ["liveStreamChunkSec", "liveUnfixedChunkNum", "liveUnfixedTokenNum"].forEach((id) => { + $("liveMinPartialSec").closest(".field").hidden = useFunASR || useR2T2; + $("liveUnfixedChunkNum").closest(".field").hidden = useFunASR || useR2T2; + ["liveStreamChunkSec", "liveUnfixedTokenNum"].forEach((id) => { $(id).closest(".field").hidden = useFunASR; }); - $("liveEnableNativePartialStream").closest(".switch").hidden = useFunASR; - $("liveEnableRealtimeLongform").closest(".switch").hidden = useFunASR; + $("liveEnableNativePartialStream").closest(".switch").hidden = useFunASR || useR2T2; + $("liveEnableRealtimeLongform").closest(".switch").hidden = useFunASR || useR2T2; $("livePreRollMs").closest(".field").querySelector("label").textContent = "项目 VAD 句首预读缓冲(pre_roll_ms)"; - $("liveProtocolBadge").textContent = useFunASR ? "FunASR 流式识别" : "Qwen3-ASR"; + $("liveProtocolBadge").textContent = useFunASR + ? "FunASR 流式识别" + : useR2T2 ? "Confucius4-R2T2 流式识别" : "Qwen3-ASR"; $("liveProtocolNote").textContent = useFunASR ? "Paraformer 固定按 600ms 分块;静音收段、项目 VAD 预读和说话人选项仍生效。" - : "Qwen3-ASR 使用可调解码块和 partial 参数;静音收段、VAD 预读和说话人选项仍生效。"; + : useR2T2 + ? "R2T2 使用 vLLM 累积音频滑窗和可调解码块;静音收段、项目 VAD 预读和说话人选项仍生效。" + : "Qwen3-ASR 使用可调解码块和 partial 参数;静音收段、VAD 预读和说话人选项仍生效。"; $("liveQwenParams").hidden = false; } @@ -3947,6 +3954,13 @@ ${transcript} return; } + if (nextModel === "r2t2-streaming") { + // Match the upstream R2T2 demo's 160 ms streaming chunk as a starting point. + $("liveStreamChunkSec").value = "0.16"; + $("liveUnfixedTokenNum").value = "1"; + $("liveMinPartialSec").value = "0.25"; + } + state.liveModelSwitching = true; selector.disabled = true; $("liveStartBtn").disabled = true; @@ -3983,6 +3997,7 @@ ${transcript} || createClientMessageId().slice(0, 16); $("liveSessionId").value = sessionId; state.liveSessionId = sessionId; + const useR2T2 = state.liveAsrModel === "r2t2-streaming"; const hotwords = parseHotwords($("liveHotwords").value); const contextParts = [$("liveContext").value.trim()].filter(Boolean); if (hotwords.length) { @@ -4002,11 +4017,11 @@ ${transcript} context: contextParts.join("\n"), enable_inverse_text_normalization: $("liveEnableItn").checked, // Favor a quick provisional preview; the final segment still replaces it with re-transcribed text. - chunk_size_sec: Math.max(Number($("liveStreamChunkSec").value || 0.6), 0.6), + chunk_size_sec: Math.max(Number($("liveStreamChunkSec").value || 0.6), useR2T2 ? 0.08 : 0.6), unfixed_chunk_num: Math.max(Number($("liveUnfixedChunkNum").value || 2), 1), unfixed_token_num: Math.max(Number($("liveUnfixedTokenNum").value || 5), 1), silence_duration_ms: Number($("liveSilenceDurationMs").value || 800), - min_partial_sec: Number($("liveMinPartialSec").value || 0.6), + min_partial_sec: useR2T2 ? 0.25 : Number($("liveMinPartialSec").value || 0.6), pre_roll_ms: Number($("livePreRollMs").value || 600), // FunASR emits the current non-empty partial without a tail holdback. partial_holdback_chars: 0,