将Confucius4-R2T2 接入测试页面的实时模型列表
parent
539e699915
commit
541c800fb4
|
|
@ -0,0 +1,193 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship made available under
|
||||
the License, as indicated by a copyright notice that is included in
|
||||
or attached to the work (an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean, as submitted to the Licensor for inclusion
|
||||
in the Work by the copyright owner or by an individual or Legal Entity
|
||||
authorized to submit on behalf of the copyright owner. For the purposes
|
||||
of this definition, "submitted" means any form of electronic, verbal,
|
||||
or written communication sent to the Licensor or its representatives,
|
||||
including but not limited to communication on electronic mailing lists,
|
||||
source code control systems, and issue tracking systems that are managed
|
||||
by, or on behalf of, the Licensor for the purpose of discussing and
|
||||
improving the Work, but excluding communication that is conspicuously
|
||||
marked or otherwise designated in writing by the copyright owner as
|
||||
"Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any Legal Entity on behalf of
|
||||
whom a Contribution has been received by the Licensor and subsequently
|
||||
incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a cross-claim
|
||||
or counterclaim in a lawsuit) alleging that the Work or a Contribution
|
||||
incorporated within the Work constitutes direct or contributory patent
|
||||
infringement, then any patent licenses granted to You under this License
|
||||
for that Work shall terminate as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the Work
|
||||
or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You meet
|
||||
the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or Derivative Works
|
||||
a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices stating
|
||||
that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works that
|
||||
You distribute, all copyright, patent, trademark, and attribution
|
||||
notices from the Source form of the Work, excluding those notices
|
||||
that do not pertain to any part of the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, You must include a readable copy of the attribution
|
||||
notices contained within such NOTICE file, in at least one of the
|
||||
following places: within a NOTICE text file distributed as part of
|
||||
the Derivative Works; within the Source form or documentation, if
|
||||
provided along with the Derivative Works; or, within a display
|
||||
generated by the Derivative Works, if and wherever such third-party
|
||||
notices normally appear. The contents of the NOTICE file are for
|
||||
informational purposes only and do not modify the License. You may
|
||||
add Your own attribution notices within Derivative Works that You
|
||||
distribute, alongside or as an addendum to the NOTICE text from the
|
||||
Work, provided that such additional attribution notices cannot be
|
||||
construed as modifying the License.
|
||||
|
||||
You may add Your own license statement for Your modifications and may
|
||||
provide additional grant of rights to use, reproduce, modify, display,
|
||||
perform, sublicense and distribute Your modifications and such
|
||||
Derivative Works as a whole, provided Your use, reproduction, and
|
||||
distribution of the Work otherwise complies with the conditions stated
|
||||
in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or exemplary damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or all other
|
||||
commercial damages or losses), even if such Contributor has been
|
||||
advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format in use. We also recommend that
|
||||
a file or directory name and a description of the purpose of the
|
||||
file be included on the same "first page" as the copyright notice
|
||||
for easier identification within third-party archives.
|
||||
|
||||
Copyright 2026 NetEase Youdao
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied. See the License for the specific language governing
|
||||
permissions and limitations under the License.
|
||||
|
|
@ -40,6 +40,17 @@ _VAD_ASSETS = (
|
|||
)
|
||||
|
||||
_REALTIME_ASR_ASSETS = (
|
||||
ModelAsset(
|
||||
source="modelscope",
|
||||
model_id="netease-youdao/Confucius4-R2T2",
|
||||
description="Confucius4-R2T2 Realtime ASR",
|
||||
required_patterns=("config.json",),
|
||||
alternative_required_patterns=(
|
||||
("model.safetensors",),
|
||||
("model-*.safetensors",),
|
||||
),
|
||||
min_total_size_bytes=1_000_000_000,
|
||||
),
|
||||
ModelAsset(
|
||||
source="modelscope",
|
||||
model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,836 @@
|
|||
# coding=utf-8
|
||||
# Copyright 2026 The NetEase Youdao team.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Tuple, List, Optional
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.services.asr.qwen3_vllm import (
|
||||
_build_chat_prompt,
|
||||
_normalize_language_name,
|
||||
_parse_asr_output,
|
||||
)
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
_ASR_TEXT_TAG = "<asr_text>"
|
||||
_LANG_PREFIX = "language "
|
||||
normalize_language_name = _normalize_language_name
|
||||
|
||||
|
||||
def parse_asr_output(raw: str, user_language: Optional[str] = None) -> tuple[str, str]:
|
||||
"""Adapt the project's parser to the signature expected by the upstream decoder."""
|
||||
return _parse_asr_output(raw, user_language)
|
||||
|
||||
|
||||
def validate_language(language: str) -> None:
|
||||
"""Reject empty forced-language values; aliases are normalized by Qwen utilities."""
|
||||
if not language or not str(language).strip():
|
||||
raise ValueError("language must not be empty")
|
||||
|
||||
|
||||
def _build_text_prompt(context: str = "", force_language: Optional[str] = None) -> str:
|
||||
"""Build the same Qwen audio prompt used by the project's vLLM adapter."""
|
||||
return _build_chat_prompt(context=context, language=force_language)
|
||||
|
||||
_EN2ZH_PUNCT = {
|
||||
",": "?", ".": "?", "!": "?", "?": "?",
|
||||
";": "?", ":": "?", "(": "?", ")": "?",
|
||||
}
|
||||
_ZH2EN_PUNCT = {v: k for k, v in _EN2ZH_PUNCT.items()}
|
||||
_ALL_PUNCT_PAT = re.compile(r'[,\.!?;:()\uff0c\u3002\uff01\uff1f\uff1b\uff1a\uff08\uff09]')
|
||||
|
||||
|
||||
def _normalize_punct_by_context(text: str) -> str:
|
||||
"""根据每个标点前字符的类型规范化标点:
|
||||
- 如果前一个字符是中文,则替换为中文标点。
|
||||
- 如果前一个字符是英文字母或数字,则替换为英文标点。
|
||||
- 其他情况保留原标点。
|
||||
"""
|
||||
def _replace(m):
|
||||
punct = m.group()
|
||||
pos = m.start()
|
||||
prev_char = ""
|
||||
for i in range(pos - 1, -1, -1):
|
||||
if not text[i].isspace():
|
||||
prev_char = text[i]
|
||||
break
|
||||
if not prev_char:
|
||||
return punct
|
||||
if '\u4e00' <= prev_char <= '\u9fff':
|
||||
return _EN2ZH_PUNCT.get(punct, punct)
|
||||
elif prev_char.isascii() and (prev_char.isalnum() or prev_char in '"\''):
|
||||
return _ZH2EN_PUNCT.get(punct, punct)
|
||||
return punct
|
||||
return _ALL_PUNCT_PAT.sub(_replace, text)
|
||||
|
||||
def parse_language_output(
|
||||
raw: str,
|
||||
user_language: Optional[str] = None,
|
||||
) -> Tuple[str, str]:
|
||||
"""
|
||||
将 Qwen3-ASR 原始输出解析为 (language, text)。
|
||||
|
||||
情况:
|
||||
- 带标签:“language Chinese<asr_text>....”
|
||||
- 带换行:“language Chinese\\n...\\n<asr_text>....”
|
||||
- 无标签:将整个字符串视为文本。
|
||||
- “language None<asr_text>”:视为空音频 -> ("", "")
|
||||
|
||||
如果提供了 user_language,则强制使用该语言,并将 raw 视为纯文本
|
||||
(模型应输出不含元数据的纯转写文本)。
|
||||
|
||||
参数:
|
||||
raw:原始解码字符串。
|
||||
user_language:用户强制指定语言时使用的规范语言名称。
|
||||
|
||||
返回:
|
||||
Tuple[str, str]:(language, text)
|
||||
"""
|
||||
if raw is None:
|
||||
return "", ""
|
||||
if user_language == "English":
|
||||
s = str(raw).rstrip()
|
||||
else:
|
||||
s = str(raw).strip()
|
||||
if not s:
|
||||
return "", ""
|
||||
|
||||
if user_language:
|
||||
# 用户显式指定了语言,因此将模型输出视为纯文本
|
||||
return user_language, s
|
||||
|
||||
meta_part = s
|
||||
text_part = ""
|
||||
has_tag = _ASR_TEXT_TAG in s
|
||||
if has_tag:
|
||||
meta_part, text_part = s.split(_ASR_TEXT_TAG, 1)
|
||||
else:
|
||||
# 没有标签,视为纯文本
|
||||
return "", s.strip()
|
||||
|
||||
meta_lower = meta_part.lower()
|
||||
|
||||
# 空音频启发式判断
|
||||
if "language none" in meta_lower:
|
||||
t = text_part.strip()
|
||||
if not t:
|
||||
return "", ""
|
||||
# 如果模型仍然返回了内容,则保留内容,但语言设为未知
|
||||
return "", t
|
||||
|
||||
# 从 meta 中提取“language xxx”
|
||||
lang = ""
|
||||
for line in meta_part.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
low = line.lower()
|
||||
if low.startswith(_LANG_PREFIX):
|
||||
val = line[len(_LANG_PREFIX):].strip()
|
||||
if val:
|
||||
lang = normalize_language_name(val)
|
||||
break
|
||||
|
||||
return lang, text_part.strip()
|
||||
|
||||
@dataclass
|
||||
class ASRStreamingState:
|
||||
"""
|
||||
单个音频流(单句)的流式 ASR 状态。
|
||||
|
||||
属性:
|
||||
unfixed_chunk_num (int):前 N 个分块不使用之前的 ASR 结果作为前缀提示(将前缀重置为 "")。
|
||||
unfixed_token_num (int):当 chunk_id >= unfixed_chunk_num 时,使用累积文本作为前缀提示前,
|
||||
回退最后 K 个 token,以减少边界抖动。
|
||||
chunk_size_sec (float):分块大小(秒)。音频将按此长度递增输入模型。
|
||||
chunk_size_samples (int):16kHz 下的分块采样点数(由 chunk_size_sec 推导)。
|
||||
chunk_id (int):当前分块索引(从 0 开始)。
|
||||
buffer (np.ndarray):尚未消费为完整分块的 PCM 采样点缓冲区。
|
||||
audio_accum (np.ndarray):从流开始到当前时刻累积的音频(不填充)。
|
||||
prompt_raw (str):由聊天模板生成的基础提示(包含生成提示),不附加前缀文本。
|
||||
context (str):上下文字符串。
|
||||
force_language (Optional[str]):如果提供,则在 prompt_raw 中追加 "language X<asr_text>",
|
||||
强制输出纯文本,与非流式 transcribe() 保持一致。
|
||||
language (str):最近一次解析出的语言(每个分块解码后更新)。未知/静音时为空。
|
||||
text (str):最近一次解析出的转写文本(每个分块解码后更新)。
|
||||
_raw_decoded (str):内部累积的原始解码文本(parse_asr_output 规范化之前)。
|
||||
用于回退/token 裁剪,并作为提示前缀。
|
||||
"""
|
||||
unfixed_chunk_num: int
|
||||
unfixed_token_num: int
|
||||
chunk_size_sec: float
|
||||
chunk_size_samples: int
|
||||
|
||||
chunk_id: int
|
||||
buffer: np.ndarray
|
||||
audio_accum: np.ndarray
|
||||
|
||||
prompt_raw: str
|
||||
context: str
|
||||
force_language: Optional[str]
|
||||
|
||||
language: str
|
||||
text: str
|
||||
_raw_decoded: str
|
||||
|
||||
chunk_text: List
|
||||
last_fixed_text: str
|
||||
_first_chunk_discarded: bool
|
||||
|
||||
class R2T2ASRModel:
|
||||
def __init__(
|
||||
self,
|
||||
backend: str,
|
||||
model: Any,
|
||||
processor: Any,
|
||||
sampling_params: Optional[Any] = None,
|
||||
forced_aligner: Optional[Any] = None,
|
||||
max_inference_batch_size: int = -1,
|
||||
max_new_tokens: int = 512,
|
||||
):
|
||||
self.backend = backend
|
||||
self.model = model
|
||||
self.processor = processor
|
||||
self.sampling_params = sampling_params
|
||||
self.forced_aligner = forced_aligner
|
||||
self.max_inference_batch_size = max_inference_batch_size
|
||||
self.max_new_tokens = max_new_tokens
|
||||
|
||||
def _build_text_prompt(self, context: str = "", force_language: Optional[str] = None) -> str:
|
||||
return _build_text_prompt(context=context, force_language=force_language)
|
||||
|
||||
def init_streaming_state(
|
||||
self,
|
||||
context: str = "",
|
||||
language: Optional[str] = None,
|
||||
unfixed_chunk_num: int = 2,
|
||||
unfixed_token_num: int = 5,
|
||||
chunk_size_sec: float = 2.0,
|
||||
) -> ASRStreamingState:
|
||||
"""
|
||||
初始化单个流的 ASR 状态。
|
||||
|
||||
注意:
|
||||
- 流式 ASR 仅支持 vLLM 后端。
|
||||
- 流式 ASR 不支持时间戳(不使用强制对齐器)。
|
||||
- 不支持批量推理。
|
||||
|
||||
参数:
|
||||
context:上下文字符串。
|
||||
language:可选的强制语言。提供时必须属于支持的语言。
|
||||
行为与 transcribe() 相同:通过提示后缀强制输出纯文本。
|
||||
unfixed_chunk_num:前 N 个分块不使用之前的输出作为前缀提示(将前缀重置为 "")。
|
||||
unfixed_token_num:在 unfixed_chunk_num 之后将累积输出作为前缀提示时,回退最后 K 个 token。
|
||||
chunk_size_sec:分块大小(秒,音频为 16k PCM)。函数会在内部将其转换为 16kHz 下的采样点数。
|
||||
|
||||
返回:
|
||||
ASRStreamingState:可变状态对象,传递给 streaming_transcribe() 和 finish_streaming_transcribe()。
|
||||
|
||||
异常:
|
||||
ValueError:
|
||||
- backend 不是 "vllm"。
|
||||
- chunk_size_sec <= 0。
|
||||
- 强制语言无效(校验规则与 transcribe() 相同)。
|
||||
"""
|
||||
if self.backend != "vllm":
|
||||
raise ValueError("Streaming ASR is supported only for vLLM backend (backend='vllm').")
|
||||
if chunk_size_sec is None or float(chunk_size_sec) <= 0:
|
||||
raise ValueError(f"chunk_size_sec must be > 0, got: {chunk_size_sec}")
|
||||
|
||||
force_language = None
|
||||
if language is not None and str(language).strip() != "":
|
||||
ln = normalize_language_name(str(language))
|
||||
validate_language(ln)
|
||||
force_language = ln
|
||||
|
||||
chunk_size_samples = int(round(float(chunk_size_sec) * SAMPLE_RATE))
|
||||
chunk_size_samples = max(1, chunk_size_samples)
|
||||
|
||||
prompt_raw = self._build_text_prompt(context=context, force_language=force_language)
|
||||
|
||||
return ASRStreamingState(
|
||||
unfixed_chunk_num=int(unfixed_chunk_num),
|
||||
unfixed_token_num=int(unfixed_token_num),
|
||||
chunk_size_sec=float(chunk_size_sec),
|
||||
chunk_size_samples=int(chunk_size_samples),
|
||||
chunk_id=0,
|
||||
buffer=np.zeros((0,), dtype=np.float32),
|
||||
audio_accum=np.zeros((0,), dtype=np.float32),
|
||||
prompt_raw=prompt_raw,
|
||||
context=context or "",
|
||||
force_language=force_language,
|
||||
language="",
|
||||
text="",
|
||||
_raw_decoded="",
|
||||
chunk_text=[],
|
||||
last_fixed_text="",
|
||||
_first_chunk_discarded=False
|
||||
)
|
||||
|
||||
def streaming_transcribe(
|
||||
self,
|
||||
pcm16k: np.ndarray,
|
||||
state: ASRStreamingState,
|
||||
max_new_tokens=None,
|
||||
rollback_punctuation: bool = False,
|
||||
) -> Tuple[str, str]:
|
||||
"""
|
||||
流式 ASR 解码步骤。
|
||||
|
||||
此函数接收任意长度的 16k PCM float numpy 数组(单声道)。
|
||||
它会缓冲输入采样点;每当累积的采样点足以组成一个完整分块(chunk_size_sec)时,
|
||||
就执行一次增量解码并更新:
|
||||
|
||||
- state.language
|
||||
- state.text
|
||||
|
||||
调用方只需持续向此函数传入音频,并读取 state.language/state.text。
|
||||
|
||||
实现细节:
|
||||
- 每当新分块就绪时,将其追加到 audio_accum,并将截至目前的*全部*音频重新输入模型
|
||||
(不进行填充)。
|
||||
- 按以下方式更新提示:state.prompt_raw + prefix_text
|
||||
- 前缀回退策略:
|
||||
* 如果 chunk_id < unfixed_chunk_num:prefix_text = ""
|
||||
* 否则:从之前累积的解码文本中回退最后 unfixed_token_num 个 token。
|
||||
|
||||
注意:
|
||||
- 仅支持 vLLM 后端。
|
||||
- 不提供时间戳。
|
||||
- 仅支持单流(不支持批处理)。
|
||||
|
||||
参数:
|
||||
pcm16k:16kHz 单声道 PCM 波形(np.ndarray)。长度可以是任意非负整数。
|
||||
dtype 可以是 float32/float64/int16;将转换为 float32。
|
||||
state:init_streaming_state() 返回的流式状态。
|
||||
|
||||
返回:
|
||||
Tuple[str, str]:包含最新转写文本(``state.text``)和当前更新中已固定文本
|
||||
(``fixed_text``)的元组。状态对象会原地修改。
|
||||
|
||||
异常:
|
||||
ValueError:backend 不是 "vllm" 或状态无效。
|
||||
"""
|
||||
if max_new_tokens is not None:
|
||||
from vllm import SamplingParams
|
||||
if self.backend != "vllm":
|
||||
raise ValueError("streaming_transcribe() is supported only for vLLM backend (backend='vllm').")
|
||||
if state is None:
|
||||
raise ValueError("state must not be None. Call init_streaming_state() first.")
|
||||
if pcm16k is None:
|
||||
raise ValueError("pcm16k must not be None.")
|
||||
|
||||
# 确保为一维单声道
|
||||
x = np.asarray(pcm16k)
|
||||
if x.ndim != 1:
|
||||
x = x.reshape(-1)
|
||||
|
||||
# 如果输入为 int16,则转换为 [-1, 1] 范围内的 float32 PCM
|
||||
if x.dtype == np.int16:
|
||||
x = (x.astype(np.float32) / 32768.0)
|
||||
else:
|
||||
x = x.astype(np.float32, copy=False)
|
||||
|
||||
# 追加到缓冲区
|
||||
if x.shape[0] > 0:
|
||||
state.buffer = np.concatenate([state.buffer, x], axis=0)
|
||||
|
||||
# 消费完整分块
|
||||
fixed_text = "" # 初始化,避免循环未执行时出现 UnboundLocalError
|
||||
while state.buffer.shape[0] >= state.chunk_size_samples:
|
||||
chunk = state.buffer[: state.chunk_size_samples]
|
||||
state.buffer = state.buffer[state.chunk_size_samples :]
|
||||
|
||||
# 累积音频(从头重新输入,不进行填充)
|
||||
if state.audio_accum.shape[0] == 0:
|
||||
state.audio_accum = chunk
|
||||
else:
|
||||
state.audio_accum = np.concatenate([state.audio_accum, chunk], axis=0)
|
||||
|
||||
# 使用回退策略构建前缀
|
||||
prefix = ""
|
||||
if state.chunk_id < state.unfixed_chunk_num:
|
||||
prefix = ""
|
||||
else:
|
||||
state._raw_decoded = state._raw_decoded.split("|")[0]
|
||||
cur_ids = self.processor.tokenizer.encode(state._raw_decoded)
|
||||
if rollback_punctuation == True:
|
||||
_punct = ",。!?、;:.!?;:"
|
||||
raw_stripped = state._raw_decoded.strip()
|
||||
if raw_stripped and raw_stripped[-1] in _punct:
|
||||
k = 0
|
||||
else:
|
||||
k = int(state.unfixed_token_num)
|
||||
else:
|
||||
k = int(state.unfixed_token_num)
|
||||
while True:
|
||||
end_idx = max(0, len(cur_ids) - k)
|
||||
prefix = self.processor.tokenizer.decode(cur_ids[:end_idx]) if end_idx > 0 else ""
|
||||
if '\ufffd' not in prefix:
|
||||
break
|
||||
else:
|
||||
if end_idx == 0:
|
||||
prefix = ""
|
||||
break
|
||||
k += 1
|
||||
|
||||
prefix = prefix.split("|")[0]
|
||||
|
||||
prompt = state.prompt_raw + prefix
|
||||
|
||||
# vLLM 输入:单个项目
|
||||
inp = {"prompt": prompt, "multi_modal_data": {"audio": [state.audio_accum]}}
|
||||
if max_new_tokens is not None:
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=max_new_tokens,
|
||||
skip_special_tokens=True,
|
||||
)
|
||||
outputs = self.model.generate([inp], sampling_params=sampling_params, use_tqdm=False)
|
||||
else:
|
||||
outputs = self.model.generate([inp], sampling_params=self.sampling_params, use_tqdm=False)
|
||||
gen_text = outputs[0].outputs[0].text
|
||||
gen_text = _normalize_punct_by_context(gen_text).replace('\ufffd', '')
|
||||
#print(f"prefix={prefix}")
|
||||
state._raw_decoded = (prefix + gen_text) if prefix is not None else gen_text
|
||||
#print(f"state._raw_decoded={state._raw_decoded}")
|
||||
|
||||
lang = None
|
||||
if state.force_language == None:
|
||||
lang,_ = parse_language_output(state._raw_decoded, user_language=state.force_language)
|
||||
|
||||
if state.force_language == "Chinese" or lang == "Chinese":
|
||||
# 移除相邻中文字符之间的多余空格;在中英文混合文本中保留英文空格
|
||||
state._raw_decoded = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', state._raw_decoded)
|
||||
lang, txt = parse_asr_output(state._raw_decoded, user_language=state.force_language)
|
||||
|
||||
has_tag = "<asr_text>" in state._raw_decoded
|
||||
if has_tag:
|
||||
state._raw_decoded = state._raw_decoded.split("<asr_text>")[0] + "<asr_text>" + txt
|
||||
else:
|
||||
state._raw_decoded = txt
|
||||
|
||||
state._raw_decoded = state._raw_decoded.split("|")[0]
|
||||
cur_ids = self.processor.tokenizer.encode(state._raw_decoded)
|
||||
if rollback_punctuation == True:
|
||||
_punct = ",。!?、;:,.!?;:"
|
||||
raw_stripped = state._raw_decoded.strip()
|
||||
if raw_stripped and raw_stripped[-1] in _punct:
|
||||
k = 0
|
||||
else:
|
||||
k = int(state.unfixed_token_num)
|
||||
else:
|
||||
k = int(state.unfixed_token_num)
|
||||
|
||||
has_tag = "<asr_text>" in state._raw_decoded
|
||||
if has_tag and state._raw_decoded.split('<asr_text>')[1] == "":
|
||||
k = 0
|
||||
while True:
|
||||
end_idx = max(0, len(cur_ids) - k)
|
||||
fixed_text = self.processor.tokenizer.decode(cur_ids[:end_idx]) if end_idx > 0 else ""
|
||||
if '\ufffd' not in fixed_text:
|
||||
break
|
||||
else:
|
||||
if end_idx == 0:
|
||||
fixed_text = ""
|
||||
break
|
||||
k += 1
|
||||
has_tag = "<asr_text>" in fixed_text
|
||||
if has_tag:
|
||||
meta_part, fixed_text = fixed_text.split("<asr_text>", 1)
|
||||
fixed_text = fixed_text.split("|")[0]
|
||||
|
||||
has_tag = "<asr_text>" in state._raw_decoded
|
||||
if not has_tag and state.force_language == None:
|
||||
state.text = ""
|
||||
fixed_text = ""
|
||||
continue
|
||||
|
||||
state.language = lang
|
||||
state.text = txt.split("|")[0]
|
||||
|
||||
state.chunk_id += 1
|
||||
|
||||
return state.text,fixed_text
|
||||
|
||||
def finish_streaming_transcribe(
|
||||
self,
|
||||
state: ASRStreamingState,
|
||||
max_new_tokens=None,
|
||||
) -> str:
|
||||
"""
|
||||
完成流式 ASR。
|
||||
|
||||
此函数会刷新 state.buffer 中剩余的缓冲音频(尾部音频)。
|
||||
即使剩余采样点少于 chunk_size_sec,也会在不填充的情况下发送给模型。
|
||||
然后最后一次更新 state.language/state.text。
|
||||
|
||||
注意:
|
||||
- 仅支持 vLLM 后端。
|
||||
- 不提供时间戳。
|
||||
- 仅支持单流。
|
||||
|
||||
参数:
|
||||
state:init_streaming_state() 返回的流式状态。
|
||||
max_new_tokens:最终解码时可选的最大生成 token 数。
|
||||
|
||||
返回:
|
||||
str:最新的转写文本(``state.text``)。状态对象会原地修改,且可复用于后续流式调用。
|
||||
|
||||
异常:
|
||||
ValueError:backend 不是 "vllm" 或状态无效。
|
||||
"""
|
||||
if max_new_tokens is not None:
|
||||
from vllm import SamplingParams
|
||||
if self.backend != "vllm":
|
||||
raise ValueError("finish_streaming_transcribe() is supported only for vLLM backend (backend='vllm').")
|
||||
if state is None:
|
||||
raise ValueError("state must not be None.")
|
||||
|
||||
# 如果没有剩余缓冲区,仍按原样返回状态。
|
||||
if state.buffer is None or state.buffer.shape[0] == 0:
|
||||
return state.text
|
||||
|
||||
tail = state.buffer
|
||||
state.buffer = np.zeros((0,), dtype=np.float32)
|
||||
|
||||
# 将尾部追加到累积音频
|
||||
if state.audio_accum.shape[0] == 0:
|
||||
state.audio_accum = tail
|
||||
else:
|
||||
state.audio_accum = np.concatenate([state.audio_accum, tail], axis=0)
|
||||
|
||||
# 前缀回退策略(与逐分块处理相同)
|
||||
prefix = ""
|
||||
if state.chunk_id < state.unfixed_chunk_num:
|
||||
prefix = ""
|
||||
else:
|
||||
cur_ids = self.processor.tokenizer.encode(state._raw_decoded)
|
||||
end_idx = max(1, len(cur_ids) - int(state.unfixed_token_num))
|
||||
prefix = self.processor.tokenizer.decode(cur_ids[:end_idx])
|
||||
|
||||
prefix = prefix.split("|")[0]
|
||||
prompt = state.prompt_raw + prefix
|
||||
inp = {"prompt": prompt, "multi_modal_data": {"audio": [state.audio_accum]}}
|
||||
|
||||
if max_new_tokens is not None:
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=max_new_tokens,
|
||||
skip_special_tokens=True,
|
||||
)
|
||||
outputs = self.model.generate([inp], sampling_params=sampling_params, use_tqdm=False)
|
||||
else:
|
||||
outputs = self.model.generate([inp], sampling_params=self.sampling_params, use_tqdm=False)
|
||||
gen_text = outputs[0].outputs[0].text
|
||||
gen_text = _normalize_punct_by_context(gen_text).replace('\ufffd', '')
|
||||
|
||||
state._raw_decoded = (prefix + gen_text) if prefix is not None else gen_text
|
||||
state._raw_decoded = state._raw_decoded.split("|")[0]
|
||||
lang, txt = parse_asr_output(state._raw_decoded, user_language=state.force_language)
|
||||
|
||||
state.language = lang
|
||||
state.text = txt.split("|")[0]
|
||||
state.chunk_id += 1
|
||||
return state.text
|
||||
|
||||
def streaming_transcribe_no_reset(
|
||||
self,
|
||||
pcm16k: np.ndarray,
|
||||
state: ASRStreamingState,
|
||||
max_new_tokens=None,
|
||||
rollback_punctuation: bool = False,
|
||||
) -> Tuple[str, str]:
|
||||
"""不重置累积上下文的流式 ASR 解码步骤。
|
||||
|
||||
此变体维护滚动音频窗口,并将已固定文本累积在 ``state.last_fixed_text`` 中。
|
||||
它会缓冲输入采样点,解码每个完整分块,并原地更新流式状态。
|
||||
|
||||
注意:
|
||||
- 仅支持 vLLM 后端。
|
||||
- 不提供时间戳。
|
||||
- 仅支持单流(不支持批处理)。
|
||||
|
||||
参数:
|
||||
pcm16k:16kHz 单声道 PCM 波形(np.ndarray)。长度可以是任意非负整数,
|
||||
函数内部会将其转换为 float32。
|
||||
state:init_streaming_state() 返回的流式状态。
|
||||
max_new_tokens:每个分块可生成的最大 token 数。
|
||||
rollback_punctuation:如果为 True,则当前输出以标点结尾时不回退 token。
|
||||
|
||||
返回:
|
||||
Tuple[str, str]:包含最新转写文本(``state.text``)和累积已固定转写文本
|
||||
(``state.last_fixed_text``)的元组。状态对象会原地修改。
|
||||
|
||||
异常:
|
||||
ValueError:backend 不是 "vllm"、状态无效或 pcm16k 为 None。
|
||||
"""
|
||||
if max_new_tokens is not None:
|
||||
from vllm import SamplingParams
|
||||
if self.backend != "vllm":
|
||||
raise ValueError("streaming_transcribe() is supported only for vLLM backend (backend='vllm').")
|
||||
if state is None:
|
||||
raise ValueError("state must not be None. Call init_streaming_state() first.")
|
||||
if pcm16k is None:
|
||||
raise ValueError("pcm16k must not be None.")
|
||||
|
||||
# 确保为一维单声道
|
||||
x = np.asarray(pcm16k)
|
||||
if x.ndim != 1:
|
||||
x = x.reshape(-1)
|
||||
|
||||
# 如果输入为 int16,则转换为 [-1, 1] 范围内的 float32 PCM
|
||||
if x.dtype == np.int16:
|
||||
x = (x.astype(np.float32) / 32768.0)
|
||||
else:
|
||||
x = x.astype(np.float32, copy=False)
|
||||
|
||||
# 追加到缓冲区
|
||||
if x.shape[0] > 0:
|
||||
state.buffer = np.concatenate([state.buffer, x], axis=0)
|
||||
|
||||
# 消费完整分块
|
||||
fixed_text = "" # 初始化,避免循环未执行时出现 UnboundLocalError
|
||||
omit_new_asr_text = ""
|
||||
while state.buffer.shape[0] >= state.chunk_size_samples:
|
||||
chunk = state.buffer[: state.chunk_size_samples]
|
||||
state.buffer = state.buffer[state.chunk_size_samples :]
|
||||
|
||||
# 累积音频(从头重新输入,不进行填充)
|
||||
if state.audio_accum.shape[0] == 0:
|
||||
state.audio_accum = chunk
|
||||
else:
|
||||
state.audio_accum = np.concatenate([state.audio_accum, chunk], axis=0)
|
||||
# 当累积音频达到 16 秒时,丢弃最早的 8 秒及其对应文本
|
||||
max_samples = 16 * SAMPLE_RATE
|
||||
discard_samples = 8 * SAMPLE_RATE
|
||||
if state.audio_accum.shape[0] > max_samples:
|
||||
keep_samples = state.audio_accum.shape[0] - discard_samples
|
||||
# 计算要丢弃的准确分块数:第一个分块为 320 毫秒(5,120 个采样点),后续分块为 160 毫秒(2,560 个采样点)
|
||||
normal_chunk_samples = 2560 # 160 毫秒
|
||||
if not state._first_chunk_discarded:
|
||||
# 第一次丢弃:第一个分块为 320 毫秒
|
||||
first_chunk_samples = 5120 # 320 毫秒
|
||||
discard_chunks = 1 + (discard_samples - first_chunk_samples) // normal_chunk_samples # = 49
|
||||
state._first_chunk_discarded = True
|
||||
else:
|
||||
# 后续丢弃:所有分块均为 160 毫秒
|
||||
discard_chunks = discard_samples // normal_chunk_samples # = 50
|
||||
state.audio_accum = state.audio_accum[-keep_samples:]
|
||||
state.chunk_text = state.chunk_text[discard_chunks:]
|
||||
|
||||
prefix_text = "".join(state.chunk_text)
|
||||
if state.force_language == None and state.language != "":
|
||||
prefix = f"language {state.language}<asr_text>" + prefix_text
|
||||
else:
|
||||
prefix = prefix_text
|
||||
prefix = prefix.split("|")[0]
|
||||
|
||||
prompt = state.prompt_raw + prefix
|
||||
|
||||
# vLLM 输入:单个项目
|
||||
inp = {"prompt": prompt, "multi_modal_data": {"audio": [state.audio_accum]}}
|
||||
if max_new_tokens is not None:
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=max_new_tokens,
|
||||
skip_special_tokens=True,
|
||||
)
|
||||
outputs = self.model.generate([inp], sampling_params=sampling_params, use_tqdm=False)
|
||||
else:
|
||||
outputs = self.model.generate([inp], sampling_params=self.sampling_params, use_tqdm=False)
|
||||
gen_text = outputs[0].outputs[0].text
|
||||
# gen_text = gen_text.replace("#","|")
|
||||
gen_text = _normalize_punct_by_context(gen_text).replace('\ufffd', '')
|
||||
|
||||
state._raw_decoded = (prefix + gen_text) if prefix is not None else gen_text
|
||||
|
||||
lang = None
|
||||
if state.force_language == None:
|
||||
lang,_ = parse_language_output(state._raw_decoded, user_language=state.force_language)
|
||||
|
||||
if state.force_language == "Chinese" or lang == "Chinese":
|
||||
# 移除相邻中文字符之间的多余空格;在中英文混合文本中保留英文空格
|
||||
state._raw_decoded = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', state._raw_decoded)
|
||||
lang, txt = parse_asr_output(state._raw_decoded, user_language=state.force_language)
|
||||
has_tag = "<asr_text>" in state._raw_decoded
|
||||
if has_tag:
|
||||
state._raw_decoded = state._raw_decoded.split("<asr_text>")[0] + "<asr_text>" + txt
|
||||
else:
|
||||
state._raw_decoded = txt
|
||||
|
||||
state._raw_decoded = state._raw_decoded.split("|")[0]
|
||||
cur_ids = self.processor.tokenizer.encode(state._raw_decoded)
|
||||
if rollback_punctuation == True:
|
||||
_punct = ",。!?、;:,.!?;:"
|
||||
raw_stripped = state._raw_decoded.strip()
|
||||
if raw_stripped and raw_stripped[-1] in _punct:
|
||||
k = 0
|
||||
else:
|
||||
k = int(state.unfixed_token_num)
|
||||
else:
|
||||
k = int(state.unfixed_token_num)
|
||||
|
||||
has_tag = "<asr_text>" in state._raw_decoded
|
||||
if has_tag and state._raw_decoded.split('<asr_text>')[1] == "":
|
||||
k = 0
|
||||
while True:
|
||||
end_idx = max(0, len(cur_ids) - k)
|
||||
fixed_text = self.processor.tokenizer.decode(cur_ids[:end_idx]) if end_idx > 0 else ""
|
||||
if '\ufffd' not in fixed_text:
|
||||
break
|
||||
else:
|
||||
if end_idx == 0:
|
||||
fixed_text = ""
|
||||
break
|
||||
k += 1
|
||||
has_tag = "<asr_text>" in fixed_text
|
||||
if has_tag:
|
||||
meta_part, fixed_text = fixed_text.split("<asr_text>", 1)
|
||||
|
||||
has_tag = "<asr_text>" in state._raw_decoded
|
||||
if not has_tag and state.force_language == None:
|
||||
state.text = ""
|
||||
fixed_text = ""
|
||||
continue
|
||||
|
||||
state.language = lang
|
||||
state.text = txt.split("|")[0]
|
||||
|
||||
state.chunk_id += 1
|
||||
prefix_stripped = prefix_text.strip()
|
||||
fixed_text_stripped = fixed_text.strip()
|
||||
if fixed_text_stripped.startswith(prefix_stripped):
|
||||
new_asr_text = fixed_text_stripped[len(prefix_stripped):]
|
||||
else:
|
||||
new_asr_text = ""
|
||||
new_asr_text = new_asr_text.split("|")[0]
|
||||
state.chunk_text.append(new_asr_text)
|
||||
state.last_fixed_text = state.last_fixed_text + new_asr_text
|
||||
|
||||
|
||||
return state.text, state.last_fixed_text
|
||||
|
||||
def finish_streaming_transcribe_no_reset(
|
||||
self,
|
||||
state: ASRStreamingState,
|
||||
max_new_tokens=None,
|
||||
) -> str:
|
||||
"""
|
||||
完成流式 ASR,但不重置累积上下文。
|
||||
|
||||
此函数会刷新 state.buffer 中剩余的缓冲音频(尾部音频)。即使剩余采样点少于
|
||||
chunk_size_sec,也会在不填充的情况下发送给模型,并将新固定的文本追加到
|
||||
``state.last_fixed_text``。
|
||||
|
||||
注意:
|
||||
- 仅支持 vLLM 后端。
|
||||
- 不提供时间戳。
|
||||
- 仅支持单流。
|
||||
|
||||
参数:
|
||||
state:init_streaming_state() 返回的流式状态。
|
||||
max_new_tokens:最终解码时可选的最大生成 token 数。
|
||||
|
||||
返回:
|
||||
str:累积的已固定转写文本(``state.last_fixed_text``)。状态对象会原地修改。
|
||||
如果没有剩余尾部音频,则返回当前已固定文本,不再发起模型请求。
|
||||
|
||||
异常:
|
||||
ValueError:backend 不是 "vllm" 或状态无效。
|
||||
"""
|
||||
if max_new_tokens is not None:
|
||||
from vllm import SamplingParams
|
||||
if self.backend != "vllm":
|
||||
raise ValueError("finish_streaming_transcribe() is supported only for vLLM backend (backend='vllm').")
|
||||
if state is None:
|
||||
raise ValueError("state must not be None.")
|
||||
|
||||
# 如果没有剩余缓冲区,仍按原样返回状态。
|
||||
if state.buffer is None or state.buffer.shape[0] == 0:
|
||||
# print(f"22text={state.text}")
|
||||
return state.last_fixed_text
|
||||
|
||||
tail = state.buffer
|
||||
state.buffer = np.zeros((0,), dtype=np.float32)
|
||||
|
||||
# 将尾部追加到累积音频
|
||||
if state.audio_accum.shape[0] == 0:
|
||||
state.audio_accum = tail
|
||||
else:
|
||||
state.audio_accum = np.concatenate([state.audio_accum, tail], axis=0)
|
||||
|
||||
# 当累积音频达到 16 秒时,丢弃最早的 8 秒及其对应文本
|
||||
max_samples = 16 * SAMPLE_RATE
|
||||
discard_samples = 8 * SAMPLE_RATE
|
||||
if state.audio_accum.shape[0] > max_samples:
|
||||
keep_samples = state.audio_accum.shape[0] - discard_samples
|
||||
# 计算要丢弃的准确分块数:第一个分块为 320 毫秒(5,120 个采样点),后续分块为 160 毫秒(2,560 个采样点)
|
||||
normal_chunk_samples = 2560 # 160 毫秒
|
||||
if not state._first_chunk_discarded:
|
||||
# 第一次丢弃:第一个分块为 320 毫秒
|
||||
first_chunk_samples = 5120 # 320 毫秒
|
||||
discard_chunks = 1 + (discard_samples - first_chunk_samples) // normal_chunk_samples # = 49
|
||||
state._first_chunk_discarded = True
|
||||
else:
|
||||
# 后续丢弃:所有分块均为 160 毫秒
|
||||
discard_chunks = discard_samples // normal_chunk_samples # = 50
|
||||
state.audio_accum = state.audio_accum[-keep_samples:]
|
||||
state.chunk_text = state.chunk_text[discard_chunks:]
|
||||
|
||||
prefix_text = "".join(state.chunk_text)
|
||||
if state.force_language == None and state.language != "":
|
||||
prefix = f"language {state.language}<asr_text>" + prefix_text
|
||||
else:
|
||||
prefix = prefix_text
|
||||
|
||||
prefix = prefix.split("|")[0]
|
||||
prompt = state.prompt_raw + prefix
|
||||
|
||||
inp = {"prompt": prompt, "multi_modal_data": {"audio": [state.audio_accum]}}
|
||||
|
||||
if max_new_tokens is not None:
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=max_new_tokens,
|
||||
skip_special_tokens=True,
|
||||
)
|
||||
outputs = self.model.generate([inp], sampling_params=sampling_params, use_tqdm=False)
|
||||
else:
|
||||
outputs = self.model.generate([inp], sampling_params=self.sampling_params, use_tqdm=False)
|
||||
gen_text = outputs[0].outputs[0].text
|
||||
|
||||
gen_text = _normalize_punct_by_context(gen_text).replace('\ufffd', '')
|
||||
|
||||
state._raw_decoded = (prefix + gen_text) if prefix is not None else gen_text
|
||||
state._raw_decoded = state._raw_decoded.split("|")[0]
|
||||
lang, txt = parse_asr_output(state._raw_decoded, user_language=state.force_language)
|
||||
txt = txt.split("|")[0]
|
||||
state.language = lang
|
||||
state.text = txt
|
||||
state.chunk_id += 1
|
||||
prefix_stripped = prefix_text.strip()
|
||||
fixed_text_stripped = txt.strip()
|
||||
new_asr_text = fixed_text_stripped[len(prefix_stripped):]
|
||||
state.chunk_text.append(new_asr_text)
|
||||
state.last_fixed_text = state.last_fixed_text + new_asr_text
|
||||
return state.last_fixed_text
|
||||
|
||||
|
||||
|
||||
__all__ = ["R2T2ASRModel"]
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""Lazy lifecycle wrapper for the R2T2 vLLM streaming model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.core.accelerator import get_accelerator_info
|
||||
from app.core.config import settings
|
||||
from app.core.device import detect_device
|
||||
from app.services.asr.qwen3_engine import Qwen3ASREngine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
R2T2_MODEL_ID = "netease-youdao/Confucius4-R2T2"
|
||||
_model: Optional[Any] = None
|
||||
_engine: Optional[Qwen3ASREngine] = None
|
||||
_model_lock = threading.RLock()
|
||||
_inference_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_r2t2_model_path() -> Path:
|
||||
"""Resolve the downloaded ModelScope checkpoint for the selected route."""
|
||||
return Path(settings.MODELSCOPE_PATH) / R2T2_MODEL_ID
|
||||
|
||||
|
||||
def get_r2t2_model() -> Any:
|
||||
"""Load R2T2 weights with the app's vLLM backend and upstream stream decoder."""
|
||||
global _model, _engine
|
||||
with _model_lock:
|
||||
if _model is not None:
|
||||
return _model
|
||||
|
||||
accelerator = get_accelerator_info()
|
||||
device = detect_device(settings.DEVICE)
|
||||
if not accelerator.is_gpu or not device.startswith("cuda"):
|
||||
raise RuntimeError("Confucius4-R2T2 streaming requires a CUDA GPU and vLLM")
|
||||
|
||||
model_path = get_r2t2_model_path()
|
||||
if not model_path.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"Confucius4-R2T2 checkpoint is missing: {model_path}; "
|
||||
"run the model download check before starting the service"
|
||||
)
|
||||
|
||||
# Keep R2T2's vLLM-only decoder lazy so CPU-only installations do not
|
||||
# import its runtime until this route is selected.
|
||||
from app.services.asr.r2t2_model import R2T2ASRModel
|
||||
|
||||
logger.info(
|
||||
"Loading Confucius4-R2T2: model=%s device=%s",
|
||||
model_path,
|
||||
device,
|
||||
)
|
||||
_engine = Qwen3ASREngine(
|
||||
model_path=str(model_path),
|
||||
device=device,
|
||||
max_model_len=16384,
|
||||
max_new_tokens=64,
|
||||
)
|
||||
backend = _engine.model
|
||||
from types import SimpleNamespace
|
||||
|
||||
_model = R2T2ASRModel(
|
||||
backend=_engine.backend,
|
||||
model=backend._llm,
|
||||
processor=SimpleNamespace(tokenizer=backend._tokenizer),
|
||||
sampling_params=backend._sampling_params,
|
||||
max_new_tokens=64,
|
||||
)
|
||||
# Retain the engine so its batching worker and vLLM references close on switch.
|
||||
_model._engine_owner = _engine
|
||||
# Match the upstream server's eager chunk warmup so the first live
|
||||
# utterance does not pay vLLM's initial scheduler/model setup cost.
|
||||
warmup_state = _model.init_streaming_state(
|
||||
context="",
|
||||
language=None,
|
||||
unfixed_chunk_num=0,
|
||||
unfixed_token_num=1,
|
||||
chunk_size_sec=0.16,
|
||||
)
|
||||
with _inference_lock:
|
||||
_model.streaming_transcribe_no_reset(
|
||||
np.zeros((5120,), dtype=np.float32),
|
||||
warmup_state,
|
||||
4,
|
||||
)
|
||||
_model.finish_streaming_transcribe_no_reset(warmup_state, 4)
|
||||
logger.info("Confucius4-R2T2 streaming model loaded")
|
||||
return _model
|
||||
|
||||
|
||||
def unload_r2t2_model() -> bool:
|
||||
"""Release the R2T2 vLLM instance when switching realtime model routes."""
|
||||
global _model, _engine
|
||||
# Use the same lock order as get_r2t2_model (model, then inference) to
|
||||
# prevent a model switch from deadlocking with a first-use warmup.
|
||||
with _model_lock:
|
||||
if not _inference_lock.acquire(blocking=False):
|
||||
return False
|
||||
try:
|
||||
model = _model
|
||||
if model is None:
|
||||
return True
|
||||
engine = _engine
|
||||
if engine is not None:
|
||||
engine.close()
|
||||
_model = None
|
||||
_engine = None
|
||||
finally:
|
||||
_inference_lock.release()
|
||||
del model
|
||||
gc.collect()
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
except ImportError:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def get_r2t2_inference_lock() -> threading.Lock:
|
||||
"""Serialize calls because each R2T2 stream mutates shared vLLM model state."""
|
||||
return _inference_lock
|
||||
|
|
@ -41,6 +41,12 @@ from app.services.asr.funasr_streaming import (
|
|||
get_streaming_model as get_funasr_streaming_model,
|
||||
unload_models as unload_funasr_models,
|
||||
)
|
||||
from app.services.asr.r2t2_runtime import (
|
||||
R2T2_MODEL_ID,
|
||||
get_r2t2_inference_lock,
|
||||
get_r2t2_model,
|
||||
unload_r2t2_model,
|
||||
)
|
||||
from app.services.asr.engines.global_models import (
|
||||
get_global_punc_realtime_model,
|
||||
get_global_vad_model,
|
||||
|
|
@ -286,6 +292,11 @@ class ConnectionContext:
|
|||
funasr_punc_cache: Dict[str, Any] = field(default_factory=dict)
|
||||
funasr_pending_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
|
||||
funasr_transcript: RealtimeTranscriptAssembler = field(default_factory=RealtimeTranscriptAssembler)
|
||||
# R2T2 maintains a dedicated decode state for each active speech turn.
|
||||
r2t2_model: Any = None
|
||||
r2t2_stream_state: Any = None
|
||||
r2t2_max_new_tokens: int = 0
|
||||
r2t2_last_fixed_text: str = ""
|
||||
pre_roll_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
|
||||
pre_roll_start_sample: int = 0
|
||||
audio_samples_received: int = 0
|
||||
|
|
@ -386,7 +397,7 @@ class Qwen3ASRService:
|
|||
|
||||
async def prepare_realtime_model(self, model_id: str) -> Dict[str, str]:
|
||||
"""Swap and warm the selected realtime ASR route before capture begins."""
|
||||
if model_id not in {"qwen3-asr", "funasr-paraformer-streaming"}:
|
||||
if model_id not in {"qwen3-asr", "funasr-paraformer-streaming", "r2t2-streaming"}:
|
||||
raise ValueError(f"Unsupported realtime model: {model_id}")
|
||||
async with self._model_switch_lock:
|
||||
if self._active_websocket_connections and self._current_realtime_model != model_id:
|
||||
|
|
@ -415,12 +426,21 @@ class Qwen3ASRService:
|
|||
if settings.ASR_ENABLE_REALTIME_PUNC:
|
||||
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
|
||||
await run_sync(get_global_vad_model, settings.DEVICE)
|
||||
elif model_id == "r2t2-streaming":
|
||||
await run_sync(get_r2t2_model)
|
||||
if settings.ASR_ENABLE_REALTIME_PUNC:
|
||||
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
|
||||
await run_sync(get_global_vad_model, settings.DEVICE)
|
||||
else:
|
||||
qwen_model_id = validate_realtime_model_id("qwen3-asr")
|
||||
await run_sync(runtime_router.warmup_model, qwen_model_id)
|
||||
elif previous_model == "funasr-paraformer-streaming":
|
||||
unload_funasr_models()
|
||||
unload_global_punc_realtime_model()
|
||||
elif previous_model == "r2t2-streaming":
|
||||
if not unload_r2t2_model():
|
||||
raise RuntimeError("R2T2 推理仍在运行,暂时无法释放模型;请稍后重试。")
|
||||
unload_global_punc_realtime_model()
|
||||
else:
|
||||
qwen_model_id = validate_realtime_model_id("qwen3-asr")
|
||||
if not runtime_router.unload_model(qwen_model_id):
|
||||
|
|
@ -432,6 +452,11 @@ class Qwen3ASRService:
|
|||
if settings.ASR_ENABLE_REALTIME_PUNC:
|
||||
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
|
||||
await run_sync(get_global_vad_model, settings.DEVICE)
|
||||
elif model_id == "r2t2-streaming":
|
||||
await run_sync(get_r2t2_model)
|
||||
if settings.ASR_ENABLE_REALTIME_PUNC:
|
||||
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
|
||||
await run_sync(get_global_vad_model, settings.DEVICE)
|
||||
else:
|
||||
qwen_model_id = validate_realtime_model_id("qwen3-asr")
|
||||
await run_sync(runtime_router.warmup_model, qwen_model_id)
|
||||
|
|
@ -448,11 +473,17 @@ class Qwen3ASRService:
|
|||
if model_id == "funasr-paraformer-streaming":
|
||||
unload_funasr_models()
|
||||
unload_global_punc_realtime_model()
|
||||
elif model_id == "r2t2-streaming":
|
||||
unload_r2t2_model()
|
||||
if previous_model == "funasr-paraformer-streaming":
|
||||
await run_sync(get_funasr_streaming_model)
|
||||
await run_sync(get_funasr_offline_model)
|
||||
if settings.ASR_ENABLE_REALTIME_PUNC:
|
||||
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
|
||||
elif previous_model == "r2t2-streaming":
|
||||
await run_sync(get_r2t2_model)
|
||||
if settings.ASR_ENABLE_REALTIME_PUNC:
|
||||
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
|
||||
else:
|
||||
qwen_model_id = validate_realtime_model_id("qwen3-asr")
|
||||
await run_sync(get_runtime_router().warmup_model, qwen_model_id)
|
||||
|
|
@ -495,6 +526,8 @@ class Qwen3ASRService:
|
|||
ctx.engine_lease = None
|
||||
ctx.engine = None
|
||||
ctx.realtime_stream_state = None
|
||||
ctx.r2t2_model = None
|
||||
ctx.r2t2_stream_state = None
|
||||
|
||||
async def _detach_session(
|
||||
self,
|
||||
|
|
@ -1376,6 +1409,9 @@ class Qwen3ASRService:
|
|||
def _uses_funasr_streaming(self, ctx: ConnectionContext) -> bool:
|
||||
return ctx.params.get("model_id", "qwen3-asr") == "funasr-paraformer-streaming"
|
||||
|
||||
def _uses_r2t2_streaming(self, ctx: ConnectionContext) -> bool:
|
||||
return ctx.params.get("model_id", "qwen3-asr") == "r2t2-streaming"
|
||||
|
||||
async def _ensure_engine(self, ctx: ConnectionContext) -> Optional[Qwen3ASREngine]:
|
||||
if self._uses_funasr_streaming(ctx):
|
||||
if ctx.funasr_model is None:
|
||||
|
|
@ -1399,6 +1435,21 @@ class Qwen3ASRService:
|
|||
)
|
||||
return None
|
||||
|
||||
if self._uses_r2t2_streaming(ctx):
|
||||
if ctx.r2t2_model is None:
|
||||
ctx.r2t2_model = await run_sync(get_r2t2_model)
|
||||
logger.info(
|
||||
"[%s] Using Confucius4-R2T2 streaming model: %s",
|
||||
ctx.diagnostic_session_id,
|
||||
R2T2_MODEL_ID,
|
||||
)
|
||||
if settings.ASR_ENABLE_REALTIME_PUNC and ctx.funasr_punc_model is None:
|
||||
ctx.funasr_punc_model = await run_sync(
|
||||
get_global_punc_realtime_model,
|
||||
settings.DEVICE,
|
||||
)
|
||||
return None
|
||||
|
||||
if ctx.engine is not None:
|
||||
return ctx.engine
|
||||
|
||||
|
|
@ -1587,6 +1638,31 @@ class Qwen3ASRService:
|
|||
new_text += after
|
||||
return new_text
|
||||
|
||||
@staticmethod
|
||||
def _run_r2t2_stream_chunk(
|
||||
model: Any,
|
||||
audio: np.ndarray,
|
||||
state: Any,
|
||||
max_new_tokens: int,
|
||||
) -> tuple[str, str]:
|
||||
"""Decode one R2T2 chunk under the shared model lock, off the event loop."""
|
||||
with get_r2t2_inference_lock():
|
||||
return model.streaming_transcribe_no_reset(
|
||||
np.asarray(audio, dtype=np.float32),
|
||||
state,
|
||||
max_new_tokens,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _run_r2t2_stream_finish(model: Any, state: Any, max_new_tokens: int) -> str:
|
||||
"""Flush one R2T2 stream while keeping shared model access serialized."""
|
||||
with get_r2t2_inference_lock():
|
||||
return model.finish_streaming_transcribe_no_reset(state, max_new_tokens)
|
||||
|
||||
@staticmethod
|
||||
def _r2t2_last_character_is_chinese(text: str) -> bool:
|
||||
return bool(text) and "\u4e00" <= text[-1] <= "\u9fff"
|
||||
|
||||
def _reset_realtime_speaker_change_state(self, ctx: ConnectionContext) -> None:
|
||||
"""Discard pending online speaker evidence when an ASR turn is cut."""
|
||||
task = ctx.speaker_change_probe_task
|
||||
|
|
@ -1890,6 +1966,8 @@ class Qwen3ASRService:
|
|||
Qwen vLLM refreshes a bounded audio-window snapshot per complete model
|
||||
chunk, so polling for a partial more frequently cannot produce new text.
|
||||
"""
|
||||
if self._uses_r2t2_streaming(ctx):
|
||||
return int(self._stream_chunk_size_sec(ctx) * 16000)
|
||||
interval_sec = max(
|
||||
float(
|
||||
getattr(
|
||||
|
|
@ -2115,18 +2193,21 @@ class Qwen3ASRService:
|
|||
return False
|
||||
|
||||
def _stream_chunk_size_sec(self, ctx: ConnectionContext) -> float:
|
||||
return max(
|
||||
float(
|
||||
ctx.params.get(
|
||||
"chunk_size_sec",
|
||||
settings.REALTIME_STREAM_CHUNK_SEC,
|
||||
)
|
||||
or settings.REALTIME_STREAM_CHUNK_SEC
|
||||
),
|
||||
0.6,
|
||||
configured = float(
|
||||
ctx.params.get(
|
||||
"chunk_size_sec",
|
||||
settings.REALTIME_STREAM_CHUNK_SEC,
|
||||
)
|
||||
or settings.REALTIME_STREAM_CHUNK_SEC
|
||||
)
|
||||
if self._uses_r2t2_streaming(ctx):
|
||||
return min(max(configured, 0.08), 2.0)
|
||||
return max(configured, 0.6)
|
||||
|
||||
def _stream_unfixed_chunk_num(self, ctx: ConnectionContext) -> int:
|
||||
if self._uses_r2t2_streaming(ctx):
|
||||
# The upstream R2T2 WebSocket route fixes text from its first chunk.
|
||||
return 0
|
||||
return max(
|
||||
int(
|
||||
ctx.params.get(
|
||||
|
|
@ -2139,7 +2220,8 @@ class Qwen3ASRService:
|
|||
)
|
||||
|
||||
def _stream_unfixed_token_num(self, ctx: ConnectionContext) -> int:
|
||||
return max(int(ctx.params.get("unfixed_token_num", 5) or 5), 1)
|
||||
default = 1 if self._uses_r2t2_streaming(ctx) else 5
|
||||
return max(int(ctx.params.get("unfixed_token_num", default) or default), 1)
|
||||
|
||||
def _should_use_native_partial_stream(
|
||||
self,
|
||||
|
|
@ -2147,6 +2229,8 @@ class Qwen3ASRService:
|
|||
engine: Qwen3ASREngine,
|
||||
) -> bool:
|
||||
_ = engine
|
||||
if self._uses_r2t2_streaming(ctx):
|
||||
return True
|
||||
configured = ctx.params.get("enable_native_partial_stream")
|
||||
if configured is not None:
|
||||
return bool(configured)
|
||||
|
|
@ -2216,6 +2300,58 @@ class Qwen3ASRService:
|
|||
finally:
|
||||
ctx.last_stream_push_ms = (time.perf_counter() - push_started) * 1000.0
|
||||
return
|
||||
if self._uses_r2t2_streaming(ctx):
|
||||
push_started = time.perf_counter()
|
||||
try:
|
||||
if ctx.r2t2_stream_state is None:
|
||||
if ctx.r2t2_model is None:
|
||||
await self._ensure_engine(ctx)
|
||||
ctx.r2t2_stream_state = await run_sync(
|
||||
ctx.r2t2_model.init_streaming_state,
|
||||
context=ctx.params.get("context", ""),
|
||||
language=ctx.params.get("language"),
|
||||
chunk_size_sec=self._stream_chunk_size_sec(ctx),
|
||||
unfixed_chunk_num=self._stream_unfixed_chunk_num(ctx),
|
||||
unfixed_token_num=self._stream_unfixed_token_num(ctx),
|
||||
)
|
||||
initial_chunk_tokens = max(
|
||||
1,
|
||||
int(self._stream_chunk_size_sec(ctx) * 2 * 16000 / 1280),
|
||||
)
|
||||
ctx.r2t2_max_new_tokens = initial_chunk_tokens
|
||||
chunk_tokens = max(1, int(self._stream_chunk_size_sec(ctx) * 16000 / 1280))
|
||||
max_token_floor = min(32, max(4, 2 * chunk_tokens))
|
||||
previous_fixed_text = ctx.r2t2_last_fixed_text
|
||||
current_text, fixed_text = await run_sync(
|
||||
self._run_r2t2_stream_chunk,
|
||||
ctx.r2t2_model,
|
||||
np.asarray(audio, dtype=np.float32),
|
||||
ctx.r2t2_stream_state,
|
||||
ctx.r2t2_max_new_tokens,
|
||||
)
|
||||
ctx.r2t2_stream_state.text = str(current_text or "")
|
||||
ctx.r2t2_last_fixed_text = str(fixed_text or "")
|
||||
if len(ctx.r2t2_last_fixed_text) > len(previous_fixed_text):
|
||||
next_max_tokens = chunk_tokens
|
||||
elif self._r2t2_last_character_is_chinese(ctx.r2t2_last_fixed_text):
|
||||
next_max_tokens = chunk_tokens
|
||||
else:
|
||||
next_max_tokens = ctx.r2t2_max_new_tokens + 0.5
|
||||
if self._r2t2_last_character_is_chinese(ctx.r2t2_last_fixed_text):
|
||||
next_max_tokens *= 2
|
||||
ctx.r2t2_max_new_tokens = min(max_token_floor, int(next_max_tokens))
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"[%s] R2T2 stream push failed: segment=%d chunk_samples=%d error=%s",
|
||||
ctx.diagnostic_session_id,
|
||||
ctx.segment_index,
|
||||
int(audio.size),
|
||||
exc,
|
||||
)
|
||||
ctx.r2t2_stream_state = None
|
||||
finally:
|
||||
ctx.last_stream_push_ms = (time.perf_counter() - push_started) * 1000.0
|
||||
return
|
||||
if engine is None:
|
||||
return
|
||||
if not self._should_use_native_partial_stream(ctx, engine):
|
||||
|
|
@ -2286,11 +2422,77 @@ class Qwen3ASRService:
|
|||
finally:
|
||||
ctx.realtime_stream_state = None
|
||||
|
||||
async def _finish_r2t2_stream_text(
|
||||
self,
|
||||
ctx: ConnectionContext,
|
||||
finalized_audio: np.ndarray,
|
||||
) -> tuple[str, str]:
|
||||
"""Flush R2T2's fixed-prefix decoder, replaying the turn if needed."""
|
||||
if ctx.r2t2_model is None:
|
||||
ctx.r2t2_model = await run_sync(get_r2t2_model)
|
||||
state = ctx.r2t2_stream_state
|
||||
try:
|
||||
if state is None:
|
||||
state = await run_sync(
|
||||
ctx.r2t2_model.init_streaming_state,
|
||||
context=ctx.params.get("context", ""),
|
||||
language=ctx.params.get("language"),
|
||||
chunk_size_sec=self._stream_chunk_size_sec(ctx),
|
||||
unfixed_chunk_num=self._stream_unfixed_chunk_num(ctx),
|
||||
unfixed_token_num=self._stream_unfixed_token_num(ctx),
|
||||
)
|
||||
await run_sync(
|
||||
self._run_r2t2_stream_chunk,
|
||||
ctx.r2t2_model,
|
||||
np.asarray(finalized_audio, dtype=np.float32),
|
||||
state,
|
||||
max(1, int(self._stream_chunk_size_sec(ctx) * 2 * 16000 / 1280)),
|
||||
)
|
||||
finish_max_tokens = max(
|
||||
1,
|
||||
int(self._stream_chunk_size_sec(ctx) * 2 * 16000 / 1280),
|
||||
)
|
||||
text = await run_sync(
|
||||
self._run_r2t2_stream_finish,
|
||||
ctx.r2t2_model,
|
||||
state,
|
||||
finish_max_tokens,
|
||||
)
|
||||
text = self._normalize_output_text(str(text or ""), ctx, enable_itn=False)
|
||||
language = self._normalize_output_language(
|
||||
getattr(state, "language", ""),
|
||||
ctx,
|
||||
)
|
||||
return text, language
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"[%s] Finalize R2T2 stream failed: segment=%d error=%s",
|
||||
ctx.diagnostic_session_id,
|
||||
ctx.segment_index,
|
||||
exc,
|
||||
)
|
||||
return "", ""
|
||||
finally:
|
||||
ctx.r2t2_stream_state = None
|
||||
|
||||
async def _decode_turn_partial_text(
|
||||
self,
|
||||
engine: Qwen3ASREngine,
|
||||
ctx: ConnectionContext,
|
||||
) -> tuple[str, str]:
|
||||
if self._uses_r2t2_streaming(ctx):
|
||||
state = ctx.r2t2_stream_state
|
||||
if state is None:
|
||||
return "", ""
|
||||
text = self._normalize_output_text(
|
||||
str(getattr(state, "text", "") or ""),
|
||||
ctx,
|
||||
enable_itn=False,
|
||||
)
|
||||
return text, self._normalize_output_language(
|
||||
getattr(state, "language", ""),
|
||||
ctx,
|
||||
)
|
||||
if self._should_use_native_partial_stream(ctx, engine) and ctx.realtime_stream_state is not None:
|
||||
stream_text = self._normalize_output_text(
|
||||
str(ctx.realtime_stream_state.last_text or ""),
|
||||
|
|
@ -2409,6 +2611,9 @@ class Qwen3ASRService:
|
|||
ctx.best_partial_language = ""
|
||||
ctx.last_partial_decode_samples = 0
|
||||
ctx.realtime_stream_state = None
|
||||
ctx.r2t2_stream_state = None
|
||||
ctx.r2t2_max_new_tokens = 0
|
||||
ctx.r2t2_last_fixed_text = ""
|
||||
ctx.realtime_stream_failure_logged = False
|
||||
ctx.funasr_cache = {}
|
||||
ctx.funasr_pending_audio = np.array([], dtype=np.float32)
|
||||
|
|
@ -3269,6 +3474,7 @@ class Qwen3ASRService:
|
|||
commit_started = time.perf_counter()
|
||||
engine = await self._ensure_engine(ctx)
|
||||
funasr_streaming = self._uses_funasr_streaming(ctx)
|
||||
r2t2_streaming = self._uses_r2t2_streaming(ctx)
|
||||
|
||||
current_audio = np.asarray(ctx.segment_audio_buffer, dtype=np.float32)
|
||||
finalized_audio = current_audio
|
||||
|
|
@ -3285,8 +3491,14 @@ class Qwen3ASRService:
|
|||
has_native_stream_snapshot = (
|
||||
not funasr_streaming
|
||||
and carry_audio.size == 0
|
||||
and self._should_use_native_partial_stream(ctx, engine)
|
||||
and ctx.realtime_stream_state is not None
|
||||
and (
|
||||
(r2t2_streaming and ctx.r2t2_stream_state is not None)
|
||||
or (
|
||||
not r2t2_streaming
|
||||
and self._should_use_native_partial_stream(ctx, engine)
|
||||
and ctx.realtime_stream_state is not None
|
||||
)
|
||||
)
|
||||
)
|
||||
funasr_final_inference_ms = 0.0
|
||||
if funasr_streaming:
|
||||
|
|
@ -3300,6 +3512,24 @@ class Qwen3ASRService:
|
|||
stream_text = await self._finish_funasr_stream(ctx, finalized_audio)
|
||||
funasr_final_inference_ms = (time.perf_counter() - funasr_final_started) * 1000.0
|
||||
stream_language = self._infer_text_language(stream_text)
|
||||
elif r2t2_streaming:
|
||||
if carry_audio.size > 0:
|
||||
# Replay only the clipped prefix; the existing state also holds
|
||||
# post-boundary samples that belong to the next turn.
|
||||
ctx.r2t2_stream_state = None
|
||||
stream_text, stream_language = await self._finish_r2t2_stream_text(ctx, finalized_audio)
|
||||
if stream_text.strip():
|
||||
observed_text = self._replace_segment_observed_text(
|
||||
ctx,
|
||||
stream_text,
|
||||
stream_language or self._infer_text_language(stream_text),
|
||||
)
|
||||
self._update_best_partial(
|
||||
ctx,
|
||||
observed_text,
|
||||
stream_language or self._infer_text_language(observed_text),
|
||||
replace_snapshot=True,
|
||||
)
|
||||
elif carry_audio.size == 0:
|
||||
stream_text, stream_language = await self._finish_realtime_stream_text(ctx, engine)
|
||||
if stream_text.strip():
|
||||
|
|
@ -3319,9 +3549,10 @@ class Qwen3ASRService:
|
|||
)
|
||||
else:
|
||||
ctx.realtime_stream_state = None
|
||||
ctx.r2t2_stream_state = None
|
||||
|
||||
final_transcribe_started = time.perf_counter()
|
||||
if funasr_streaming:
|
||||
if funasr_streaming or r2t2_streaming:
|
||||
# Paraformer's AutoModel has no Qwen-style ITN switch; apply the
|
||||
# existing project normalizer to final text when the UI enables it.
|
||||
segment_text = self._normalize_output_text(
|
||||
|
|
@ -3371,6 +3602,7 @@ class Qwen3ASRService:
|
|||
)
|
||||
if (
|
||||
not funasr_streaming
|
||||
and not r2t2_streaming
|
||||
and engine is not None
|
||||
and suspicious
|
||||
and finalized_audio.size > 0
|
||||
|
|
@ -3407,7 +3639,7 @@ class Qwen3ASRService:
|
|||
segment_text,
|
||||
preserve_boundary_punctuation=True,
|
||||
)
|
||||
if funasr_streaming:
|
||||
if funasr_streaming or r2t2_streaming:
|
||||
# Keep CT-Transformer inference off the asyncio event loop so other
|
||||
# WebSocket sessions can continue sending and decoding audio.
|
||||
segment_text = await run_sync(self._finalize_funasr_text, ctx, segment_text)
|
||||
|
|
@ -3415,7 +3647,7 @@ class Qwen3ASRService:
|
|||
ctx,
|
||||
segment_text,
|
||||
)
|
||||
if funasr_streaming and reason == "speaker_change":
|
||||
if (funasr_streaming or r2t2_streaming) and reason == "speaker_change":
|
||||
# A confirmed speaker transition is a sentence boundary even when
|
||||
# the punctuation model did not emit a terminal mark.
|
||||
boundary_segment = {"text": segment_text}
|
||||
|
|
@ -3681,7 +3913,7 @@ class Qwen3ASRService:
|
|||
)
|
||||
)
|
||||
model_id = str(payload.get("model_id", "qwen3-asr") or "qwen3-asr").strip()
|
||||
if model_id not in {"qwen3-asr", "funasr-paraformer-streaming"}:
|
||||
if model_id not in {"qwen3-asr", "funasr-paraformer-streaming", "r2t2-streaming"}:
|
||||
await self._send_error(
|
||||
websocket,
|
||||
f"Unsupported realtime model: {model_id}",
|
||||
|
|
@ -3696,9 +3928,8 @@ class Qwen3ASRService:
|
|||
hotword_context = format_hotword_prompt_context(
|
||||
hotwords
|
||||
)
|
||||
if model_id == "qwen3-asr" and hotword_context:
|
||||
# Keep Qwen's existing prompt-based hotword behavior while
|
||||
# exposing the same hotword field for both realtime models.
|
||||
if model_id in {"qwen3-asr", "r2t2-streaming"} and hotword_context:
|
||||
# Both Qwen-derived decoders accept hotwords through context.
|
||||
context = "\n".join(part for part in (context, hotword_context) if part)
|
||||
ctx.params = {
|
||||
"model_id": model_id,
|
||||
|
|
@ -3903,8 +4134,12 @@ class Qwen3ASRService:
|
|||
if not self._uses_funasr_streaming(ctx) and self._should_decode_turn_partial(ctx):
|
||||
partial_decode_started = time.perf_counter()
|
||||
has_native_stream_snapshot = (
|
||||
self._should_use_native_partial_stream(ctx, engine)
|
||||
and ctx.realtime_stream_state is not None
|
||||
(self._uses_r2t2_streaming(ctx) and ctx.r2t2_stream_state is not None)
|
||||
or (
|
||||
not self._uses_r2t2_streaming(ctx)
|
||||
and self._should_use_native_partial_stream(ctx, engine)
|
||||
and ctx.realtime_stream_state is not None
|
||||
)
|
||||
)
|
||||
current, current_language = await self._decode_turn_partial_text(
|
||||
engine,
|
||||
|
|
@ -4011,11 +4246,19 @@ class Qwen3ASRService:
|
|||
# path, without logging transcript contents or changing events.
|
||||
partial_source = "window_fallback"
|
||||
if has_native_stream_snapshot:
|
||||
partial_source = (
|
||||
"qwen_vllm_window_snapshot"
|
||||
if getattr(engine, "backend", "") == "vllm"
|
||||
else "qwen_native_stream"
|
||||
)
|
||||
if self._uses_r2t2_streaming(ctx):
|
||||
partial_source = "r2t2_streaming_snapshot"
|
||||
else:
|
||||
partial_source = (
|
||||
"qwen_vllm_window_snapshot"
|
||||
if getattr(engine, "backend", "") == "vllm"
|
||||
else "qwen_native_stream"
|
||||
)
|
||||
stream_state = (
|
||||
ctx.r2t2_stream_state
|
||||
if self._uses_r2t2_streaming(ctx)
|
||||
else ctx.realtime_stream_state
|
||||
)
|
||||
logger.info(
|
||||
"[%s] Realtime partial result: segment=%d source=%s outcome=%s "
|
||||
"vad_push_ms=%.1f stream_push_ms=%.1f partial_resolve_ms=%.1f "
|
||||
|
|
@ -4031,9 +4274,9 @@ class Qwen3ASRService:
|
|||
partial_decode_ms,
|
||||
len(ctx.segment_audio_buffer) / 16000.0,
|
||||
len(ctx.stream_window_buffer) / 16000.0,
|
||||
getattr(ctx.realtime_stream_state, "chunk_count", None),
|
||||
getattr(stream_state, "chunk_id", getattr(stream_state, "chunk_count", None)),
|
||||
getattr(
|
||||
getattr(ctx.realtime_stream_state, "internal_state", None),
|
||||
getattr(stream_state, "internal_state", None),
|
||||
"window_rolled",
|
||||
None,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -29,6 +29,16 @@ class ModelAsset:
|
|||
|
||||
|
||||
MODEL_ASSETS: tuple[ModelAsset, ...] = (
|
||||
ModelAsset(
|
||||
model_id="netease-youdao/Confucius4-R2T2",
|
||||
description="Confucius4-R2T2 Realtime ASR",
|
||||
required_patterns=("config.json",),
|
||||
alternative_required_patterns=(
|
||||
("model.safetensors",),
|
||||
("model-*.safetensors",),
|
||||
),
|
||||
min_total_size_bytes=1_000_000_000,
|
||||
),
|
||||
ModelAsset(
|
||||
model_id="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404",
|
||||
description="FunASR Contextual Offline Final ASR",
|
||||
|
|
|
|||
|
|
@ -2183,6 +2183,7 @@
|
|||
<select id="liveAsrModel">
|
||||
<option value="qwen3-asr" selected>Qwen3-ASR(现有实时路线)</option>
|
||||
<option value="funasr-paraformer-streaming">FunASR Paraformer Streaming(在线模型)</option>
|
||||
<option value="r2t2-streaming">Confucius4-R2T2(vLLM 实时流)</option>
|
||||
</select>
|
||||
</div>
|
||||
<label class="switch">数字归一化 <input id="liveEnableItn" type="checkbox" checked></label>
|
||||
|
|
@ -2236,8 +2237,8 @@
|
|||
</div>
|
||||
<div class="mini-grid" style="margin-top: 10px;">
|
||||
<div class="field">
|
||||
<label for="liveStreamChunkSec">在线解码块长(FunASR 约 600 ms)</label>
|
||||
<input id="liveStreamChunkSec" type="number" min="0.6" step="0.1" value="0.6">
|
||||
<label for="liveStreamChunkSec">在线解码块长(Qwen/R2T2 可调;FunASR 固定 600 ms)</label>
|
||||
<input id="liveStreamChunkSec" type="number" min="0.08" step="0.08" value="0.6">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="liveUnfixedChunkNum">Qwen 前缀启用块数(FunASR look-back 无直接对应)</label>
|
||||
|
|
@ -3916,19 +3917,25 @@ ${transcript}
|
|||
function syncLiveProtocolOptions() {
|
||||
state.liveAsrModel = $("liveAsrModel").value || "qwen3-asr";
|
||||
const useFunASR = state.liveAsrModel === "funasr-paraformer-streaming";
|
||||
const useR2T2 = state.liveAsrModel === "r2t2-streaming";
|
||||
// Keep protocol-specific controls visible only when the selected backend reads them.
|
||||
$("liveMinPartialSec").closest(".field").hidden = useFunASR;
|
||||
["liveStreamChunkSec", "liveUnfixedChunkNum", "liveUnfixedTokenNum"].forEach((id) => {
|
||||
$("liveMinPartialSec").closest(".field").hidden = useFunASR || useR2T2;
|
||||
$("liveUnfixedChunkNum").closest(".field").hidden = useFunASR || useR2T2;
|
||||
["liveStreamChunkSec", "liveUnfixedTokenNum"].forEach((id) => {
|
||||
$(id).closest(".field").hidden = useFunASR;
|
||||
});
|
||||
$("liveEnableNativePartialStream").closest(".switch").hidden = useFunASR;
|
||||
$("liveEnableRealtimeLongform").closest(".switch").hidden = useFunASR;
|
||||
$("liveEnableNativePartialStream").closest(".switch").hidden = useFunASR || useR2T2;
|
||||
$("liveEnableRealtimeLongform").closest(".switch").hidden = useFunASR || useR2T2;
|
||||
$("livePreRollMs").closest(".field").querySelector("label").textContent =
|
||||
"项目 VAD 句首预读缓冲(pre_roll_ms)";
|
||||
$("liveProtocolBadge").textContent = useFunASR ? "FunASR 流式识别" : "Qwen3-ASR";
|
||||
$("liveProtocolBadge").textContent = useFunASR
|
||||
? "FunASR 流式识别"
|
||||
: useR2T2 ? "Confucius4-R2T2 流式识别" : "Qwen3-ASR";
|
||||
$("liveProtocolNote").textContent = useFunASR
|
||||
? "Paraformer 固定按 600ms 分块;静音收段、项目 VAD 预读和说话人选项仍生效。"
|
||||
: "Qwen3-ASR 使用可调解码块和 partial 参数;静音收段、VAD 预读和说话人选项仍生效。";
|
||||
: useR2T2
|
||||
? "R2T2 使用 vLLM 累积音频滑窗和可调解码块;静音收段、项目 VAD 预读和说话人选项仍生效。"
|
||||
: "Qwen3-ASR 使用可调解码块和 partial 参数;静音收段、VAD 预读和说话人选项仍生效。";
|
||||
$("liveQwenParams").hidden = false;
|
||||
}
|
||||
|
||||
|
|
@ -3947,6 +3954,13 @@ ${transcript}
|
|||
return;
|
||||
}
|
||||
|
||||
if (nextModel === "r2t2-streaming") {
|
||||
// Match the upstream R2T2 demo's 160 ms streaming chunk as a starting point.
|
||||
$("liveStreamChunkSec").value = "0.16";
|
||||
$("liveUnfixedTokenNum").value = "1";
|
||||
$("liveMinPartialSec").value = "0.25";
|
||||
}
|
||||
|
||||
state.liveModelSwitching = true;
|
||||
selector.disabled = true;
|
||||
$("liveStartBtn").disabled = true;
|
||||
|
|
@ -3983,6 +3997,7 @@ ${transcript}
|
|||
|| createClientMessageId().slice(0, 16);
|
||||
$("liveSessionId").value = sessionId;
|
||||
state.liveSessionId = sessionId;
|
||||
const useR2T2 = state.liveAsrModel === "r2t2-streaming";
|
||||
const hotwords = parseHotwords($("liveHotwords").value);
|
||||
const contextParts = [$("liveContext").value.trim()].filter(Boolean);
|
||||
if (hotwords.length) {
|
||||
|
|
@ -4002,11 +4017,11 @@ ${transcript}
|
|||
context: contextParts.join("\n"),
|
||||
enable_inverse_text_normalization: $("liveEnableItn").checked,
|
||||
// Favor a quick provisional preview; the final segment still replaces it with re-transcribed text.
|
||||
chunk_size_sec: Math.max(Number($("liveStreamChunkSec").value || 0.6), 0.6),
|
||||
chunk_size_sec: Math.max(Number($("liveStreamChunkSec").value || 0.6), useR2T2 ? 0.08 : 0.6),
|
||||
unfixed_chunk_num: Math.max(Number($("liveUnfixedChunkNum").value || 2), 1),
|
||||
unfixed_token_num: Math.max(Number($("liveUnfixedTokenNum").value || 5), 1),
|
||||
silence_duration_ms: Number($("liveSilenceDurationMs").value || 800),
|
||||
min_partial_sec: Number($("liveMinPartialSec").value || 0.6),
|
||||
min_partial_sec: useR2T2 ? 0.25 : Number($("liveMinPartialSec").value || 0.6),
|
||||
pre_roll_ms: Number($("livePreRollMs").value || 600),
|
||||
// FunASR emits the current non-empty partial without a tail holdback.
|
||||
partial_holdback_chars: 0,
|
||||
|
|
|
|||
Loading…
Reference in New Issue