调整日志结构,排查阻塞问题

main
Bifang 2026-10-09 15:09:58 +08:00
parent 8e8d1ef44f
commit 9e87bcc2e0
2 changed files with 120 additions and 6 deletions

View File

@ -174,6 +174,8 @@ class _GenerateRequest:
prompts: list[dict[str, Any]]
sampling_params: Any
batch_key: tuple[Any, ...]
operation: str = "transcribe"
trace_id: str = ""
queued_at: float = field(default_factory=time.monotonic)
done: threading.Event = field(default_factory=threading.Event)
outputs: Optional[list[Any]] = None
@ -210,11 +212,15 @@ class _VLLMGenerateBatcher:
*,
sampling_params: Any,
batch_key: tuple[Any, ...],
operation: str = "transcribe",
trace_id: str = "",
) -> list[Any]:
request = _GenerateRequest(
prompts=prompts,
sampling_params=sampling_params,
batch_key=batch_key,
operation=operation,
trace_id=trace_id,
)
self._requests.put(request)
request.done.wait()
@ -276,10 +282,21 @@ class _VLLMGenerateBatcher:
)
generate_ms = (time.monotonic() - generate_started) * 1000.0
lock_wait_ms = (generate_started - lock_wait_started) * 1000.0
if len(requests) > 1 or generate_ms >= 500.0 or lock_wait_ms >= 100.0:
operations = ",".join(sorted({request.operation for request in requests}))
trace_ids = ",".join(sorted({request.trace_id for request in requests if request.trace_id})) or "-"
# Log every realtime decode so partial inference latency can be
# separated from final/offline transcription and queue contention.
if (
"realtime_partial" in operations
or len(requests) > 1
or generate_ms >= 500.0
or lock_wait_ms >= 100.0
):
logger.info(
"vLLM decode batch completed: requests=%d prompts=%d max_tokens=%s "
"vLLM decode batch completed: operation=%s sessions=%s requests=%d prompts=%d max_tokens=%s "
"queue_wait_ms=%.1f engine_lock_wait_ms=%.1f generate_ms=%.1f",
operations,
trace_ids,
len(requests),
len(prompts),
getattr(requests[0].sampling_params, "max_tokens", None),
@ -329,6 +346,7 @@ class VLLMRealtimeState:
unfixed_chunk_num: int
unfixed_token_num: int
max_new_tokens: int
trace_id: str = ""
max_audio_window_sec: float = 8.0
chunk_id: int = 0
text: str = ""
@ -524,6 +542,7 @@ class Qwen3VLLMBackend:
prompts[start:start + self._decode_max_batch_size],
sampling_params=self._sampling_params,
batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature),
operation="transcribe",
)
)
@ -689,6 +708,7 @@ class Qwen3VLLMBackend:
unfixed_token_num: int = 5,
max_new_tokens: int = 32,
max_audio_window_sec: float = 8.0,
trace_id: str = "",
) -> VLLMRealtimeState:
normalized_language = _normalize_language_name(language) or ""
return VLLMRealtimeState(
@ -698,6 +718,7 @@ class Qwen3VLLMBackend:
unfixed_chunk_num=unfixed_chunk_num,
unfixed_token_num=unfixed_token_num,
max_new_tokens=max_new_tokens,
trace_id=trace_id,
max_audio_window_sec=max(1.0, max_audio_window_sec),
audio_buffer=np.array([], dtype=np.float32),
audio_accum=np.array([], dtype=np.float32),
@ -726,6 +747,8 @@ class Qwen3VLLMBackend:
],
sampling_params=sampling_params,
batch_key=(state.max_new_tokens, 0.01),
operation="realtime_partial",
trace_id=state.trace_id,
)[0]
generated = str(output.outputs[0].text if output.outputs else "")
parsed_language, parsed_text = _parse_asr_output(prefix + generated, state.language or None)

View File

@ -240,6 +240,7 @@ class ConnectionState(IntEnum):
@dataclass
class ConnectionContext:
state: ConnectionState = ConnectionState.READY
diagnostic_session_id: str = ""
params: Dict[str, Any] = field(default_factory=dict)
engine_lease: Optional[RuntimeEngineLease] = None
engine: Optional[Qwen3ASREngine] = None
@ -259,6 +260,8 @@ class ConnectionContext:
stream_window_buffer: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
# realtime_stream_state 只负责低延迟 partial,不参与最终 segment 定稿。
realtime_stream_state: Optional[Qwen3StreamingState] = None
# Keep stream failure diagnostics to one event per turn to avoid log flooding.
realtime_stream_failure_logged: bool = False
sentence_active: bool = False
silence_samples: int = 0
total_samples: int = 0
@ -1406,6 +1409,7 @@ class Qwen3ASRService:
try:
# Native stream 只用来拿低延迟 partial,最终句子仍走整段重转写定稿。
init_started = time.perf_counter()
ctx.realtime_stream_state = await run_sync(
engine.init_streaming_state,
ctx.params.get("context", ""),
@ -1418,9 +1422,26 @@ class Qwen3ASRService:
float(getattr(settings, "REALTIME_STREAM_WINDOW_SEC", 8.0) or 8.0),
1.0,
),
trace_id=ctx.diagnostic_session_id,
)
logger.info(
"[%s] Realtime partial stream initialized: segment=%d init_ms=%.1f "
"max_new_tokens=48 chunk_size_sec=%.2f",
ctx.diagnostic_session_id,
ctx.segment_index,
(time.perf_counter() - init_started) * 1000.0,
self._stream_chunk_size_sec(ctx),
)
except Exception as exc:
logger.debug("Init realtime partial stream failed: %s", exc)
if not ctx.realtime_stream_failure_logged:
logger.warning(
"[%s] Realtime partial stream initialization failed: segment=%d error=%s; "
"partial decoding will use window transcription fallback",
ctx.diagnostic_session_id,
ctx.segment_index,
exc,
)
ctx.realtime_stream_failure_logged = True
ctx.realtime_stream_state = None
async def _push_realtime_stream_audio(
@ -1446,7 +1467,16 @@ class Qwen3ASRService:
ctx.realtime_stream_state,
)
except Exception as exc:
logger.debug("Realtime partial stream push failed: %s", exc)
if not ctx.realtime_stream_failure_logged:
logger.warning(
"[%s] Realtime partial stream push failed: segment=%d chunk_samples=%d error=%s; "
"partial decoding will use window transcription fallback",
ctx.diagnostic_session_id,
ctx.segment_index,
int(audio.size),
exc,
)
ctx.realtime_stream_failure_logged = True
ctx.realtime_stream_state = None
async def _finish_realtime_stream_text(
@ -1475,7 +1505,12 @@ class Qwen3ASRService:
)
return text, language
except Exception as exc:
logger.debug("Finalize realtime partial stream failed: %s", exc)
logger.warning(
"[%s] Finalize realtime partial stream failed: segment=%d error=%s",
ctx.diagnostic_session_id,
ctx.segment_index,
exc,
)
return "", ""
finally:
ctx.realtime_stream_state = None
@ -1608,6 +1643,7 @@ class Qwen3ASRService:
ctx.last_partial_decode_samples = 0
ctx.partial_stable_rounds = 0
ctx.realtime_stream_state = None
ctx.realtime_stream_failure_logged = False
def _update_partial_stability(
self,
@ -2280,7 +2316,15 @@ class Qwen3ASRService:
)
return normalized
except Exception as exc:
logger.warning("Realtime segment audio transcription failed: %s", exc)
logger.warning(
"[%s] Realtime segment transcription failed: segment=%d final_pass=%s "
"audio_sec=%.2f error=%s",
ctx.diagnostic_session_id,
ctx.segment_index,
final_pass,
len(audio) / 16000.0,
exc,
)
return ""
finally:
get_speaker_registry_service().cleanup_file(temp_path)
@ -2608,6 +2652,7 @@ class Qwen3ASRService:
segment_end_ms_override: Optional[int] = None,
emit_segment_start: bool = True,
) -> None:
commit_started = time.perf_counter()
engine = await self._ensure_engine(ctx)
current_audio = np.asarray(ctx.segment_audio_buffer, dtype=np.float32)
@ -2647,6 +2692,7 @@ class Qwen3ASRService:
else:
ctx.realtime_stream_state = None
final_transcribe_started = time.perf_counter()
if self._enable_realtime_longform(ctx):
segment_text = await self._transcribe_audio_longform_text(
engine,
@ -2661,6 +2707,7 @@ class Qwen3ASRService:
finalized_audio,
final_pass=True,
)
final_transcribe_ms = (time.perf_counter() - final_transcribe_started) * 1000.0
segment_text = self._sanitize_candidate_text(segment_text)
if not segment_text.strip():
segment_text = self._prefer_segment_text(
@ -2713,6 +2760,20 @@ class Qwen3ASRService:
segment_text = self._trim_previous_segment_overlap(ctx, segment_text)
is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec)
logger.info(
"[%s] Realtime segment finalized: segment=%d reason=%s duration_sec=%.2f "
"stream_snapshot=%s final_transcribe_ms=%.1f commit_elapsed_ms=%.1f "
"text_chars=%d valid=%s",
task_id,
ctx.segment_index,
reason,
duration_sec,
bool(has_native_stream_snapshot),
final_transcribe_ms,
(time.perf_counter() - commit_started) * 1000.0,
len(segment_text),
is_valid,
)
segment_duration_ms = int(finalized_audio.size / 16)
segment_start_ms = int(
ctx.timeline_cursor_ms
@ -2899,6 +2960,7 @@ class Qwen3ASRService:
ctx = resumed_ctx
session_id = requested_session_id
task_id = session_id
ctx.diagnostic_session_id = task_id
keep_for_resume = True
if resumed and ctx.state != ConnectionState.READY:
@ -3128,6 +3190,7 @@ class Qwen3ASRService:
continue
if self._should_decode_turn_partial(ctx):
partial_decode_started = time.perf_counter()
has_native_stream_snapshot = (
self._should_use_native_partial_stream(ctx, engine)
and ctx.realtime_stream_state is not None
@ -3136,8 +3199,11 @@ class Qwen3ASRService:
engine,
ctx,
)
partial_decode_ms = (time.perf_counter() - partial_decode_started) * 1000.0
ctx.last_partial_decode_samples = int(ctx.segment_audio_buffer.size)
current = self._sanitize_candidate_text(current)
partial_display = ""
partial_outcome = "empty"
if current and not self._is_degenerate_repetition(current):
current_language = current_language or self._infer_text_language(current)
update_observed_text = (
@ -3160,6 +3226,10 @@ class Qwen3ASRService:
native_stream_snapshot=has_native_stream_snapshot,
)
partial_display = self._clip_partial_text(partial_display, ctx)
if not partial_display:
partial_outcome = "filtered_empty"
else:
partial_outcome = "unchanged"
if (
partial_display
and (
@ -3167,6 +3237,7 @@ class Qwen3ASRService:
or current_language != ctx.last_partial_language
)
):
partial_outcome = "emitted"
ctx.last_partial_chunk_id += 1
ctx.last_partial_text = visible_observed.strip()
ctx.last_partial_display_text = partial_display.strip()
@ -3222,6 +3293,26 @@ class Qwen3ASRService:
},
)
ctx.state = ConnectionState.STREAMING
elif current:
partial_outcome = "degenerate_repetition"
# This records whether the partial reached the WebSocket send
# path, without logging transcript contents or changing events.
logger.info(
"[%s] Realtime partial result: segment=%d source=%s outcome=%s "
"decode_ms=%.1f turn_audio_sec=%.2f window_audio_sec=%.2f "
"stream_chunk_id=%s candidate_chars=%d display_chars=%d",
task_id,
ctx.segment_index,
"native_stream" if has_native_stream_snapshot else "window_fallback",
partial_outcome,
partial_decode_ms,
len(ctx.segment_audio_buffer) / 16000.0,
len(ctx.stream_window_buffer) / 16000.0,
getattr(ctx.realtime_stream_state, "chunk_count", None),
len(current),
len(partial_display),
)
# Match FunASR's streaming lifecycle: the active turn ends at a VAD
# endpoint, not because a separate wall-clock segment timer elapsed.