[Bugfix] Fix Qwen3-ASR transcription streaming postprocessing (#42478)

Signed-off-by: JooHo Lee <BWAAEEEK@users.noreply.github.com>
Signed-off-by: JooHo Lee <jooho414@gmail.com>
Co-authored-by: JooHo Lee <BWAAEEEK@users.noreply.github.com>
This commit is contained in:
JooHo Lee
2026-07-09 00:33:27 -07:00
committed by GitHub
co-authored by JooHo Lee
parent 1171467e91
commit a07765c6bd
4 changed files with 207 additions and 17 deletions
+20
View File
@@ -70,6 +70,13 @@ The output embeddings must be one of the following formats:
"""
class StreamingTranscriptionPostProcessor:
"""Stateful streaming post-processor for transcription deltas."""
def process_delta(self, text_delta: str, finished: bool) -> str:
return text_delta
def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
"""
A helper function to be used in the context of
@@ -1198,6 +1205,19 @@ class SupportsTranscription(Protocol):
"""
return text
@classmethod
def get_streaming_post_processor_cls(
cls,
) -> type[StreamingTranscriptionPostProcessor]:
"""
Return a stateful post-processor class for streaming output deltas.
Each instance receives the next decoded text delta and whether the
request output is final. It returns the cleaned delta that should be
sent to the client.
"""
return StreamingTranscriptionPostProcessor
@classmethod
def get_language_detection_prompt(
cls,
+54 -8
View File
@@ -38,6 +38,7 @@ from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensProm
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
StreamingTranscriptionPostProcessor,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
@@ -93,6 +94,53 @@ logger = init_logger(__name__)
_ASR_TEXT_TAG = "<asr_text>"
# User-supplied `prompt` / `response_prefix` must not inject extra ChatML turns.
_CHATML_LIKE_TOKEN = re.compile(r"<\|[^|]+\|>")
_LANGUAGE_PREFIX = "language "
_MAX_STREAMING_PREFIX_CHARS = 50
def _post_process_qwen3_asr_output(text: str) -> str:
if not text or _ASR_TEXT_TAG not in text:
return text
_, text_part = text.rsplit(_ASR_TEXT_TAG, 1)
return text_part
class Qwen3ASRStreamingPostProcessor(StreamingTranscriptionPostProcessor):
def __init__(self) -> None:
self.raw_text = ""
self.emitted_text = ""
self.is_structured_output: bool | None = None
def process_delta(self, text_delta: str, finished: bool) -> str:
self.raw_text += text_delta
if self.is_structured_output is None:
text_without_leading_space = self.raw_text.lstrip()
maybe_structured_output = (
text_without_leading_space == ""
or _LANGUAGE_PREFIX.startswith(text_without_leading_space)
or (
text_without_leading_space.startswith(_LANGUAGE_PREFIX)
and len(text_without_leading_space) < _MAX_STREAMING_PREFIX_CHARS
and "\n" not in text_without_leading_space
)
)
if maybe_structured_output and _ASR_TEXT_TAG not in self.raw_text:
if not finished:
return ""
self.is_structured_output = False
else:
self.is_structured_output = _ASR_TEXT_TAG in self.raw_text
processed_text = (
_post_process_qwen3_asr_output(self.raw_text)
if self.is_structured_output
else self.raw_text
)
new_text = processed_text[len(self.emitted_text) :]
self.emitted_text = processed_text
return new_text
def _sanitize_transcription_user_text(text: str) -> str:
@@ -632,12 +680,10 @@ class Qwen3ASRForConditionalGeneration(
The model outputs in format: "language {lang}<asr_text>{transcription}"
This method strips the language prefix and asr_text tags.
"""
if not text:
return ""
return _post_process_qwen3_asr_output(text)
if _ASR_TEXT_TAG not in text:
return text
# Split on <asr_text> tag and take the transcription part
_, text_part = text.rsplit(_ASR_TEXT_TAG, 1)
return text_part
@classmethod
def get_streaming_post_processor_cls(
cls,
) -> type[StreamingTranscriptionPostProcessor]:
return Qwen3ASRStreamingPostProcessor