[Bugfix] Fix Qwen3-ASR transcription streaming postprocessing (#42478)

Signed-off-by: JooHo Lee <BWAAEEEK@users.noreply.github.com>
Signed-off-by: JooHo Lee <jooho414@gmail.com>
Co-authored-by: JooHo Lee <BWAAEEEK@users.noreply.github.com>
This commit is contained in:
JooHo Lee
2026-07-09 00:33:27 -07:00
committed by GitHub
co-authored by JooHo Lee
parent 1171467e91
commit a07765c6bd
4 changed files with 207 additions and 17 deletions
@@ -114,6 +114,9 @@ class SpeechToTextBaseServing(GenerateBaseServing):
self.asr_config = self.model_cls.get_speech_to_text_config(
self.model_config, task_type
)
self.streaming_post_processor_cls = (
self.model_cls.get_streaming_post_processor_cls()
)
self.enable_force_include_usage = enable_force_include_usage
@@ -692,6 +695,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
try:
for result_generator in list_result_generator:
beginning_of_chunk = True
post_processor = self.streaming_post_processor_cls()
async for res in result_generator:
# On first result.
if res.prompt_token_ids is not None:
@@ -709,19 +713,24 @@ class SpeechToTextBaseServing(GenerateBaseServing):
assert len(res.outputs) == 1
output = res.outputs[0]
output_text = post_processor.process_delta(
output.text, output.finish_reason is not None
)
# dont add separator to the first chunk
if (
result_generator is not list_result_generator[0]
and beginning_of_chunk
and output_text
):
output.text = separator + output.text
output_text = separator + output_text
beginning_of_chunk = False
# TODO: For models that output structured formats (e.g.,
# Qwen3-ASR with "language X<asr_text>" prefix), streaming
# would need buffering to strip the prefix properly since
# deltas may split the tag across chunks.
delta_message = DeltaMessage(content=output.text)
if output.finish_reason is None and not output_text:
completion_tokens += len(output.token_ids)
continue
delta_message = DeltaMessage(content=output_text)
completion_tokens += len(output.token_ids)
if output.finish_reason is None: