[Bugfix] Fix Qwen3-ASR transcription streaming postprocessing (#42478)
Signed-off-by: JooHo Lee <BWAAEEEK@users.noreply.github.com> Signed-off-by: JooHo Lee <jooho414@gmail.com> Co-authored-by: JooHo Lee <BWAAEEEK@users.noreply.github.com>
This commit is contained in:
@@ -114,6 +114,9 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
self.asr_config = self.model_cls.get_speech_to_text_config(
|
||||
self.model_config, task_type
|
||||
)
|
||||
self.streaming_post_processor_cls = (
|
||||
self.model_cls.get_streaming_post_processor_cls()
|
||||
)
|
||||
|
||||
self.enable_force_include_usage = enable_force_include_usage
|
||||
|
||||
@@ -692,6 +695,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
try:
|
||||
for result_generator in list_result_generator:
|
||||
beginning_of_chunk = True
|
||||
post_processor = self.streaming_post_processor_cls()
|
||||
async for res in result_generator:
|
||||
# On first result.
|
||||
if res.prompt_token_ids is not None:
|
||||
@@ -709,19 +713,24 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
assert len(res.outputs) == 1
|
||||
output = res.outputs[0]
|
||||
|
||||
output_text = post_processor.process_delta(
|
||||
output.text, output.finish_reason is not None
|
||||
)
|
||||
|
||||
# dont add separator to the first chunk
|
||||
if (
|
||||
result_generator is not list_result_generator[0]
|
||||
and beginning_of_chunk
|
||||
and output_text
|
||||
):
|
||||
output.text = separator + output.text
|
||||
output_text = separator + output_text
|
||||
beginning_of_chunk = False
|
||||
|
||||
# TODO: For models that output structured formats (e.g.,
|
||||
# Qwen3-ASR with "language X<asr_text>" prefix), streaming
|
||||
# would need buffering to strip the prefix properly since
|
||||
# deltas may split the tag across chunks.
|
||||
delta_message = DeltaMessage(content=output.text)
|
||||
if output.finish_reason is None and not output_text:
|
||||
completion_tokens += len(output.token_ids)
|
||||
continue
|
||||
|
||||
delta_message = DeltaMessage(content=output_text)
|
||||
completion_tokens += len(output.token_ids)
|
||||
|
||||
if output.finish_reason is None:
|
||||
|
||||
Reference in New Issue
Block a user