[Frontend] Add Support for MM Encoder/Decoder Beam Search (Online Transcriptions) (#36160)

Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
Alex Brooks
2026-03-09 05:46:23 +00:00
committed by GitHub
parent 217f27598d
commit 65a4da1504
6 changed files with 216 additions and 25 deletions
@@ -129,6 +129,11 @@ class OpenAIServingCompletion(OpenAIServing):
- suffix (the language models we currently support do not support
suffix)
"""
if request.stream and request.use_beam_search:
return self.create_error_response(
"Streaming is not currently supported with beam search"
)
result = await self.render_completion_request(request)
if isinstance(result, ErrorResponse):
return result
@@ -211,13 +216,10 @@ class OpenAIServingCompletion(OpenAIServing):
model_name = self.models.model_name(lora_request)
num_prompts = len(engine_prompts)
# We do not stream the results when using beam search.
stream = request.stream and not request.use_beam_search
# Streaming response
tokenizer = self.renderer.tokenizer
if stream:
if request.stream:
return self.completion_stream_generator(
request,
engine_prompts,