[Bugfix][Frontend] Avoid generation for empty streaming input (#59015)

Signed-off-by: Benedikt Falk <bfalk@nvidia.com>
This commit is contained in:
Benedikt Falk
2026-10-02 18:48:43 +00:00
committed by GitHub
parent 10f6ab01a9
commit c4973d94fe
3 changed files with 41 additions and 8 deletions
+10 -3
View File
@@ -599,6 +599,8 @@ class AsyncLLM(EngineClient):
async def handle_inputs():
cancelled = False
errored = False
any_added = False
try:
async for input_chunk in input_stream:
sp = input_chunk.sampling_params
@@ -623,18 +625,23 @@ class AsyncLLM(EngineClient):
self.model_config, input_chunk.prompt
)
await self._add_request(req, prompt_text, None, 0, queue)
any_added = True
except (asyncio.CancelledError, GeneratorExit):
cancelled = True
except Exception as error:
# Wrap in InputStreamError so generate() can propagate it
# without wrapping in EngineGenerateError.
queue.put(InputStreamError(error))
errored = True
finally:
queue._input_stream_task = None
if not cancelled:
# Send empty final request to indicate that inputs have
# finished. Don't send if cancelled (session was aborted).
await self._add_request(final_req, None, None, 0, queue)
if any_added:
# Send empty final request to indicate that inputs have
# finished. Don't send if cancelled (session was aborted).
await self._add_request(final_req, None, None, 0, queue)
elif not errored:
queue.put(STREAM_FINISHED)
# Ensure output handler is running.
self._run_output_handler()