[Bugfix][Frontend] Avoid generation for empty streaming input (#59015)
Signed-off-by: Benedikt Falk <bfalk@nvidia.com>
This commit is contained in:
@@ -599,6 +599,8 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
async def handle_inputs():
|
||||
cancelled = False
|
||||
errored = False
|
||||
any_added = False
|
||||
try:
|
||||
async for input_chunk in input_stream:
|
||||
sp = input_chunk.sampling_params
|
||||
@@ -623,18 +625,23 @@ class AsyncLLM(EngineClient):
|
||||
self.model_config, input_chunk.prompt
|
||||
)
|
||||
await self._add_request(req, prompt_text, None, 0, queue)
|
||||
any_added = True
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
cancelled = True
|
||||
except Exception as error:
|
||||
# Wrap in InputStreamError so generate() can propagate it
|
||||
# without wrapping in EngineGenerateError.
|
||||
queue.put(InputStreamError(error))
|
||||
errored = True
|
||||
finally:
|
||||
queue._input_stream_task = None
|
||||
if not cancelled:
|
||||
# Send empty final request to indicate that inputs have
|
||||
# finished. Don't send if cancelled (session was aborted).
|
||||
await self._add_request(final_req, None, None, 0, queue)
|
||||
if any_added:
|
||||
# Send empty final request to indicate that inputs have
|
||||
# finished. Don't send if cancelled (session was aborted).
|
||||
await self._add_request(final_req, None, None, 0, queue)
|
||||
elif not errored:
|
||||
queue.put(STREAM_FINISHED)
|
||||
|
||||
# Ensure output handler is running.
|
||||
self._run_output_handler()
|
||||
|
||||
Reference in New Issue
Block a user