[Bugfix] Terminate streaming sessions on ABORT/ERROR and prevent zombie sessions (#58942)
Signed-off-by: David Cheung <d7cheung@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Benedikt Falk <bfalk@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Benedikt Falk
Nick Hill
parent
766c837bd5
commit
1463501553
@@ -628,6 +628,13 @@ class AsyncLLM(EngineClient):
|
||||
any_added = False
|
||||
try:
|
||||
async for input_chunk in input_stream:
|
||||
# Stop once the session has finished (aborted or errored).
|
||||
# There is no await between this check and the ADD send,
|
||||
# so any chunk already sent precedes the engine abort.
|
||||
if any_added and not self.output_processor.has_request(
|
||||
internal_req_id
|
||||
):
|
||||
break
|
||||
sp = input_chunk.sampling_params
|
||||
if sp:
|
||||
self._validate_streaming_input_sampling_params(sp)
|
||||
@@ -663,8 +670,10 @@ class AsyncLLM(EngineClient):
|
||||
if not cancelled:
|
||||
if any_added:
|
||||
# Send empty final request to indicate that inputs have
|
||||
# finished. Don't send if cancelled (session was aborted).
|
||||
await self._add_request(final_req, None, None, 0, queue)
|
||||
# finished. Don't send if cancelled or if the session
|
||||
# already finished (aborted or errored).
|
||||
if self.output_processor.has_request(internal_req_id):
|
||||
await self._add_request(final_req, None, None, 0, queue)
|
||||
elif not errored:
|
||||
queue.put(STREAM_FINISHED)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user