[Bugfix] Terminate streaming sessions on ABORT/ERROR and prevent zombie sessions (#58942)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Benedikt Falk <bfalk@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
David Cheung
2026-10-15 13:32:19 +00:00
committed by GitHub
co-authored by Benedikt Falk Nick Hill
parent 766c837bd5
commit 1463501553
6 changed files with 213 additions and 11 deletions
+11 -2
View File
@@ -628,6 +628,13 @@ class AsyncLLM(EngineClient):
any_added = False
try:
async for input_chunk in input_stream:
# Stop once the session has finished (aborted or errored).
# There is no await between this check and the ADD send,
# so any chunk already sent precedes the engine abort.
if any_added and not self.output_processor.has_request(
internal_req_id
):
break
sp = input_chunk.sampling_params
if sp:
self._validate_streaming_input_sampling_params(sp)
@@ -663,8 +670,10 @@ class AsyncLLM(EngineClient):
if not cancelled:
if any_added:
# Send empty final request to indicate that inputs have
# finished. Don't send if cancelled (session was aborted).
await self._add_request(final_req, None, None, 0, queue)
# finished. Don't send if cancelled or if the session
# already finished (aborted or errored).
if self.output_processor.has_request(internal_req_id):
await self._add_request(final_req, None, None, 0, queue)
elif not errored:
queue.put(STREAM_FINISHED)