[Bugfix] Terminate streaming sessions on ABORT/ERROR and prevent zombie sessions (#58942)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Benedikt Falk <bfalk@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
David Cheung
2026-10-15 13:32:19 +00:00
committed by GitHub
co-authored by Benedikt Falk Nick Hill
parent 766c837bd5
commit 1463501553
6 changed files with 213 additions and 11 deletions
+10 -5
View File
@@ -784,6 +784,10 @@ class OutputProcessor:
# if required.
req_state.logprobs_processor.update_from_output(engine_core_output)
streaming_chunk_boundary = req_state.streaming_input and (
finish_reason not in (FinishReason.ABORT, FinishReason.ERROR)
)
# 4) Create and handle RequestOutput objects.
if request_output := req_state.make_request_output(
new_token_ids,
@@ -794,7 +798,7 @@ class OutputProcessor:
ec_transfer_params,
error=request_error,
):
if req_state.streaming_input:
if streaming_chunk_boundary:
request_output.finished = False
if req_state.queue is not None:
@@ -806,7 +810,7 @@ class OutputProcessor:
# Free completed requests.
if finish_reason is not None:
if req_state.streaming_input:
if streaming_chunk_boundary:
if req_state.input_chunk_queue:
update = req_state.input_chunk_queue.popleft()
req_state.apply_streaming_update(update)
@@ -814,9 +818,10 @@ class OutputProcessor:
req_state.input_chunk_queue = None
else:
self._finish_request(req_state)
if not engine_core_output.finished:
# If req not finished in EngineCore, but Detokenizer
# detected stop string, abort needed in EngineCore.
if req_state.streaming_input or not engine_core_output.finished:
# EngineCore may still have the request: a stop string
# was detected here, or an in-flight streaming-input
# chunk may re-create the session.
reqs_to_abort.append(req_id)
# Track per-request stats