[Bugfix] Terminate streaming sessions on ABORT/ERROR and prevent zombie sessions (#58942)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Benedikt Falk <bfalk@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
David Cheung
2026-10-15 13:32:19 +00:00
committed by GitHub
co-authored by Benedikt Falk Nick Hill
parent 766c837bd5
commit 1463501553
6 changed files with 213 additions and 11 deletions
+11 -2
View File
@@ -628,6 +628,13 @@ class AsyncLLM(EngineClient):
any_added = False
try:
async for input_chunk in input_stream:
# Stop once the session has finished (aborted or errored).
# There is no await between this check and the ADD send,
# so any chunk already sent precedes the engine abort.
if any_added and not self.output_processor.has_request(
internal_req_id
):
break
sp = input_chunk.sampling_params
if sp:
self._validate_streaming_input_sampling_params(sp)
@@ -663,8 +670,10 @@ class AsyncLLM(EngineClient):
if not cancelled:
if any_added:
# Send empty final request to indicate that inputs have
# finished. Don't send if cancelled (session was aborted).
await self._add_request(final_req, None, None, 0, queue)
# finished. Don't send if cancelled or if the session
# already finished (aborted or errored).
if self.output_processor.has_request(internal_req_id):
await self._add_request(final_req, None, None, 0, queue)
elif not errored:
queue.put(STREAM_FINISHED)
+21 -4
View File
@@ -1618,6 +1618,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
# To route aborts to the correct engine.
self.reqs_in_flight: dict[str, EngineIdentity] = {}
# Open streaming-input sessions, whose aborts go to all engines.
self.streaming_reqs: set[str] = set()
# Exact per-engine count of this client's unfinished requests.
self.engine_inflight: Counter[EngineIdentity] = Counter()
@@ -1640,6 +1642,12 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
) // client_count
def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity:
if request.resumable:
self.streaming_reqs.add(request.request_id)
else:
# Also removes a session on its final (end-of-stream) request.
self.streaming_reqs.discard(request.request_id)
if (engine := self.reqs_in_flight.get(request.request_id)) is not None:
# This is for streaming-input session affinity.
return engine
@@ -1759,15 +1767,24 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
if not request_ids or self.resources.engine_dead:
return
if len(request_ids) == 1:
if (
len(request_ids) == 1
and (req_id := request_ids[0]) not in self.streaming_reqs
and (engine := self.reqs_in_flight.get(req_id))
):
# Fast-path common case.
if engine := self.reqs_in_flight.get(request_ids[0]):
await self._abort_requests(request_ids, engine)
await self._abort_requests(request_ids, engine)
return
by_engine = defaultdict[EngineIdentity, list[str]](list)
for req_id in request_ids:
if engine := self.reqs_in_flight.get(req_id):
if req_id in self.streaming_reqs:
# A chunk in flight when the session finished may have
# re-created it on any engine.
self.streaming_reqs.discard(req_id)
for engine in self.core_engines:
by_engine[engine].append(req_id)
elif engine := self.reqs_in_flight.get(req_id):
by_engine[engine].append(req_id)
for engine, req_ids in by_engine.items():
await self._abort_requests(req_ids, engine)
+10 -5
View File
@@ -784,6 +784,10 @@ class OutputProcessor:
# if required.
req_state.logprobs_processor.update_from_output(engine_core_output)
streaming_chunk_boundary = req_state.streaming_input and (
finish_reason not in (FinishReason.ABORT, FinishReason.ERROR)
)
# 4) Create and handle RequestOutput objects.
if request_output := req_state.make_request_output(
new_token_ids,
@@ -794,7 +798,7 @@ class OutputProcessor:
ec_transfer_params,
error=request_error,
):
if req_state.streaming_input:
if streaming_chunk_boundary:
request_output.finished = False
if req_state.queue is not None:
@@ -806,7 +810,7 @@ class OutputProcessor:
# Free completed requests.
if finish_reason is not None:
if req_state.streaming_input:
if streaming_chunk_boundary:
if req_state.input_chunk_queue:
update = req_state.input_chunk_queue.popleft()
req_state.apply_streaming_update(update)
@@ -814,9 +818,10 @@ class OutputProcessor:
req_state.input_chunk_queue = None
else:
self._finish_request(req_state)
if not engine_core_output.finished:
# If req not finished in EngineCore, but Detokenizer
# detected stop string, abort needed in EngineCore.
if req_state.streaming_input or not engine_core_output.finished:
# EngineCore may still have the request: a stop string
# was detected here, or an in-flight streaming-input
# chunk may re-create the session.
reqs_to_abort.append(req_id)
# Track per-request stats