[Bugfix] Terminate streaming sessions on ABORT/ERROR and prevent zombie sessions (#58942)
Signed-off-by: David Cheung <d7cheung@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Benedikt Falk <bfalk@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Benedikt Falk
Nick Hill
parent
766c837bd5
commit
1463501553
@@ -628,6 +628,13 @@ class AsyncLLM(EngineClient):
|
||||
any_added = False
|
||||
try:
|
||||
async for input_chunk in input_stream:
|
||||
# Stop once the session has finished (aborted or errored).
|
||||
# There is no await between this check and the ADD send,
|
||||
# so any chunk already sent precedes the engine abort.
|
||||
if any_added and not self.output_processor.has_request(
|
||||
internal_req_id
|
||||
):
|
||||
break
|
||||
sp = input_chunk.sampling_params
|
||||
if sp:
|
||||
self._validate_streaming_input_sampling_params(sp)
|
||||
@@ -663,8 +670,10 @@ class AsyncLLM(EngineClient):
|
||||
if not cancelled:
|
||||
if any_added:
|
||||
# Send empty final request to indicate that inputs have
|
||||
# finished. Don't send if cancelled (session was aborted).
|
||||
await self._add_request(final_req, None, None, 0, queue)
|
||||
# finished. Don't send if cancelled or if the session
|
||||
# already finished (aborted or errored).
|
||||
if self.output_processor.has_request(internal_req_id):
|
||||
await self._add_request(final_req, None, None, 0, queue)
|
||||
elif not errored:
|
||||
queue.put(STREAM_FINISHED)
|
||||
|
||||
|
||||
@@ -1618,6 +1618,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
|
||||
# To route aborts to the correct engine.
|
||||
self.reqs_in_flight: dict[str, EngineIdentity] = {}
|
||||
# Open streaming-input sessions, whose aborts go to all engines.
|
||||
self.streaming_reqs: set[str] = set()
|
||||
|
||||
# Exact per-engine count of this client's unfinished requests.
|
||||
self.engine_inflight: Counter[EngineIdentity] = Counter()
|
||||
@@ -1640,6 +1642,12 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
) // client_count
|
||||
|
||||
def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity:
|
||||
if request.resumable:
|
||||
self.streaming_reqs.add(request.request_id)
|
||||
else:
|
||||
# Also removes a session on its final (end-of-stream) request.
|
||||
self.streaming_reqs.discard(request.request_id)
|
||||
|
||||
if (engine := self.reqs_in_flight.get(request.request_id)) is not None:
|
||||
# This is for streaming-input session affinity.
|
||||
return engine
|
||||
@@ -1759,15 +1767,24 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
if not request_ids or self.resources.engine_dead:
|
||||
return
|
||||
|
||||
if len(request_ids) == 1:
|
||||
if (
|
||||
len(request_ids) == 1
|
||||
and (req_id := request_ids[0]) not in self.streaming_reqs
|
||||
and (engine := self.reqs_in_flight.get(req_id))
|
||||
):
|
||||
# Fast-path common case.
|
||||
if engine := self.reqs_in_flight.get(request_ids[0]):
|
||||
await self._abort_requests(request_ids, engine)
|
||||
await self._abort_requests(request_ids, engine)
|
||||
return
|
||||
|
||||
by_engine = defaultdict[EngineIdentity, list[str]](list)
|
||||
for req_id in request_ids:
|
||||
if engine := self.reqs_in_flight.get(req_id):
|
||||
if req_id in self.streaming_reqs:
|
||||
# A chunk in flight when the session finished may have
|
||||
# re-created it on any engine.
|
||||
self.streaming_reqs.discard(req_id)
|
||||
for engine in self.core_engines:
|
||||
by_engine[engine].append(req_id)
|
||||
elif engine := self.reqs_in_flight.get(req_id):
|
||||
by_engine[engine].append(req_id)
|
||||
for engine, req_ids in by_engine.items():
|
||||
await self._abort_requests(req_ids, engine)
|
||||
|
||||
@@ -784,6 +784,10 @@ class OutputProcessor:
|
||||
# if required.
|
||||
req_state.logprobs_processor.update_from_output(engine_core_output)
|
||||
|
||||
streaming_chunk_boundary = req_state.streaming_input and (
|
||||
finish_reason not in (FinishReason.ABORT, FinishReason.ERROR)
|
||||
)
|
||||
|
||||
# 4) Create and handle RequestOutput objects.
|
||||
if request_output := req_state.make_request_output(
|
||||
new_token_ids,
|
||||
@@ -794,7 +798,7 @@ class OutputProcessor:
|
||||
ec_transfer_params,
|
||||
error=request_error,
|
||||
):
|
||||
if req_state.streaming_input:
|
||||
if streaming_chunk_boundary:
|
||||
request_output.finished = False
|
||||
|
||||
if req_state.queue is not None:
|
||||
@@ -806,7 +810,7 @@ class OutputProcessor:
|
||||
|
||||
# Free completed requests.
|
||||
if finish_reason is not None:
|
||||
if req_state.streaming_input:
|
||||
if streaming_chunk_boundary:
|
||||
if req_state.input_chunk_queue:
|
||||
update = req_state.input_chunk_queue.popleft()
|
||||
req_state.apply_streaming_update(update)
|
||||
@@ -814,9 +818,10 @@ class OutputProcessor:
|
||||
req_state.input_chunk_queue = None
|
||||
else:
|
||||
self._finish_request(req_state)
|
||||
if not engine_core_output.finished:
|
||||
# If req not finished in EngineCore, but Detokenizer
|
||||
# detected stop string, abort needed in EngineCore.
|
||||
if req_state.streaming_input or not engine_core_output.finished:
|
||||
# EngineCore may still have the request: a stop string
|
||||
# was detected here, or an in-flight streaming-input
|
||||
# chunk may re-create the session.
|
||||
reqs_to_abort.append(req_id)
|
||||
|
||||
# Track per-request stats
|
||||
|
||||
Reference in New Issue
Block a user