[Bugfix] Terminate streaming sessions on ABORT/ERROR and prevent zombie sessions (#58942)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Benedikt Falk <bfalk@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
David Cheung
2026-10-15 13:32:19 +00:00
committed by GitHub
co-authored by Benedikt Falk Nick Hill
parent 766c837bd5
commit 1463501553
6 changed files with 213 additions and 11 deletions
+84
View File
@@ -1642,6 +1642,90 @@ async def test_cumulative_output_collector_n():
assert third[0].text == "c"
def _make_streaming_input_request(
request_id: str, prompt_token_ids: list[int]
) -> EngineCoreRequest:
"""One chunk of a streaming-input (resumable) request."""
return EngineCoreRequest(
request_id=request_id,
prompt_token_ids=prompt_token_ids,
mm_features=None,
sampling_params=SamplingParams(
detokenize=False, output_kind=RequestOutputKind.DELTA
),
pooling_params=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
resumable=True,
external_req_id=f"{request_id}-ext",
)
def _add_streaming_request_with_queued_chunk(
processor: OutputProcessor, request_id: str
) -> RequestOutputCollector:
queue = RequestOutputCollector(RequestOutputKind.DELTA, request_id=request_id)
processor.add_request(
_make_streaming_input_request(request_id, [1, 2, 3]), None, queue=queue
)
processor.add_request(_make_streaming_input_request(request_id, [7, 8]), None)
return queue
def test_streaming_input_stop_is_chunk_boundary():
"""STOP is a per-chunk boundary: finished is forced False, the queued
chunk is applied, and the request state stays alive."""
processor = OutputProcessor(None, log_stats=False)
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
processed = processor.process_outputs(
[
EngineCoreOutput(
request_id="stream-req",
new_token_ids=[4],
finish_reason=FinishReason.STOP,
)
]
)
assert processed.reqs_to_abort == []
assert processor.get_num_unfinished_requests() == 1
output = queue.get_nowait()
assert output is not None
assert output.finished is False
assert output.outputs[0].finish_reason == "stop"
req_state = processor.request_states["stream-req"]
assert req_state.prompt_token_ids is not None
assert req_state.prompt_token_ids[-2:] == [7, 8]
@pytest.mark.parametrize("finish_reason", [FinishReason.ABORT, FinishReason.ERROR])
def test_streaming_input_abort_error_is_terminal(finish_reason: FinishReason):
"""ABORT/ERROR is terminal, not a chunk boundary: finished stays True,
the state is freed, and an abort is routed back to the engine core."""
processor = OutputProcessor(None, log_stats=False)
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
processed = processor.process_outputs(
[
EngineCoreOutput(
request_id="stream-req",
new_token_ids=[],
finish_reason=finish_reason,
)
]
)
assert processor.get_num_unfinished_requests() == 0
assert processed.reqs_to_abort == ["stream-req"]
output = queue.get_nowait()
assert output is not None
assert output.finished is True
assert output.outputs[0].finish_reason == str(finish_reason)
@pytest.mark.parametrize("runner", ["generate", "pooling"])
@pytest.mark.parametrize("abort_by", ["internal", "external"])
def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):