[Bugfix] Terminate streaming sessions on ABORT/ERROR and prevent zombie sessions (#58942)
Signed-off-by: David Cheung <d7cheung@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Benedikt Falk <bfalk@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Benedikt Falk
Nick Hill
parent
766c837bd5
commit
1463501553
@@ -210,6 +210,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
|
||||
client = object.__new__(DPLBAsyncMPClient)
|
||||
client.client_count = client_count
|
||||
client.reqs_in_flight = {}
|
||||
client.streaming_reqs = set()
|
||||
client.engine_inflight = Counter()
|
||||
client.core_engines = [bytes([i, 0]) for i in range(num_engines)]
|
||||
client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)]
|
||||
@@ -407,6 +408,42 @@ def test_dplb_finished_requests_release_inflight():
|
||||
assert req.request_id not in client.reqs_in_flight
|
||||
|
||||
|
||||
@pytest.mark.parametrize("final_sent", [False, True])
|
||||
def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
|
||||
"""A streaming-input chunk in flight when its session finished may
|
||||
re-create it on any engine, so an open session's abort reaches every
|
||||
engine. Other aborts go only to the routed engine."""
|
||||
client = _make_dplb_client(num_engines=3)
|
||||
client.resources = SimpleNamespace(engine_dead=False)
|
||||
sent: dict[bytes, list[str]] = {}
|
||||
|
||||
async def abort_requests(request_ids, engine):
|
||||
sent[engine] = request_ids
|
||||
|
||||
client._abort_requests = abort_requests
|
||||
|
||||
chunk = make_request(SamplingParams(max_tokens=1))
|
||||
chunk.resumable = True
|
||||
client.get_core_engine_for_request(chunk)
|
||||
if final_sent:
|
||||
chunk.resumable = False
|
||||
client.get_core_engine_for_request(chunk)
|
||||
# The engine finished the session, dropping its routing entry.
|
||||
outputs = EngineCoreOutputs(finished_requests={chunk.request_id})
|
||||
asyncio.run(DPLBAsyncMPClient.process_engine_outputs(client, outputs))
|
||||
client.reqs_in_flight["live"] = client.core_engines[1]
|
||||
|
||||
asyncio.run(client.abort_requests_async([chunk.request_id, "live"]))
|
||||
|
||||
if final_sent:
|
||||
expected = {client.core_engines[1]: ["live"]}
|
||||
else:
|
||||
expected = {engine: [chunk.request_id] for engine in client.core_engines}
|
||||
expected[client.core_engines[1]].append("live")
|
||||
assert sent == expected
|
||||
assert not client.streaming_reqs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("effective_size", "other_size"),
|
||||
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
|
||||
|
||||
@@ -1642,6 +1642,90 @@ async def test_cumulative_output_collector_n():
|
||||
assert third[0].text == "c"
|
||||
|
||||
|
||||
def _make_streaming_input_request(
|
||||
request_id: str, prompt_token_ids: list[int]
|
||||
) -> EngineCoreRequest:
|
||||
"""One chunk of a streaming-input (resumable) request."""
|
||||
return EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(
|
||||
detokenize=False, output_kind=RequestOutputKind.DELTA
|
||||
),
|
||||
pooling_params=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
resumable=True,
|
||||
external_req_id=f"{request_id}-ext",
|
||||
)
|
||||
|
||||
|
||||
def _add_streaming_request_with_queued_chunk(
|
||||
processor: OutputProcessor, request_id: str
|
||||
) -> RequestOutputCollector:
|
||||
queue = RequestOutputCollector(RequestOutputKind.DELTA, request_id=request_id)
|
||||
processor.add_request(
|
||||
_make_streaming_input_request(request_id, [1, 2, 3]), None, queue=queue
|
||||
)
|
||||
processor.add_request(_make_streaming_input_request(request_id, [7, 8]), None)
|
||||
return queue
|
||||
|
||||
|
||||
def test_streaming_input_stop_is_chunk_boundary():
|
||||
"""STOP is a per-chunk boundary: finished is forced False, the queued
|
||||
chunk is applied, and the request state stays alive."""
|
||||
processor = OutputProcessor(None, log_stats=False)
|
||||
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
|
||||
|
||||
processed = processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id="stream-req",
|
||||
new_token_ids=[4],
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processed.reqs_to_abort == []
|
||||
assert processor.get_num_unfinished_requests() == 1
|
||||
output = queue.get_nowait()
|
||||
assert output is not None
|
||||
assert output.finished is False
|
||||
assert output.outputs[0].finish_reason == "stop"
|
||||
req_state = processor.request_states["stream-req"]
|
||||
assert req_state.prompt_token_ids is not None
|
||||
assert req_state.prompt_token_ids[-2:] == [7, 8]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("finish_reason", [FinishReason.ABORT, FinishReason.ERROR])
|
||||
def test_streaming_input_abort_error_is_terminal(finish_reason: FinishReason):
|
||||
"""ABORT/ERROR is terminal, not a chunk boundary: finished stays True,
|
||||
the state is freed, and an abort is routed back to the engine core."""
|
||||
processor = OutputProcessor(None, log_stats=False)
|
||||
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
|
||||
|
||||
processed = processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id="stream-req",
|
||||
new_token_ids=[],
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processor.get_num_unfinished_requests() == 0
|
||||
assert processed.reqs_to_abort == ["stream-req"]
|
||||
output = queue.get_nowait()
|
||||
assert output is not None
|
||||
assert output.finished is True
|
||||
assert output.outputs[0].finish_reason == str(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner", ["generate", "pooling"])
|
||||
@pytest.mark.parametrize("abort_by", ["internal", "external"])
|
||||
def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest import mock
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -198,3 +199,52 @@ async def test_empty_input_stream_finishes_without_engine_request():
|
||||
|
||||
assert queue.get_nowait() is STREAM_FINISHED
|
||||
llm._add_request.assert_not_awaited()
|
||||
|
||||
|
||||
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
|
||||
llm = AsyncLLM.__new__(AsyncLLM)
|
||||
llm._validate_streaming_input_sampling_params = MagicMock()
|
||||
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm._add_request = AsyncMock()
|
||||
llm.model_config = MagicMock()
|
||||
llm.output_processor = MagicMock()
|
||||
llm.output_processor.has_request.return_value = has_request
|
||||
|
||||
# The first processed request is the final (end-of-stream) request.
|
||||
reqs = [MagicMock(prompt_embeds=None) for _ in range(num_chunks + 1)]
|
||||
reqs[0].request_id = "r1-int"
|
||||
llm.input_processor = MagicMock()
|
||||
llm.input_processor.process_inputs.side_effect = reqs
|
||||
return llm, reqs
|
||||
|
||||
|
||||
async def _stream_chunks(llm: AsyncLLM, num_chunks: int) -> None:
|
||||
async def chunks():
|
||||
for i in range(num_chunks):
|
||||
yield StreamingInput(prompt=f"frame-{i}")
|
||||
|
||||
with mock.patch(
|
||||
"vllm.v1.engine.async_llm.extract_prompt_components",
|
||||
return_value=("", None, None),
|
||||
):
|
||||
queue = await llm._add_streaming_input_request(
|
||||
"r1", chunks(), SamplingParams(max_tokens=4)
|
||||
)
|
||||
task = queue._input_stream_task
|
||||
assert task is not None
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("session_live", [True, False])
|
||||
async def test_handle_inputs_stops_after_session_finished(session_live: bool):
|
||||
"""Once the session's request state is gone (aborted or errored), no
|
||||
further chunks or final request are sent: they would recreate a request
|
||||
state that is never cleaned up."""
|
||||
llm, reqs = _make_streaming_llm(num_chunks=2, has_request=session_live)
|
||||
await _stream_chunks(llm, num_chunks=2)
|
||||
|
||||
sent = [call.args[0] for call in llm._add_request.await_args_list]
|
||||
# Both chunks then the final request, or only the first chunk.
|
||||
assert sent == ([reqs[1], reqs[2], reqs[0]] if session_live else [reqs[1]])
|
||||
|
||||
Reference in New Issue
Block a user