From c68eb98b69f4863d94318fbe9dedf654be0c5945 Mon Sep 17 00:00:00 2001 From: Benedikt Falk Date: Tue, 29 Sep 2026 01:52:29 +0200 Subject: [PATCH] [Bugfix][Frontend] Keep in-flight requests on the same DP engine (#59017) Signed-off-by: Benedikt Falk --- tests/v1/engine/test_engine_core_client.py | 13 +++++++++++++ vllm/v1/engine/core_client.py | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index 15dec72a41..e7b065c565 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -248,6 +248,19 @@ def test_dplb_non_late_interaction_still_uses_lb(): assert client.lb_engines[1][0] == 1 +def test_dplb_in_flight_request_stays_on_same_engine(): + client = _make_dplb_client() + request = make_request(SamplingParams(max_tokens=1)) + engine = client.core_engines[2] + client.reqs_in_flight[request.request_id] = engine + client.engine_inflight[engine] = 1 + + chosen_engine = client.get_core_engine_for_request(request) + + assert chosen_engine == engine + assert client.engine_inflight[engine] == 1 + + def test_dplb_burst_round_robins_despite_snapshot_rebinds(): """A stats snapshot rebind wipes the optimistic lb_engines increments; the exact in-flight floor must keep a burst spreading round-robin.""" diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 5d59bdeaea..669af3caaf 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -1587,6 +1587,10 @@ class DPLBAsyncMPClient(DPAsyncMPClient): ) // client_count def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity: + if (engine := self.reqs_in_flight.get(request.request_id)) is not None: + # This is for streaming-input session affinity. + return engine + # Engines are in rank order. if (eng_index := request.data_parallel_rank) is None and ( eng_index := get_late_interaction_engine_index(