[Bugfix][Frontend] Keep in-flight requests on the same DP engine (#59017)

Signed-off-by: Benedikt Falk <bfalk@nvidia.com>
This commit is contained in:
Benedikt Falk
2026-09-28 23:52:29 +00:00
committed by GitHub
parent 75fad5bbef
commit c68eb98b69
2 changed files with 17 additions and 0 deletions
@@ -248,6 +248,19 @@ def test_dplb_non_late_interaction_still_uses_lb():
assert client.lb_engines[1][0] == 1
def test_dplb_in_flight_request_stays_on_same_engine():
client = _make_dplb_client()
request = make_request(SamplingParams(max_tokens=1))
engine = client.core_engines[2]
client.reqs_in_flight[request.request_id] = engine
client.engine_inflight[engine] = 1
chosen_engine = client.get_core_engine_for_request(request)
assert chosen_engine == engine
assert client.engine_inflight[engine] == 1
def test_dplb_burst_round_robins_despite_snapshot_rebinds():
"""A stats snapshot rebind wipes the optimistic lb_engines increments;
the exact in-flight floor must keep a burst spreading round-robin."""
+4
View File
@@ -1587,6 +1587,10 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
) // client_count
def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity:
if (engine := self.reqs_in_flight.get(request.request_id)) is not None:
# This is for streaming-input session affinity.
return engine
# Engines are in rank order.
if (eng_index := request.data_parallel_rank) is None and (
eng_index := get_late_interaction_engine_index(