[Bugfix][Frontend] Keep in-flight requests on the same DP engine (#59017)
Signed-off-by: Benedikt Falk <bfalk@nvidia.com>
This commit is contained in:
@@ -248,6 +248,19 @@ def test_dplb_non_late_interaction_still_uses_lb():
|
||||
assert client.lb_engines[1][0] == 1
|
||||
|
||||
|
||||
def test_dplb_in_flight_request_stays_on_same_engine():
|
||||
client = _make_dplb_client()
|
||||
request = make_request(SamplingParams(max_tokens=1))
|
||||
engine = client.core_engines[2]
|
||||
client.reqs_in_flight[request.request_id] = engine
|
||||
client.engine_inflight[engine] = 1
|
||||
|
||||
chosen_engine = client.get_core_engine_for_request(request)
|
||||
|
||||
assert chosen_engine == engine
|
||||
assert client.engine_inflight[engine] == 1
|
||||
|
||||
|
||||
def test_dplb_burst_round_robins_despite_snapshot_rebinds():
|
||||
"""A stats snapshot rebind wipes the optimistic lb_engines increments;
|
||||
the exact in-flight floor must keep a burst spreading round-robin."""
|
||||
|
||||
@@ -1587,6 +1587,10 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
) // client_count
|
||||
|
||||
def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity:
|
||||
if (engine := self.reqs_in_flight.get(request.request_id)) is not None:
|
||||
# This is for streaming-input session affinity.
|
||||
return engine
|
||||
|
||||
# Engines are in rank order.
|
||||
if (eng_index := request.data_parallel_rank) is None and (
|
||||
eng_index := get_late_interaction_engine_index(
|
||||
|
||||
Reference in New Issue
Block a user