[Bugfix][Frontend] Keep in-flight requests on the same DP engine (#59017)

Signed-off-by: Benedikt Falk <bfalk@nvidia.com>
This commit is contained in:
Benedikt Falk
2026-09-28 23:52:29 +00:00
committed by GitHub
parent 75fad5bbef
commit c68eb98b69
2 changed files with 17 additions and 0 deletions
+4
View File
@@ -1587,6 +1587,10 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
) // client_count
def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity:
if (engine := self.reqs_in_flight.get(request.request_id)) is not None:
# This is for streaming-input session affinity.
return engine
# Engines are in rank order.
if (eng_index := request.data_parallel_rank) is None and (
eng_index := get_late_interaction_engine_index(