Allow AsyncLLMEngine.generate to target a specific DP rank (#19102)

Signed-off-by: Jon Swenson <jmswen@gmail.com>
This commit is contained in:
jmswen
2025-06-04 08:26:47 -07:00
committed by GitHub
parent 8f4ffbd373
commit c8dcc15921
10 changed files with 97 additions and 5 deletions
+12 -2
View File
@@ -982,7 +982,16 @@ class DPAsyncMPClient(AsyncMPClient):
resources.stats_update_task = asyncio.create_task(
run_engine_stats_update_task())
def get_core_engine_for_request(self) -> CoreEngine:
def get_core_engine_for_request(self,
dp_rank: Optional[int] = None
) -> CoreEngine:
if dp_rank is not None:
# engines are already in rank order
if dp_rank < 0 or dp_rank >= len(self.core_engines):
raise ValueError(f"Requested DP rank {dp_rank} is out of "
f"range [0, {len(self.core_engines)})")
return self.core_engines[dp_rank]
if not self.lb_engines:
return self.core_engines[0]
# TODO use P2C alg for larger DP sizes
@@ -1018,7 +1027,8 @@ class DPAsyncMPClient(AsyncMPClient):
request.current_wave = self.current_wave
request.client_index = self.client_index
chosen_engine = self.get_core_engine_for_request()
chosen_engine = self.get_core_engine_for_request(
request.data_parallel_rank)
self.reqs_in_flight[request.request_id] = chosen_engine
to_await = self._send_input(EngineCoreRequestType.ADD, request,