Elastic Expert Parallel Initial Support (#20775)

Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
This commit is contained in:
Rui Qiao
2025-07-18 17:46:09 -07:00
committed by GitHub
parent 5782581acf
commit 217937221b
24 changed files with 1659 additions and 68 deletions
+182 -7
View File
@@ -21,9 +21,11 @@ import zmq.asyncio
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.utils import get_open_zmq_inproc_path, make_zmq_socket
from vllm.utils import get_open_port, get_open_zmq_inproc_path, make_zmq_socket
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
EngineCoreRequestType, UtilityOutput)
EngineCoreRequestType,
ReconfigureDistributedRequest, ReconfigureRankType,
UtilityOutput)
from vllm.v1.engine.coordinator import DPCoordinator
from vllm.v1.engine.core import EngineCore, EngineCoreProc
from vllm.v1.engine.exceptions import EngineDeadError
@@ -162,6 +164,9 @@ class EngineCoreClient(ABC):
running state."""
raise NotImplementedError
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
raise NotImplementedError
async def get_output_async(self) -> EngineCoreOutputs:
raise NotImplementedError
@@ -910,14 +915,30 @@ class DPAsyncMPClient(AsyncMPClient):
events = await poller.poll()
if not self.engines_running and len(events) == 2 or (
events[0][0] == first_req_rcv_socket):
# Send a message to notify the coordinator that
# Check if this is a regular request notification or
# scale up notification
buf = first_req_rcv_socket.recv(
flags=zmq.NOBLOCK).result()
decoded = msgspec.msgpack.decode(buf)
if isinstance(
decoded,
(list, tuple)) and len(decoded) == 2 and decoded[
0] == "SCALE_ELASTIC_EP":
# Extract new engine count from the decoded message
new_engine_count = decoded[1]
# Send scale up notification to coordinator
scale_msg = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_engine_count))
await socket.send(scale_msg)
continue
# we're sending a request while the engines are
# paused, so that it can wake the others up
# (to run dummy EP loop).
assert decoded[0] == "FIRST_REQ"
target_eng_index = decoded[1]
self.engines_running = True
buf = first_req_rcv_socket.recv(
flags=zmq.NOBLOCK).result()
target_eng_index = int.from_bytes(buf, "little")
msg = msgspec.msgpack.encode(
(target_eng_index, self.current_wave))
await socket.send(msg)
@@ -953,7 +974,8 @@ class DPAsyncMPClient(AsyncMPClient):
chosen_engine)
if not self.engines_running:
# Notify coordinator that we're sending a request
await self.first_req_send_socket.send(chosen_engine)
req_msg = msgspec.msgpack.encode(("FIRST_REQ", chosen_engine))
await self.first_req_send_socket.send(req_msg)
await to_await
@@ -1047,3 +1069,156 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
engine: EngineIdentity) -> None:
await self._send_input(EngineCoreRequestType.ABORT, request_ids,
engine)
async def _send_reconfig_message(
self, reconfig_request: ReconfigureDistributedRequest,
engine: EngineIdentity) -> asyncio.Future:
"""Send reconfiguration message and return the result future without
waiting for completion."""
call_id = uuid.uuid1().int >> 64
future = asyncio.get_running_loop().create_future()
self.utility_results[call_id] = future
message = (EngineCoreRequestType.UTILITY.value, *self.encoder.encode(
(self.client_index, call_id, "reinitialize_distributed",
(reconfig_request, ))))
await self._send_input_message(message, engine, reconfig_request)
self._ensure_output_queue_task()
return future
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
"""Scale elastic EP data parallel size"""
cur_data_parallel_size = len(self.core_engines)
assert new_data_parallel_size != cur_data_parallel_size, (
f"new_data_parallel_size {new_data_parallel_size} must be "
f"different from cur_data_parallel_size {cur_data_parallel_size}")
assert self.vllm_config.parallel_config.data_parallel_backend == \
"ray", ("Only ray DP backend supports scaling elastic EP")
scale_up = new_data_parallel_size > cur_data_parallel_size
if scale_up:
await self._scale_up_elastic_ep(cur_data_parallel_size,
new_data_parallel_size)
else:
await self._scale_down_elastic_ep(cur_data_parallel_size,
new_data_parallel_size)
async def _scale_up_elastic_ep(self, cur_data_parallel_size: int,
new_data_parallel_size: int) -> None:
"""Scale up the data parallel size by creating new engine cores
and reconfiguring existing ones."""
cur_data_parallel_size = len(self.core_engines)
# Phase 1: Send reconfigure messages to all existing engines and wait
# for them to be sent
reconfig_futures = []
self.vllm_config.parallel_config.data_parallel_master_port = \
get_open_port()
for engine in self.core_engines:
reconfig_request = ReconfigureDistributedRequest(
new_data_parallel_size=new_data_parallel_size,
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_rank_local=\
ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_master_ip=self.vllm_config.parallel_config.
data_parallel_master_ip,
new_data_parallel_master_port=self.vllm_config.parallel_config.
data_parallel_master_port)
future = await self._send_reconfig_message(reconfig_request,
engine)
reconfig_futures.append(future)
logger.info("All reconfigure messages sent, starting engine creation")
# Phase 2: Create new engines now that reconfig messages have been sent
# self.resources.engine_manager is guaranteed to be
# CoreEngineActorManager for RayDPClient
assert isinstance(self.resources.engine_manager,
CoreEngineActorManager)
self.resources.engine_manager.scale_up_elastic_ep(
self.vllm_config, new_data_parallel_size)
# Create new CoreEngine objects for the new engines
new_engine_identities = set()
for i in range(cur_data_parallel_size, new_data_parallel_size):
new_engine = i.to_bytes(2, "little")
self.core_engines.append(new_engine)
new_engine_identities.add(new_engine)
# Wait for ready messages from new engines on the input socket
sync_input_socket = zmq.Socket.shadow(self.input_socket)
while new_engine_identities:
if not sync_input_socket.poll(timeout=600_000):
raise TimeoutError(
"Timed out waiting for new engines to send initial "
"message on input socket.")
identity, _ = sync_input_socket.recv_multipart()
new_engine_identities.discard(identity)
# Phase 3: Wait for all existing engines to complete reconfiguration
logger.info("Waiting for existing engines to complete reconfiguration")
await asyncio.gather(*reconfig_futures)
# Notify coordinator about scale up through existing
# stats_update_task connection
self._ensure_stats_update_task()
scale_up_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size))
await self.first_req_send_socket.send(scale_up_marker)
# Update the parallel config
self.vllm_config.parallel_config.data_parallel_size = \
new_data_parallel_size
logger.info(
"[Elastic EP] Scale up completed, new data parallel size: %s",
new_data_parallel_size)
async def _scale_down_elastic_ep(self, cur_data_parallel_size: int,
new_data_parallel_size: int) -> None:
"""Scale down the data parallel size by shutting down and
reconfiguring existing engine cores."""
cur_data_parallel_size = len(self.core_engines)
self.vllm_config.parallel_config.data_parallel_master_port = \
get_open_port()
reconfig_futures = []
for cur_dp_rank, engine in enumerate(self.core_engines):
reconfig_request = ReconfigureDistributedRequest(
new_data_parallel_size=new_data_parallel_size,
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_rank_local=\
ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_master_ip=self.vllm_config.parallel_config.
data_parallel_master_ip,
new_data_parallel_master_port=self.vllm_config.parallel_config.
data_parallel_master_port)
if cur_dp_rank >= new_data_parallel_size:
reconfig_request.new_data_parallel_rank = \
ReconfigureRankType.SHUTDOWN_CURRENT_RANK
future = await self._send_reconfig_message(reconfig_request,
engine)
reconfig_futures.append(future)
for _ in range(new_data_parallel_size, cur_data_parallel_size):
self.core_engines.pop()
await asyncio.gather(*reconfig_futures)
assert isinstance(self.resources.engine_manager,
CoreEngineActorManager)
self.resources.engine_manager.scale_down_elastic_ep(
cur_data_parallel_size, new_data_parallel_size)
self._ensure_stats_update_task()
scale_down_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size))
await self.first_req_send_socket.send(scale_down_marker)
self.vllm_config.parallel_config.data_parallel_size = \
new_data_parallel_size
logger.info(
"[Elastic EP] Scale down completed, new data parallel size: %s",
new_data_parallel_size)