[Elastic EP] Async preparation (#47288)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
+174
-136
@@ -225,7 +225,10 @@ class EngineCoreClient(ABC):
|
||||
running state."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
async def commit_elastic_ep(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_output_async(self) -> EngineCoreOutputs:
|
||||
@@ -1490,6 +1493,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
)
|
||||
|
||||
assert len(self.core_engines) > 1
|
||||
self._prepared_elastic_ep: tuple[int, int] | None = None
|
||||
|
||||
self.eng_start_index = (
|
||||
len(self.core_engines) * self.client_index
|
||||
@@ -1603,31 +1607,16 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
if len(cache.pending_notifications[notification_type]) >= abs(
|
||||
cache.num_new_core_engines
|
||||
):
|
||||
if notification_type == EEPNotificationType.SHUTDOWN_COMPLETE:
|
||||
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
|
||||
assert cache.num_new_core_engines < 0
|
||||
old_dp_size = len(cache.existing_core_engines)
|
||||
new_dp_size = old_dp_size + cache.num_new_core_engines
|
||||
self.resources.engine_manager.scale_down_elastic_ep(
|
||||
old_dp_size, new_dp_size
|
||||
)
|
||||
else:
|
||||
await asyncio.gather(
|
||||
*[
|
||||
self._call_utility_async(
|
||||
"eep_handle_engine_core_notification",
|
||||
notification_type,
|
||||
engine=engine,
|
||||
)
|
||||
for engine in cache.existing_core_engines
|
||||
]
|
||||
)
|
||||
cache.pending_notifications[notification_type] = set()
|
||||
if notification_type in [
|
||||
EEPNotificationType.SHUTDOWN_COMPLETE,
|
||||
EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY,
|
||||
]:
|
||||
self.eep_scaling_cache = None
|
||||
engine_manager = self.resources.engine_manager
|
||||
assert isinstance(engine_manager, CoreEngineActorManager)
|
||||
assert cache.num_new_core_engines < 0
|
||||
old_dp_size = len(cache.existing_core_engines)
|
||||
new_dp_size = old_dp_size + cache.num_new_core_engines
|
||||
engine_manager.scale_down_elastic_ep(old_dp_size, new_dp_size)
|
||||
self.vllm_config.parallel_config.data_parallel_size_local = len(
|
||||
engine_manager.local_engine_actors
|
||||
)
|
||||
self.eep_scaling_cache = None
|
||||
|
||||
async def abort_requests_async(self, request_ids: list[str]) -> None:
|
||||
if not request_ids or self.resources.engine_dead:
|
||||
@@ -1651,31 +1640,46 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
) -> None:
|
||||
await self._send_input(EngineCoreRequestType.ABORT, request_ids, engine)
|
||||
|
||||
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
"""Scale elastic EP data parallel size"""
|
||||
async def commit_elastic_ep(self) -> None:
|
||||
"""Commit prepared elastic EP scaling."""
|
||||
prepared = self._prepared_elastic_ep
|
||||
if prepared is None:
|
||||
raise RuntimeError("Elastic EP scaling has not been prepared")
|
||||
new_data_parallel_size, num_redundant_experts = prepared
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
|
||||
assert new_data_parallel_size != cur_data_parallel_size, (
|
||||
f"new_data_parallel_size {new_data_parallel_size} must be "
|
||||
f"different from cur_data_parallel_size {cur_data_parallel_size}"
|
||||
if new_data_parallel_size > cur_data_parallel_size:
|
||||
await self._commit_scale_up_elastic_ep(new_data_parallel_size)
|
||||
else:
|
||||
await self._commit_scale_down_elastic_ep(new_data_parallel_size)
|
||||
self.vllm_config.parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_redundant_experts
|
||||
)
|
||||
self._prepared_elastic_ep = None
|
||||
|
||||
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
"""Prepare elastic EP scaling without routing requests to new engines."""
|
||||
if (prepared := self._prepared_elastic_ep) is not None:
|
||||
if prepared[0] == new_data_parallel_size:
|
||||
return
|
||||
raise RuntimeError("Elastic EP scaling is already prepared")
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
assert self.vllm_config.parallel_config.data_parallel_backend == "ray", (
|
||||
"Only ray DP backend supports scaling elastic EP"
|
||||
)
|
||||
|
||||
scale_up = new_data_parallel_size > cur_data_parallel_size
|
||||
|
||||
if scale_up:
|
||||
await self._scale_up_elastic_ep(
|
||||
cur_data_parallel_size, new_data_parallel_size
|
||||
)
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
num_experts = self.vllm_config.model_config.get_num_experts()
|
||||
num_redundant_experts = (
|
||||
num_experts + parallel_config.eplb_config.num_redundant_experts
|
||||
) * new_data_parallel_size // cur_data_parallel_size - num_experts
|
||||
if new_data_parallel_size < cur_data_parallel_size:
|
||||
await self._prepare_scale_down_elastic_ep(new_data_parallel_size)
|
||||
else:
|
||||
await self._scale_down_elastic_ep(
|
||||
cur_data_parallel_size, new_data_parallel_size
|
||||
await self._prepare_scale_up_elastic_ep(
|
||||
new_data_parallel_size, num_redundant_experts
|
||||
)
|
||||
self._prepared_elastic_ep = new_data_parallel_size, num_redundant_experts
|
||||
|
||||
async def _eep_wait_for_setup_switch_complete(self) -> None:
|
||||
def _eep_wait_for_setup_switch_complete(self) -> asyncio.Future:
|
||||
"""
|
||||
Wait for core engines to switch to the new setup.
|
||||
|
||||
@@ -1687,9 +1691,26 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
self.utility_results[EEP_NOTIFICATION_CALL_ID] = future
|
||||
self._ensure_output_queue_task()
|
||||
await future
|
||||
return future
|
||||
|
||||
def _setup_elastic_ep_reconfig_bootstrap(self) -> tuple[str, int]:
|
||||
def _wait_for_new_engine_ready(self, new_core_engines: list[bytes]) -> None:
|
||||
new_engine_identities = set(new_core_engines)
|
||||
sync_input_socket = zmq.Socket.shadow(self.input_socket)
|
||||
while new_engine_identities:
|
||||
if not sync_input_socket.poll(timeout=VLLM_ENGINE_READY_TIMEOUT_S * 1000):
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for new engine core processes to "
|
||||
f"start. Waited "
|
||||
f"{VLLM_ENGINE_READY_TIMEOUT_S}s (configured by "
|
||||
f"VLLM_ENGINE_READY_TIMEOUT_S). To increase the "
|
||||
f"timeout, set the environment variable: "
|
||||
f"VLLM_ENGINE_READY_TIMEOUT_S=<seconds>"
|
||||
)
|
||||
identity, payload = sync_input_socket.recv_multipart()
|
||||
new_engine_identities.discard(identity)
|
||||
self._apply_ready_response(payload)
|
||||
|
||||
def _setup_elastic_ep_reconfig_bootstrap(self) -> None:
|
||||
from vllm.distributed.utils import create_tcp_store
|
||||
from vllm.utils.network_utils import get_open_ports_list
|
||||
|
||||
@@ -1709,36 +1730,36 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
)
|
||||
parallel_config._coord_store_port = store.port
|
||||
self._coord_store = store
|
||||
return ip, store.port
|
||||
|
||||
async def _scale_up_elastic_ep(
|
||||
self, cur_data_parallel_size: int, new_data_parallel_size: int
|
||||
) -> None:
|
||||
"""Scale up the data parallel size by creating new engine cores
|
||||
and reconfiguring existing ones."""
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
|
||||
self.eep_scaling_cache = ElasticScalingCache(
|
||||
existing_core_engines=self.core_engines.copy(),
|
||||
num_new_core_engines=new_data_parallel_size - cur_data_parallel_size,
|
||||
pending_notifications=dict(),
|
||||
def _make_reconfig_request(
|
||||
self,
|
||||
new_data_parallel_size: int,
|
||||
rank_type: ReconfigureRankType = ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
) -> ReconfigureDistributedRequest:
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
return ReconfigureDistributedRequest(
|
||||
new_data_parallel_size=new_data_parallel_size,
|
||||
new_data_parallel_rank=rank_type,
|
||||
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_master_ip=parallel_config.data_parallel_master_ip,
|
||||
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
|
||||
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
|
||||
coord_store_port=parallel_config._coord_store_port,
|
||||
)
|
||||
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap()
|
||||
async def _prepare_scale_up_elastic_ep(
|
||||
self,
|
||||
new_data_parallel_size: int,
|
||||
num_redundant_experts: int,
|
||||
) -> None:
|
||||
"""Prepare scale up by creating new engine cores and reconfiguring
|
||||
existing ones."""
|
||||
self._setup_elastic_ep_reconfig_bootstrap()
|
||||
|
||||
# Phase 1: Send reconfig messages to existing engines
|
||||
reconfig_futures = []
|
||||
for engine in self.core_engines:
|
||||
reconfig_request = ReconfigureDistributedRequest(
|
||||
new_data_parallel_size=new_data_parallel_size,
|
||||
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_master_ip=ip,
|
||||
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
|
||||
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
|
||||
coord_store_port=coord_store_port,
|
||||
)
|
||||
reconfig_request = self._make_reconfig_request(new_data_parallel_size)
|
||||
coro = self._call_utility_async(
|
||||
"reinitialize_distributed", reconfig_request, engine=engine
|
||||
)
|
||||
@@ -1746,51 +1767,54 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
|
||||
# Phase 2: Create new engines
|
||||
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
|
||||
parallel_config.eplb_config.num_redundant_experts = 0
|
||||
start_new_worker_future = asyncio.to_thread(
|
||||
self.resources.engine_manager.scale_up_elastic_ep,
|
||||
self.vllm_config,
|
||||
new_data_parallel_size,
|
||||
num_redundant_experts,
|
||||
)
|
||||
wait_future = self._eep_wait_for_setup_switch_complete()
|
||||
|
||||
# Phase 3: Wait for new engines to be created
|
||||
# and reconfig messages to be received
|
||||
await asyncio.gather(start_new_worker_future, *reconfig_futures)
|
||||
ready_keys = [future.result() for future in reconfig_futures]
|
||||
ready_keys.extend(
|
||||
f"eep_ready/{rank}"
|
||||
for rank in range(len(self.core_engines), new_data_parallel_size)
|
||||
)
|
||||
await asyncio.to_thread(self._coord_store.wait, ready_keys)
|
||||
logger.info("[Elastic EP] Successfully started new engines")
|
||||
|
||||
# Create new CoreEngine objects for the new engines
|
||||
new_engine_identities = set()
|
||||
for i in range(cur_data_parallel_size, new_data_parallel_size):
|
||||
new_engine = i.to_bytes(2, "little")
|
||||
self.core_engines.append(new_engine)
|
||||
# NOTE(yongji): we don't update lb_engines here,
|
||||
# we let run_engine_stats_update_task to update it.
|
||||
new_engine_identities.add(new_engine)
|
||||
async def _commit_scale_up_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
new_core_engines = [
|
||||
rank.to_bytes(2, "little")
|
||||
for rank in range(len(self.core_engines), new_data_parallel_size)
|
||||
]
|
||||
|
||||
# Wait for ready messages from new engines on the input socket
|
||||
sync_input_socket = zmq.Socket.shadow(self.input_socket)
|
||||
while new_engine_identities:
|
||||
if not sync_input_socket.poll(
|
||||
timeout=VLLM_ENGINE_READY_TIMEOUT_S * 1000 # convert to ms
|
||||
):
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for new engine core processes to "
|
||||
f"start. Waited "
|
||||
f"{VLLM_ENGINE_READY_TIMEOUT_S}s (configured by "
|
||||
f"VLLM_ENGINE_READY_TIMEOUT_S). To increase the "
|
||||
f"timeout, set the environment variable: "
|
||||
f"VLLM_ENGINE_READY_TIMEOUT_S=<seconds>"
|
||||
)
|
||||
identity, payload = sync_input_socket.recv_multipart()
|
||||
new_engine_identities.discard(identity)
|
||||
self._apply_ready_response(payload)
|
||||
await self.pause_scheduler_async(mode="keep", clear_cache=False)
|
||||
wait_future = self._eep_wait_for_setup_switch_complete()
|
||||
finish_futures = [
|
||||
asyncio.create_task(
|
||||
self._call_utility_async("commit_prepared_elastic_ep", engine=engine)
|
||||
)
|
||||
for engine in self.core_engines
|
||||
]
|
||||
try:
|
||||
await asyncio.gather(*finish_futures)
|
||||
await wait_future
|
||||
self._wait_for_new_engine_ready(new_core_engines)
|
||||
except Exception:
|
||||
wait_future.cancel()
|
||||
raise
|
||||
|
||||
# NOTE(yongji): Before we schedule any requests on the new workers,
|
||||
# we should wait for them to switch to the new setup.
|
||||
await wait_future
|
||||
self.core_engines.extend(new_core_engines)
|
||||
# Update the parallel config
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
parallel_config.data_parallel_size = new_data_parallel_size
|
||||
if isinstance(self.resources.engine_manager, CoreEngineActorManager):
|
||||
parallel_config.data_parallel_size_local = len(
|
||||
self.resources.engine_manager.local_engine_actors
|
||||
)
|
||||
# Notify coordinator about scale up through existing
|
||||
# stats_update_task connection
|
||||
self._ensure_stats_update_task()
|
||||
@@ -1803,10 +1827,23 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
"[Elastic EP] Scale up completed, new data parallel size: %s",
|
||||
new_data_parallel_size,
|
||||
)
|
||||
await self.resume_scheduler_async()
|
||||
|
||||
async def _scale_down_elastic_ep(
|
||||
self, cur_data_parallel_size: int, new_data_parallel_size: int
|
||||
) -> None:
|
||||
async def _prepare_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
self._setup_elastic_ep_reconfig_bootstrap()
|
||||
|
||||
reconfig_futures = []
|
||||
for engine in self.core_engines[:new_data_parallel_size]:
|
||||
reconfig_request = self._make_reconfig_request(new_data_parallel_size)
|
||||
coro = self._call_utility_async(
|
||||
"reinitialize_distributed", reconfig_request, engine=engine
|
||||
)
|
||||
reconfig_futures.append(asyncio.create_task(coro))
|
||||
|
||||
ready_keys = await asyncio.gather(*reconfig_futures)
|
||||
await asyncio.to_thread(self._coord_store.wait, ready_keys)
|
||||
|
||||
async def _commit_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
"""Scale down the data parallel size by shutting down and
|
||||
reconfiguring existing engine cores."""
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
@@ -1817,50 +1854,51 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
pending_notifications=dict(),
|
||||
)
|
||||
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap()
|
||||
|
||||
old_core_engines = self.core_engines
|
||||
# NOTE(yongji): Immediately stop sending requests to the removing engines.
|
||||
self.core_engines = old_core_engines[:new_data_parallel_size]
|
||||
self.lb_engines = self.lb_engines[:new_data_parallel_size]
|
||||
removed_dp_size = cur_data_parallel_size - new_data_parallel_size
|
||||
pause_modes = ["keep"] * new_data_parallel_size + ["abort"] * removed_dp_size
|
||||
pause_futures = [
|
||||
self._call_utility_async("pause_scheduler", mode, False, engine=engine)
|
||||
for mode, engine in zip(pause_modes, old_core_engines)
|
||||
]
|
||||
await asyncio.gather(*pause_futures)
|
||||
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
|
||||
self.resources.engine_manager.remove_run_refs_for_scale_down(removed_dp_size)
|
||||
wait_future = self._eep_wait_for_setup_switch_complete()
|
||||
reconfig_futures = []
|
||||
for cur_dp_rank, engine in enumerate(self.core_engines):
|
||||
reconfig_request = ReconfigureDistributedRequest(
|
||||
new_data_parallel_size=new_data_parallel_size,
|
||||
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_master_ip=ip,
|
||||
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
|
||||
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
|
||||
coord_store_port=coord_store_port,
|
||||
)
|
||||
if cur_dp_rank >= new_data_parallel_size:
|
||||
reconfig_request.new_data_parallel_rank = (
|
||||
ReconfigureRankType.SHUTDOWN_CURRENT_RANK
|
||||
for cur_dp_rank, engine in enumerate(old_core_engines):
|
||||
if cur_dp_rank < new_data_parallel_size:
|
||||
coro = self._call_utility_async(
|
||||
"commit_prepared_elastic_ep", engine=engine
|
||||
)
|
||||
else:
|
||||
reconfig_request = self._make_reconfig_request(
|
||||
new_data_parallel_size,
|
||||
ReconfigureRankType.SHUTDOWN_CURRENT_RANK,
|
||||
)
|
||||
coro = self._call_utility_async(
|
||||
"reinitialize_distributed", reconfig_request, engine=engine
|
||||
)
|
||||
coro = self._call_utility_async(
|
||||
"reinitialize_distributed", reconfig_request, engine=engine
|
||||
)
|
||||
reconfig_futures.append(asyncio.create_task(coro))
|
||||
|
||||
# NOTE(yongji): Immediately stop sending requests to the removing engines.
|
||||
self.core_engines = self.core_engines[:new_data_parallel_size]
|
||||
self.lb_engines = self.lb_engines[:new_data_parallel_size]
|
||||
wait_future = self._eep_wait_for_setup_switch_complete()
|
||||
try:
|
||||
await asyncio.gather(*reconfig_futures)
|
||||
|
||||
await asyncio.gather(*reconfig_futures)
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
self._ensure_stats_update_task()
|
||||
scale_down_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_down_marker)
|
||||
await wait_future
|
||||
await self.resume_scheduler_async()
|
||||
except Exception:
|
||||
wait_future.cancel()
|
||||
raise
|
||||
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
self._ensure_stats_update_task()
|
||||
scale_down_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_down_marker)
|
||||
|
||||
# NOTE(yongji): Unlike scaling up,
|
||||
# here we don't actually need to wait for the setup switch to complete.
|
||||
# We may want to remove it in the future.
|
||||
await wait_future
|
||||
logger.info(
|
||||
"[Elastic EP] Scale down completed, new data parallel size: %s",
|
||||
new_data_parallel_size,
|
||||
|
||||
Reference in New Issue
Block a user