Signed-off-by: fh3 <a@b> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
898 lines
36 KiB
Python
898 lines
36 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import enum
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from threading import Event, Thread
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import msgspec
|
|
import msgspec.msgpack
|
|
import zmq
|
|
|
|
from vllm.logger import init_logger
|
|
from vllm.utils.network_utils import make_zmq_socket
|
|
from vllm.v1.engine import (
|
|
EEPNotificationType,
|
|
ReconfigureDistributedRequest,
|
|
ReconfigureRankType,
|
|
)
|
|
from vllm.v1.engine.utils import EngineHandshakeMetadata, EngineZmqAddresses
|
|
from vllm.v1.utils import get_engine_client_zmq_addr
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.v1.engine.core_client import DPAsyncMPClient
|
|
|
|
logger = init_logger(__name__)
|
|
POLL_BACKOFF_STEPS_S = (0.1, 0.2, 0.4, 0.8)
|
|
|
|
|
|
class ExternalElasticEPScalePhase(str, enum.Enum):
|
|
"""Lifecycle phases shared for an epoch-scoped external EEP operation.
|
|
|
|
These values are stored in the control and reconfiguration stores and
|
|
exposed through the status API so external orchestrators can observe the
|
|
same operation across API processes and reconnects. COMPLETED means the
|
|
topology commit and rank completion barrier have finished; API processes
|
|
may clear their process-local request gate shortly afterward.
|
|
"""
|
|
|
|
IDLE = "idle"
|
|
PREPARING = "preparing"
|
|
COMMITTING = "committing"
|
|
FAILED = "failed"
|
|
COMPLETED = "completed"
|
|
|
|
|
|
@dataclass
|
|
class ExternalElasticEPScaleUpHandshakeMetadata:
|
|
engine_metadata: EngineHandshakeMetadata
|
|
num_redundant_experts: int
|
|
elastic_ep_max_dp_size: int
|
|
|
|
|
|
class ExternalElasticEPScaleUpHandshakeServer:
|
|
"""Temporary rank-0 handshake server for external EEP scale-up.
|
|
|
|
During normal external startup the global handshake listener only exists
|
|
while the rank starts. Scale-up needs the same handshake contract again for
|
|
newly launched ranks, so rank 0 re-opens a temporary listener for the
|
|
duration of the current scale operation.
|
|
|
|
The handshake gives new ranks the information they cannot infer locally:
|
|
front-end and DP coordinator ZMQ addresses, the new DP master address/ports,
|
|
target DP size, target expert redundancy, and the coordination store port
|
|
used for EEP reconfiguration.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
handshake_address: str,
|
|
expected_new_ranks: list[int],
|
|
addresses: EngineZmqAddresses,
|
|
bootstrap: ReconfigureDistributedRequest,
|
|
num_redundant_experts: int,
|
|
elastic_ep_max_dp_size: int,
|
|
) -> None:
|
|
self.handshake_address = handshake_address
|
|
self.expected_new_ranks = set(expected_new_ranks)
|
|
self.addresses = addresses
|
|
self.bootstrap = bootstrap
|
|
self.num_redundant_experts = num_redundant_experts
|
|
self.elastic_ep_max_dp_size = elastic_ep_max_dp_size
|
|
self.started_event = Event()
|
|
self._stop_event = Event()
|
|
self._thread = Thread(
|
|
target=self._run,
|
|
name="ExternalElasticEPHandshakeServer",
|
|
daemon=True,
|
|
)
|
|
self._error: Exception | None = None
|
|
|
|
def start(self) -> None:
|
|
self._thread.start()
|
|
started = self.started_event.wait(timeout=5)
|
|
if self._error is not None:
|
|
raise self._error
|
|
if not started:
|
|
raise TimeoutError(
|
|
"Timed out waiting for external EEP handshake server to "
|
|
f"start listening at {self.handshake_address}"
|
|
)
|
|
|
|
def stop(self) -> None:
|
|
self._stop_event.set()
|
|
if self._thread.is_alive():
|
|
self._thread.join(timeout=5)
|
|
if self._error is not None:
|
|
raise self._error
|
|
|
|
def raise_if_failed(self) -> None:
|
|
if self._error is not None:
|
|
raise self._error
|
|
|
|
def _run(self) -> None:
|
|
pending_ready_ranks = self.expected_new_ranks.copy()
|
|
try:
|
|
with (
|
|
zmq.Context() as ctx,
|
|
make_zmq_socket(
|
|
ctx,
|
|
self.handshake_address,
|
|
zmq.ROUTER,
|
|
bind=True,
|
|
linger=0,
|
|
router_handover=True,
|
|
) as handshake_socket,
|
|
):
|
|
self.started_event.set()
|
|
poller = zmq.Poller()
|
|
poller.register(handshake_socket, zmq.POLLIN)
|
|
|
|
while not self._stop_event.is_set():
|
|
if not pending_ready_ranks:
|
|
return
|
|
|
|
events = dict(poller.poll(timeout=1000))
|
|
if handshake_socket not in events:
|
|
continue
|
|
|
|
engine_identity, payload = handshake_socket.recv_multipart()
|
|
engine_rank = int.from_bytes(engine_identity, "little")
|
|
if engine_rank not in self.expected_new_ranks:
|
|
raise RuntimeError(
|
|
"Received scale-up handshake from unexpected "
|
|
f"dp rank {engine_rank}"
|
|
)
|
|
|
|
message = msgspec.msgpack.decode(payload)
|
|
status = message["status"]
|
|
if status == "HELLO":
|
|
b = self.bootstrap
|
|
parallel_config: dict[str, int | str | list[int]] = {
|
|
"data_parallel_master_ip": b.new_data_parallel_master_ip,
|
|
"data_parallel_master_port": (
|
|
b.new_data_parallel_master_port
|
|
),
|
|
"_data_parallel_master_port_list": (
|
|
b.new_data_parallel_master_port_list
|
|
),
|
|
"data_parallel_size": b.new_data_parallel_size,
|
|
"_coord_store_port": b.coord_store_port,
|
|
}
|
|
init_message = msgspec.msgpack.encode(
|
|
ExternalElasticEPScaleUpHandshakeMetadata(
|
|
engine_metadata=EngineHandshakeMetadata(
|
|
addresses=self.addresses,
|
|
parallel_config=parallel_config,
|
|
),
|
|
num_redundant_experts=self.num_redundant_experts,
|
|
elastic_ep_max_dp_size=self.elastic_ep_max_dp_size,
|
|
)
|
|
)
|
|
handshake_socket.send_multipart(
|
|
(engine_identity, init_message), copy=False
|
|
)
|
|
elif status == "READY":
|
|
pending_ready_ranks.discard(engine_rank)
|
|
else:
|
|
raise RuntimeError(
|
|
f"Unexpected handshake status {status} from dp rank "
|
|
f"{engine_rank}"
|
|
)
|
|
except Exception as e:
|
|
self._error = e
|
|
self.started_event.set()
|
|
|
|
|
|
@dataclass
|
|
class _PreparedExternalElasticEPScale:
|
|
control_store: Any
|
|
reconfig_store: Any
|
|
epoch: str
|
|
bootstrap: ReconfigureDistributedRequest
|
|
dp_rank: int
|
|
cur_data_parallel_size: int
|
|
num_redundant_experts: int
|
|
scale_up: bool
|
|
handshake_server: ExternalElasticEPScaleUpHandshakeServer | None
|
|
|
|
|
|
class ExternalElasticEPScaleCoordinator:
|
|
def __init__(self, client: "DPAsyncMPClient") -> None:
|
|
self.client = client
|
|
self.active_reconfig_store: tuple[str, int] | None = None
|
|
# Keep the previous rank-0 TCPStore server alive while the next scale
|
|
# operation publishes bootstrap metadata through it. Overwriting
|
|
# client._coord_store with the new store can otherwise close the old
|
|
# server while other ranks are still polling it.
|
|
self.control_store_ref: Any | None = None
|
|
self.reconfig_store_ref: Any | None = None
|
|
self.active_epoch: str | None = None
|
|
self.prepared_scale: _PreparedExternalElasticEPScale | None = None
|
|
|
|
@staticmethod
|
|
def key(*parts: str | int) -> str:
|
|
return "/".join(["elastic_ep/external", *[str(part) for part in parts]])
|
|
|
|
@staticmethod
|
|
async def _sleep_with_backoff(backoff_step: int) -> int:
|
|
max_step = len(POLL_BACKOFF_STEPS_S) - 1
|
|
backoff_step = max(0, min(backoff_step, max_step))
|
|
await asyncio.sleep(POLL_BACKOFF_STEPS_S[backoff_step])
|
|
return min(backoff_step + 1, max_step)
|
|
|
|
def _update_parallel_config(
|
|
self,
|
|
bootstrap: ReconfigureDistributedRequest,
|
|
num_redundant_experts: int,
|
|
) -> None:
|
|
parallel_config = self.client.vllm_config.parallel_config
|
|
parallel_config.data_parallel_size = bootstrap.new_data_parallel_size
|
|
parallel_config.data_parallel_master_ip = bootstrap.new_data_parallel_master_ip
|
|
parallel_config.data_parallel_master_port = (
|
|
bootstrap.new_data_parallel_master_port
|
|
)
|
|
parallel_config._data_parallel_master_port_list = (
|
|
bootstrap.new_data_parallel_master_port_list.copy()
|
|
)
|
|
parallel_config._coord_store_port = bootstrap.coord_store_port
|
|
parallel_config.eplb_config.num_redundant_experts = num_redundant_experts
|
|
|
|
def _calculate_num_redundant_experts(
|
|
self,
|
|
cur_data_parallel_size: int,
|
|
new_data_parallel_size: int,
|
|
) -> int:
|
|
parallel_config = self.client.vllm_config.parallel_config
|
|
num_experts = self.client.vllm_config.model_config.get_num_experts()
|
|
num_redundant_experts = (
|
|
num_experts + parallel_config.eplb_config.num_redundant_experts
|
|
) * new_data_parallel_size // cur_data_parallel_size - num_experts
|
|
if num_redundant_experts < 0:
|
|
raise ValueError(
|
|
"Cannot scale external Elastic EP from DP size "
|
|
f"{cur_data_parallel_size} to {new_data_parallel_size}: "
|
|
"the target topology cannot retain all model experts."
|
|
)
|
|
return num_redundant_experts
|
|
|
|
def _get_reconfig_store(self):
|
|
from vllm.distributed.utils import get_cached_tcp_store_client
|
|
|
|
if self.active_reconfig_store is not None:
|
|
store_addr = self.active_reconfig_store
|
|
else:
|
|
parallel_config = self.client.vllm_config.parallel_config
|
|
if not parallel_config._coord_store_port:
|
|
raise RuntimeError(
|
|
"External Elastic EP requires an active reconfiguration "
|
|
"coordination store."
|
|
)
|
|
store_addr = (
|
|
parallel_config.data_parallel_master_ip,
|
|
parallel_config._coord_store_port,
|
|
)
|
|
|
|
return get_cached_tcp_store_client(*store_addr)
|
|
|
|
def _get_existing_engine_zmq_address(self) -> EngineZmqAddresses:
|
|
coordinator_input = self.client.coordinator_input_address
|
|
coordinator_output = self.client.coordinator_output_address
|
|
if coordinator_input is None or coordinator_output is None:
|
|
raise RuntimeError(
|
|
"External Elastic EP scale-up requires DP coordinator addresses."
|
|
)
|
|
|
|
return EngineZmqAddresses(
|
|
inputs=[self.client.input_address],
|
|
outputs=[self.client.output_address],
|
|
coordinator_input=coordinator_input,
|
|
coordinator_output=coordinator_output,
|
|
frontend_stats_publish_address=self.client.stats_update_address,
|
|
)
|
|
|
|
def _setup_reconfig_bootstrap(self) -> tuple[str, int]:
|
|
from vllm.distributed.utils import create_tcp_store
|
|
|
|
self.control_store_ref = getattr(self.client, "_coord_store", None)
|
|
ip = self.client.vllm_config.parallel_config.data_parallel_master_ip
|
|
# Keep the serving configuration on the old store until commit succeeds.
|
|
self.reconfig_store_ref = create_tcp_store(
|
|
ip, 0, is_master=True, world_size=-1, wait_for_workers=False
|
|
)
|
|
return ip, self.reconfig_store_ref.port
|
|
|
|
def _get_error(self, store: Any, epoch: str) -> str | None:
|
|
error_key = self.key(epoch, "error")
|
|
if not store.check([error_key]):
|
|
return None
|
|
return store.get(error_key).decode()
|
|
|
|
def get_status(self) -> dict[str, str | int | None]:
|
|
store = self._get_reconfig_store()
|
|
current_epoch_key = self.key("current_epoch")
|
|
if not store.check([current_epoch_key]):
|
|
return {
|
|
"phase": ExternalElasticEPScalePhase.IDLE.value,
|
|
"epoch": None,
|
|
"error": None,
|
|
"requested_data_parallel_size": None,
|
|
}
|
|
|
|
epoch = store.get(current_epoch_key).decode()
|
|
error = self._get_error(store, epoch)
|
|
if error is not None:
|
|
phase = ExternalElasticEPScalePhase.FAILED
|
|
elif store.check([self.key(epoch, "completed")]):
|
|
phase = ExternalElasticEPScalePhase.COMPLETED
|
|
elif store.check([self.key(epoch, "phase")]):
|
|
phase = ExternalElasticEPScalePhase(
|
|
store.get(self.key(epoch, "phase")).decode()
|
|
)
|
|
else:
|
|
phase = ExternalElasticEPScalePhase.IDLE
|
|
requested_dp_size = None
|
|
bootstrap_key = self.key(epoch, "bootstrap")
|
|
if store.check([bootstrap_key]):
|
|
bootstrap = msgspec.msgpack.decode(
|
|
store.get(bootstrap_key), type=ReconfigureDistributedRequest
|
|
)
|
|
requested_dp_size = bootstrap.new_data_parallel_size
|
|
return {
|
|
"phase": phase.value,
|
|
"epoch": epoch,
|
|
"error": error,
|
|
"requested_data_parallel_size": requested_dp_size,
|
|
}
|
|
|
|
def _set_phase(
|
|
self, store: Any, epoch: str, phase: ExternalElasticEPScalePhase
|
|
) -> None:
|
|
store.set(self.key(epoch, "phase"), phase.value.encode())
|
|
|
|
async def _wait_for_bootstrap(
|
|
self,
|
|
store: Any,
|
|
requested_new_dp_size: int,
|
|
timeout_s: float = 300,
|
|
) -> tuple[str, ReconfigureDistributedRequest]:
|
|
"""Wait for rank 0 to publish matching scale bootstrap metadata.
|
|
Non-zero ranks poll the old control store until prepared or errored."""
|
|
loop = asyncio.get_running_loop()
|
|
start = loop.time()
|
|
current_epoch_key = self.key("current_epoch")
|
|
previous_epoch = self.active_epoch
|
|
if store.check([current_epoch_key]):
|
|
current_epoch = store.get(current_epoch_key).decode()
|
|
if store.check([self.key(current_epoch, "completed")]):
|
|
previous_epoch = current_epoch
|
|
backoff_step = 0
|
|
while True:
|
|
if store.check([current_epoch_key]):
|
|
epoch = store.get(current_epoch_key).decode()
|
|
if epoch != previous_epoch:
|
|
error = self._get_error(store, epoch)
|
|
if error is not None:
|
|
self.active_epoch = epoch
|
|
raise RuntimeError(error)
|
|
|
|
bootstrap_key = self.key(epoch, "bootstrap")
|
|
if store.check([bootstrap_key]):
|
|
bootstrap = msgspec.msgpack.decode(
|
|
store.get(bootstrap_key),
|
|
type=ReconfigureDistributedRequest,
|
|
)
|
|
prepared_key = self.key(epoch, "prepared")
|
|
completed = store.check([self.key(epoch, "completed")])
|
|
if bootstrap.new_data_parallel_size != requested_new_dp_size:
|
|
if store.check([prepared_key]) and not completed:
|
|
raise RuntimeError(
|
|
"A different external Elastic EP scaling "
|
|
"operation is already in progress for target "
|
|
f"dp size {bootstrap.new_data_parallel_size}."
|
|
)
|
|
elif store.check([prepared_key]):
|
|
return epoch, bootstrap
|
|
|
|
now = loop.time()
|
|
if now - start > timeout_s:
|
|
raise TimeoutError(
|
|
"Timed out waiting for rank 0 to publish external Elastic EP "
|
|
"bootstrap metadata."
|
|
)
|
|
backoff_step = await self._sleep_with_backoff(backoff_step)
|
|
|
|
def _prepare_reconfig_bootstrap(
|
|
self,
|
|
store: Any,
|
|
cur_data_parallel_size: int,
|
|
new_data_parallel_size: int,
|
|
) -> tuple[str, ReconfigureDistributedRequest]:
|
|
from vllm.utils.network_utils import get_open_ports_list
|
|
|
|
current_epoch_key = self.key("current_epoch")
|
|
if store.check([current_epoch_key]):
|
|
current_epoch = store.get(current_epoch_key).decode()
|
|
current_error = self._get_error(store, current_epoch)
|
|
if current_error is None and not store.check(
|
|
[self.key(current_epoch, "completed")]
|
|
):
|
|
raise RuntimeError(
|
|
"Another external Elastic EP scaling operation is already active."
|
|
)
|
|
|
|
ip, coord_store_port = self._setup_reconfig_bootstrap()
|
|
epoch = uuid.uuid4().hex
|
|
master_ports = get_open_ports_list(5)
|
|
bootstrap = ReconfigureDistributedRequest(
|
|
new_data_parallel_size=new_data_parallel_size,
|
|
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
|
|
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
|
|
new_data_parallel_master_ip=ip,
|
|
new_data_parallel_master_port=master_ports.pop(),
|
|
new_data_parallel_master_port_list=master_ports,
|
|
coord_store_port=coord_store_port,
|
|
)
|
|
|
|
bootstrap_key = self.key(epoch, "bootstrap")
|
|
bootstrap_payload = msgspec.msgpack.encode(bootstrap)
|
|
stores = [store]
|
|
if self.reconfig_store_ref is not None:
|
|
stores.append(self.reconfig_store_ref)
|
|
for target_store in stores:
|
|
target_store.set(bootstrap_key, bootstrap_payload)
|
|
self._set_phase(target_store, epoch, ExternalElasticEPScalePhase.PREPARING)
|
|
# Publish the epoch only after its metadata is ready for readers.
|
|
target_store.set(current_epoch_key, epoch.encode())
|
|
return epoch, bootstrap
|
|
|
|
def _start_scale_up_handshake_server(
|
|
self,
|
|
bootstrap: ReconfigureDistributedRequest,
|
|
cur_data_parallel_size: int,
|
|
num_redundant_experts: int,
|
|
elastic_ep_max_dp_size: int,
|
|
) -> ExternalElasticEPScaleUpHandshakeServer:
|
|
handshake_server = ExternalElasticEPScaleUpHandshakeServer(
|
|
handshake_address=get_engine_client_zmq_addr(
|
|
False,
|
|
bootstrap.new_data_parallel_master_ip,
|
|
self.client.vllm_config.parallel_config.data_parallel_rpc_port,
|
|
),
|
|
expected_new_ranks=list(
|
|
range(
|
|
cur_data_parallel_size,
|
|
bootstrap.new_data_parallel_size,
|
|
)
|
|
),
|
|
addresses=self._get_existing_engine_zmq_address(),
|
|
bootstrap=bootstrap,
|
|
num_redundant_experts=num_redundant_experts,
|
|
elastic_ep_max_dp_size=elastic_ep_max_dp_size,
|
|
)
|
|
handshake_server.start()
|
|
return handshake_server
|
|
|
|
async def _wait_for_ready_ranks(
|
|
self,
|
|
control_store: Any,
|
|
reconfig_store: Any,
|
|
epoch: str,
|
|
new_data_parallel_size: int,
|
|
handshake_server: ExternalElasticEPScaleUpHandshakeServer | None = None,
|
|
timeout_s: float = 300,
|
|
) -> None:
|
|
"""Wait until every rank in the new DP group is ready to commit."""
|
|
loop = asyncio.get_running_loop()
|
|
start = loop.time()
|
|
ready_keys = [f"eep_ready/{rank}" for rank in range(new_data_parallel_size)]
|
|
|
|
backoff_step = 0
|
|
while True:
|
|
if handshake_server is not None:
|
|
handshake_server.raise_if_failed()
|
|
|
|
error = self._get_error(control_store, epoch)
|
|
if error is not None:
|
|
raise RuntimeError(
|
|
"External Elastic EP scaling failed while waiting for "
|
|
f"ranks to prepare: {error}"
|
|
)
|
|
|
|
if reconfig_store.check(ready_keys):
|
|
return
|
|
|
|
now = loop.time()
|
|
if now - start > timeout_s:
|
|
raise TimeoutError(
|
|
"Timed out waiting for external Elastic EP ranks to prepare."
|
|
)
|
|
backoff_step = await self._sleep_with_backoff(backoff_step)
|
|
|
|
async def _wait_for_local_reconfig_finished(
|
|
self,
|
|
control_store: Any,
|
|
reconfig_store: Any,
|
|
epoch: str,
|
|
bootstrap: ReconfigureDistributedRequest,
|
|
dp_rank: int,
|
|
scale_up: bool,
|
|
timeout_s: float = 600,
|
|
) -> None:
|
|
"""Wait until this rank's EngineCore finishes the scale transition."""
|
|
if not scale_up and dp_rank >= bootstrap.new_data_parallel_size:
|
|
wait_key = self.key(epoch, "shutdown_complete", dp_rank)
|
|
timeout_msg = (
|
|
"Timed out waiting for local external Elastic EP scale-down "
|
|
"shutdown to finish."
|
|
)
|
|
else:
|
|
wait_key = self.key(epoch, "old_rank_finished", dp_rank)
|
|
timeout_msg = (
|
|
"Timed out waiting for local external Elastic EP "
|
|
"reconfiguration to finish."
|
|
)
|
|
|
|
loop = asyncio.get_running_loop()
|
|
start = loop.time()
|
|
backoff_step = 0
|
|
while True:
|
|
error = self._get_error(control_store, epoch)
|
|
if error is not None:
|
|
raise RuntimeError(
|
|
error or "External Elastic EP scaling failed on another rank."
|
|
)
|
|
|
|
if reconfig_store.check([wait_key]):
|
|
return
|
|
|
|
now = loop.time()
|
|
if now - start > timeout_s:
|
|
raise TimeoutError(timeout_msg)
|
|
backoff_step = await self._sleep_with_backoff(backoff_step)
|
|
|
|
async def _wait_for_all_old_ranks(
|
|
self,
|
|
prepared: _PreparedExternalElasticEPScale,
|
|
timeout_s: float = 600,
|
|
) -> None:
|
|
bootstrap = prepared.bootstrap
|
|
keys = [
|
|
self.key(
|
|
prepared.epoch,
|
|
(
|
|
"shutdown_complete"
|
|
if not prepared.scale_up
|
|
and rank >= bootstrap.new_data_parallel_size
|
|
else "old_rank_finished"
|
|
),
|
|
rank,
|
|
)
|
|
for rank in range(prepared.cur_data_parallel_size)
|
|
]
|
|
loop = asyncio.get_running_loop()
|
|
start = loop.time()
|
|
backoff_step = 0
|
|
while True:
|
|
error = self._get_error(prepared.control_store, prepared.epoch)
|
|
if error is not None:
|
|
raise RuntimeError(error)
|
|
if prepared.reconfig_store.check(keys):
|
|
return
|
|
if loop.time() - start > timeout_s:
|
|
raise TimeoutError(
|
|
"Timed out waiting for all old ranks to finish external "
|
|
"Elastic EP scaling."
|
|
)
|
|
backoff_step = await self._sleep_with_backoff(backoff_step)
|
|
|
|
@staticmethod
|
|
def _stop_handshake_server(
|
|
server: ExternalElasticEPScaleUpHandshakeServer | None,
|
|
suppress_errors: bool,
|
|
) -> None:
|
|
if server is None:
|
|
return
|
|
if suppress_errors:
|
|
with contextlib.suppress(Exception):
|
|
server.stop()
|
|
else:
|
|
server.stop()
|
|
|
|
def _publish_error(
|
|
self,
|
|
prepared: _PreparedExternalElasticEPScale,
|
|
error: BaseException,
|
|
) -> None:
|
|
error_key = self.key(prepared.epoch, "error")
|
|
error_payload = (str(error) or type(error).__name__).encode()
|
|
for store in (prepared.control_store, prepared.reconfig_store):
|
|
with contextlib.suppress(Exception):
|
|
store.set(error_key, error_payload)
|
|
self._set_phase(
|
|
store, prepared.epoch, ExternalElasticEPScalePhase.FAILED
|
|
)
|
|
|
|
def abort(self, error: str) -> None:
|
|
prepared = self.prepared_scale
|
|
if prepared is not None:
|
|
self._publish_error(prepared, RuntimeError(error))
|
|
self._stop_handshake_server(prepared.handshake_server, suppress_errors=True)
|
|
self.prepared_scale = None
|
|
|
|
async def _wait_for_commit_ready(
|
|
self, prepared: _PreparedExternalElasticEPScale, timeout_s: float = 600
|
|
) -> None:
|
|
# No engine may enter the collective pause until all old API ranks
|
|
# have finished draining; a drain failure must reach waiting peers.
|
|
store = prepared.reconfig_store
|
|
await asyncio.to_thread(
|
|
store.set, self.key(prepared.epoch, "commit_ready", prepared.dp_rank), b"1"
|
|
)
|
|
keys = [
|
|
self.key(prepared.epoch, "commit_ready", rank)
|
|
for rank in range(prepared.cur_data_parallel_size)
|
|
]
|
|
|
|
def is_ready() -> bool:
|
|
error = self._get_error(prepared.control_store, prepared.epoch)
|
|
if error is not None:
|
|
raise RuntimeError(error)
|
|
return store.check(keys)
|
|
|
|
start = asyncio.get_running_loop().time()
|
|
backoff_step = 0
|
|
while True:
|
|
if await asyncio.to_thread(is_ready):
|
|
return
|
|
if asyncio.get_running_loop().time() - start > timeout_s:
|
|
raise TimeoutError("Timed out waiting for external Elastic EP drains.")
|
|
backoff_step = await self._sleep_with_backoff(backoff_step)
|
|
|
|
async def prepare(
|
|
self, cur_data_parallel_size: int, new_data_parallel_size: int
|
|
) -> None:
|
|
from vllm.distributed.utils import get_cached_tcp_store_client
|
|
|
|
if self.prepared_scale is not None:
|
|
if (
|
|
self.prepared_scale.bootstrap.new_data_parallel_size
|
|
== new_data_parallel_size
|
|
):
|
|
return
|
|
raise RuntimeError("External Elastic EP scaling is already prepared.")
|
|
|
|
parallel_config = self.client.vllm_config.parallel_config
|
|
dp_rank = parallel_config.data_parallel_rank
|
|
scale_up = new_data_parallel_size > cur_data_parallel_size
|
|
if new_data_parallel_size > parallel_config.elastic_ep_max_dp_size:
|
|
raise ValueError(
|
|
f"Cannot scale to data_parallel_size {new_data_parallel_size}; "
|
|
"--elastic-ep-max-dp-size is "
|
|
f"{parallel_config.elastic_ep_max_dp_size}."
|
|
)
|
|
# Validate expert capacity on every rank: rank 0 publishes no epoch
|
|
# when validation fails, so peers must reject before polling for one.
|
|
num_redundant_experts = self._calculate_num_redundant_experts(
|
|
cur_data_parallel_size,
|
|
new_data_parallel_size,
|
|
)
|
|
if not parallel_config._coord_store_port:
|
|
raise RuntimeError(
|
|
"External Elastic EP requires a runtime coordination store port."
|
|
)
|
|
control_store = get_cached_tcp_store_client(
|
|
parallel_config.data_parallel_master_ip,
|
|
parallel_config._coord_store_port,
|
|
)
|
|
handshake_server: ExternalElasticEPScaleUpHandshakeServer | None = None
|
|
bootstrap: ReconfigureDistributedRequest | None = None
|
|
epoch: str | None = None
|
|
reconfig_store: Any | None = None
|
|
|
|
try:
|
|
if dp_rank == 0:
|
|
epoch, bootstrap = self._prepare_reconfig_bootstrap(
|
|
control_store,
|
|
cur_data_parallel_size,
|
|
new_data_parallel_size,
|
|
)
|
|
control_store.set(
|
|
self.key(epoch, "num_redundant_experts"),
|
|
str(num_redundant_experts).encode(),
|
|
)
|
|
if scale_up:
|
|
handshake_server = self._start_scale_up_handshake_server(
|
|
bootstrap,
|
|
cur_data_parallel_size,
|
|
num_redundant_experts,
|
|
parallel_config.elastic_ep_max_dp_size,
|
|
)
|
|
control_store.set(self.key(epoch, "prepared"), b"1")
|
|
else:
|
|
epoch, bootstrap = await self._wait_for_bootstrap(
|
|
control_store, new_data_parallel_size
|
|
)
|
|
num_redundant_experts = int(
|
|
control_store.get(self.key(epoch, "num_redundant_experts")).decode()
|
|
)
|
|
|
|
assert epoch is not None
|
|
assert bootstrap is not None
|
|
self.active_epoch = epoch
|
|
self.active_reconfig_store = (
|
|
bootstrap.new_data_parallel_master_ip,
|
|
bootstrap.coord_store_port,
|
|
)
|
|
reconfig_store = self._get_reconfig_store()
|
|
reconfig_store.set(self.key("current_epoch"), epoch.encode())
|
|
self._set_phase(
|
|
reconfig_store, epoch, ExternalElasticEPScalePhase.PREPARING
|
|
)
|
|
|
|
if scale_up or dp_rank < bootstrap.new_data_parallel_size:
|
|
await self.client.call_utility_async(
|
|
"reinitialize_distributed", bootstrap
|
|
)
|
|
|
|
await self._wait_for_ready_ranks(
|
|
control_store,
|
|
reconfig_store,
|
|
epoch,
|
|
bootstrap.new_data_parallel_size,
|
|
handshake_server,
|
|
)
|
|
self.prepared_scale = _PreparedExternalElasticEPScale(
|
|
control_store=control_store,
|
|
reconfig_store=reconfig_store,
|
|
epoch=epoch,
|
|
bootstrap=bootstrap,
|
|
dp_rank=dp_rank,
|
|
cur_data_parallel_size=cur_data_parallel_size,
|
|
num_redundant_experts=num_redundant_experts,
|
|
scale_up=scale_up,
|
|
handshake_server=handshake_server,
|
|
)
|
|
except BaseException as e:
|
|
if epoch is not None:
|
|
self.active_epoch = epoch
|
|
error_key = self.key(epoch, "error")
|
|
error_payload = (str(e) or type(e).__name__).encode()
|
|
with contextlib.suppress(Exception):
|
|
control_store.set(error_key, error_payload)
|
|
self._set_phase(
|
|
control_store, epoch, ExternalElasticEPScalePhase.FAILED
|
|
)
|
|
failed_reconfig_store = (
|
|
reconfig_store
|
|
if reconfig_store is not None
|
|
else self.reconfig_store_ref
|
|
)
|
|
if failed_reconfig_store is not None:
|
|
with contextlib.suppress(Exception):
|
|
failed_reconfig_store.set(error_key, error_payload)
|
|
self._set_phase(
|
|
failed_reconfig_store,
|
|
epoch,
|
|
ExternalElasticEPScalePhase.FAILED,
|
|
)
|
|
self.active_reconfig_store = None
|
|
self._stop_handshake_server(handshake_server, suppress_errors=True)
|
|
raise
|
|
|
|
async def commit(self) -> None:
|
|
prepared = self.prepared_scale
|
|
if prepared is None:
|
|
raise RuntimeError("External Elastic EP scaling has not been prepared.")
|
|
|
|
bootstrap = prepared.bootstrap
|
|
remaining = prepared.scale_up or (
|
|
prepared.dp_rank < bootstrap.new_data_parallel_size
|
|
)
|
|
try:
|
|
await self._wait_for_commit_ready(prepared)
|
|
for store in (prepared.control_store, prepared.reconfig_store):
|
|
self._set_phase(
|
|
store, prepared.epoch, ExternalElasticEPScalePhase.COMMITTING
|
|
)
|
|
await self.client.pause_scheduler_async(
|
|
mode=self.client._eep_commit_pause_mode() if remaining else "abort",
|
|
clear_cache=False,
|
|
)
|
|
if remaining:
|
|
await self.client.call_utility_async("commit_prepared_elastic_ep")
|
|
else:
|
|
reconfig_request = msgspec.structs.replace(
|
|
bootstrap,
|
|
new_data_parallel_rank=ReconfigureRankType.SHUTDOWN_CURRENT_RANK,
|
|
)
|
|
await self.client.call_utility_async(
|
|
"reinitialize_distributed", reconfig_request
|
|
)
|
|
|
|
await self._wait_for_local_reconfig_finished(
|
|
prepared.control_store,
|
|
prepared.reconfig_store,
|
|
prepared.epoch,
|
|
bootstrap,
|
|
prepared.dp_rank,
|
|
prepared.scale_up,
|
|
)
|
|
if prepared.dp_rank == 0:
|
|
await self._wait_for_all_old_ranks(prepared)
|
|
completed_key = self.key(prepared.epoch, "completed")
|
|
prepared.control_store.set(completed_key, b"1")
|
|
prepared.reconfig_store.set(completed_key, b"1")
|
|
for store in (prepared.control_store, prepared.reconfig_store):
|
|
self._set_phase(
|
|
store, prepared.epoch, ExternalElasticEPScalePhase.COMPLETED
|
|
)
|
|
|
|
if remaining:
|
|
self._update_parallel_config(bootstrap, prepared.num_redundant_experts)
|
|
if prepared.dp_rank == 0:
|
|
self.client._coord_store = self.reconfig_store_ref
|
|
self.client._ensure_stats_update_task()
|
|
await self.client.scale_send_socket.send(
|
|
msgspec.msgpack.encode(
|
|
("SCALE_ELASTIC_EP", bootstrap.new_data_parallel_size)
|
|
)
|
|
)
|
|
await self.client.resume_scheduler_async()
|
|
self._stop_handshake_server(
|
|
prepared.handshake_server, suppress_errors=False
|
|
)
|
|
except BaseException as e:
|
|
self._publish_error(prepared, e)
|
|
self._stop_handshake_server(prepared.handshake_server, suppress_errors=True)
|
|
raise
|
|
finally:
|
|
self.prepared_scale = None
|
|
|
|
async def process_engine_core_notification(
|
|
self, notification_data: tuple[str, int]
|
|
) -> None:
|
|
"""Record scale notifications emitted by EngineCore processes.
|
|
The stored keys are later polled by the external scale coordinator."""
|
|
parallel_config = self.client.vllm_config.parallel_config
|
|
if not (
|
|
parallel_config.enable_elastic_ep
|
|
and parallel_config.data_parallel_external_lb
|
|
):
|
|
return
|
|
|
|
notification_type_str, dp_rank = notification_data
|
|
notification_type = EEPNotificationType(notification_type_str)
|
|
if not parallel_config._coord_store_port:
|
|
logger.warning(
|
|
"Ignoring external Elastic EP notification %s because coord "
|
|
"store metadata is not available yet.",
|
|
notification_type.value,
|
|
)
|
|
return
|
|
|
|
epoch = self.active_epoch
|
|
reconfig_store = self._get_reconfig_store()
|
|
if epoch is None:
|
|
current_epoch_key = self.key("current_epoch")
|
|
if not reconfig_store.check([current_epoch_key]):
|
|
logger.warning(
|
|
"Ignoring external Elastic EP notification %s because "
|
|
"active epoch metadata is not available yet.",
|
|
notification_type.value,
|
|
)
|
|
return
|
|
epoch = reconfig_store.get(current_epoch_key).decode()
|
|
|
|
if notification_type == EEPNotificationType.RECONFIGURE_FINISHED:
|
|
key = self.key(epoch, "old_rank_finished", dp_rank)
|
|
reconfig_store.set(key, b"1")
|
|
elif notification_type == EEPNotificationType.SHUTDOWN_COMPLETE:
|
|
key = self.key(epoch, "shutdown_complete", dp_rank)
|
|
reconfig_store.set(key, b"1")
|