Files
a94c315711 [DP] Engines own wave wake-ups; drop the front-end FIRST_REQ (#59559)
Signed-off-by: fh3 <a@b>
Co-authored-by: fh3 <a@b>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 19:41:54 -04:00

898 lines
36 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import contextlib
import enum
import uuid
from dataclasses import dataclass
from threading import Event, Thread
from typing import TYPE_CHECKING, Any
import msgspec
import msgspec.msgpack
import zmq
from vllm.logger import init_logger
from vllm.utils.network_utils import make_zmq_socket
from vllm.v1.engine import (
EEPNotificationType,
ReconfigureDistributedRequest,
ReconfigureRankType,
)
from vllm.v1.engine.utils import EngineHandshakeMetadata, EngineZmqAddresses
from vllm.v1.utils import get_engine_client_zmq_addr
if TYPE_CHECKING:
from vllm.v1.engine.core_client import DPAsyncMPClient
logger = init_logger(__name__)
POLL_BACKOFF_STEPS_S = (0.1, 0.2, 0.4, 0.8)
class ExternalElasticEPScalePhase(str, enum.Enum):
"""Lifecycle phases shared for an epoch-scoped external EEP operation.
These values are stored in the control and reconfiguration stores and
exposed through the status API so external orchestrators can observe the
same operation across API processes and reconnects. COMPLETED means the
topology commit and rank completion barrier have finished; API processes
may clear their process-local request gate shortly afterward.
"""
IDLE = "idle"
PREPARING = "preparing"
COMMITTING = "committing"
FAILED = "failed"
COMPLETED = "completed"
@dataclass
class ExternalElasticEPScaleUpHandshakeMetadata:
engine_metadata: EngineHandshakeMetadata
num_redundant_experts: int
elastic_ep_max_dp_size: int
class ExternalElasticEPScaleUpHandshakeServer:
"""Temporary rank-0 handshake server for external EEP scale-up.
During normal external startup the global handshake listener only exists
while the rank starts. Scale-up needs the same handshake contract again for
newly launched ranks, so rank 0 re-opens a temporary listener for the
duration of the current scale operation.
The handshake gives new ranks the information they cannot infer locally:
front-end and DP coordinator ZMQ addresses, the new DP master address/ports,
target DP size, target expert redundancy, and the coordination store port
used for EEP reconfiguration.
"""
def __init__(
self,
*,
handshake_address: str,
expected_new_ranks: list[int],
addresses: EngineZmqAddresses,
bootstrap: ReconfigureDistributedRequest,
num_redundant_experts: int,
elastic_ep_max_dp_size: int,
) -> None:
self.handshake_address = handshake_address
self.expected_new_ranks = set(expected_new_ranks)
self.addresses = addresses
self.bootstrap = bootstrap
self.num_redundant_experts = num_redundant_experts
self.elastic_ep_max_dp_size = elastic_ep_max_dp_size
self.started_event = Event()
self._stop_event = Event()
self._thread = Thread(
target=self._run,
name="ExternalElasticEPHandshakeServer",
daemon=True,
)
self._error: Exception | None = None
def start(self) -> None:
self._thread.start()
started = self.started_event.wait(timeout=5)
if self._error is not None:
raise self._error
if not started:
raise TimeoutError(
"Timed out waiting for external EEP handshake server to "
f"start listening at {self.handshake_address}"
)
def stop(self) -> None:
self._stop_event.set()
if self._thread.is_alive():
self._thread.join(timeout=5)
if self._error is not None:
raise self._error
def raise_if_failed(self) -> None:
if self._error is not None:
raise self._error
def _run(self) -> None:
pending_ready_ranks = self.expected_new_ranks.copy()
try:
with (
zmq.Context() as ctx,
make_zmq_socket(
ctx,
self.handshake_address,
zmq.ROUTER,
bind=True,
linger=0,
router_handover=True,
) as handshake_socket,
):
self.started_event.set()
poller = zmq.Poller()
poller.register(handshake_socket, zmq.POLLIN)
while not self._stop_event.is_set():
if not pending_ready_ranks:
return
events = dict(poller.poll(timeout=1000))
if handshake_socket not in events:
continue
engine_identity, payload = handshake_socket.recv_multipart()
engine_rank = int.from_bytes(engine_identity, "little")
if engine_rank not in self.expected_new_ranks:
raise RuntimeError(
"Received scale-up handshake from unexpected "
f"dp rank {engine_rank}"
)
message = msgspec.msgpack.decode(payload)
status = message["status"]
if status == "HELLO":
b = self.bootstrap
parallel_config: dict[str, int | str | list[int]] = {
"data_parallel_master_ip": b.new_data_parallel_master_ip,
"data_parallel_master_port": (
b.new_data_parallel_master_port
),
"_data_parallel_master_port_list": (
b.new_data_parallel_master_port_list
),
"data_parallel_size": b.new_data_parallel_size,
"_coord_store_port": b.coord_store_port,
}
init_message = msgspec.msgpack.encode(
ExternalElasticEPScaleUpHandshakeMetadata(
engine_metadata=EngineHandshakeMetadata(
addresses=self.addresses,
parallel_config=parallel_config,
),
num_redundant_experts=self.num_redundant_experts,
elastic_ep_max_dp_size=self.elastic_ep_max_dp_size,
)
)
handshake_socket.send_multipart(
(engine_identity, init_message), copy=False
)
elif status == "READY":
pending_ready_ranks.discard(engine_rank)
else:
raise RuntimeError(
f"Unexpected handshake status {status} from dp rank "
f"{engine_rank}"
)
except Exception as e:
self._error = e
self.started_event.set()
@dataclass
class _PreparedExternalElasticEPScale:
control_store: Any
reconfig_store: Any
epoch: str
bootstrap: ReconfigureDistributedRequest
dp_rank: int
cur_data_parallel_size: int
num_redundant_experts: int
scale_up: bool
handshake_server: ExternalElasticEPScaleUpHandshakeServer | None
class ExternalElasticEPScaleCoordinator:
def __init__(self, client: "DPAsyncMPClient") -> None:
self.client = client
self.active_reconfig_store: tuple[str, int] | None = None
# Keep the previous rank-0 TCPStore server alive while the next scale
# operation publishes bootstrap metadata through it. Overwriting
# client._coord_store with the new store can otherwise close the old
# server while other ranks are still polling it.
self.control_store_ref: Any | None = None
self.reconfig_store_ref: Any | None = None
self.active_epoch: str | None = None
self.prepared_scale: _PreparedExternalElasticEPScale | None = None
@staticmethod
def key(*parts: str | int) -> str:
return "/".join(["elastic_ep/external", *[str(part) for part in parts]])
@staticmethod
async def _sleep_with_backoff(backoff_step: int) -> int:
max_step = len(POLL_BACKOFF_STEPS_S) - 1
backoff_step = max(0, min(backoff_step, max_step))
await asyncio.sleep(POLL_BACKOFF_STEPS_S[backoff_step])
return min(backoff_step + 1, max_step)
def _update_parallel_config(
self,
bootstrap: ReconfigureDistributedRequest,
num_redundant_experts: int,
) -> None:
parallel_config = self.client.vllm_config.parallel_config
parallel_config.data_parallel_size = bootstrap.new_data_parallel_size
parallel_config.data_parallel_master_ip = bootstrap.new_data_parallel_master_ip
parallel_config.data_parallel_master_port = (
bootstrap.new_data_parallel_master_port
)
parallel_config._data_parallel_master_port_list = (
bootstrap.new_data_parallel_master_port_list.copy()
)
parallel_config._coord_store_port = bootstrap.coord_store_port
parallel_config.eplb_config.num_redundant_experts = num_redundant_experts
def _calculate_num_redundant_experts(
self,
cur_data_parallel_size: int,
new_data_parallel_size: int,
) -> int:
parallel_config = self.client.vllm_config.parallel_config
num_experts = self.client.vllm_config.model_config.get_num_experts()
num_redundant_experts = (
num_experts + parallel_config.eplb_config.num_redundant_experts
) * new_data_parallel_size // cur_data_parallel_size - num_experts
if num_redundant_experts < 0:
raise ValueError(
"Cannot scale external Elastic EP from DP size "
f"{cur_data_parallel_size} to {new_data_parallel_size}: "
"the target topology cannot retain all model experts."
)
return num_redundant_experts
def _get_reconfig_store(self):
from vllm.distributed.utils import get_cached_tcp_store_client
if self.active_reconfig_store is not None:
store_addr = self.active_reconfig_store
else:
parallel_config = self.client.vllm_config.parallel_config
if not parallel_config._coord_store_port:
raise RuntimeError(
"External Elastic EP requires an active reconfiguration "
"coordination store."
)
store_addr = (
parallel_config.data_parallel_master_ip,
parallel_config._coord_store_port,
)
return get_cached_tcp_store_client(*store_addr)
def _get_existing_engine_zmq_address(self) -> EngineZmqAddresses:
coordinator_input = self.client.coordinator_input_address
coordinator_output = self.client.coordinator_output_address
if coordinator_input is None or coordinator_output is None:
raise RuntimeError(
"External Elastic EP scale-up requires DP coordinator addresses."
)
return EngineZmqAddresses(
inputs=[self.client.input_address],
outputs=[self.client.output_address],
coordinator_input=coordinator_input,
coordinator_output=coordinator_output,
frontend_stats_publish_address=self.client.stats_update_address,
)
def _setup_reconfig_bootstrap(self) -> tuple[str, int]:
from vllm.distributed.utils import create_tcp_store
self.control_store_ref = getattr(self.client, "_coord_store", None)
ip = self.client.vllm_config.parallel_config.data_parallel_master_ip
# Keep the serving configuration on the old store until commit succeeds.
self.reconfig_store_ref = create_tcp_store(
ip, 0, is_master=True, world_size=-1, wait_for_workers=False
)
return ip, self.reconfig_store_ref.port
def _get_error(self, store: Any, epoch: str) -> str | None:
error_key = self.key(epoch, "error")
if not store.check([error_key]):
return None
return store.get(error_key).decode()
def get_status(self) -> dict[str, str | int | None]:
store = self._get_reconfig_store()
current_epoch_key = self.key("current_epoch")
if not store.check([current_epoch_key]):
return {
"phase": ExternalElasticEPScalePhase.IDLE.value,
"epoch": None,
"error": None,
"requested_data_parallel_size": None,
}
epoch = store.get(current_epoch_key).decode()
error = self._get_error(store, epoch)
if error is not None:
phase = ExternalElasticEPScalePhase.FAILED
elif store.check([self.key(epoch, "completed")]):
phase = ExternalElasticEPScalePhase.COMPLETED
elif store.check([self.key(epoch, "phase")]):
phase = ExternalElasticEPScalePhase(
store.get(self.key(epoch, "phase")).decode()
)
else:
phase = ExternalElasticEPScalePhase.IDLE
requested_dp_size = None
bootstrap_key = self.key(epoch, "bootstrap")
if store.check([bootstrap_key]):
bootstrap = msgspec.msgpack.decode(
store.get(bootstrap_key), type=ReconfigureDistributedRequest
)
requested_dp_size = bootstrap.new_data_parallel_size
return {
"phase": phase.value,
"epoch": epoch,
"error": error,
"requested_data_parallel_size": requested_dp_size,
}
def _set_phase(
self, store: Any, epoch: str, phase: ExternalElasticEPScalePhase
) -> None:
store.set(self.key(epoch, "phase"), phase.value.encode())
async def _wait_for_bootstrap(
self,
store: Any,
requested_new_dp_size: int,
timeout_s: float = 300,
) -> tuple[str, ReconfigureDistributedRequest]:
"""Wait for rank 0 to publish matching scale bootstrap metadata.
Non-zero ranks poll the old control store until prepared or errored."""
loop = asyncio.get_running_loop()
start = loop.time()
current_epoch_key = self.key("current_epoch")
previous_epoch = self.active_epoch
if store.check([current_epoch_key]):
current_epoch = store.get(current_epoch_key).decode()
if store.check([self.key(current_epoch, "completed")]):
previous_epoch = current_epoch
backoff_step = 0
while True:
if store.check([current_epoch_key]):
epoch = store.get(current_epoch_key).decode()
if epoch != previous_epoch:
error = self._get_error(store, epoch)
if error is not None:
self.active_epoch = epoch
raise RuntimeError(error)
bootstrap_key = self.key(epoch, "bootstrap")
if store.check([bootstrap_key]):
bootstrap = msgspec.msgpack.decode(
store.get(bootstrap_key),
type=ReconfigureDistributedRequest,
)
prepared_key = self.key(epoch, "prepared")
completed = store.check([self.key(epoch, "completed")])
if bootstrap.new_data_parallel_size != requested_new_dp_size:
if store.check([prepared_key]) and not completed:
raise RuntimeError(
"A different external Elastic EP scaling "
"operation is already in progress for target "
f"dp size {bootstrap.new_data_parallel_size}."
)
elif store.check([prepared_key]):
return epoch, bootstrap
now = loop.time()
if now - start > timeout_s:
raise TimeoutError(
"Timed out waiting for rank 0 to publish external Elastic EP "
"bootstrap metadata."
)
backoff_step = await self._sleep_with_backoff(backoff_step)
def _prepare_reconfig_bootstrap(
self,
store: Any,
cur_data_parallel_size: int,
new_data_parallel_size: int,
) -> tuple[str, ReconfigureDistributedRequest]:
from vllm.utils.network_utils import get_open_ports_list
current_epoch_key = self.key("current_epoch")
if store.check([current_epoch_key]):
current_epoch = store.get(current_epoch_key).decode()
current_error = self._get_error(store, current_epoch)
if current_error is None and not store.check(
[self.key(current_epoch, "completed")]
):
raise RuntimeError(
"Another external Elastic EP scaling operation is already active."
)
ip, coord_store_port = self._setup_reconfig_bootstrap()
epoch = uuid.uuid4().hex
master_ports = get_open_ports_list(5)
bootstrap = ReconfigureDistributedRequest(
new_data_parallel_size=new_data_parallel_size,
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_master_ip=ip,
new_data_parallel_master_port=master_ports.pop(),
new_data_parallel_master_port_list=master_ports,
coord_store_port=coord_store_port,
)
bootstrap_key = self.key(epoch, "bootstrap")
bootstrap_payload = msgspec.msgpack.encode(bootstrap)
stores = [store]
if self.reconfig_store_ref is not None:
stores.append(self.reconfig_store_ref)
for target_store in stores:
target_store.set(bootstrap_key, bootstrap_payload)
self._set_phase(target_store, epoch, ExternalElasticEPScalePhase.PREPARING)
# Publish the epoch only after its metadata is ready for readers.
target_store.set(current_epoch_key, epoch.encode())
return epoch, bootstrap
def _start_scale_up_handshake_server(
self,
bootstrap: ReconfigureDistributedRequest,
cur_data_parallel_size: int,
num_redundant_experts: int,
elastic_ep_max_dp_size: int,
) -> ExternalElasticEPScaleUpHandshakeServer:
handshake_server = ExternalElasticEPScaleUpHandshakeServer(
handshake_address=get_engine_client_zmq_addr(
False,
bootstrap.new_data_parallel_master_ip,
self.client.vllm_config.parallel_config.data_parallel_rpc_port,
),
expected_new_ranks=list(
range(
cur_data_parallel_size,
bootstrap.new_data_parallel_size,
)
),
addresses=self._get_existing_engine_zmq_address(),
bootstrap=bootstrap,
num_redundant_experts=num_redundant_experts,
elastic_ep_max_dp_size=elastic_ep_max_dp_size,
)
handshake_server.start()
return handshake_server
async def _wait_for_ready_ranks(
self,
control_store: Any,
reconfig_store: Any,
epoch: str,
new_data_parallel_size: int,
handshake_server: ExternalElasticEPScaleUpHandshakeServer | None = None,
timeout_s: float = 300,
) -> None:
"""Wait until every rank in the new DP group is ready to commit."""
loop = asyncio.get_running_loop()
start = loop.time()
ready_keys = [f"eep_ready/{rank}" for rank in range(new_data_parallel_size)]
backoff_step = 0
while True:
if handshake_server is not None:
handshake_server.raise_if_failed()
error = self._get_error(control_store, epoch)
if error is not None:
raise RuntimeError(
"External Elastic EP scaling failed while waiting for "
f"ranks to prepare: {error}"
)
if reconfig_store.check(ready_keys):
return
now = loop.time()
if now - start > timeout_s:
raise TimeoutError(
"Timed out waiting for external Elastic EP ranks to prepare."
)
backoff_step = await self._sleep_with_backoff(backoff_step)
async def _wait_for_local_reconfig_finished(
self,
control_store: Any,
reconfig_store: Any,
epoch: str,
bootstrap: ReconfigureDistributedRequest,
dp_rank: int,
scale_up: bool,
timeout_s: float = 600,
) -> None:
"""Wait until this rank's EngineCore finishes the scale transition."""
if not scale_up and dp_rank >= bootstrap.new_data_parallel_size:
wait_key = self.key(epoch, "shutdown_complete", dp_rank)
timeout_msg = (
"Timed out waiting for local external Elastic EP scale-down "
"shutdown to finish."
)
else:
wait_key = self.key(epoch, "old_rank_finished", dp_rank)
timeout_msg = (
"Timed out waiting for local external Elastic EP "
"reconfiguration to finish."
)
loop = asyncio.get_running_loop()
start = loop.time()
backoff_step = 0
while True:
error = self._get_error(control_store, epoch)
if error is not None:
raise RuntimeError(
error or "External Elastic EP scaling failed on another rank."
)
if reconfig_store.check([wait_key]):
return
now = loop.time()
if now - start > timeout_s:
raise TimeoutError(timeout_msg)
backoff_step = await self._sleep_with_backoff(backoff_step)
async def _wait_for_all_old_ranks(
self,
prepared: _PreparedExternalElasticEPScale,
timeout_s: float = 600,
) -> None:
bootstrap = prepared.bootstrap
keys = [
self.key(
prepared.epoch,
(
"shutdown_complete"
if not prepared.scale_up
and rank >= bootstrap.new_data_parallel_size
else "old_rank_finished"
),
rank,
)
for rank in range(prepared.cur_data_parallel_size)
]
loop = asyncio.get_running_loop()
start = loop.time()
backoff_step = 0
while True:
error = self._get_error(prepared.control_store, prepared.epoch)
if error is not None:
raise RuntimeError(error)
if prepared.reconfig_store.check(keys):
return
if loop.time() - start > timeout_s:
raise TimeoutError(
"Timed out waiting for all old ranks to finish external "
"Elastic EP scaling."
)
backoff_step = await self._sleep_with_backoff(backoff_step)
@staticmethod
def _stop_handshake_server(
server: ExternalElasticEPScaleUpHandshakeServer | None,
suppress_errors: bool,
) -> None:
if server is None:
return
if suppress_errors:
with contextlib.suppress(Exception):
server.stop()
else:
server.stop()
def _publish_error(
self,
prepared: _PreparedExternalElasticEPScale,
error: BaseException,
) -> None:
error_key = self.key(prepared.epoch, "error")
error_payload = (str(error) or type(error).__name__).encode()
for store in (prepared.control_store, prepared.reconfig_store):
with contextlib.suppress(Exception):
store.set(error_key, error_payload)
self._set_phase(
store, prepared.epoch, ExternalElasticEPScalePhase.FAILED
)
def abort(self, error: str) -> None:
prepared = self.prepared_scale
if prepared is not None:
self._publish_error(prepared, RuntimeError(error))
self._stop_handshake_server(prepared.handshake_server, suppress_errors=True)
self.prepared_scale = None
async def _wait_for_commit_ready(
self, prepared: _PreparedExternalElasticEPScale, timeout_s: float = 600
) -> None:
# No engine may enter the collective pause until all old API ranks
# have finished draining; a drain failure must reach waiting peers.
store = prepared.reconfig_store
await asyncio.to_thread(
store.set, self.key(prepared.epoch, "commit_ready", prepared.dp_rank), b"1"
)
keys = [
self.key(prepared.epoch, "commit_ready", rank)
for rank in range(prepared.cur_data_parallel_size)
]
def is_ready() -> bool:
error = self._get_error(prepared.control_store, prepared.epoch)
if error is not None:
raise RuntimeError(error)
return store.check(keys)
start = asyncio.get_running_loop().time()
backoff_step = 0
while True:
if await asyncio.to_thread(is_ready):
return
if asyncio.get_running_loop().time() - start > timeout_s:
raise TimeoutError("Timed out waiting for external Elastic EP drains.")
backoff_step = await self._sleep_with_backoff(backoff_step)
async def prepare(
self, cur_data_parallel_size: int, new_data_parallel_size: int
) -> None:
from vllm.distributed.utils import get_cached_tcp_store_client
if self.prepared_scale is not None:
if (
self.prepared_scale.bootstrap.new_data_parallel_size
== new_data_parallel_size
):
return
raise RuntimeError("External Elastic EP scaling is already prepared.")
parallel_config = self.client.vllm_config.parallel_config
dp_rank = parallel_config.data_parallel_rank
scale_up = new_data_parallel_size > cur_data_parallel_size
if new_data_parallel_size > parallel_config.elastic_ep_max_dp_size:
raise ValueError(
f"Cannot scale to data_parallel_size {new_data_parallel_size}; "
"--elastic-ep-max-dp-size is "
f"{parallel_config.elastic_ep_max_dp_size}."
)
# Validate expert capacity on every rank: rank 0 publishes no epoch
# when validation fails, so peers must reject before polling for one.
num_redundant_experts = self._calculate_num_redundant_experts(
cur_data_parallel_size,
new_data_parallel_size,
)
if not parallel_config._coord_store_port:
raise RuntimeError(
"External Elastic EP requires a runtime coordination store port."
)
control_store = get_cached_tcp_store_client(
parallel_config.data_parallel_master_ip,
parallel_config._coord_store_port,
)
handshake_server: ExternalElasticEPScaleUpHandshakeServer | None = None
bootstrap: ReconfigureDistributedRequest | None = None
epoch: str | None = None
reconfig_store: Any | None = None
try:
if dp_rank == 0:
epoch, bootstrap = self._prepare_reconfig_bootstrap(
control_store,
cur_data_parallel_size,
new_data_parallel_size,
)
control_store.set(
self.key(epoch, "num_redundant_experts"),
str(num_redundant_experts).encode(),
)
if scale_up:
handshake_server = self._start_scale_up_handshake_server(
bootstrap,
cur_data_parallel_size,
num_redundant_experts,
parallel_config.elastic_ep_max_dp_size,
)
control_store.set(self.key(epoch, "prepared"), b"1")
else:
epoch, bootstrap = await self._wait_for_bootstrap(
control_store, new_data_parallel_size
)
num_redundant_experts = int(
control_store.get(self.key(epoch, "num_redundant_experts")).decode()
)
assert epoch is not None
assert bootstrap is not None
self.active_epoch = epoch
self.active_reconfig_store = (
bootstrap.new_data_parallel_master_ip,
bootstrap.coord_store_port,
)
reconfig_store = self._get_reconfig_store()
reconfig_store.set(self.key("current_epoch"), epoch.encode())
self._set_phase(
reconfig_store, epoch, ExternalElasticEPScalePhase.PREPARING
)
if scale_up or dp_rank < bootstrap.new_data_parallel_size:
await self.client.call_utility_async(
"reinitialize_distributed", bootstrap
)
await self._wait_for_ready_ranks(
control_store,
reconfig_store,
epoch,
bootstrap.new_data_parallel_size,
handshake_server,
)
self.prepared_scale = _PreparedExternalElasticEPScale(
control_store=control_store,
reconfig_store=reconfig_store,
epoch=epoch,
bootstrap=bootstrap,
dp_rank=dp_rank,
cur_data_parallel_size=cur_data_parallel_size,
num_redundant_experts=num_redundant_experts,
scale_up=scale_up,
handshake_server=handshake_server,
)
except BaseException as e:
if epoch is not None:
self.active_epoch = epoch
error_key = self.key(epoch, "error")
error_payload = (str(e) or type(e).__name__).encode()
with contextlib.suppress(Exception):
control_store.set(error_key, error_payload)
self._set_phase(
control_store, epoch, ExternalElasticEPScalePhase.FAILED
)
failed_reconfig_store = (
reconfig_store
if reconfig_store is not None
else self.reconfig_store_ref
)
if failed_reconfig_store is not None:
with contextlib.suppress(Exception):
failed_reconfig_store.set(error_key, error_payload)
self._set_phase(
failed_reconfig_store,
epoch,
ExternalElasticEPScalePhase.FAILED,
)
self.active_reconfig_store = None
self._stop_handshake_server(handshake_server, suppress_errors=True)
raise
async def commit(self) -> None:
prepared = self.prepared_scale
if prepared is None:
raise RuntimeError("External Elastic EP scaling has not been prepared.")
bootstrap = prepared.bootstrap
remaining = prepared.scale_up or (
prepared.dp_rank < bootstrap.new_data_parallel_size
)
try:
await self._wait_for_commit_ready(prepared)
for store in (prepared.control_store, prepared.reconfig_store):
self._set_phase(
store, prepared.epoch, ExternalElasticEPScalePhase.COMMITTING
)
await self.client.pause_scheduler_async(
mode=self.client._eep_commit_pause_mode() if remaining else "abort",
clear_cache=False,
)
if remaining:
await self.client.call_utility_async("commit_prepared_elastic_ep")
else:
reconfig_request = msgspec.structs.replace(
bootstrap,
new_data_parallel_rank=ReconfigureRankType.SHUTDOWN_CURRENT_RANK,
)
await self.client.call_utility_async(
"reinitialize_distributed", reconfig_request
)
await self._wait_for_local_reconfig_finished(
prepared.control_store,
prepared.reconfig_store,
prepared.epoch,
bootstrap,
prepared.dp_rank,
prepared.scale_up,
)
if prepared.dp_rank == 0:
await self._wait_for_all_old_ranks(prepared)
completed_key = self.key(prepared.epoch, "completed")
prepared.control_store.set(completed_key, b"1")
prepared.reconfig_store.set(completed_key, b"1")
for store in (prepared.control_store, prepared.reconfig_store):
self._set_phase(
store, prepared.epoch, ExternalElasticEPScalePhase.COMPLETED
)
if remaining:
self._update_parallel_config(bootstrap, prepared.num_redundant_experts)
if prepared.dp_rank == 0:
self.client._coord_store = self.reconfig_store_ref
self.client._ensure_stats_update_task()
await self.client.scale_send_socket.send(
msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", bootstrap.new_data_parallel_size)
)
)
await self.client.resume_scheduler_async()
self._stop_handshake_server(
prepared.handshake_server, suppress_errors=False
)
except BaseException as e:
self._publish_error(prepared, e)
self._stop_handshake_server(prepared.handshake_server, suppress_errors=True)
raise
finally:
self.prepared_scale = None
async def process_engine_core_notification(
self, notification_data: tuple[str, int]
) -> None:
"""Record scale notifications emitted by EngineCore processes.
The stored keys are later polled by the external scale coordinator."""
parallel_config = self.client.vllm_config.parallel_config
if not (
parallel_config.enable_elastic_ep
and parallel_config.data_parallel_external_lb
):
return
notification_type_str, dp_rank = notification_data
notification_type = EEPNotificationType(notification_type_str)
if not parallel_config._coord_store_port:
logger.warning(
"Ignoring external Elastic EP notification %s because coord "
"store metadata is not available yet.",
notification_type.value,
)
return
epoch = self.active_epoch
reconfig_store = self._get_reconfig_store()
if epoch is None:
current_epoch_key = self.key("current_epoch")
if not reconfig_store.check([current_epoch_key]):
logger.warning(
"Ignoring external Elastic EP notification %s because "
"active epoch metadata is not available yet.",
notification_type.value,
)
return
epoch = reconfig_store.get(current_epoch_key).decode()
if notification_type == EEPNotificationType.RECONFIGURE_FINISHED:
key = self.key(epoch, "old_rank_finished", dp_rank)
reconfig_store.set(key, b"1")
elif notification_type == EEPNotificationType.SHUTDOWN_COMPLETE:
key = self.key(epoch, "shutdown_complete", dp_rank)
reconfig_store.set(key, b"1")