[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)

Signed-off-by: lqh-hh <liuqihang6@huawei.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: lqh-hh <liuqihang6@huawei.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: 2017403603 <705328646@qq.com>
Co-authored-by: Julien Mancuso <jmancuso@nvidia.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fangyuchu
2026-10-15 21:29:02 -04:00
committed by GitHub
co-authored by lqh-hh Tyler Michael Smith OpenAI Codex 2017403603 Julien Mancuso Itay Alroy Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fb3fec970e
commit dd9e07a25f
18 changed files with 1638 additions and 167 deletions
+3
View File
@@ -104,6 +104,9 @@ class EngineCoreReadyResponse:
supports_draft_weight_updates: bool = False
# Full-attention block size in tokens after initialization, or unavailable.
effective_attention_block_size: int | None = None
coord_store_port: int = 0
coordinator_input_address: str | None = None
coordinator_output_address: str | None = None
class EngineCoreRequest(
+56 -7
View File
@@ -22,7 +22,11 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient, StreamingInput
from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep
from vllm.entrypoints.serve.elastic_ep.middleware import (
get_elastic_ep_rank_retired,
set_elastic_ep_rank_retired,
set_scaling_elastic_ep,
)
from vllm.exceptions import (
GracefulHTTPError,
MaxQueuedTokensError,
@@ -918,6 +922,12 @@ class AsyncLLM(EngineClient):
mm_cache_stats=renderer.stat_mm_cache(),
)
except Exception as e:
if get_elastic_ep_rank_retired() and isinstance(e, EngineDeadError):
logger.info(
"[Elastic EP] Retired rank stopped receiving EngineCore outputs"
)
output_processor.propagate_error(e)
return
logger.exception("AsyncLLM output_handler failed.")
output_processor.propagate_error(e)
@@ -1103,6 +1113,8 @@ class AsyncLLM(EngineClient):
async def check_health(self) -> None:
logger.debug("Called check_health.")
if get_elastic_ep_rank_retired():
return
if self.errored:
raise self.dead_error
@@ -1252,8 +1264,13 @@ class AsyncLLM(EngineClient):
"waiting for requests to drain before scaling"
)
await self.wait_for_requests_to_drain(drain_timeout)
except BaseException:
set_scaling_elastic_ep(False)
except BaseException as e:
if self.vllm_config.parallel_config.data_parallel_external_lb:
await self.engine_core.abort_external_elastic_ep(
str(e) or type(e).__name__
)
else:
set_scaling_elastic_ep(False)
raise
async def scale_elastic_ep(
@@ -1266,7 +1283,8 @@ class AsyncLLM(EngineClient):
async def _scale_elastic_ep(
self, new_data_parallel_size: int, drain_timeout: int
) -> None:
old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
parallel_config = self.vllm_config.parallel_config
old_data_parallel_size = parallel_config.data_parallel_size
if old_data_parallel_size == new_data_parallel_size:
logger.info(
"Data parallel size is already %s, skipping scale",
@@ -1297,9 +1315,36 @@ class AsyncLLM(EngineClient):
if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS:
await self._drain_requests_for_elastic_ep(drain_timeout)
await self.engine_core.commit_elastic_ep()
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
set_scaling_elastic_ep(False)
rank_will_retire = (
parallel_config.data_parallel_external_lb
and new_data_parallel_size < old_data_parallel_size
and parallel_config.data_parallel_rank >= new_data_parallel_size
)
if rank_will_retire:
set_elastic_ep_rank_retired(True)
commit_succeeded = False
try:
await self.engine_core.commit_elastic_ep()
parallel_config.data_parallel_size = new_data_parallel_size
commit_succeeded = True
if rank_will_retire:
logger.info(
"[Elastic EP] Data-parallel rank %s is retired and awaiting "
"external process shutdown",
parallel_config.data_parallel_rank,
)
finally:
if rank_will_retire and not commit_succeeded:
set_elastic_ep_rank_retired(False)
# A failed commit may leave engines partially reconfigured or paused.
if commit_succeeded:
set_scaling_elastic_ep(False)
async def get_external_elastic_ep_status(
self,
) -> dict[str, str | int | None] | None:
return await self.engine_core.get_external_elastic_ep_status()
async def handle_fault(
self, fault_tolerance_request: FaultToleranceRequest
@@ -1323,6 +1368,10 @@ class AsyncLLM(EngineClient):
def errored(self) -> bool:
return self.engine_core.resources.engine_dead or not self.is_running
@property
def should_keep_api_server_alive(self) -> bool:
return get_elastic_ep_rank_retired()
@property
def dead_error(self) -> BaseException:
return EngineDeadError()
+1
View File
@@ -126,6 +126,7 @@ class DPCoordinator:
self.stats_publish_address = front_publish_address
self.coord_in_address = back_publish_address
self.coord_out_address = back_output_address
self._coord_store: object | None = None
self._finalizer = weakref.finalize(self, shutdown, [self.proc])
def get_stats_publish_address(self) -> str:
+50 -9
View File
@@ -1346,6 +1346,16 @@ class EngineCoreProc(EngineCore):
ready_msg["parallel_config_hash"] = (
vllm_config.parallel_config.compute_hash()
)
if vllm_config.parallel_config.enable_elastic_ep:
ready_msg["coord_store_port"] = (
vllm_config.parallel_config._coord_store_port
)
ready_msg["elastic_ep_max_dp_size"] = (
vllm_config.parallel_config.elastic_ep_max_dp_size
)
ready_msg["num_redundant_experts"] = (
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
handshake_socket.send(msgspec.msgpack.encode(ready_msg))
@@ -1376,14 +1386,37 @@ class EngineCoreProc(EngineCore):
f"minutes"
)
init_bytes = handshake_socket.recv()
init_message: EngineHandshakeMetadata = msgspec.msgpack.decode(
init_bytes, type=EngineHandshakeMetadata
)
num_redundant_experts: int | None = None
if (
envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH
and parallel_config is not None
and parallel_config.data_parallel_external_lb
):
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleUpHandshakeMetadata,
)
scale_up_message = msgspec.msgpack.decode(
init_bytes, type=ExternalElasticEPScaleUpHandshakeMetadata
)
init_message = scale_up_message.engine_metadata
num_redundant_experts = scale_up_message.num_redundant_experts
parallel_config.elastic_ep_max_dp_size = (
scale_up_message.elastic_ep_max_dp_size
)
else:
init_message = msgspec.msgpack.decode(
init_bytes, type=EngineHandshakeMetadata
)
logger.debug("Received init message: %s", init_message)
if parallel_config is not None:
for key, value in init_message.parallel_config.items():
setattr(parallel_config, key, value)
if num_redundant_experts is not None:
parallel_config.eplb_config.num_redundant_experts = (
num_redundant_experts
)
return init_message.addresses
@@ -1826,6 +1859,9 @@ class EngineCoreProc(EngineCore):
if self.vllm_config.weight_transfer_config is not None
else False
),
coord_store_port=parallel_config._coord_store_port,
coordinator_input_address=self.addresses.coordinator_input,
coordinator_output_address=self.addresses.coordinator_output,
)
def process_input_sockets(
@@ -2342,6 +2378,13 @@ class DPEngineCoreProc(EngineCoreProc):
if state.is_complete():
if state.worker_type == "removing":
raise SystemExit
if (
state.worker_type == "new"
and self.vllm_config.parallel_config.data_parallel_external_lb
):
# An independently launched rank has no external utility
# caller to join the post-commit resume barrier.
self.resume_scheduler()
self.process_input_queue_block = True
self.eep_scaling_state = None
elif not state.commit_requested and state.is_ready_for_switch():
@@ -2512,12 +2555,10 @@ class DPEngineCoreProc(EngineCoreProc):
self.output_queue.put_nowait((0, outputs))
else:
encoder = MsgpackEncoder()
with (
zmq.Context() as ctx,
make_zmq_socket(
ctx, self.addresses.outputs[0], zmq.PUSH, linger=4000
) as socket,
):
ctx = zmq.Context.instance()
with make_zmq_socket(
ctx, self.addresses.outputs[0], zmq.PUSH, linger=4000
) as socket:
socket.send_multipart(encoder.encode(outputs))
def _eep_scale_up_before_kv_init(self):
+146 -54
View File
@@ -23,6 +23,9 @@ import zmq.asyncio
from vllm import envs
from vllm.config import VllmConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleCoordinator,
)
from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
@@ -271,6 +274,14 @@ class EngineCoreClient(ABC):
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
raise NotImplementedError
async def abort_external_elastic_ep(self, error: str) -> None:
raise NotImplementedError
async def get_external_elastic_ep_status(
self,
) -> dict[str, str | int | None] | None:
return None
async def get_output_async(self) -> EngineCoreOutputs:
raise NotImplementedError
@@ -626,6 +637,8 @@ class MPClient(EngineCoreClient):
enable_input_socket_handover = parallel_config.enable_elastic_ep
self.stats_update_address: str | None = None
self.coordinator_input_address: str | None = None
self.coordinator_output_address: str | None = None
tensor_queue: Queue | None = None
if client_addresses:
# Engines are managed externally to this client.
@@ -702,6 +715,11 @@ class MPClient(EngineCoreClient):
coordinator.get_stats_publish_address()
)
self.input_address = self.input_socket.getsockopt_string(zmq.LAST_ENDPOINT)
output_socket = self.resources.output_socket
assert output_socket is not None
self.output_address = output_socket.getsockopt_string(zmq.LAST_ENDPOINT)
# Serialization setup with tensor queues for multimodal tensor IPC.
tensor_ipc_sender: TensorIpcSender | None = None
model_config = getattr(vllm_config, "model_config", None)
@@ -885,6 +903,25 @@ class MPClient(EngineCoreClient):
response.kv_events_config
)
parallel_config = vllm_config.parallel_config
if response.coord_store_port:
if parallel_config._coord_store_port:
assert response.coord_store_port == parallel_config._coord_store_port
else:
parallel_config._coord_store_port = response.coord_store_port
for attribute in (
"coordinator_input_address",
"coordinator_output_address",
):
address = getattr(response, attribute)
if address is not None:
existing_address = getattr(self, attribute)
if existing_address is None:
setattr(self, attribute, address)
else:
assert address == existing_address
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
return dict(self._kv_event_sources)
@@ -1431,6 +1468,12 @@ class DPAsyncMPClient(AsyncMPClient):
renderer: BaseRenderer | None = None,
):
self.current_wave = 0
parallel_config = vllm_config.parallel_config
self.external_eep_coordinator: ExternalElasticEPScaleCoordinator | None = None
if parallel_config.enable_elastic_ep and (
parallel_config.data_parallel_external_lb
):
self.external_eep_coordinator = ExternalElasticEPScaleCoordinator(self)
super().__init__(
vllm_config,
@@ -1514,23 +1557,19 @@ class DPAsyncMPClient(AsyncMPClient):
dp_rank = parallel_config.data_parallel_rank
assert dp_rank == 0
assert dp_size == new_engine_count
assert not (
parallel_config.data_parallel_hybrid_lb
or parallel_config.data_parallel_external_lb
)
num_ranks = dp_size
self.engine_ranks_managed = list(
range(dp_rank, dp_rank + num_ranks)
)
if len(self.lb_engines) < new_engine_count:
self.lb_engines = self.lb_engines + [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
assert not parallel_config.data_parallel_hybrid_lb
# External LB clients continue to manage only local ranks.
if not parallel_config.data_parallel_external_lb:
self.engine_ranks_managed = list(range(dp_size))
if len(self.lb_engines) < new_engine_count:
self.lb_engines += [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
# Send scale up notification to coordinator
scale_msg = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_engine_count)
@@ -1599,6 +1638,93 @@ class DPAsyncMPClient(AsyncMPClient):
def get_core_engine_for_request(self, request: EngineCoreRequest):
return self.core_engine
def _eep_commit_pause_mode(self) -> PauseMode:
from vllm.distributed.elastic_ep.elastic_execute import (
can_reuse_fused_moe_kernel,
)
# MRV2 re-warms through the request pool, so running requests must
# finish first while new ones stay queued in the scheduler.
parallel_config = self.vllm_config.parallel_config
if self.vllm_config.use_v2_model_runner and not can_reuse_fused_moe_kernel(
parallel_config
):
return "wait"
return "keep"
def _setup_elastic_ep_reconfig_bootstrap(self) -> tuple[str, int]:
from vllm.distributed.utils import create_tcp_store
from vllm.utils.network_utils import get_open_ports_list
parallel_config = self.vllm_config.parallel_config
parallel_config._data_parallel_master_port_list = get_open_ports_list(5)
parallel_config.data_parallel_master_port = (
parallel_config._data_parallel_master_port_list.pop()
)
ip = parallel_config.data_parallel_master_ip
store = create_tcp_store(
ip,
0,
is_master=True,
world_size=-1,
wait_for_workers=False,
)
parallel_config._coord_store_port = store.port
self._coord_store = store
return ip, store.port
@staticmethod
async def eep_process_engine_core_notification(
self: "DPAsyncMPClient", notification_data: tuple[str, int]
) -> None:
if self.external_eep_coordinator is not None:
await self.external_eep_coordinator.process_engine_core_notification(
notification_data
)
def _get_external_eep_coordinator(self) -> ExternalElasticEPScaleCoordinator:
parallel_config = self.vllm_config.parallel_config
if not parallel_config.enable_elastic_ep:
raise NotImplementedError(
"Elastic EP scaling requires enable_elastic_ep=True."
)
if not parallel_config.data_parallel_external_lb:
raise NotImplementedError(
"DPAsyncMPClient only supports Elastic EP scaling in external "
"load-balancer mode."
)
coordinator = self.external_eep_coordinator
assert coordinator is not None
return coordinator
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
cur_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
assert new_data_parallel_size != cur_data_parallel_size, (
f"new_data_parallel_size {new_data_parallel_size} must be "
f"different from cur_data_parallel_size {cur_data_parallel_size}"
)
await self._get_external_eep_coordinator().prepare(
cur_data_parallel_size, new_data_parallel_size
)
async def commit_elastic_ep(self) -> None:
await self._get_external_eep_coordinator().commit()
async def abort_external_elastic_ep(self, error: str) -> None:
self._get_external_eep_coordinator().abort(error)
async def get_external_elastic_ep_status(
self,
) -> dict[str, str | int | None] | None:
coordinator = self.external_eep_coordinator
return (
None
if coordinator is None
else await asyncio.to_thread(coordinator.get_status)
)
class DPLBAsyncMPClient(DPAsyncMPClient):
"""Asyncio-compatible client for multi-proc, multi-engine (data parallel)
@@ -1719,8 +1845,9 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
@staticmethod
async def eep_process_engine_core_notification(
self: "DPLBAsyncMPClient", notification_data: tuple[str, int]
):
self: "DPAsyncMPClient", notification_data: tuple[str, int]
) -> None:
assert isinstance(self, DPLBAsyncMPClient)
cache = self.eep_scaling_cache
notification_type_str, dp_rank = notification_data
try:
@@ -1881,27 +2008,6 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
new_engine_identities.discard(identity)
self._apply_ready_response(payload)
def _setup_elastic_ep_reconfig_bootstrap(self) -> None:
from vllm.distributed.utils import create_tcp_store
from vllm.utils.network_utils import get_open_ports_list
parallel_config = self.vllm_config.parallel_config
parallel_config._data_parallel_master_port_list = get_open_ports_list(5)
parallel_config.data_parallel_master_port = (
parallel_config._data_parallel_master_port_list.pop()
)
ip = parallel_config.data_parallel_master_ip
store = create_tcp_store(
ip,
0,
is_master=True,
world_size=-1,
wait_for_workers=False,
)
parallel_config._coord_store_port = store.port
self._coord_store = store
def _make_reconfig_request(
self,
new_data_parallel_size: int,
@@ -1956,20 +2062,6 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
await asyncio.to_thread(self._coord_store.wait, ready_keys)
logger.info("[Elastic EP] Successfully started new engines")
def _eep_commit_pause_mode(self) -> PauseMode:
from vllm.distributed.elastic_ep.elastic_execute import (
can_reuse_fused_moe_kernel,
)
# MRV2 re-warms through the request pool, so running requests must
# finish first while new ones stay queued in the scheduler.
parallel_config = self.vllm_config.parallel_config
if self.vllm_config.use_v2_model_runner and not can_reuse_fused_moe_kernel(
parallel_config
):
return "wait"
return "keep"
async def _commit_scale_up_elastic_ep(self, new_data_parallel_size: int) -> None:
new_core_engines = [
rank.to_bytes(2, "little")
+75 -14
View File
@@ -290,6 +290,7 @@ class CoreEngineProcManager:
self._finalizer = weakref.finalize(self, shutdown, self.processes)
self.manager_stopped = threading.Event()
self.failed_proc_name: str | None = None
self._coord_store: object | None = None
# All ranks share this config object: capture the user-provided
# --device-ids list before the per-rank shard overwrites it. Mutating
@@ -1139,7 +1140,9 @@ def get_engine_zmq_bind_addresses(
offline_mode or local_engines_only or (local_engine_count == dp_size)
)
# NOTE(yongji): handling scaling from intra-node to inter-node
if parallel_config.enable_elastic_ep:
if parallel_config.enable_elastic_ep and (
not local_engines_only or parallel_config.data_parallel_rank == 0
):
client_local_only = False
def _addr() -> str:
@@ -1228,6 +1231,32 @@ def launch_core_engines(
vllm_config.needs_dp_coordinator and not offline_mode and dp_rank == 0
)
coord_store = None
create_coord_store = (
parallel_config.enable_elastic_ep
and parallel_config.data_parallel_backend != "ray"
and dp_rank == 0
and not parallel_config._coord_store_port
)
if create_coord_store:
from vllm.distributed.utils import create_tcp_store
coord_store = create_tcp_store(
host,
0,
is_master=True,
world_size=-1,
wait_for_workers=False,
)
parallel_config._coord_store_port = coord_store.port
logger.info(
"Created Elastic EP coordination TCPStore on %s:%d",
host,
coord_store.port,
)
if run_coordinator:
coordinator = DPCoordinator(
parallel_config,
@@ -1243,12 +1272,13 @@ def launch_core_engines(
)
logger.info("Started DP Coordinator process (PID: %d)", coordinator.proc.pid)
if coord_store is not None:
coordinator._coord_store = coord_store
else:
coordinator = None
# Hold a coordination TCPStore (alive for this frame) so engines pick DP
# master ports at bind time; pre-allocated ports can be taken before use.
coord_store = None
if (
dp_size > 1
and not offline_mode
@@ -1344,6 +1374,13 @@ def launch_core_engines(
else:
local_engine_manager = None
if (
coord_store is not None
and coordinator is None
and (local_engine_manager is not None)
):
local_engine_manager._coord_store = coord_store
launch = CoreEngineLaunch(
local_engine_manager, coordinator, addresses, tensor_queue
)
@@ -1474,21 +1511,33 @@ def wait_for_engine_startup(
if status == "HELLO" and engine.state == CoreEngineState.NEW:
# Send init message with DP config info.
parallel_config_payload: dict[str, int | str | list[int]] = {}
if coordinated_dp and not (
parallel_config.local_engines_only
and parallel_config.data_parallel_rank > 0
):
parallel_config_payload = {
k: getattr(parallel_config, k)
for k in (
"data_parallel_master_ip",
"data_parallel_master_port",
"_data_parallel_master_port_list",
"_coord_store_port",
"data_parallel_size",
)
}
if (
parallel_config.enable_elastic_ep
and not parallel_config._coord_store_port
):
raise RuntimeError(
"Elastic EP startup requires a coordination TCPStore "
"port before engine handshake."
)
init_message = msgspec.msgpack.encode(
EngineHandshakeMetadata(
addresses=launch.addresses,
parallel_config={
k: getattr(parallel_config, k)
for k in (
"data_parallel_master_ip",
"data_parallel_master_port",
"_data_parallel_master_port_list",
"_coord_store_port",
"data_parallel_size",
)
}
if coordinated_dp
else {},
parallel_config=parallel_config_payload,
)
)
handshake_socket.send_multipart((eng_identity, init_message), copy=False)
@@ -1496,6 +1545,18 @@ def wait_for_engine_startup(
start_pending[0 if local else 1] += 1
engine.state = CoreEngineState.CONNECTED
elif status == "READY" and engine.state == CoreEngineState.CONNECTED:
coord_store_port = msg.get("coord_store_port")
if coord_store_port is not None and local:
parallel_config._coord_store_port = coord_store_port
num_redundant_experts = msg.get("num_redundant_experts")
if num_redundant_experts is not None and local:
parallel_config.eplb_config.num_redundant_experts = (
num_redundant_experts
)
elastic_ep_max_dp_size = msg.get("elastic_ep_max_dp_size")
if elastic_ep_max_dp_size is not None and local:
parallel_config.elastic_ep_max_dp_size = elastic_ep_max_dp_size
# Validate config hash consistency across DP workers for MoE models.
if coordinated_dp:
worker_config_hash = msg.get("parallel_config_hash")