[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)

Signed-off-by: lqh-hh <liuqihang6@huawei.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: lqh-hh <liuqihang6@huawei.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: 2017403603 <705328646@qq.com>
Co-authored-by: Julien Mancuso <jmancuso@nvidia.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fangyuchu
2026-10-15 21:29:02 -04:00
committed by GitHub
co-authored by lqh-hh Tyler Michael Smith OpenAI Codex 2017403603 Julien Mancuso Itay Alroy Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fb3fec970e
commit dd9e07a25f
18 changed files with 1638 additions and 167 deletions
+50 -9
View File
@@ -1346,6 +1346,16 @@ class EngineCoreProc(EngineCore):
ready_msg["parallel_config_hash"] = (
vllm_config.parallel_config.compute_hash()
)
if vllm_config.parallel_config.enable_elastic_ep:
ready_msg["coord_store_port"] = (
vllm_config.parallel_config._coord_store_port
)
ready_msg["elastic_ep_max_dp_size"] = (
vllm_config.parallel_config.elastic_ep_max_dp_size
)
ready_msg["num_redundant_experts"] = (
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
handshake_socket.send(msgspec.msgpack.encode(ready_msg))
@@ -1376,14 +1386,37 @@ class EngineCoreProc(EngineCore):
f"minutes"
)
init_bytes = handshake_socket.recv()
init_message: EngineHandshakeMetadata = msgspec.msgpack.decode(
init_bytes, type=EngineHandshakeMetadata
)
num_redundant_experts: int | None = None
if (
envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH
and parallel_config is not None
and parallel_config.data_parallel_external_lb
):
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleUpHandshakeMetadata,
)
scale_up_message = msgspec.msgpack.decode(
init_bytes, type=ExternalElasticEPScaleUpHandshakeMetadata
)
init_message = scale_up_message.engine_metadata
num_redundant_experts = scale_up_message.num_redundant_experts
parallel_config.elastic_ep_max_dp_size = (
scale_up_message.elastic_ep_max_dp_size
)
else:
init_message = msgspec.msgpack.decode(
init_bytes, type=EngineHandshakeMetadata
)
logger.debug("Received init message: %s", init_message)
if parallel_config is not None:
for key, value in init_message.parallel_config.items():
setattr(parallel_config, key, value)
if num_redundant_experts is not None:
parallel_config.eplb_config.num_redundant_experts = (
num_redundant_experts
)
return init_message.addresses
@@ -1826,6 +1859,9 @@ class EngineCoreProc(EngineCore):
if self.vllm_config.weight_transfer_config is not None
else False
),
coord_store_port=parallel_config._coord_store_port,
coordinator_input_address=self.addresses.coordinator_input,
coordinator_output_address=self.addresses.coordinator_output,
)
def process_input_sockets(
@@ -2342,6 +2378,13 @@ class DPEngineCoreProc(EngineCoreProc):
if state.is_complete():
if state.worker_type == "removing":
raise SystemExit
if (
state.worker_type == "new"
and self.vllm_config.parallel_config.data_parallel_external_lb
):
# An independently launched rank has no external utility
# caller to join the post-commit resume barrier.
self.resume_scheduler()
self.process_input_queue_block = True
self.eep_scaling_state = None
elif not state.commit_requested and state.is_ready_for_switch():
@@ -2512,12 +2555,10 @@ class DPEngineCoreProc(EngineCoreProc):
self.output_queue.put_nowait((0, outputs))
else:
encoder = MsgpackEncoder()
with (
zmq.Context() as ctx,
make_zmq_socket(
ctx, self.addresses.outputs[0], zmq.PUSH, linger=4000
) as socket,
):
ctx = zmq.Context.instance()
with make_zmq_socket(
ctx, self.addresses.outputs[0], zmq.PUSH, linger=4000
) as socket:
socket.send_multipart(encoder.encode(outputs))
def _eep_scale_up_before_kv_init(self):