[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)
Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
lqh-hh
Tyler Michael Smith
OpenAI Codex
2017403603
Julien Mancuso
Itay Alroy
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fb3fec970e
commit
dd9e07a25f
+75
-14
@@ -290,6 +290,7 @@ class CoreEngineProcManager:
|
||||
self._finalizer = weakref.finalize(self, shutdown, self.processes)
|
||||
self.manager_stopped = threading.Event()
|
||||
self.failed_proc_name: str | None = None
|
||||
self._coord_store: object | None = None
|
||||
|
||||
# All ranks share this config object: capture the user-provided
|
||||
# --device-ids list before the per-rank shard overwrites it. Mutating
|
||||
@@ -1139,7 +1140,9 @@ def get_engine_zmq_bind_addresses(
|
||||
offline_mode or local_engines_only or (local_engine_count == dp_size)
|
||||
)
|
||||
# NOTE(yongji): handling scaling from intra-node to inter-node
|
||||
if parallel_config.enable_elastic_ep:
|
||||
if parallel_config.enable_elastic_ep and (
|
||||
not local_engines_only or parallel_config.data_parallel_rank == 0
|
||||
):
|
||||
client_local_only = False
|
||||
|
||||
def _addr() -> str:
|
||||
@@ -1228,6 +1231,32 @@ def launch_core_engines(
|
||||
vllm_config.needs_dp_coordinator and not offline_mode and dp_rank == 0
|
||||
)
|
||||
|
||||
coord_store = None
|
||||
|
||||
create_coord_store = (
|
||||
parallel_config.enable_elastic_ep
|
||||
and parallel_config.data_parallel_backend != "ray"
|
||||
and dp_rank == 0
|
||||
and not parallel_config._coord_store_port
|
||||
)
|
||||
|
||||
if create_coord_store:
|
||||
from vllm.distributed.utils import create_tcp_store
|
||||
|
||||
coord_store = create_tcp_store(
|
||||
host,
|
||||
0,
|
||||
is_master=True,
|
||||
world_size=-1,
|
||||
wait_for_workers=False,
|
||||
)
|
||||
parallel_config._coord_store_port = coord_store.port
|
||||
logger.info(
|
||||
"Created Elastic EP coordination TCPStore on %s:%d",
|
||||
host,
|
||||
coord_store.port,
|
||||
)
|
||||
|
||||
if run_coordinator:
|
||||
coordinator = DPCoordinator(
|
||||
parallel_config,
|
||||
@@ -1243,12 +1272,13 @@ def launch_core_engines(
|
||||
)
|
||||
|
||||
logger.info("Started DP Coordinator process (PID: %d)", coordinator.proc.pid)
|
||||
if coord_store is not None:
|
||||
coordinator._coord_store = coord_store
|
||||
else:
|
||||
coordinator = None
|
||||
|
||||
# Hold a coordination TCPStore (alive for this frame) so engines pick DP
|
||||
# master ports at bind time; pre-allocated ports can be taken before use.
|
||||
coord_store = None
|
||||
if (
|
||||
dp_size > 1
|
||||
and not offline_mode
|
||||
@@ -1344,6 +1374,13 @@ def launch_core_engines(
|
||||
else:
|
||||
local_engine_manager = None
|
||||
|
||||
if (
|
||||
coord_store is not None
|
||||
and coordinator is None
|
||||
and (local_engine_manager is not None)
|
||||
):
|
||||
local_engine_manager._coord_store = coord_store
|
||||
|
||||
launch = CoreEngineLaunch(
|
||||
local_engine_manager, coordinator, addresses, tensor_queue
|
||||
)
|
||||
@@ -1474,21 +1511,33 @@ def wait_for_engine_startup(
|
||||
|
||||
if status == "HELLO" and engine.state == CoreEngineState.NEW:
|
||||
# Send init message with DP config info.
|
||||
parallel_config_payload: dict[str, int | str | list[int]] = {}
|
||||
if coordinated_dp and not (
|
||||
parallel_config.local_engines_only
|
||||
and parallel_config.data_parallel_rank > 0
|
||||
):
|
||||
parallel_config_payload = {
|
||||
k: getattr(parallel_config, k)
|
||||
for k in (
|
||||
"data_parallel_master_ip",
|
||||
"data_parallel_master_port",
|
||||
"_data_parallel_master_port_list",
|
||||
"_coord_store_port",
|
||||
"data_parallel_size",
|
||||
)
|
||||
}
|
||||
if (
|
||||
parallel_config.enable_elastic_ep
|
||||
and not parallel_config._coord_store_port
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Elastic EP startup requires a coordination TCPStore "
|
||||
"port before engine handshake."
|
||||
)
|
||||
init_message = msgspec.msgpack.encode(
|
||||
EngineHandshakeMetadata(
|
||||
addresses=launch.addresses,
|
||||
parallel_config={
|
||||
k: getattr(parallel_config, k)
|
||||
for k in (
|
||||
"data_parallel_master_ip",
|
||||
"data_parallel_master_port",
|
||||
"_data_parallel_master_port_list",
|
||||
"_coord_store_port",
|
||||
"data_parallel_size",
|
||||
)
|
||||
}
|
||||
if coordinated_dp
|
||||
else {},
|
||||
parallel_config=parallel_config_payload,
|
||||
)
|
||||
)
|
||||
handshake_socket.send_multipart((eng_identity, init_message), copy=False)
|
||||
@@ -1496,6 +1545,18 @@ def wait_for_engine_startup(
|
||||
start_pending[0 if local else 1] += 1
|
||||
engine.state = CoreEngineState.CONNECTED
|
||||
elif status == "READY" and engine.state == CoreEngineState.CONNECTED:
|
||||
coord_store_port = msg.get("coord_store_port")
|
||||
if coord_store_port is not None and local:
|
||||
parallel_config._coord_store_port = coord_store_port
|
||||
num_redundant_experts = msg.get("num_redundant_experts")
|
||||
if num_redundant_experts is not None and local:
|
||||
parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_redundant_experts
|
||||
)
|
||||
elastic_ep_max_dp_size = msg.get("elastic_ep_max_dp_size")
|
||||
if elastic_ep_max_dp_size is not None and local:
|
||||
parallel_config.elastic_ep_max_dp_size = elastic_ep_max_dp_size
|
||||
|
||||
# Validate config hash consistency across DP workers for MoE models.
|
||||
if coordinated_dp:
|
||||
worker_config_hash = msg.get("parallel_config_hash")
|
||||
|
||||
Reference in New Issue
Block a user