[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)

Signed-off-by: lqh-hh <liuqihang6@huawei.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: lqh-hh <liuqihang6@huawei.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: 2017403603 <705328646@qq.com>
Co-authored-by: Julien Mancuso <jmancuso@nvidia.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fangyuchu
2026-10-15 21:29:02 -04:00
committed by GitHub
co-authored by lqh-hh Tyler Michael Smith OpenAI Codex 2017403603 Julien Mancuso Itay Alroy Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fb3fec970e
commit dd9e07a25f
18 changed files with 1638 additions and 167 deletions
+270 -46
View File
@@ -4,17 +4,28 @@ import asyncio
import os
import threading
import time
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AsyncExitStack
from types import SimpleNamespace
from typing import TYPE_CHECKING, cast
import msgspec
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
import requests
from tests.utils import RemoteOpenAIServer
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleCoordinator,
)
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.engine.core_client import DPAsyncMPClient
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
# Number of data parallel ranks for external LB testing
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
@@ -22,6 +33,111 @@ DP_SIZE = int(os.getenv("DP_SIZE", "2"))
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
class DictStore:
def __init__(self, values: dict[str, bytes]):
self.values = values
def check(self, keys: list[str]) -> bool:
return all(key in self.values for key in keys)
def get(self, key: str) -> bytes:
return self.values[key]
def set(self, key: str, value: bytes) -> None:
self.values[key] = value
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
client = SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_master_port=12345,
_data_parallel_master_port_list=[12346, 12347],
)
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
control_store, reconfig_store = DictStore({}), DictStore({})
coordinator.reconfig_store_ref = reconfig_store
monkeypatch.setattr(
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
)
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
for store in (control_store, reconfig_store):
# DictStore preserves insertion order: metadata must precede the epoch.
assert list(store.values)[-1] == coordinator.key("current_epoch")
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
bootstrap
)
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
monkeypatch.setattr(
coordinator, "_get_reconfig_store", lambda store=store: store
)
assert coordinator.get_status()["requested_data_parallel_size"] == 3
def test_external_elastic_ep_calculates_target_expert_redundancy():
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
eplb_config=SimpleNamespace(num_redundant_experts=0)
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
@pytest.mark.asyncio
@pytest.mark.parametrize("dp_rank", range(4))
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
"""Validate expert capacity on every rank before waiting for an epoch."""
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
data_parallel_rank=dp_rank,
elastic_ep_max_dp_size=4,
data_parallel_master_ip="127.0.0.1",
_coord_store_port=12345,
eplb_config=SimpleNamespace(num_redundant_experts=0),
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
store = DictStore({})
monkeypatch.setattr(
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
)
with pytest.raises(ValueError, match="cannot retain all model experts"):
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
assert not store.values
assert coordinator.prepared_scale is None
@pytest.mark.asyncio
async def test_external_elastic_ep_late_rank_observes_epoch_error():
coordinator = ExternalElasticEPScaleCoordinator(
cast("DPAsyncMPClient", SimpleNamespace())
)
epoch = "failed-epoch"
store = DictStore(
{
coordinator.key("current_epoch"): epoch.encode(),
coordinator.key(epoch, "error"): b"scale failed",
}
)
with pytest.raises(RuntimeError, match="scale failed"):
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
class ExternalLBServerManager:
"""Manages data parallel vLLM server instances for external
load balancer testing."""
@@ -42,57 +158,69 @@ class ExternalLBServerManager:
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
self.server_threads: list[threading.Thread] = []
def start_rank(
self,
rank: int,
dp_size: int | None = None,
elastic_ep_scale_up_launch: bool = False,
) -> tuple[RemoteOpenAIServer, list[str]]:
"""Start one externally managed DP rank."""
dp_size = self.dp_size if dp_size is None else dp_size
server_args = self.base_server_args.copy()
server_args.extend(
[
"--data-parallel-size",
str(dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank),
"--api-server-count",
str(self.api_server_count),
]
)
env_dict = {
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
),
}
if elastic_ep_scale_up_launch:
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
server = RemoteOpenAIServer(
self.model_name,
server_args,
auto_port=False,
env_dict=env_dict,
)
print(
f"Server rank {rank} started successfully with "
f"{self.api_server_count} API servers"
)
server_and_args = (server, server_args)
self.servers.append(server_and_args)
return server_and_args
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
"""Start all server instances for external LB mode."""
"""Start all initial server instances for external LB mode."""
start_errors: list[Exception] = []
for rank in range(self.dp_size):
# Create server args for this specific rank
server_args = self.base_server_args.copy()
# Add external LB specific arguments
server_args.extend(
[
"--data-parallel-size",
str(self.dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank), # Different port for each rank
"--api-server-count",
str(self.api_server_count),
]
)
# Use a thread to start each server to allow parallel initialization
def start_server(r: int, sargs: list[str]):
def start_server(r: int):
try:
# Start the server
server = RemoteOpenAIServer(
self.model_name,
sargs,
auto_port=False,
env_dict={
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE)
),
},
)
server.__enter__()
print(
f"Server rank {r} started successfully with "
f"{self.api_server_count} API servers"
)
self.servers.append((server, sargs))
self.start_rank(r)
except Exception as e:
print(f"Failed to start server rank {r}: {e}")
raise
start_errors.append(e)
thread = threading.Thread(target=start_server, args=(rank, server_args))
thread = threading.Thread(target=start_server, args=(rank,))
thread.start()
self.server_threads.append(thread)
@@ -101,6 +229,10 @@ class ExternalLBServerManager:
for thread in self.server_threads:
thread.join()
if start_errors:
start_error = start_errors[0]
raise RuntimeError("Failed to start external LB servers") from start_error
# Give servers additional time to fully initialize and coordinate
time.sleep(2)
@@ -184,12 +316,104 @@ def _assert_expert_parallel_config(
assert parallel_config["enable_expert_parallel"] is expected
def _send_scale_command(
server: RemoteOpenAIServer, new_dp_size: int
) -> requests.Response:
return requests.post(
server.url_for("scale_elastic_ep"),
json={"new_data_parallel_size": new_dp_size},
timeout=600,
)
def _wait_for_scale_requests_to_start(
scale_requests: list[Future[requests.Response]],
timeout: float = 30,
) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
for request in scale_requests:
if request.done():
response = request.result()
pytest.fail(
"Scale request finished before the new rank was launched: "
f"{response.status_code} {response.text}"
)
if all(request.running() for request in scale_requests):
return
time.sleep(0.1)
pytest.fail("Timed out waiting for all scale requests to start")
@pytest.mark.distributed(num_gpus=3)
@pytest.mark.skipif(
TP_SIZE != 1 or current_platform.device_count() < 3,
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
)
@pytest.mark.skipif(
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
reason="Elastic EP scaling is not supported by the Rust frontend",
)
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
server_args = [
*default_server_args,
"--enable-expert-parallel",
"--all2all-backend",
"allgather_reducescatter",
"--attention-backend",
"TRITON_MLA",
"--enable-elastic-ep",
"--elastic-ep-max-dp-size",
"3",
"--enable-eplb",
"--eplb-config",
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
]
server_manager = ExternalLBServerManager(
ELASTIC_EP_MODEL_NAME,
dp_size=2,
api_server_count=1,
base_server_args=server_args,
)
with server_manager:
existing_servers = [server for server, _ in server_manager.servers]
with ThreadPoolExecutor(max_workers=2) as executor:
scale_requests = [
executor.submit(_send_scale_command, server, 3)
for server in existing_servers
]
_wait_for_scale_requests_to_start(scale_requests)
server_manager.start_rank(
rank=2,
dp_size=3,
elastic_ep_scale_up_launch=True,
)
for request in scale_requests:
response = request.result(timeout=600)
assert response.status_code == 200, response.text
for server, _ in server_manager.servers:
parallel_config = _get_parallel_config(server)
assert parallel_config["data_parallel_size"] == 3
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
with server.get_client() as client:
completion = client.completions.create(
model=ELASTIC_EP_MODEL_NAME,
prompt="Hello, my name is",
max_tokens=5,
temperature=0.0,
)
assert completion.choices[0].finish_reason in ("length", "stop")
def test_external_lb_server_info(server_manager):
servers = server_manager.servers
api_server_count = server_manager.api_server_count
_assert_expert_parallel_config(servers)
for i, (server, _) in enumerate(servers):
print(f"Testing {i=}")