[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)

Signed-off-by: lqh-hh <liuqihang6@huawei.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: lqh-hh <liuqihang6@huawei.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: 2017403603 <705328646@qq.com>
Co-authored-by: Julien Mancuso <jmancuso@nvidia.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fangyuchu
2026-10-15 21:29:02 -04:00
committed by GitHub
co-authored by lqh-hh Tyler Michael Smith OpenAI Codex 2017403603 Julien Mancuso Itay Alroy Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fb3fec970e
commit dd9e07a25f
18 changed files with 1638 additions and 167 deletions
+22 -13
View File
@@ -51,7 +51,7 @@ def _traffic_loop(
stop: threading.Event,
finished: threading.Event,
is_probe: bool = False,
) -> list[tuple[float, float, int | None]]:
) -> list[tuple[float, float, int | None, str | None]]:
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
@@ -60,15 +60,18 @@ def _traffic_loop(
is_ready = False
while not stop.is_set():
request_start = time.perf_counter()
phase = None
try:
response = requests.post(
url, json=request_payload, headers=headers, timeout=120
)
status_code = response.status_code
if is_probe and status_code == 200:
phase = response.json()["phase"]
except requests.exceptions.RequestException:
status_code = None
responses.append((request_start, time.perf_counter(), status_code))
if status_code == 200:
responses.append((request_start, time.perf_counter(), status_code, phase))
if status_code == 200 and (not is_probe or phase in ("idle", "completed")):
if not is_ready:
ready.wait(timeout=120)
is_ready = True
@@ -78,14 +81,16 @@ def _traffic_loop(
return responses
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
rejected = [end for _, end, status in responses if status == 503]
if not rejected:
def _downtime(responses: list[tuple[float, float, int | None, str | None]]) -> float:
committing = [end for _, end, _, phase in responses if phase == "committing"]
if not committing:
return 0
recovered = next(
end for _, end, status in responses if status == 200 and end > rejected[-1]
end
for _, end, status, phase in responses
if status == 200 and phase in ("idle", "completed") and end > committing[-1]
)
return recovered - rejected[0]
return recovered - committing[0]
def _scale_with_traffic(
@@ -124,16 +129,20 @@ def _scale_with_traffic(
bad_statuses = {
status
for responses in [probe_result, *results]
for _, _, status in responses
for _, _, status, _ in responses
if status not in (200, 503)
}
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
probe_503 = [start for start, _, status in probe_result if status == 503]
assert probe_503, "Scaling probe did not observe commit"
probe_committing = [
start for start, _, _, phase in probe_result if phase == "committing"
]
assert probe_committing, "Scaling probe did not observe commit"
assert not results or any(
status == 200 and start_time <= request_start and request_end < probe_503[0]
status == 200
and start_time <= request_start
and request_end < probe_committing[0]
for responses in results
for request_start, request_end, status in responses
for request_start, request_end, status, _ in responses
), "No request completed successfully during preparation"
print(
+270 -46
View File
@@ -4,17 +4,28 @@ import asyncio
import os
import threading
import time
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AsyncExitStack
from types import SimpleNamespace
from typing import TYPE_CHECKING, cast
import msgspec
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
import requests
from tests.utils import RemoteOpenAIServer
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleCoordinator,
)
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.engine.core_client import DPAsyncMPClient
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
# Number of data parallel ranks for external LB testing
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
@@ -22,6 +33,111 @@ DP_SIZE = int(os.getenv("DP_SIZE", "2"))
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
class DictStore:
def __init__(self, values: dict[str, bytes]):
self.values = values
def check(self, keys: list[str]) -> bool:
return all(key in self.values for key in keys)
def get(self, key: str) -> bytes:
return self.values[key]
def set(self, key: str, value: bytes) -> None:
self.values[key] = value
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
client = SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_master_port=12345,
_data_parallel_master_port_list=[12346, 12347],
)
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
control_store, reconfig_store = DictStore({}), DictStore({})
coordinator.reconfig_store_ref = reconfig_store
monkeypatch.setattr(
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
)
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
for store in (control_store, reconfig_store):
# DictStore preserves insertion order: metadata must precede the epoch.
assert list(store.values)[-1] == coordinator.key("current_epoch")
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
bootstrap
)
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
monkeypatch.setattr(
coordinator, "_get_reconfig_store", lambda store=store: store
)
assert coordinator.get_status()["requested_data_parallel_size"] == 3
def test_external_elastic_ep_calculates_target_expert_redundancy():
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
eplb_config=SimpleNamespace(num_redundant_experts=0)
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
@pytest.mark.asyncio
@pytest.mark.parametrize("dp_rank", range(4))
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
"""Validate expert capacity on every rank before waiting for an epoch."""
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
data_parallel_rank=dp_rank,
elastic_ep_max_dp_size=4,
data_parallel_master_ip="127.0.0.1",
_coord_store_port=12345,
eplb_config=SimpleNamespace(num_redundant_experts=0),
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
store = DictStore({})
monkeypatch.setattr(
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
)
with pytest.raises(ValueError, match="cannot retain all model experts"):
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
assert not store.values
assert coordinator.prepared_scale is None
@pytest.mark.asyncio
async def test_external_elastic_ep_late_rank_observes_epoch_error():
coordinator = ExternalElasticEPScaleCoordinator(
cast("DPAsyncMPClient", SimpleNamespace())
)
epoch = "failed-epoch"
store = DictStore(
{
coordinator.key("current_epoch"): epoch.encode(),
coordinator.key(epoch, "error"): b"scale failed",
}
)
with pytest.raises(RuntimeError, match="scale failed"):
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
class ExternalLBServerManager:
"""Manages data parallel vLLM server instances for external
load balancer testing."""
@@ -42,57 +158,69 @@ class ExternalLBServerManager:
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
self.server_threads: list[threading.Thread] = []
def start_rank(
self,
rank: int,
dp_size: int | None = None,
elastic_ep_scale_up_launch: bool = False,
) -> tuple[RemoteOpenAIServer, list[str]]:
"""Start one externally managed DP rank."""
dp_size = self.dp_size if dp_size is None else dp_size
server_args = self.base_server_args.copy()
server_args.extend(
[
"--data-parallel-size",
str(dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank),
"--api-server-count",
str(self.api_server_count),
]
)
env_dict = {
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
),
}
if elastic_ep_scale_up_launch:
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
server = RemoteOpenAIServer(
self.model_name,
server_args,
auto_port=False,
env_dict=env_dict,
)
print(
f"Server rank {rank} started successfully with "
f"{self.api_server_count} API servers"
)
server_and_args = (server, server_args)
self.servers.append(server_and_args)
return server_and_args
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
"""Start all server instances for external LB mode."""
"""Start all initial server instances for external LB mode."""
start_errors: list[Exception] = []
for rank in range(self.dp_size):
# Create server args for this specific rank
server_args = self.base_server_args.copy()
# Add external LB specific arguments
server_args.extend(
[
"--data-parallel-size",
str(self.dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank), # Different port for each rank
"--api-server-count",
str(self.api_server_count),
]
)
# Use a thread to start each server to allow parallel initialization
def start_server(r: int, sargs: list[str]):
def start_server(r: int):
try:
# Start the server
server = RemoteOpenAIServer(
self.model_name,
sargs,
auto_port=False,
env_dict={
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE)
),
},
)
server.__enter__()
print(
f"Server rank {r} started successfully with "
f"{self.api_server_count} API servers"
)
self.servers.append((server, sargs))
self.start_rank(r)
except Exception as e:
print(f"Failed to start server rank {r}: {e}")
raise
start_errors.append(e)
thread = threading.Thread(target=start_server, args=(rank, server_args))
thread = threading.Thread(target=start_server, args=(rank,))
thread.start()
self.server_threads.append(thread)
@@ -101,6 +229,10 @@ class ExternalLBServerManager:
for thread in self.server_threads:
thread.join()
if start_errors:
start_error = start_errors[0]
raise RuntimeError("Failed to start external LB servers") from start_error
# Give servers additional time to fully initialize and coordinate
time.sleep(2)
@@ -184,12 +316,104 @@ def _assert_expert_parallel_config(
assert parallel_config["enable_expert_parallel"] is expected
def _send_scale_command(
server: RemoteOpenAIServer, new_dp_size: int
) -> requests.Response:
return requests.post(
server.url_for("scale_elastic_ep"),
json={"new_data_parallel_size": new_dp_size},
timeout=600,
)
def _wait_for_scale_requests_to_start(
scale_requests: list[Future[requests.Response]],
timeout: float = 30,
) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
for request in scale_requests:
if request.done():
response = request.result()
pytest.fail(
"Scale request finished before the new rank was launched: "
f"{response.status_code} {response.text}"
)
if all(request.running() for request in scale_requests):
return
time.sleep(0.1)
pytest.fail("Timed out waiting for all scale requests to start")
@pytest.mark.distributed(num_gpus=3)
@pytest.mark.skipif(
TP_SIZE != 1 or current_platform.device_count() < 3,
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
)
@pytest.mark.skipif(
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
reason="Elastic EP scaling is not supported by the Rust frontend",
)
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
server_args = [
*default_server_args,
"--enable-expert-parallel",
"--all2all-backend",
"allgather_reducescatter",
"--attention-backend",
"TRITON_MLA",
"--enable-elastic-ep",
"--elastic-ep-max-dp-size",
"3",
"--enable-eplb",
"--eplb-config",
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
]
server_manager = ExternalLBServerManager(
ELASTIC_EP_MODEL_NAME,
dp_size=2,
api_server_count=1,
base_server_args=server_args,
)
with server_manager:
existing_servers = [server for server, _ in server_manager.servers]
with ThreadPoolExecutor(max_workers=2) as executor:
scale_requests = [
executor.submit(_send_scale_command, server, 3)
for server in existing_servers
]
_wait_for_scale_requests_to_start(scale_requests)
server_manager.start_rank(
rank=2,
dp_size=3,
elastic_ep_scale_up_launch=True,
)
for request in scale_requests:
response = request.result(timeout=600)
assert response.status_code == 200, response.text
for server, _ in server_manager.servers:
parallel_config = _get_parallel_config(server)
assert parallel_config["data_parallel_size"] == 3
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
with server.get_client() as client:
completion = client.completions.create(
model=ELASTIC_EP_MODEL_NAME,
prompt="Hello, my name is",
max_tokens=5,
temperature=0.0,
)
assert completion.choices[0].finish_reason in ("length", "stop")
def test_external_lb_server_info(server_manager):
servers = server_manager.servers
api_server_count = server_manager.api_server_count
_assert_expert_parallel_config(servers)
for i, (server, _) in enumerate(servers):
print(f"Testing {i=}")
@@ -141,7 +141,7 @@ def _make_addresses() -> EngineZmqAddresses:
False,
),
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, True),
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
],
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
@@ -150,8 +150,9 @@ def test_coordination_store_held_only_by_the_online_dp_master(
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
) -> None:
"""The DP master node holds a coordination store for both engine backends,
so engines there pick world-group ports at bind time. Every other
deployment keeps the pre-allocated ports and must not hold one."""
so engines there pick world-group ports at bind time. Elastic EP also
creates its coordination store before startup. The remaining cases
below keep the pre-allocated ports and must not hold a store."""
class FakeManager:
def __init__(self, **kwargs) -> None:
+15 -1
View File
@@ -142,6 +142,9 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
def setsockopt(self, *_args, **_kwargs):
pass
def getsockopt_string(self, _option):
return "inproc://dummy"
monkeypatch.setattr(core_client_mod.zmq.Socket, "shadow", lambda *_: ShadowSocket())
monkeypatch.setattr(
core_client_mod, "make_zmq_socket", lambda *_, **__: DummySocket()
@@ -448,7 +451,7 @@ def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
("effective_size", "other_size"),
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
)
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
def test_apply_ready_response_syncs_engine_metadata(effective_size, other_size):
import msgspec
client = object.__new__(MPClient)
@@ -456,8 +459,11 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
client.stats_update_address = None
client.coordinator_input_address = None
client.coordinator_output_address = None
payload = msgspec.msgpack.encode(
EngineCoreReadyResponse(
@@ -478,6 +484,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
instance_id="test-instance",
supports_lora=False,
max_loras=0,
coord_store_port=1234,
coordinator_input_address="tcp://127.0.0.1:1235",
coordinator_output_address="tcp://127.0.0.1:1236",
)
)
fields = msgspec.msgpack.decode(payload)
@@ -498,6 +507,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
client._apply_ready_response(b"")
assert cache_config.effective_attention_block_size is None
assert client.vllm_config.parallel_config._coord_store_port == 1234
assert client.coordinator_input_address == "tcp://127.0.0.1:1235"
assert client.coordinator_output_address == "tcp://127.0.0.1:1236"
def test_apply_ready_response_syncs_mamba_block_size():
@@ -508,6 +520,7 @@ def test_apply_ready_response_syncs_mamba_block_size():
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
client.stats_update_address = None
@@ -1578,6 +1591,7 @@ def test_apply_ready_response_retains_kv_event_sources():
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
@@ -281,6 +281,7 @@ class _FinishedProcess:
def test_wait_for_engine_startup_reports_watched_process_exit():
"""Update the startup test argument so it reaches frontend failure handling."""
ctx = zmq.Context()
handshake_socket = ctx.socket(zmq.ROUTER)
recv, send = connection.Pipe(duplex=False)