[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)
Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
lqh-hh
Tyler Michael Smith
OpenAI Codex
2017403603
Julien Mancuso
Itay Alroy
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fb3fec970e
commit
dd9e07a25f
@@ -4,17 +4,28 @@ import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AsyncExitStack
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import msgspec
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import (
|
||||
ExternalElasticEPScaleCoordinator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine.core_client import DPAsyncMPClient
|
||||
|
||||
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
|
||||
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
|
||||
# Number of data parallel ranks for external LB testing
|
||||
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
@@ -22,6 +33,111 @@ DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
|
||||
|
||||
|
||||
class DictStore:
|
||||
def __init__(self, values: dict[str, bytes]):
|
||||
self.values = values
|
||||
|
||||
def check(self, keys: list[str]) -> bool:
|
||||
return all(key in self.values for key in keys)
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
return self.values[key]
|
||||
|
||||
def set(self, key: str, value: bytes) -> None:
|
||||
self.values[key] = value
|
||||
|
||||
|
||||
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_master_port=12345,
|
||||
_data_parallel_master_port_list=[12346, 12347],
|
||||
)
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
control_store, reconfig_store = DictStore({}), DictStore({})
|
||||
coordinator.reconfig_store_ref = reconfig_store
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
|
||||
)
|
||||
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
|
||||
|
||||
for store in (control_store, reconfig_store):
|
||||
# DictStore preserves insertion order: metadata must precede the epoch.
|
||||
assert list(store.values)[-1] == coordinator.key("current_epoch")
|
||||
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
|
||||
bootstrap
|
||||
)
|
||||
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
|
||||
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_get_reconfig_store", lambda store=store: store
|
||||
)
|
||||
assert coordinator.get_status()["requested_data_parallel_size"] == 3
|
||||
|
||||
|
||||
def test_external_elastic_ep_calculates_target_expert_redundancy():
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0)
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
|
||||
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("dp_rank", range(4))
|
||||
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
|
||||
"""Validate expert capacity on every rank before waiting for an epoch."""
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_rank=dp_rank,
|
||||
elastic_ep_max_dp_size=4,
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
_coord_store_port=12345,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0),
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
store = DictStore({})
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot retain all model experts"):
|
||||
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
|
||||
|
||||
assert not store.values
|
||||
assert coordinator.prepared_scale is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_elastic_ep_late_rank_observes_epoch_error():
|
||||
coordinator = ExternalElasticEPScaleCoordinator(
|
||||
cast("DPAsyncMPClient", SimpleNamespace())
|
||||
)
|
||||
epoch = "failed-epoch"
|
||||
store = DictStore(
|
||||
{
|
||||
coordinator.key("current_epoch"): epoch.encode(),
|
||||
coordinator.key(epoch, "error"): b"scale failed",
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="scale failed"):
|
||||
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
|
||||
|
||||
|
||||
class ExternalLBServerManager:
|
||||
"""Manages data parallel vLLM server instances for external
|
||||
load balancer testing."""
|
||||
@@ -42,57 +158,69 @@ class ExternalLBServerManager:
|
||||
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
|
||||
self.server_threads: list[threading.Thread] = []
|
||||
|
||||
def start_rank(
|
||||
self,
|
||||
rank: int,
|
||||
dp_size: int | None = None,
|
||||
elastic_ep_scale_up_launch: bool = False,
|
||||
) -> tuple[RemoteOpenAIServer, list[str]]:
|
||||
"""Start one externally managed DP rank."""
|
||||
dp_size = self.dp_size if dp_size is None else dp_size
|
||||
server_args = self.base_server_args.copy()
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank),
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
env_dict = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
|
||||
),
|
||||
}
|
||||
if elastic_ep_scale_up_launch:
|
||||
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
|
||||
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
server_args,
|
||||
auto_port=False,
|
||||
env_dict=env_dict,
|
||||
)
|
||||
print(
|
||||
f"Server rank {rank} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
server_and_args = (server, server_args)
|
||||
self.servers.append(server_and_args)
|
||||
return server_and_args
|
||||
|
||||
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
|
||||
"""Start all server instances for external LB mode."""
|
||||
"""Start all initial server instances for external LB mode."""
|
||||
start_errors: list[Exception] = []
|
||||
for rank in range(self.dp_size):
|
||||
# Create server args for this specific rank
|
||||
server_args = self.base_server_args.copy()
|
||||
|
||||
# Add external LB specific arguments
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(self.dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank), # Different port for each rank
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
# Use a thread to start each server to allow parallel initialization
|
||||
def start_server(r: int, sargs: list[str]):
|
||||
def start_server(r: int):
|
||||
try:
|
||||
# Start the server
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
sargs,
|
||||
auto_port=False,
|
||||
env_dict={
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE)
|
||||
),
|
||||
},
|
||||
)
|
||||
server.__enter__()
|
||||
print(
|
||||
f"Server rank {r} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
self.servers.append((server, sargs))
|
||||
self.start_rank(r)
|
||||
except Exception as e:
|
||||
print(f"Failed to start server rank {r}: {e}")
|
||||
raise
|
||||
start_errors.append(e)
|
||||
|
||||
thread = threading.Thread(target=start_server, args=(rank, server_args))
|
||||
thread = threading.Thread(target=start_server, args=(rank,))
|
||||
thread.start()
|
||||
|
||||
self.server_threads.append(thread)
|
||||
@@ -101,6 +229,10 @@ class ExternalLBServerManager:
|
||||
for thread in self.server_threads:
|
||||
thread.join()
|
||||
|
||||
if start_errors:
|
||||
start_error = start_errors[0]
|
||||
raise RuntimeError("Failed to start external LB servers") from start_error
|
||||
|
||||
# Give servers additional time to fully initialize and coordinate
|
||||
time.sleep(2)
|
||||
|
||||
@@ -184,12 +316,104 @@ def _assert_expert_parallel_config(
|
||||
assert parallel_config["enable_expert_parallel"] is expected
|
||||
|
||||
|
||||
def _send_scale_command(
|
||||
server: RemoteOpenAIServer, new_dp_size: int
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
server.url_for("scale_elastic_ep"),
|
||||
json={"new_data_parallel_size": new_dp_size},
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_scale_requests_to_start(
|
||||
scale_requests: list[Future[requests.Response]],
|
||||
timeout: float = 30,
|
||||
) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
for request in scale_requests:
|
||||
if request.done():
|
||||
response = request.result()
|
||||
pytest.fail(
|
||||
"Scale request finished before the new rank was launched: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
if all(request.running() for request in scale_requests):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
pytest.fail("Timed out waiting for all scale requests to start")
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=3)
|
||||
@pytest.mark.skipif(
|
||||
TP_SIZE != 1 or current_platform.device_count() < 3,
|
||||
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
|
||||
reason="Elastic EP scaling is not supported by the Rust frontend",
|
||||
)
|
||||
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
|
||||
server_args = [
|
||||
*default_server_args,
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
"--attention-backend",
|
||||
"TRITON_MLA",
|
||||
"--enable-elastic-ep",
|
||||
"--elastic-ep-max-dp-size",
|
||||
"3",
|
||||
"--enable-eplb",
|
||||
"--eplb-config",
|
||||
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
|
||||
]
|
||||
server_manager = ExternalLBServerManager(
|
||||
ELASTIC_EP_MODEL_NAME,
|
||||
dp_size=2,
|
||||
api_server_count=1,
|
||||
base_server_args=server_args,
|
||||
)
|
||||
|
||||
with server_manager:
|
||||
existing_servers = [server for server, _ in server_manager.servers]
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
scale_requests = [
|
||||
executor.submit(_send_scale_command, server, 3)
|
||||
for server in existing_servers
|
||||
]
|
||||
_wait_for_scale_requests_to_start(scale_requests)
|
||||
server_manager.start_rank(
|
||||
rank=2,
|
||||
dp_size=3,
|
||||
elastic_ep_scale_up_launch=True,
|
||||
)
|
||||
|
||||
for request in scale_requests:
|
||||
response = request.result(timeout=600)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
for server, _ in server_manager.servers:
|
||||
parallel_config = _get_parallel_config(server)
|
||||
assert parallel_config["data_parallel_size"] == 3
|
||||
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
|
||||
with server.get_client() as client:
|
||||
completion = client.completions.create(
|
||||
model=ELASTIC_EP_MODEL_NAME,
|
||||
prompt="Hello, my name is",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert completion.choices[0].finish_reason in ("length", "stop")
|
||||
|
||||
|
||||
def test_external_lb_server_info(server_manager):
|
||||
servers = server_manager.servers
|
||||
api_server_count = server_manager.api_server_count
|
||||
|
||||
_assert_expert_parallel_config(servers)
|
||||
|
||||
for i, (server, _) in enumerate(servers):
|
||||
print(f"Testing {i=}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user