[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)

This commit is contained in:
Itay Alroy
2026-09-12 10:26:39 -04:00
committed by GitHub
parent 13e221f830
commit 658c8131c7
22 changed files with 372 additions and 270 deletions
+33 -65
View File
@@ -27,7 +27,7 @@ MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
NUM_GSM8K_QUESTIONS = 256
EXPECTED_ACCURACY = 0.58
ACCURACY_TOL = 0.08
MAX_NUM_SEQS = 32
DEFAULT_ALL2ALL_BACKEND = "allgather_reducescatter"
def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
@@ -161,7 +161,12 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
return accuracy
def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]:
def _base_serve_args(
dp_size: int = 2,
max_dp_size: int = 4,
enforce_eager: bool = False,
all2all_backend: str = DEFAULT_ALL2ALL_BACKEND,
) -> list[str]:
args = [
"--trust-remote-code",
"--tensor-parallel-size",
@@ -170,12 +175,12 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]
"0.8",
"--max-model-len",
"4096",
"--max-num-seqs",
str(MAX_NUM_SEQS),
"--enable-expert-parallel",
"--all2all-backend",
"allgather_reducescatter",
all2all_backend,
"--enable-elastic-ep",
"--elastic-ep-max-dp-size",
str(max_dp_size),
"--enable-eplb",
"--eplb-config.num_redundant_experts",
"0",
@@ -205,28 +210,43 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]
@pytest.mark.parametrize(
("enforce_eager", "traffic_mode"),
("enforce_eager", "traffic_mode", "all2all_backend"),
[
pytest.param(True, "none", id="enforce_eager_none"),
pytest.param(True, "light", id="enforce_eager_light"),
pytest.param(True, "heavy", id="enforce_eager_heavy"),
pytest.param(False, "heavy", id="cuda_graphs_heavy"),
pytest.param(True, "none", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_none"),
pytest.param(True, "light", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_light"),
pytest.param(True, "heavy", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_heavy"),
pytest.param(
False,
"heavy",
DEFAULT_ALL2ALL_BACKEND,
id="cuda_graphs_heavy",
),
pytest.param(False, "heavy", "nixl_ep", id="cuda_graphs_heavy_nixl_ep"),
],
)
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
def test_elastic_ep_scaling(
enforce_eager: bool, traffic_mode: str, all2all_backend: str
):
from vllm.distributed.eplb.eplb_communicator import has_nixl
from vllm.utils.import_utils import has_nixl_ep
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
if all2all_backend == "nixl_ep" and not has_nixl_ep():
pytest.skip("NIXL EP is not installed")
initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
assert target_dp_size > initial_dp_size
vllm_serve_args = _base_serve_args(initial_dp_size, enforce_eager)
vllm_serve_args = _base_serve_args(
initial_dp_size, target_dp_size, enforce_eager, all2all_backend
)
with RemoteOpenAIServer(
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
MODEL_NAME,
vllm_serve_args,
max_wait_seconds=1200,
) as server:
initial_accuracy = _run_gsm8k_eval(server, "Initial")
@@ -255,55 +275,3 @@ def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
f"(diff: {scale_down_accuracy - initial_accuracy:+.3f})"
)
print(f" Tolerance: {ACCURACY_TOL:.3f}")
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling_uneven():
"""Test scale up with uneven worker distribution.
This tests the case where num_new_workers % old_dp_size != 0,
specifically 2 -> 3 where remainder = 1 % 2 = 1.
This exercises the remainder handling in sender-receiver pairing.
"""
from vllm.distributed.eplb.eplb_communicator import has_nixl
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
vllm_serve_args = _base_serve_args()
with RemoteOpenAIServer(
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
) as server:
initial_accuracy = _run_gsm8k_eval(server, "Initial (2 GPUs)")
# Scale 2 -> 3: This has remainder = 1 % 2 = 1
# Tests uneven sender-receiver pairing
assert _send_scale_command(server, 3)
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (3 GPUs)")
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
)
# Scale back down to 2
assert _send_scale_command(server, 2)
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
)
print("\nAccuracy Summary (Uneven Scaling):")
print(f" Initial: {initial_accuracy:.3f}")
print(
f" Scale up: {scale_up_accuracy:.3f} "
f"(diff: {scale_up_accuracy - initial_accuracy:+.3f})"
)
print(
f" Scale down: {scale_down_accuracy:.3f} "
f"(diff: {scale_down_accuracy - initial_accuracy:+.3f})"
)
print(f" Tolerance: {ACCURACY_TOL:.3f}")
+11 -4
View File
@@ -16,10 +16,14 @@ def _make_model_state(
phy2log: torch.Tensor,
log2phy: torch.Tensor,
logcnt: torch.Tensor,
phy2log_storage: torch.Tensor | None = None,
) -> MagicMock:
"""Build a minimal EplbModelState mock with only the three map tensors."""
"""Build a minimal EplbModelState mock with its map tensors."""
state = MagicMock()
state.physical_to_logical_map = phy2log
state.physical_to_logical_map_buffer = (
phy2log if phy2log_storage is None else phy2log_storage
)
state.logical_to_physical_map = log2phy
state.logical_replica_count = logcnt
return state
@@ -27,15 +31,17 @@ def _make_model_state(
def test_commit_eplb_maps_shape_change():
"""
The normal path copies the physical_to_logical map in-place. When the number of
physical experts changes, the old map should be replaced entirely.
When the number of physical experts changes, resize the active map within its
preallocated storage.
"""
num_layers, num_logical, num_physical = 2, 4, 6
max_replicas = 3
# Build current state tensors
storage = torch.full((num_layers, num_physical + 2), -1, dtype=torch.long)
model_state = _make_model_state(
phy2log=torch.zeros(num_layers, num_physical, dtype=torch.long),
phy2log=storage[:, :num_physical],
phy2log_storage=storage,
log2phy=torch.full(
(num_layers, num_logical, max_replicas), -1, dtype=torch.long
),
@@ -54,6 +60,7 @@ def test_commit_eplb_maps_shape_change():
# Check that the number of physical experts has been updated and that the values
# match
assert model_state.physical_to_logical_map.shape[1] == num_physical + 2
assert model_state.physical_to_logical_map.data_ptr() == storage.data_ptr()
assert torch.equal(model_state.physical_to_logical_map, new_phy2log_larger)