[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)
This commit is contained in:
@@ -27,7 +27,7 @@ MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
NUM_GSM8K_QUESTIONS = 256
|
||||
EXPECTED_ACCURACY = 0.58
|
||||
ACCURACY_TOL = 0.08
|
||||
MAX_NUM_SEQS = 32
|
||||
DEFAULT_ALL2ALL_BACKEND = "allgather_reducescatter"
|
||||
|
||||
|
||||
def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
|
||||
@@ -161,7 +161,12 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
|
||||
return accuracy
|
||||
|
||||
|
||||
def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]:
|
||||
def _base_serve_args(
|
||||
dp_size: int = 2,
|
||||
max_dp_size: int = 4,
|
||||
enforce_eager: bool = False,
|
||||
all2all_backend: str = DEFAULT_ALL2ALL_BACKEND,
|
||||
) -> list[str]:
|
||||
args = [
|
||||
"--trust-remote-code",
|
||||
"--tensor-parallel-size",
|
||||
@@ -170,12 +175,12 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]
|
||||
"0.8",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--max-num-seqs",
|
||||
str(MAX_NUM_SEQS),
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
all2all_backend,
|
||||
"--enable-elastic-ep",
|
||||
"--elastic-ep-max-dp-size",
|
||||
str(max_dp_size),
|
||||
"--enable-eplb",
|
||||
"--eplb-config.num_redundant_experts",
|
||||
"0",
|
||||
@@ -205,28 +210,43 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enforce_eager", "traffic_mode"),
|
||||
("enforce_eager", "traffic_mode", "all2all_backend"),
|
||||
[
|
||||
pytest.param(True, "none", id="enforce_eager_none"),
|
||||
pytest.param(True, "light", id="enforce_eager_light"),
|
||||
pytest.param(True, "heavy", id="enforce_eager_heavy"),
|
||||
pytest.param(False, "heavy", id="cuda_graphs_heavy"),
|
||||
pytest.param(True, "none", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_none"),
|
||||
pytest.param(True, "light", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_light"),
|
||||
pytest.param(True, "heavy", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_heavy"),
|
||||
pytest.param(
|
||||
False,
|
||||
"heavy",
|
||||
DEFAULT_ALL2ALL_BACKEND,
|
||||
id="cuda_graphs_heavy",
|
||||
),
|
||||
pytest.param(False, "heavy", "nixl_ep", id="cuda_graphs_heavy_nixl_ep"),
|
||||
],
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
|
||||
def test_elastic_ep_scaling(
|
||||
enforce_eager: bool, traffic_mode: str, all2all_backend: str
|
||||
):
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
from vllm.utils.import_utils import has_nixl_ep
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
if all2all_backend == "nixl_ep" and not has_nixl_ep():
|
||||
pytest.skip("NIXL EP is not installed")
|
||||
|
||||
initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
|
||||
target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
|
||||
assert target_dp_size > initial_dp_size
|
||||
vllm_serve_args = _base_serve_args(initial_dp_size, enforce_eager)
|
||||
vllm_serve_args = _base_serve_args(
|
||||
initial_dp_size, target_dp_size, enforce_eager, all2all_backend
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
MODEL_NAME,
|
||||
vllm_serve_args,
|
||||
max_wait_seconds=1200,
|
||||
) as server:
|
||||
initial_accuracy = _run_gsm8k_eval(server, "Initial")
|
||||
|
||||
@@ -255,55 +275,3 @@ def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
|
||||
f"(diff: {scale_down_accuracy - initial_accuracy:+.3f})"
|
||||
)
|
||||
print(f" Tolerance: {ACCURACY_TOL:.3f}")
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling_uneven():
|
||||
"""Test scale up with uneven worker distribution.
|
||||
|
||||
This tests the case where num_new_workers % old_dp_size != 0,
|
||||
specifically 2 -> 3 where remainder = 1 % 2 = 1.
|
||||
This exercises the remainder handling in sender-receiver pairing.
|
||||
"""
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args()
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
) as server:
|
||||
initial_accuracy = _run_gsm8k_eval(server, "Initial (2 GPUs)")
|
||||
|
||||
# Scale 2 -> 3: This has remainder = 1 % 2 = 1
|
||||
# Tests uneven sender-receiver pairing
|
||||
assert _send_scale_command(server, 3)
|
||||
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (3 GPUs)")
|
||||
|
||||
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
|
||||
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
|
||||
)
|
||||
|
||||
# Scale back down to 2
|
||||
assert _send_scale_command(server, 2)
|
||||
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
|
||||
|
||||
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
|
||||
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
|
||||
)
|
||||
|
||||
print("\nAccuracy Summary (Uneven Scaling):")
|
||||
print(f" Initial: {initial_accuracy:.3f}")
|
||||
print(
|
||||
f" Scale up: {scale_up_accuracy:.3f} "
|
||||
f"(diff: {scale_up_accuracy - initial_accuracy:+.3f})"
|
||||
)
|
||||
print(
|
||||
f" Scale down: {scale_down_accuracy:.3f} "
|
||||
f"(diff: {scale_down_accuracy - initial_accuracy:+.3f})"
|
||||
)
|
||||
print(f" Tolerance: {ACCURACY_TOL:.3f}")
|
||||
|
||||
@@ -16,10 +16,14 @@ def _make_model_state(
|
||||
phy2log: torch.Tensor,
|
||||
log2phy: torch.Tensor,
|
||||
logcnt: torch.Tensor,
|
||||
phy2log_storage: torch.Tensor | None = None,
|
||||
) -> MagicMock:
|
||||
"""Build a minimal EplbModelState mock with only the three map tensors."""
|
||||
"""Build a minimal EplbModelState mock with its map tensors."""
|
||||
state = MagicMock()
|
||||
state.physical_to_logical_map = phy2log
|
||||
state.physical_to_logical_map_buffer = (
|
||||
phy2log if phy2log_storage is None else phy2log_storage
|
||||
)
|
||||
state.logical_to_physical_map = log2phy
|
||||
state.logical_replica_count = logcnt
|
||||
return state
|
||||
@@ -27,15 +31,17 @@ def _make_model_state(
|
||||
|
||||
def test_commit_eplb_maps_shape_change():
|
||||
"""
|
||||
The normal path copies the physical_to_logical map in-place. When the number of
|
||||
physical experts changes, the old map should be replaced entirely.
|
||||
When the number of physical experts changes, resize the active map within its
|
||||
preallocated storage.
|
||||
"""
|
||||
num_layers, num_logical, num_physical = 2, 4, 6
|
||||
max_replicas = 3
|
||||
|
||||
# Build current state tensors
|
||||
storage = torch.full((num_layers, num_physical + 2), -1, dtype=torch.long)
|
||||
model_state = _make_model_state(
|
||||
phy2log=torch.zeros(num_layers, num_physical, dtype=torch.long),
|
||||
phy2log=storage[:, :num_physical],
|
||||
phy2log_storage=storage,
|
||||
log2phy=torch.full(
|
||||
(num_layers, num_logical, max_replicas), -1, dtype=torch.long
|
||||
),
|
||||
@@ -54,6 +60,7 @@ def test_commit_eplb_maps_shape_change():
|
||||
# Check that the number of physical experts has been updated and that the values
|
||||
# match
|
||||
assert model_state.physical_to_logical_map.shape[1] == num_physical + 2
|
||||
assert model_state.physical_to_logical_map.data_ptr() == storage.data_ptr()
|
||||
assert torch.equal(model_state.physical_to_logical_map, new_phy2log_larger)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user