[EPLB] Enable nixl eplb communicator for elastic ep (#45013)
Signed-off-by: Markov Ilya <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
parent
f2069b005b
commit
ac614587f5
@@ -59,9 +59,8 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
|
||||
return accuracy
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling():
|
||||
vllm_serve_args = [
|
||||
def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
|
||||
args = [
|
||||
"--trust-remote-code",
|
||||
"--tensor-parallel-size",
|
||||
"1",
|
||||
@@ -79,7 +78,11 @@ def test_elastic_ep_scaling():
|
||||
"--eplb-config.num_redundant_experts",
|
||||
"0",
|
||||
"--eplb-config.use_async",
|
||||
"false",
|
||||
"true" if use_async_eplb else "false",
|
||||
"--eplb-config.step_interval",
|
||||
"10",
|
||||
"--eplb-config.window_size",
|
||||
"5",
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
"--data-parallel-size",
|
||||
@@ -90,7 +93,23 @@ def test_elastic_ep_scaling():
|
||||
|
||||
leader_address = os.environ.get("LEADER_ADDRESS")
|
||||
if leader_address:
|
||||
vllm_serve_args.extend(["--data-parallel-address", leader_address])
|
||||
args.extend(["--data-parallel-address", leader_address])
|
||||
|
||||
return args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling(use_async_eplb: bool):
|
||||
if use_async_eplb:
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args(use_async_eplb)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
@@ -128,44 +147,24 @@ def test_elastic_ep_scaling():
|
||||
print(f" Tolerance: {ACCURACY_TOL:.3f}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling_uneven():
|
||||
def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
|
||||
"""Test scale up with uneven worker distribution.
|
||||
|
||||
This tests the case where num_new_workers % old_dp_size != 0,
|
||||
specifically 2 -> 3 where remainder = 1 % 2 = 1.
|
||||
This exercises the remainder handling in sender-receiver pairing.
|
||||
"""
|
||||
vllm_serve_args = [
|
||||
"--trust-remote-code",
|
||||
"--tensor-parallel-size",
|
||||
"1",
|
||||
"--gpu-memory-utilization",
|
||||
"0.8",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--max-num-seqs",
|
||||
str(MAX_NUM_SEQS),
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
"--enable-elastic-ep",
|
||||
"--enable-eplb",
|
||||
"--eplb-config.num_redundant_experts",
|
||||
"0",
|
||||
"--eplb-config.use_async",
|
||||
"false",
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
"--data-parallel-size",
|
||||
"2",
|
||||
"--api-server-count",
|
||||
"1",
|
||||
]
|
||||
if use_async_eplb:
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
leader_address = os.environ.get("LEADER_ADDRESS")
|
||||
if leader_address:
|
||||
vllm_serve_args.extend(["--data-parallel-address", leader_address])
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args(use_async_eplb)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
|
||||
@@ -782,3 +782,125 @@ def test_rearrange_expert_weights_profile_mode(world_size):
|
||||
_test_rearrange_expert_weights_profile_mode,
|
||||
world_size,
|
||||
)
|
||||
|
||||
|
||||
def _test_nixl_deferred_init_worker(
|
||||
env,
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
num_local_experts: int,
|
||||
num_logical_experts: int,
|
||||
) -> None:
|
||||
"""Exercise NixlEplbCommunicator with defer_remote_setup=True (elastic EP path)."""
|
||||
from vllm.distributed.eplb.eplb_communicator import NixlEplbCommunicator
|
||||
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
hidden_sizes = [32, 64]
|
||||
|
||||
redundancy_config = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
old_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
redundancy_config,
|
||||
)
|
||||
|
||||
new_redundancy_config = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
new_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
new_redundancy_config,
|
||||
)
|
||||
|
||||
expert_weights = create_expert_weights(
|
||||
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
|
||||
)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
|
||||
communicator = NixlEplbCommunicator(
|
||||
cpu_group=ep_group_coordinator.cpu_group,
|
||||
all_expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
defer_remote_setup=True,
|
||||
)
|
||||
assert not communicator._remote_state_initialized
|
||||
|
||||
rearrange_expert_weights_inplace(
|
||||
old_indices,
|
||||
new_indices,
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
)
|
||||
|
||||
assert communicator._remote_state_initialized
|
||||
|
||||
local_ok = verify_expert_weights_after_shuffle(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
num_local_experts,
|
||||
)
|
||||
|
||||
local_ok = (
|
||||
verify_redundant_experts_have_same_weights(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Deferred NIXL init verification failed on at least one rank.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_nixl(), reason="NIXL is not available")
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,num_layers,num_local_experts,num_logical_experts",
|
||||
[(2, 2, 3, 4)],
|
||||
)
|
||||
def test_nixl_deferred_init(
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
):
|
||||
"""Test NixlEplbCommunicator with defer_remote_setup=True (elastic EP path)."""
|
||||
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
_test_nixl_deferred_init_worker,
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user