[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -782,125 +782,3 @@ def test_rearrange_expert_weights_profile_mode(world_size):
|
||||
_test_rearrange_expert_weights_profile_mode,
|
||||
world_size,
|
||||
)
|
||||
|
||||
|
||||
def _test_nixl_deferred_init_worker(
|
||||
env,
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
num_local_experts: int,
|
||||
num_logical_experts: int,
|
||||
) -> None:
|
||||
"""Exercise NixlEplbCommunicator with defer_remote_setup=True (elastic EP path)."""
|
||||
from vllm.distributed.eplb.eplb_communicator import NixlEplbCommunicator
|
||||
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
hidden_sizes = [32, 64]
|
||||
|
||||
redundancy_config = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
old_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
redundancy_config,
|
||||
)
|
||||
|
||||
new_redundancy_config = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
new_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
new_redundancy_config,
|
||||
)
|
||||
|
||||
expert_weights = create_expert_weights(
|
||||
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
|
||||
)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
|
||||
communicator = NixlEplbCommunicator(
|
||||
cpu_group=ep_group_coordinator.cpu_group,
|
||||
all_expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
defer_remote_setup=True,
|
||||
)
|
||||
assert not communicator._remote_state_initialized
|
||||
|
||||
rearrange_expert_weights_inplace(
|
||||
old_indices,
|
||||
new_indices,
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
)
|
||||
|
||||
assert communicator._remote_state_initialized
|
||||
|
||||
local_ok = verify_expert_weights_after_shuffle(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
num_local_experts,
|
||||
)
|
||||
|
||||
local_ok = (
|
||||
verify_redundant_experts_have_same_weights(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Deferred NIXL init verification failed on at least one rank.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_nixl(), reason="NIXL is not available")
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,num_layers,num_local_experts,num_logical_experts",
|
||||
[(2, 2, 3, 4)],
|
||||
)
|
||||
def test_nixl_deferred_init(
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
):
|
||||
"""Test NixlEplbCommunicator with defer_remote_setup=True (elastic EP path)."""
|
||||
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
_test_nixl_deferred_init_worker,
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user