[XPU] Enable XPU eplb (#44987)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com> Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com>
This commit is contained in:
@@ -23,6 +23,7 @@ from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
@@ -208,10 +209,14 @@ def verify_redundant_experts_have_same_weights(
|
||||
ep_rank: int,
|
||||
world_size: int,
|
||||
num_local_experts: int,
|
||||
cpu_group,
|
||||
) -> bool:
|
||||
"""Verify that all replicas of the same logical expert have the same weights."""
|
||||
num_layers = len(expert_weights)
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
# XPU: gloo does not support XPU tensors, so gather on CPU then move back.
|
||||
# CUDA: gloo supports CUDA tensors directly, keep original device.
|
||||
use_cpu_gather = current_platform.is_xpu()
|
||||
|
||||
ok = True
|
||||
for layer in range(num_layers):
|
||||
@@ -219,31 +224,35 @@ def verify_redundant_experts_have_same_weights(
|
||||
all_weights: list[torch.Tensor] = []
|
||||
|
||||
for weight_idx, hidden_size in enumerate(hidden_sizes):
|
||||
orig_device = expert_weights[layer][weight_idx].device
|
||||
|
||||
if use_cpu_gather:
|
||||
local_weights = expert_weights[layer][weight_idx].cpu()
|
||||
gather_device = "cpu"
|
||||
else:
|
||||
local_weights = expert_weights[layer][weight_idx]
|
||||
gather_device = orig_device
|
||||
|
||||
# Create tensor to store all expert weights
|
||||
# Shape: [total_physical_experts, hidden_size]
|
||||
gathered_weights = torch.zeros(
|
||||
total_physical_experts,
|
||||
hidden_size,
|
||||
device=expert_weights[layer][weight_idx].device,
|
||||
dtype=expert_weights[layer][weight_idx].dtype,
|
||||
device=gather_device,
|
||||
dtype=local_weights.dtype,
|
||||
)
|
||||
|
||||
# Use all_gather to collect expert weights from current node
|
||||
# expert_weights[layer][weight_idx] shape:
|
||||
# [num_local_experts, hidden_size]
|
||||
local_weights = expert_weights[layer][
|
||||
weight_idx
|
||||
] # [num_local_experts, hidden_size]
|
||||
|
||||
# Split tensor along dim 0 into a list for all_gather
|
||||
gathered_weights_list = torch.chunk(gathered_weights, world_size, dim=0)
|
||||
|
||||
torch.distributed.all_gather(
|
||||
# Output list: each element corresponds to one rank's weights
|
||||
list(gathered_weights_list),
|
||||
local_weights, # Input: current rank's local weights
|
||||
local_weights,
|
||||
group=cpu_group,
|
||||
)
|
||||
|
||||
if use_cpu_gather:
|
||||
gathered_weights = gathered_weights.to(orig_device)
|
||||
all_weights.append(gathered_weights)
|
||||
|
||||
# Verify that all replicas of the same logical expert have the same
|
||||
@@ -294,9 +303,23 @@ def verify_redundant_experts_have_same_weights(
|
||||
return ok
|
||||
|
||||
|
||||
def assert_verification_synced(local_ok: bool, msg: str) -> None:
|
||||
ok_tensor = torch.tensor([1 if local_ok else 0], device="cuda", dtype=torch.int32)
|
||||
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
|
||||
def assert_verification_synced(local_ok: bool, msg: str, cpu_group) -> None:
|
||||
if current_platform.is_xpu():
|
||||
# XPU: default group backend (gloo) does not support XPU tensors;
|
||||
# use CPU tensor with cpu_group instead.
|
||||
ok_tensor = torch.tensor(
|
||||
[1 if local_ok else 0], device="cpu", dtype=torch.int32
|
||||
)
|
||||
torch.distributed.all_reduce(
|
||||
ok_tensor, op=torch.distributed.ReduceOp.MIN, group=cpu_group
|
||||
)
|
||||
else:
|
||||
# CUDA: nccl (default group) supports device tensors directly.
|
||||
device = torch.accelerator.current_accelerator()
|
||||
ok_tensor = torch.tensor(
|
||||
[1 if local_ok else 0], device=device, dtype=torch.int32
|
||||
)
|
||||
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
|
||||
assert bool(ok_tensor.item()), msg
|
||||
|
||||
|
||||
@@ -336,8 +359,10 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
hidden_sizes = [16, 32]
|
||||
@@ -376,7 +401,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
new_indices_cpu = new_indices.cpu()
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
cuda_stream = torch.cuda.Stream(device=device)
|
||||
stream = torch.Stream(device=device)
|
||||
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
@@ -384,7 +409,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
communicator.set_stream(cuda_stream)
|
||||
communicator.set_stream(stream)
|
||||
|
||||
for layer_idx in range(num_layers):
|
||||
transfer_metadata = transfer_layer(
|
||||
@@ -394,10 +419,10 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
expert_weights_buffer=expert_buffer,
|
||||
ep_group=ep_group,
|
||||
communicator=communicator,
|
||||
cuda_stream=cuda_stream,
|
||||
stream=stream,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
cuda_stream.synchronize()
|
||||
stream.synchronize()
|
||||
move_from_buffer(
|
||||
expert_weights=expert_weights[layer_idx],
|
||||
expert_weights_buffers=expert_buffer,
|
||||
@@ -421,6 +446,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
@@ -428,6 +454,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
local_ok,
|
||||
"Async transfer verification failed on at least one rank. "
|
||||
"See logs for details.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -452,9 +479,11 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
# Test parameters
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
@@ -523,12 +552,14 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Rearrange verification failed on at least one rank. See logs for details.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -556,7 +587,7 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"eplb_communicator", ["torch_nccl", "torch_gloo", "pynccl", "nixl"]
|
||||
"eplb_communicator", ["torch_nccl", "torch_gloo", "torch_xccl", "pynccl", "nixl"]
|
||||
)
|
||||
def test_rearrange_expert_weights_with_redundancy(
|
||||
world_size,
|
||||
@@ -568,6 +599,12 @@ def test_rearrange_expert_weights_with_redundancy(
|
||||
"""Test the functionality of rearranging expert weights with redundancy."""
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if eplb_communicator == "nixl" and current_platform.is_xpu():
|
||||
pytest.skip("NIXL does not support XPU")
|
||||
if eplb_communicator in ("torch_nccl", "pynccl") and not torch.cuda.is_available():
|
||||
pytest.skip(f"{eplb_communicator} requires CUDA")
|
||||
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
|
||||
pytest.skip("torch_xccl requires XPU")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
@@ -592,9 +629,11 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
num_layers = 2
|
||||
num_local_experts = 2
|
||||
@@ -623,9 +662,10 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend="torch_nccl",
|
||||
backend=default_backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
@@ -657,6 +697,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"No-change EPLB verification failed on at least one rank.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -666,7 +707,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
(2, 2, 2, 3),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "nixl"])
|
||||
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "torch_xccl", "nixl"])
|
||||
def test_async_transfer_layer_without_mtp(
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
@@ -677,6 +718,10 @@ def test_async_transfer_layer_without_mtp(
|
||||
"""Exercise async EPLB transfer path without MTP/spec decode."""
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if eplb_communicator == "nixl" and current_platform.is_xpu():
|
||||
pytest.skip("NIXL does not support XPU")
|
||||
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
|
||||
pytest.skip("torch_xccl requires XPU")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
|
||||
@@ -715,9 +760,11 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
num_layers = 1
|
||||
num_local_experts = 2
|
||||
@@ -753,9 +800,10 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend="torch_nccl",
|
||||
backend=default_backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
@@ -788,6 +836,7 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Profile-mode EPLB verification failed on at least one rank.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user