[XPU] Enable XPU eplb (#44987)

Signed-off-by: mayuyuace <qiming1.zhang@intel.com>
Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com>
This commit is contained in:
Qiming Zhang
2026-09-17 13:19:13 +08:00
committed by GitHub
parent 2743dc4f92
commit e6b1a5e5e3
16 changed files with 267 additions and 109 deletions
+6 -2
View File
@@ -13,6 +13,7 @@ from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed.parallel_state import (
init_distributed_environment,
)
from vllm.platforms import current_platform
from vllm.utils.system_utils import update_environment_variables
mp.set_start_method("spawn", force=True)
@@ -73,13 +74,16 @@ def distributed_run(fn, world_size, *args):
def set_env_vars_and_device(env: dict[str, str]) -> None:
update_environment_variables(env)
local_rank = os.environ["LOCAL_RANK"]
device = torch.device(f"cuda:{local_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{local_rank}")
torch.accelerator.set_device_index(device)
backend = "xccl" if current_platform.is_xpu() else "nccl"
# Create a minimal vllm config for init_distributed_environment
vllm_config = VllmConfig()
with set_current_vllm_config(vllm_config):
init_distributed_environment()
init_distributed_environment(backend=backend)
atexit.register(_destroy_process_group_if_initialized)
# Ensure each worker process has the same random seed
random.seed(42)
+4
View File
@@ -10,6 +10,8 @@ from concurrent.futures import ThreadPoolExecutor
import pytest
import requests
from vllm.platforms import current_platform
from ..evals.gsm8k.gsm8k_eval import evaluate_gsm8k
from ..utils import RemoteOpenAIServer, multi_gpu_test
@@ -233,6 +235,8 @@ def test_elastic_ep_scaling(
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
if current_platform.is_xpu():
pytest.skip("Elastic EP with NIXL is not supported on XPU")
if all2all_backend == "nixl_ep" and not has_nixl_ep():
pytest.skip("NIXL EP is not installed")
+4 -1
View File
@@ -10,6 +10,7 @@ from vllm.distributed.eplb.eplb_state import (
compute_logical_maps,
)
from vllm.distributed.eplb.policy.default import DefaultEplbPolicy
from vllm.platforms import current_platform
def test_eplb_load_stats_reduce_across_ranks():
@@ -263,11 +264,13 @@ def test_global_load_balance_fallback():
assert torch.sum(logcnt) == num_replicas
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("device", ["cpu", "cuda", "xpu"])
def test_device_compatibility(device):
"""Test device compatibility."""
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA not available")
if device == "xpu" and not current_platform.is_xpu():
pytest.skip("XPU not available")
weight = torch.tensor([[10, 20, 30, 40]], device=device)
num_replicas = 6
+18 -11
View File
@@ -4,15 +4,17 @@
import threading
import time
import pytest
import torch
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
from vllm.platforms import current_platform
def test_wait_blocks_until_record():
event = CpuGpuEvent()
record_stream = torch.cuda.Stream()
wait_stream = torch.cuda.Stream()
record_stream = torch.Stream()
wait_stream = torch.Stream()
wait_returned = threading.Event()
def waiter():
@@ -33,8 +35,8 @@ def test_wait_blocks_until_record():
def test_reuse_across_multiple_cycles():
wrapper = CpuGpuEvent()
record_stream = torch.cuda.Stream()
wait_stream = torch.cuda.Stream()
record_stream = torch.Stream()
wait_stream = torch.Stream()
NUM_CYCLES = 8
completed_cycles = []
barriers = [threading.Barrier(2) for _ in range(NUM_CYCLES)]
@@ -56,13 +58,18 @@ def test_reuse_across_multiple_cycles():
assert len(completed_cycles) == NUM_CYCLES
def test_producer_consumer():
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared
GPU tensor on multiple CPU threads.
@pytest.mark.parametrize("device", ["cuda", "xpu"])
def test_producer_consumer(device):
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from
a shared GPU tensor on multiple CPU threads.
"""
worker_stream = torch.cuda.Stream()
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA not available")
if device == "xpu" and not current_platform.is_xpu():
pytest.skip("XPU not available")
worker_stream = torch.Stream()
# Create a single element counter that will be shared between two threads
buf = torch.zeros(1, device="cuda")
buf = torch.zeros(1, device=device)
NUM_ROUNDS = 5
ready_cpu = [threading.Event() for _ in range(NUM_ROUNDS)]
@@ -76,7 +83,7 @@ def test_producer_consumer():
if i > 0:
events[i - 1].wait(stream=worker_stream)
with torch.cuda.stream(worker_stream):
with device_stream(worker_stream):
buf.fill_(float(i + 1))
worker_stream.synchronize()
+78 -29
View File
@@ -23,6 +23,7 @@ from vllm.distributed.parallel_state import (
ensure_model_parallel_initialized,
get_tp_group,
)
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@@ -208,10 +209,14 @@ def verify_redundant_experts_have_same_weights(
ep_rank: int,
world_size: int,
num_local_experts: int,
cpu_group,
) -> bool:
"""Verify that all replicas of the same logical expert have the same weights."""
num_layers = len(expert_weights)
total_physical_experts = world_size * num_local_experts
# XPU: gloo does not support XPU tensors, so gather on CPU then move back.
# CUDA: gloo supports CUDA tensors directly, keep original device.
use_cpu_gather = current_platform.is_xpu()
ok = True
for layer in range(num_layers):
@@ -219,31 +224,35 @@ def verify_redundant_experts_have_same_weights(
all_weights: list[torch.Tensor] = []
for weight_idx, hidden_size in enumerate(hidden_sizes):
orig_device = expert_weights[layer][weight_idx].device
if use_cpu_gather:
local_weights = expert_weights[layer][weight_idx].cpu()
gather_device = "cpu"
else:
local_weights = expert_weights[layer][weight_idx]
gather_device = orig_device
# Create tensor to store all expert weights
# Shape: [total_physical_experts, hidden_size]
gathered_weights = torch.zeros(
total_physical_experts,
hidden_size,
device=expert_weights[layer][weight_idx].device,
dtype=expert_weights[layer][weight_idx].dtype,
device=gather_device,
dtype=local_weights.dtype,
)
# Use all_gather to collect expert weights from current node
# expert_weights[layer][weight_idx] shape:
# [num_local_experts, hidden_size]
local_weights = expert_weights[layer][
weight_idx
] # [num_local_experts, hidden_size]
# Split tensor along dim 0 into a list for all_gather
gathered_weights_list = torch.chunk(gathered_weights, world_size, dim=0)
torch.distributed.all_gather(
# Output list: each element corresponds to one rank's weights
list(gathered_weights_list),
local_weights, # Input: current rank's local weights
local_weights,
group=cpu_group,
)
if use_cpu_gather:
gathered_weights = gathered_weights.to(orig_device)
all_weights.append(gathered_weights)
# Verify that all replicas of the same logical expert have the same
@@ -294,9 +303,23 @@ def verify_redundant_experts_have_same_weights(
return ok
def assert_verification_synced(local_ok: bool, msg: str) -> None:
ok_tensor = torch.tensor([1 if local_ok else 0], device="cuda", dtype=torch.int32)
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
def assert_verification_synced(local_ok: bool, msg: str, cpu_group) -> None:
if current_platform.is_xpu():
# XPU: default group backend (gloo) does not support XPU tensors;
# use CPU tensor with cpu_group instead.
ok_tensor = torch.tensor(
[1 if local_ok else 0], device="cpu", dtype=torch.int32
)
torch.distributed.all_reduce(
ok_tensor, op=torch.distributed.ReduceOp.MIN, group=cpu_group
)
else:
# CUDA: nccl (default group) supports device tensors directly.
device = torch.accelerator.current_accelerator()
ok_tensor = torch.tensor(
[1 if local_ok else 0], device=device, dtype=torch.int32
)
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
assert bool(ok_tensor.item()), msg
@@ -336,8 +359,10 @@ def _test_async_transfer_layer_without_mtp_worker(
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
total_physical_experts = world_size * num_local_experts
hidden_sizes = [16, 32]
@@ -376,7 +401,7 @@ def _test_async_transfer_layer_without_mtp_worker(
new_indices_cpu = new_indices.cpu()
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
cuda_stream = torch.cuda.Stream(device=device)
stream = torch.Stream(device=device)
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
@@ -384,7 +409,7 @@ def _test_async_transfer_layer_without_mtp_worker(
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
communicator.set_stream(cuda_stream)
communicator.set_stream(stream)
for layer_idx in range(num_layers):
transfer_metadata = transfer_layer(
@@ -394,10 +419,10 @@ def _test_async_transfer_layer_without_mtp_worker(
expert_weights_buffer=expert_buffer,
ep_group=ep_group,
communicator=communicator,
cuda_stream=cuda_stream,
stream=stream,
layer_idx=layer_idx,
)
cuda_stream.synchronize()
stream.synchronize()
move_from_buffer(
expert_weights=expert_weights[layer_idx],
expert_weights_buffers=expert_buffer,
@@ -421,6 +446,7 @@ def _test_async_transfer_layer_without_mtp_worker(
ep_rank,
world_size,
num_local_experts,
cpu_group=cpu_group,
)
and local_ok
)
@@ -428,6 +454,7 @@ def _test_async_transfer_layer_without_mtp_worker(
local_ok,
"Async transfer verification failed on at least one rank. "
"See logs for details.",
cpu_group=cpu_group,
)
@@ -452,9 +479,11 @@ def _test_rearrange_expert_weights_with_redundancy(
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.cpu_group
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
# Test parameters
total_physical_experts = world_size * num_local_experts
@@ -523,12 +552,14 @@ def _test_rearrange_expert_weights_with_redundancy(
ep_rank,
world_size,
num_local_experts,
cpu_group=cpu_group,
)
and local_ok
)
assert_verification_synced(
local_ok,
"Rearrange verification failed on at least one rank. See logs for details.",
cpu_group=cpu_group,
)
@@ -556,7 +587,7 @@ def _test_rearrange_expert_weights_with_redundancy(
],
)
@pytest.mark.parametrize(
"eplb_communicator", ["torch_nccl", "torch_gloo", "pynccl", "nixl"]
"eplb_communicator", ["torch_nccl", "torch_gloo", "torch_xccl", "pynccl", "nixl"]
)
def test_rearrange_expert_weights_with_redundancy(
world_size,
@@ -568,6 +599,12 @@ def test_rearrange_expert_weights_with_redundancy(
"""Test the functionality of rearranging expert weights with redundancy."""
if eplb_communicator == "nixl" and not has_nixl():
pytest.skip("NIXL is not available")
if eplb_communicator == "nixl" and current_platform.is_xpu():
pytest.skip("NIXL does not support XPU")
if eplb_communicator in ("torch_nccl", "pynccl") and not torch.cuda.is_available():
pytest.skip(f"{eplb_communicator} requires CUDA")
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
pytest.skip("torch_xccl requires XPU")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
@@ -592,9 +629,11 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.cpu_group
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
num_layers = 2
num_local_experts = 2
@@ -623,9 +662,10 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
original_weights.append(layer_copy)
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
backend="torch_nccl",
backend=default_backend,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
@@ -657,6 +697,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
assert_verification_synced(
local_ok,
"No-change EPLB verification failed on at least one rank.",
cpu_group=cpu_group,
)
@@ -666,7 +707,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
(2, 2, 2, 3),
],
)
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "nixl"])
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "torch_xccl", "nixl"])
def test_async_transfer_layer_without_mtp(
world_size: int,
num_layers: int,
@@ -677,6 +718,10 @@ def test_async_transfer_layer_without_mtp(
"""Exercise async EPLB transfer path without MTP/spec decode."""
if eplb_communicator == "nixl" and not has_nixl():
pytest.skip("NIXL is not available")
if eplb_communicator == "nixl" and current_platform.is_xpu():
pytest.skip("NIXL does not support XPU")
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
pytest.skip("torch_xccl requires XPU")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
@@ -715,9 +760,11 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.cpu_group
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
num_layers = 1
num_local_experts = 2
@@ -753,9 +800,10 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
original_weights.append(layer_copy)
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
backend="torch_nccl",
backend=default_backend,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
@@ -788,6 +836,7 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
assert_verification_synced(
local_ok,
"Profile-mode EPLB verification failed on at least one rank.",
cpu_group=cpu_group,
)
@@ -17,6 +17,7 @@ from vllm.distributed.parallel_state import (
get_tp_group,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@@ -81,7 +82,7 @@ def make_fused_moe_layer(
)
re = fml.routed_experts
device = torch.device(f"cuda:{rank}")
device = torch.device(f"{torch.accelerator.current_accelerator().type}:{rank}")
from functools import partial
@@ -190,13 +191,14 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.parallel_config.enable_eplb = True
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
)
ep_group = get_tp_group().cpu_group
ep_group = get_tp_group().device_group
ep_rank = torch.distributed.get_rank()
fml_layers = [
@@ -216,9 +218,10 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend="torch_nccl",
backend=default_backend,
expert_weights=rank_expert_weights,
expert_buffer=expert_buffer,
)
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
ModelOptNvFp4FusedMoE,
)
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@@ -270,6 +271,8 @@ def test_eplb_fml(
):
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
if current_platform.is_xpu():
pytest.skip("NVFP4 quantization and flashinfer backends are CUDA-only")
num_local_experts = num_experts // world_size
num_topk = 4