From e6b1a5e5e3fce6b777c5f63398f87c7c911a40b3 Mon Sep 17 00:00:00 2001 From: Qiming Zhang Date: Wed, 16 Sep 2026 22:19:13 -0700 Subject: [PATCH] [XPU] Enable XPU eplb (#44987) Signed-off-by: mayuyuace Signed-off-by: Qiming Zhang --- tests/conftest.py | 2 +- tests/distributed/eplb_utils.py | 8 +- tests/distributed/test_elastic_ep.py | 4 + tests/distributed/test_eplb_algo.py | 5 +- tests/distributed/test_eplb_events.py | 29 +++-- tests/distributed/test_eplb_execute.py | 107 +++++++++++++----- .../distributed/test_eplb_fused_moe_layer.py | 9 +- .../test_eplb_fused_moe_layer_dep_nvfp4.py | 3 + vllm/config/parallel.py | 17 ++- vllm/distributed/eplb/async_worker.py | 28 ++--- vllm/distributed/eplb/eplb_communicator.py | 95 +++++++++++++--- vllm/distributed/eplb/eplb_state.py | 22 ++-- vllm/distributed/eplb/eplb_utils.py | 25 +++- vllm/distributed/eplb/rebalance_execute.py | 19 ++-- .../layers/fused_moe/experts/xpu_moe.py | 1 + .../layers/fused_moe/router/base_router.py | 2 +- 16 files changed, 267 insertions(+), 109 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5fd594a04a..a52a38f357 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -935,7 +935,7 @@ def hf_runner(): def _default_block_size() -> int: - if torch.xpu.is_available(): + if current_platform.is_xpu(): return 64 if current_platform.is_cpu(): return 128 diff --git a/tests/distributed/eplb_utils.py b/tests/distributed/eplb_utils.py index e6a41842e1..d868069e0a 100644 --- a/tests/distributed/eplb_utils.py +++ b/tests/distributed/eplb_utils.py @@ -13,6 +13,7 @@ from vllm.config import VllmConfig, set_current_vllm_config from vllm.distributed.parallel_state import ( init_distributed_environment, ) +from vllm.platforms import current_platform from vllm.utils.system_utils import update_environment_variables mp.set_start_method("spawn", force=True) @@ -73,13 +74,16 @@ def distributed_run(fn, world_size, *args): def set_env_vars_and_device(env: dict[str, str]) -> None: update_environment_variables(env) local_rank = os.environ["LOCAL_RANK"] - device = torch.device(f"cuda:{local_rank}") + accelerator_type = torch.accelerator.current_accelerator().type + device = torch.device(f"{accelerator_type}:{local_rank}") torch.accelerator.set_device_index(device) + backend = "xccl" if current_platform.is_xpu() else "nccl" + # Create a minimal vllm config for init_distributed_environment vllm_config = VllmConfig() with set_current_vllm_config(vllm_config): - init_distributed_environment() + init_distributed_environment(backend=backend) atexit.register(_destroy_process_group_if_initialized) # Ensure each worker process has the same random seed random.seed(42) diff --git a/tests/distributed/test_elastic_ep.py b/tests/distributed/test_elastic_ep.py index 8ccea86283..1dc6b5d3bb 100644 --- a/tests/distributed/test_elastic_ep.py +++ b/tests/distributed/test_elastic_ep.py @@ -10,6 +10,8 @@ from concurrent.futures import ThreadPoolExecutor import pytest import requests +from vllm.platforms import current_platform + from ..evals.gsm8k.gsm8k_eval import evaluate_gsm8k from ..utils import RemoteOpenAIServer, multi_gpu_test @@ -233,6 +235,8 @@ def test_elastic_ep_scaling( if not has_nixl(): pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)") + if current_platform.is_xpu(): + pytest.skip("Elastic EP with NIXL is not supported on XPU") if all2all_backend == "nixl_ep" and not has_nixl_ep(): pytest.skip("NIXL EP is not installed") diff --git a/tests/distributed/test_eplb_algo.py b/tests/distributed/test_eplb_algo.py index aaa987b9fc..8cd5cf9e3c 100644 --- a/tests/distributed/test_eplb_algo.py +++ b/tests/distributed/test_eplb_algo.py @@ -10,6 +10,7 @@ from vllm.distributed.eplb.eplb_state import ( compute_logical_maps, ) from vllm.distributed.eplb.policy.default import DefaultEplbPolicy +from vllm.platforms import current_platform def test_eplb_load_stats_reduce_across_ranks(): @@ -263,11 +264,13 @@ def test_global_load_balance_fallback(): assert torch.sum(logcnt) == num_replicas -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", ["cpu", "cuda", "xpu"]) def test_device_compatibility(device): """Test device compatibility.""" if device == "cuda" and not torch.cuda.is_available(): pytest.skip("CUDA not available") + if device == "xpu" and not current_platform.is_xpu(): + pytest.skip("XPU not available") weight = torch.tensor([[10, 20, 30, 40]], device=device) num_replicas = 6 diff --git a/tests/distributed/test_eplb_events.py b/tests/distributed/test_eplb_events.py index 963e3cbb77..f924b107d3 100644 --- a/tests/distributed/test_eplb_events.py +++ b/tests/distributed/test_eplb_events.py @@ -4,15 +4,17 @@ import threading import time +import pytest import torch -from vllm.distributed.eplb.eplb_utils import CpuGpuEvent +from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream +from vllm.platforms import current_platform def test_wait_blocks_until_record(): event = CpuGpuEvent() - record_stream = torch.cuda.Stream() - wait_stream = torch.cuda.Stream() + record_stream = torch.Stream() + wait_stream = torch.Stream() wait_returned = threading.Event() def waiter(): @@ -33,8 +35,8 @@ def test_wait_blocks_until_record(): def test_reuse_across_multiple_cycles(): wrapper = CpuGpuEvent() - record_stream = torch.cuda.Stream() - wait_stream = torch.cuda.Stream() + record_stream = torch.Stream() + wait_stream = torch.Stream() NUM_CYCLES = 8 completed_cycles = [] barriers = [threading.Barrier(2) for _ in range(NUM_CYCLES)] @@ -56,13 +58,18 @@ def test_reuse_across_multiple_cycles(): assert len(completed_cycles) == NUM_CYCLES -def test_producer_consumer(): - """This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared - GPU tensor on multiple CPU threads. +@pytest.mark.parametrize("device", ["cuda", "xpu"]) +def test_producer_consumer(device): + """This test uses the CpuGpuEvent to synchronize reads and writes to/from + a shared GPU tensor on multiple CPU threads. """ - worker_stream = torch.cuda.Stream() + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA not available") + if device == "xpu" and not current_platform.is_xpu(): + pytest.skip("XPU not available") + worker_stream = torch.Stream() # Create a single element counter that will be shared between two threads - buf = torch.zeros(1, device="cuda") + buf = torch.zeros(1, device=device) NUM_ROUNDS = 5 ready_cpu = [threading.Event() for _ in range(NUM_ROUNDS)] @@ -76,7 +83,7 @@ def test_producer_consumer(): if i > 0: events[i - 1].wait(stream=worker_stream) - with torch.cuda.stream(worker_stream): + with device_stream(worker_stream): buf.fill_(float(i + 1)) worker_stream.synchronize() diff --git a/tests/distributed/test_eplb_execute.py b/tests/distributed/test_eplb_execute.py index aed0decc6d..2037535e3d 100644 --- a/tests/distributed/test_eplb_execute.py +++ b/tests/distributed/test_eplb_execute.py @@ -23,6 +23,7 @@ from vllm.distributed.parallel_state import ( ensure_model_parallel_initialized, get_tp_group, ) +from vllm.platforms import current_platform from .eplb_utils import distributed_run, set_env_vars_and_device @@ -208,10 +209,14 @@ def verify_redundant_experts_have_same_weights( ep_rank: int, world_size: int, num_local_experts: int, + cpu_group, ) -> bool: """Verify that all replicas of the same logical expert have the same weights.""" num_layers = len(expert_weights) total_physical_experts = world_size * num_local_experts + # XPU: gloo does not support XPU tensors, so gather on CPU then move back. + # CUDA: gloo supports CUDA tensors directly, keep original device. + use_cpu_gather = current_platform.is_xpu() ok = True for layer in range(num_layers): @@ -219,31 +224,35 @@ def verify_redundant_experts_have_same_weights( all_weights: list[torch.Tensor] = [] for weight_idx, hidden_size in enumerate(hidden_sizes): + orig_device = expert_weights[layer][weight_idx].device + + if use_cpu_gather: + local_weights = expert_weights[layer][weight_idx].cpu() + gather_device = "cpu" + else: + local_weights = expert_weights[layer][weight_idx] + gather_device = orig_device + # Create tensor to store all expert weights # Shape: [total_physical_experts, hidden_size] gathered_weights = torch.zeros( total_physical_experts, hidden_size, - device=expert_weights[layer][weight_idx].device, - dtype=expert_weights[layer][weight_idx].dtype, + device=gather_device, + dtype=local_weights.dtype, ) - # Use all_gather to collect expert weights from current node - # expert_weights[layer][weight_idx] shape: - # [num_local_experts, hidden_size] - local_weights = expert_weights[layer][ - weight_idx - ] # [num_local_experts, hidden_size] - # Split tensor along dim 0 into a list for all_gather gathered_weights_list = torch.chunk(gathered_weights, world_size, dim=0) torch.distributed.all_gather( - # Output list: each element corresponds to one rank's weights list(gathered_weights_list), - local_weights, # Input: current rank's local weights + local_weights, + group=cpu_group, ) + if use_cpu_gather: + gathered_weights = gathered_weights.to(orig_device) all_weights.append(gathered_weights) # Verify that all replicas of the same logical expert have the same @@ -294,9 +303,23 @@ def verify_redundant_experts_have_same_weights( return ok -def assert_verification_synced(local_ok: bool, msg: str) -> None: - ok_tensor = torch.tensor([1 if local_ok else 0], device="cuda", dtype=torch.int32) - torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN) +def assert_verification_synced(local_ok: bool, msg: str, cpu_group) -> None: + if current_platform.is_xpu(): + # XPU: default group backend (gloo) does not support XPU tensors; + # use CPU tensor with cpu_group instead. + ok_tensor = torch.tensor( + [1 if local_ok else 0], device="cpu", dtype=torch.int32 + ) + torch.distributed.all_reduce( + ok_tensor, op=torch.distributed.ReduceOp.MIN, group=cpu_group + ) + else: + # CUDA: nccl (default group) supports device tensors directly. + device = torch.accelerator.current_accelerator() + ok_tensor = torch.tensor( + [1 if local_ok else 0], device=device, dtype=torch.int32 + ) + torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN) assert bool(ok_tensor.item()), msg @@ -336,8 +359,10 @@ def _test_async_transfer_layer_without_mtp_worker( ep_group_coordinator = get_tp_group() ep_group = ep_group_coordinator.device_group + cpu_group = ep_group_coordinator.cpu_group ep_rank = torch.distributed.get_rank() - device = torch.device(f"cuda:{ep_rank}") + accelerator_type = torch.accelerator.current_accelerator().type + device = torch.device(f"{accelerator_type}:{ep_rank}") total_physical_experts = world_size * num_local_experts hidden_sizes = [16, 32] @@ -376,7 +401,7 @@ def _test_async_transfer_layer_without_mtp_worker( new_indices_cpu = new_indices.cpu() expert_buffer = [torch.empty_like(w) for w in expert_weights[0]] - cuda_stream = torch.cuda.Stream(device=device) + stream = torch.Stream(device=device) communicator = create_eplb_communicator_or_raise( group_coordinator=ep_group_coordinator, @@ -384,7 +409,7 @@ def _test_async_transfer_layer_without_mtp_worker( expert_weights=expert_weights, expert_buffer=expert_buffer, ) - communicator.set_stream(cuda_stream) + communicator.set_stream(stream) for layer_idx in range(num_layers): transfer_metadata = transfer_layer( @@ -394,10 +419,10 @@ def _test_async_transfer_layer_without_mtp_worker( expert_weights_buffer=expert_buffer, ep_group=ep_group, communicator=communicator, - cuda_stream=cuda_stream, + stream=stream, layer_idx=layer_idx, ) - cuda_stream.synchronize() + stream.synchronize() move_from_buffer( expert_weights=expert_weights[layer_idx], expert_weights_buffers=expert_buffer, @@ -421,6 +446,7 @@ def _test_async_transfer_layer_without_mtp_worker( ep_rank, world_size, num_local_experts, + cpu_group=cpu_group, ) and local_ok ) @@ -428,6 +454,7 @@ def _test_async_transfer_layer_without_mtp_worker( local_ok, "Async transfer verification failed on at least one rank. " "See logs for details.", + cpu_group=cpu_group, ) @@ -452,9 +479,11 @@ def _test_rearrange_expert_weights_with_redundancy( ) ep_group_coordinator = get_tp_group() - ep_group = ep_group_coordinator.cpu_group + ep_group = ep_group_coordinator.device_group + cpu_group = ep_group_coordinator.cpu_group ep_rank = torch.distributed.get_rank() - device = torch.device(f"cuda:{ep_rank}") + accelerator_type = torch.accelerator.current_accelerator().type + device = torch.device(f"{accelerator_type}:{ep_rank}") # Test parameters total_physical_experts = world_size * num_local_experts @@ -523,12 +552,14 @@ def _test_rearrange_expert_weights_with_redundancy( ep_rank, world_size, num_local_experts, + cpu_group=cpu_group, ) and local_ok ) assert_verification_synced( local_ok, "Rearrange verification failed on at least one rank. See logs for details.", + cpu_group=cpu_group, ) @@ -556,7 +587,7 @@ def _test_rearrange_expert_weights_with_redundancy( ], ) @pytest.mark.parametrize( - "eplb_communicator", ["torch_nccl", "torch_gloo", "pynccl", "nixl"] + "eplb_communicator", ["torch_nccl", "torch_gloo", "torch_xccl", "pynccl", "nixl"] ) def test_rearrange_expert_weights_with_redundancy( world_size, @@ -568,6 +599,12 @@ def test_rearrange_expert_weights_with_redundancy( """Test the functionality of rearranging expert weights with redundancy.""" if eplb_communicator == "nixl" and not has_nixl(): pytest.skip("NIXL is not available") + if eplb_communicator == "nixl" and current_platform.is_xpu(): + pytest.skip("NIXL does not support XPU") + if eplb_communicator in ("torch_nccl", "pynccl") and not torch.cuda.is_available(): + pytest.skip(f"{eplb_communicator} requires CUDA") + if eplb_communicator == "torch_xccl" and not current_platform.is_xpu(): + pytest.skip("torch_xccl requires XPU") if torch.accelerator.device_count() < world_size: pytest.skip(f"Need at least {world_size} GPUs to run the test") distributed_run( @@ -592,9 +629,11 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None: ) ep_group_coordinator = get_tp_group() - ep_group = ep_group_coordinator.cpu_group + ep_group = ep_group_coordinator.device_group + cpu_group = ep_group_coordinator.cpu_group ep_rank = torch.distributed.get_rank() - device = torch.device(f"cuda:{ep_rank}") + accelerator_type = torch.accelerator.current_accelerator().type + device = torch.device(f"{accelerator_type}:{ep_rank}") num_layers = 2 num_local_experts = 2 @@ -623,9 +662,10 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None: original_weights.append(layer_copy) expert_buffer = [torch.empty_like(w) for w in expert_weights[0]] + default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl" communicator = create_eplb_communicator_or_raise( group_coordinator=ep_group_coordinator, - backend="torch_nccl", + backend=default_backend, expert_weights=expert_weights, expert_buffer=expert_buffer, ) @@ -657,6 +697,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None: assert_verification_synced( local_ok, "No-change EPLB verification failed on at least one rank.", + cpu_group=cpu_group, ) @@ -666,7 +707,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None: (2, 2, 2, 3), ], ) -@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "nixl"]) +@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "torch_xccl", "nixl"]) def test_async_transfer_layer_without_mtp( world_size: int, num_layers: int, @@ -677,6 +718,10 @@ def test_async_transfer_layer_without_mtp( """Exercise async EPLB transfer path without MTP/spec decode.""" if eplb_communicator == "nixl" and not has_nixl(): pytest.skip("NIXL is not available") + if eplb_communicator == "nixl" and current_platform.is_xpu(): + pytest.skip("NIXL does not support XPU") + if eplb_communicator == "torch_xccl" and not current_platform.is_xpu(): + pytest.skip("torch_xccl requires XPU") if torch.accelerator.device_count() < world_size: pytest.skip(f"Need at least {world_size} GPUs to run the test") @@ -715,9 +760,11 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None: ) ep_group_coordinator = get_tp_group() - ep_group = ep_group_coordinator.cpu_group + ep_group = ep_group_coordinator.device_group + cpu_group = ep_group_coordinator.cpu_group ep_rank = torch.distributed.get_rank() - device = torch.device(f"cuda:{ep_rank}") + accelerator_type = torch.accelerator.current_accelerator().type + device = torch.device(f"{accelerator_type}:{ep_rank}") num_layers = 1 num_local_experts = 2 @@ -753,9 +800,10 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None: original_weights.append(layer_copy) expert_buffer = [torch.empty_like(w) for w in expert_weights[0]] + default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl" communicator = create_eplb_communicator_or_raise( group_coordinator=ep_group_coordinator, - backend="torch_nccl", + backend=default_backend, expert_weights=expert_weights, expert_buffer=expert_buffer, ) @@ -788,6 +836,7 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None: assert_verification_synced( local_ok, "Profile-mode EPLB verification failed on at least one rank.", + cpu_group=cpu_group, ) diff --git a/tests/distributed/test_eplb_fused_moe_layer.py b/tests/distributed/test_eplb_fused_moe_layer.py index d8d870bc98..c6ae9e60d6 100644 --- a/tests/distributed/test_eplb_fused_moe_layer.py +++ b/tests/distributed/test_eplb_fused_moe_layer.py @@ -17,6 +17,7 @@ from vllm.distributed.parallel_state import ( get_tp_group, ) from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner +from vllm.platforms import current_platform from .eplb_utils import distributed_run, set_env_vars_and_device @@ -81,7 +82,7 @@ def make_fused_moe_layer( ) re = fml.routed_experts - device = torch.device(f"cuda:{rank}") + device = torch.device(f"{torch.accelerator.current_accelerator().type}:{rank}") from functools import partial @@ -190,13 +191,14 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig): vllm_config = VllmConfig() vllm_config.parallel_config.tensor_parallel_size = world_size vllm_config.parallel_config.enable_expert_parallel = True + vllm_config.parallel_config.enable_eplb = True with set_current_vllm_config(vllm_config): ensure_model_parallel_initialized( tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1 ) - ep_group = get_tp_group().cpu_group + ep_group = get_tp_group().device_group ep_rank = torch.distributed.get_rank() fml_layers = [ @@ -216,9 +218,10 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig): shuffled_indices[lidx] = torch.randperm(test_config.num_experts) expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]] + default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl" communicator = create_eplb_communicator( group_coordinator=get_eplb_group(), - backend="torch_nccl", + backend=default_backend, expert_weights=rank_expert_weights, expert_buffer=expert_buffer, ) diff --git a/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py b/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py index fe11e13cd6..979742d501 100644 --- a/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py +++ b/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py @@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization.modelopt import ( ModelOptNvFp4Config, ModelOptNvFp4FusedMoE, ) +from vllm.platforms import current_platform from .eplb_utils import distributed_run, set_env_vars_and_device @@ -270,6 +271,8 @@ def test_eplb_fml( ): if torch.accelerator.device_count() < world_size: pytest.skip(f"Need at least {world_size} GPUs to run the test") + if current_platform.is_xpu(): + pytest.skip("NVFP4 quantization and flashinfer backends are CUDA-only") num_local_experts = num_experts // world_size num_topk = 4 diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index f0a2b7e457..ba744e049f 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -38,7 +38,9 @@ DistributedExecutorBackend = Literal["ray", "mp", "uni", "external_launcher"] DataParallelBackend = Literal["ray", "mp"] EPLBPolicyOption = Literal["default"] DCPCommBackend = Literal["ag_rs", "a2a"] -EPLBCommunicatorBackend = Literal["torch_nccl", "torch_gloo", "nixl", "pynccl"] +EPLBCommunicatorBackend = Literal[ + "torch_nccl", "torch_gloo", "torch_xccl", "nixl", "pynccl" +] All2AllBackend = Literal[ "naive", "pplx", @@ -94,9 +96,11 @@ class EPLBConfig: Backend for EPLB expert weight communication: - "torch_nccl": Use torch.distributed on the device process group - "torch_gloo": Use torch.distributed gloo with CPU staging + - "torch_xccl": Use torch.distributed XCCL device P2P on XPU - "nixl": Use NIXL with staged send/recv buffers - "pynccl": Use PyNccl send/recv - - None: Auto-select backend (prefers "nixl", falls back to "torch_gloo") + - None: Auto-select backend ("torch_xccl" on XPU, prefers "nixl" + on CUDA, falls back to "torch_gloo") """ @model_validator(mode="after") @@ -525,10 +529,10 @@ class ParallelConfig: ) if self.enable_eplb: - if not current_platform.is_cuda_alike(): + if not current_platform.is_cuda_alike() and not current_platform.is_xpu(): raise ValueError( "Expert parallelism load balancing is only supported on " - "CUDA devices or ROCm devices now." + "CUDA devices or ROCm devices or XPU devices now." ) if not self.enable_expert_parallel: raise ValueError("enable_expert_parallel must be True to use EPLB.") @@ -1034,7 +1038,10 @@ class ParallelConfig: # See https://github.com/pytorch/pytorch/issues/174288 from vllm.distributed.nixl_utils import is_nixl_available - if is_nixl_available(): + if current_platform.is_xpu(): + # On XPU, use the device-native XCCL P2P backend. + self.eplb_config.communicator = "torch_xccl" + elif is_nixl_available(): self.eplb_config.communicator = "nixl" elif self.enable_elastic_ep: self.eplb_config.communicator = "pynccl" diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 5cf63fe5ae..3b7bc908ec 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -10,7 +10,7 @@ import torch from vllm.distributed.parallel_state import get_eplb_group from vllm.logger import init_logger -from .eplb_utils import CpuGpuEvent +from .eplb_utils import CpuGpuEvent, device_stream from .rebalance_execute import AsyncEplbLayerResult, transfer_layer if TYPE_CHECKING: @@ -24,17 +24,17 @@ def start_async_worker( is_profile: bool = False, ) -> threading.Thread: rank = get_eplb_group().device_group.rank() - device_index = state.cuda_device_index + device_index = state.device_index assert state.is_async def thread_target() -> None: assert device_index is not None torch.accelerator.set_device_index(device_index) - cuda_stream = torch.cuda.Stream(device=device_index) + stream = torch.Stream(device=torch.accelerator.current_accelerator()) try: transfer_run_periodically( state=state, - cuda_stream=cuda_stream, + stream=stream, is_profile=is_profile, ) except Exception as exc: # pragma: no cover - diagnostic path @@ -49,13 +49,13 @@ def run_rebalance_experts( model_state: "EplbModelState", eplb_state: "EplbState", physical_to_logical_map_cpu: torch.Tensor, - cuda_stream: torch.cuda.Stream, + stream: torch.Stream, ) -> torch.Tensor: assert model_state.eplb_stats is not None eplb_stats = model_state.eplb_stats # Move the global expert load window to CPU for computation. - with torch.cuda.stream(cuda_stream): + with device_stream(stream): global_expert_load_window = eplb_stats.global_expert_load_window.cpu() # Compute new expert mappings for the model new_physical_to_logical_map = eplb_state.policy.rebalance_experts( @@ -73,11 +73,11 @@ def run_rebalance_experts( def transfer_run_periodically( state: "EplbState", - cuda_stream: torch.cuda.Stream, + stream: torch.Stream, is_profile: bool = False, ) -> None: while True: - state.rearrange_event.wait(stream=cuda_stream) + state.rearrange_event.wait(stream=stream) eplb_group = get_eplb_group().device_group eplb_cpu_group = get_eplb_group().cpu_group @@ -87,16 +87,16 @@ def transfer_run_periodically( for model_state in state.model_states.values(): layer_idx = 0 # Set the async worker's CUDA stream on the communicator - model_state.communicator.set_stream(cuda_stream) + model_state.communicator.set_stream(stream) num_layers = model_state.model.num_moe_layers # Snapshot the physical_to_logical_map (synchronized with # rearrange_event) and copy it to CPU - with torch.cuda.stream(cuda_stream): + with device_stream(stream): physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu() new_physical_to_logical_map = run_rebalance_experts( - model_state, state, physical_to_logical_map_cpu, cuda_stream + model_state, state, physical_to_logical_map_cpu, stream ) # Execute one EPLB layer transfer per model forward pass. Each iteration @@ -132,13 +132,13 @@ def transfer_run_periodically( communicator=model_state.communicator, ep_group=eplb_group, is_profile=is_profile, - cuda_stream=cuda_stream, + stream=stream, layer_idx=layer_idx, ) # Wait until all writes to expert_buffer have finished before making the # AsyncEplbLayerResult visible to the main thread. - cuda_stream.synchronize() + stream.synchronize() # This event guarantees that expert_buffer will not be overwritten by # subsequent iterations of this loop until the main thread has consumed @@ -155,6 +155,6 @@ def transfer_run_periodically( # Block this thread until the main thread and main stream # finish copying model_state.expert_buffer into # model_state.model.expert_weights[layer_idx] - consumed_event.wait(stream=cuda_stream) + consumed_event.wait(stream=stream) assert model_state.pending_result is None layer_idx += 1 diff --git a/vllm/distributed/eplb/eplb_communicator.py b/vllm/distributed/eplb/eplb_communicator.py index 61fef468d3..730c01a388 100644 --- a/vllm/distributed/eplb/eplb_communicator.py +++ b/vllm/distributed/eplb/eplb_communicator.py @@ -22,6 +22,7 @@ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator from vllm.distributed.device_communicators.pynccl_wrapper import ( ncclDataTypeEnum, ) +from vllm.distributed.eplb.eplb_utils import device_stream from vllm.distributed.parallel_state import ( GroupCoordinator, get_pp_group, @@ -87,8 +88,8 @@ class EplbCommunicator(ABC): communication buffers.""" return True - def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None: - self._cuda_stream = cuda_stream + def set_stream(self, stream: torch.Stream | None) -> None: + self._stream = stream def _log_initialized(self) -> None: if is_local_first_rank(): @@ -101,10 +102,10 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator): def __init__( self, ep_group: ProcessGroup, - cuda_stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, ) -> None: self._ep_group = ep_group - self._cuda_stream = cuda_stream + self._stream = stream self._p2p_ops: list[P2POp] = [] self._log_initialized() @@ -144,7 +145,7 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator): if not self._p2p_ops: return try: - with torch.cuda.stream(self._cuda_stream): + with device_stream(self._stream): reqs = batch_isend_irecv(self._p2p_ops) for req in reqs: req.wait() @@ -158,10 +159,10 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator): def __init__( self, cpu_group: ProcessGroup, - cuda_stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, ) -> None: self._cpu_group = cpu_group - self._cuda_stream = cuda_stream + self._stream = stream self._ops: list[tuple[str, torch.Tensor, int]] = [] self._log_initialized() @@ -217,7 +218,7 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator): recv_staging.append((tensor, cpu_tensor)) try: - with torch.cuda.stream(self._cuda_stream): + with device_stream(self._stream): build_ops() finally: self._ops.clear() @@ -225,10 +226,10 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator): # Wait for all D2H copies to finish # before issuing gloo batch_isend_irecv operations. with gpu_sync_allowed(): - if self._cuda_stream is not None: - self._cuda_stream.synchronize() + if self._stream is not None: + self._stream.synchronize() else: - torch.cuda.current_stream().synchronize() + torch.accelerator.current_stream().synchronize() reqs = batch_isend_irecv(p2p_ops) for req in reqs: @@ -236,11 +237,69 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator): if not recv_staging: return - with torch.cuda.stream(self._cuda_stream): + with device_stream(self._stream): for dst_tensor, cpu_tensor in recv_staging: dst_tensor.copy_(cpu_tensor, non_blocking=True) +class TorchDistXCCLStagedEplbCommunicator(EplbCommunicator): + """EPLB communicator using XCCL device-to-device P2P on XPU.""" + + def __init__( + self, + ep_group: ProcessGroup, + stream: torch.Stream | None = None, + ) -> None: + self._ep_group = ep_group + self._stream = stream + self._ops: list[tuple[str, torch.Tensor, int]] = [] + self._log_initialized() + + def add_send( + self, + tensors: list[torch.Tensor], + dst_rank: int, + expert_id: int, # unused by this backend + ) -> None: + for tensor in tensors: + self._ops.append(("send", tensor, dst_rank)) + + def add_recv( + self, + tensors: list[torch.Tensor], + src_rank: int, + expert_id: int, # unused by this backend + ) -> None: + for tensor in tensors: + self._ops.append(("recv", tensor, src_rank)) + + def execute(self) -> None: + if not self._ops: + return + + p2p_ops: list[P2POp] = [] + try: + for op, tensor, peer_rank in self._ops: + send_or_recv = ( + torch.distributed.isend if op == "send" else torch.distributed.irecv + ) + p2p_ops.append( + P2POp( + send_or_recv, + tensor, + peer_rank, + self._ep_group, + ) + ) + finally: + self._ops.clear() + + with device_stream(self._stream): + reqs = batch_isend_irecv(p2p_ops) + for req in reqs: + req.wait() + + class NixlEplbCommunicator(EplbCommunicator): """EPLB communicator backed by NIXL READ transfers.""" @@ -343,7 +402,7 @@ class NixlEplbCommunicator(EplbCommunicator): uid = uuid.uuid4().hex[:8] return f"eplb-{self._rank}{pp_suffix}-{uid}" - def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None: + def set_stream(self, stream: torch.Stream | None) -> None: pass def add_send( @@ -602,10 +661,10 @@ class PyNcclEplbCommunicator(EplbCommunicator): def __init__( self, pynccl_comm: PyNcclCommunicator, - cuda_stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, ) -> None: self._pynccl_comm = pynccl_comm - self._cuda_stream = cuda_stream + self._stream = stream self._group_started = False self._log_initialized() @@ -622,7 +681,7 @@ class PyNcclEplbCommunicator(EplbCommunicator): ) -> None: self._ensure_group_started() for tensor in tensors: - self._pynccl_comm.send(tensor, dst_rank, stream=self._cuda_stream) + self._pynccl_comm.send(tensor, dst_rank, stream=self._stream) def add_recv( self, @@ -632,7 +691,7 @@ class PyNcclEplbCommunicator(EplbCommunicator): ) -> None: self._ensure_group_started() for tensor in tensors: - self._pynccl_comm.recv(tensor, src_rank, stream=self._cuda_stream) + self._pynccl_comm.recv(tensor, src_rank, stream=self._stream) def execute(self) -> None: if self._group_started: @@ -750,6 +809,8 @@ def create_eplb_communicator( return TorchDistGlooStagedEplbCommunicator( cpu_group=group_coordinator.cpu_group, ) + elif backend == "torch_xccl": + return TorchDistXCCLStagedEplbCommunicator(ep_group=torch_group) elif backend == "torch_nccl": return TorchDistNcclEplbCommunicator(ep_group=torch_group) elif backend == "pynccl": diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index fb5475a607..e4d805b9fe 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -203,9 +203,9 @@ class EplbModelState: """ EPLB stats for the model. """ - cuda_device_index: int | None + device_index: int | None """ - CUDA device index for the async EPLB worker thread. + Device index for the async EPLB worker thread. """ communicator: EplbCommunicator """ @@ -286,14 +286,14 @@ class EplbState: """ Background thread handling async transfers. """ - self.cuda_device_index: int | None = None + self.device_index: int | None = None """ - CUDA device index for the async EPLB worker thread. + Device index for the async EPLB worker thread. """ - if self.device.type == "cuda": - self.cuda_device_index = self.device.index - if self.cuda_device_index is None and torch.cuda.is_available(): - self.cuda_device_index = torch.accelerator.current_device_index() + if self.device.type in ("cuda", "xpu"): + self.device_index = self.device.index + if self.device_index is None and torch.accelerator.is_available(): + self.device_index = torch.accelerator.current_device_index() @staticmethod def build_initial_global_physical_to_logical_map( @@ -504,7 +504,7 @@ class EplbState: expert_buffer=expert_buffer, rebalanced=False, eplb_stats=None, - cuda_device_index=self.cuda_device_index, + device_index=self.device_index, communicator=communicator, num_unpadded_tokens_tensors=num_unpadded_tokens_tensors, ) @@ -763,8 +763,8 @@ class EplbState: is_main_rank = ep_rank == 0 if is_main_rank: if not self.is_async or is_profile: - start_event = torch.cuda.Event(enable_timing=True) - end_event = torch.cuda.Event(enable_timing=True) + start_event = torch.Event(enable_timing=True) + end_event = torch.Event(enable_timing=True) start_event.record() logger.info( "Rearranging experts %s %s...", diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 3d5be8a8ec..943f6d9bef 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Utility functions for EPLB (Expert Parallel Load Balancing).""" +import contextlib import os import threading @@ -13,6 +14,22 @@ from vllm.logger import init_logger logger = init_logger(__name__) +@contextlib.contextmanager +def device_stream(stream: torch.Stream | None): + """Platform-agnostic context manager that activates *stream* as the + current accelerator stream for the duration of the ``with`` block. + A no-op when *stream* is ``None``.""" + if stream is None: + yield + return + prev = torch.accelerator.current_stream() + torch.accelerator.set_stream(stream) + try: + yield + finally: + torch.accelerator.set_stream(prev) + + class CpuGpuEvent: """Combines a CUDA event with a CPU threading event to enforce record->wait ordering across two threads. @@ -30,10 +47,10 @@ class CpuGpuEvent: """ def __init__(self): - self._event = torch.cuda.Event() + self._event = torch.Event() self._recorded = threading.Event() - def wait(self, stream: torch.cuda.Stream | None = None): + def wait(self, stream: torch.Stream | None = None): """Blocks the calling thread until record finishes. Used to guarantee that the record kernel is called before wait. @@ -43,7 +60,7 @@ class CpuGpuEvent: self._event.wait(stream) self._recorded.clear() - def record(self, stream: torch.cuda.Stream | None = None): + def record(self, stream: torch.Stream | None = None): """Unblocks the waiting thread after calling event.record(). Should only be called by the main thread. @@ -53,7 +70,7 @@ class CpuGpuEvent: "CpuGpuEvent.record() called before the previous event was " "consumed by wait()" ) - self._event = torch.cuda.Event() + self._event = torch.Event() self._event.record(stream) self._recorded.set() diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 20ebe4f1f2..7a52dd5f56 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -13,7 +13,7 @@ import torch from torch.distributed import ProcessGroup, all_gather from vllm.distributed.eplb.eplb_communicator import EplbCommunicator -from vllm.distributed.eplb.eplb_utils import CpuGpuEvent +from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream from vllm.logger import init_logger from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -173,7 +173,7 @@ def move_to_buffer( new_indices: np.ndarray, expert_weights: Sequence[torch.Tensor], expert_weights_buffers: Sequence[torch.Tensor], - cuda_stream: torch.cuda.Stream | None, + stream: torch.Stream | None, ep_rank: int, communicator: EplbCommunicator, layer_idx: int = 0, @@ -183,12 +183,12 @@ def move_to_buffer( Args: num_local_experts: Number of local experts. old_indices: (num_experts_total,) ndarray of current (old) - global-to-local expert assignments. + global-to-locals expert assignments. new_indices: (num_experts_total,) ndarray of desired (new) global-to-local assignments after rebalance. expert_weights: Original expert weights for the layer. expert_weights_buffers: Intermediate buffers (one per tensor). - cuda_stream: CUDA stream for async copies (can be None for sync mode). + stream: CUDA/XPU stream for async copies (can be None for sync mode). ep_rank: Rank of this process in expert parallel group. communicator: EplbCommunicator instance for P2P communication. layer_idx: Index of the MoE layer being transferred. @@ -261,10 +261,9 @@ def move_to_buffer( expert = new_local_expert_ids[dst] src_local = expert_to_src_map.get(expert, -1) if src_local != -1: - with torch.cuda.stream(cuda_stream): + with device_stream(stream): for w, b in zip(expert_weights, expert_weights_buffers): b[dst].copy_(w[src_local], non_blocking=True) - communicator.set_transfer_context(old_indices, layer_idx) # 2. Post sends @@ -430,7 +429,7 @@ def transfer_layer( ep_group: ProcessGroup, communicator: EplbCommunicator, is_profile: bool = False, - cuda_stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, rank_mapping: dict[int, int] | None = None, layer_idx: int = 0, ) -> TransferMetadata: @@ -451,7 +450,7 @@ def transfer_layer( is_profile (bool): If `True`, do not perform any actual weight copy. This is used during profile run, where we only perform dummy communications to reserve enough memory for the buffers. - cuda_stream: CUDA stream for async copies (can be None for sync mode). + stream: CUDA stream for async copies (can be None for sync mode). rank_mapping: Optional rank mapping for elastic expert parallelism. layer_idx: Index of the MoE layer being transferred. @@ -499,7 +498,7 @@ def transfer_layer( new_indices=new_layer_indices_np, expert_weights=expert_weights, expert_weights_buffers=expert_weights_buffer, - cuda_stream=cuda_stream, + stream=stream, ep_rank=ep_group.rank(), communicator=communicator, layer_idx=layer_idx, @@ -601,7 +600,7 @@ def rearrange_expert_weights_inplace( new_indices=new_global_expert_indices_cpu[layer_idx], expert_weights=expert_weights[layer_idx], expert_weights_buffers=weights_buffer, - cuda_stream=None, + stream=None, ep_rank=ep_rank, communicator=communicator, layer_idx=layer_idx, diff --git a/vllm/model_executor/layers/fused_moe/experts/xpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/xpu_moe.py index dfc578b441..ddd891b1c0 100644 --- a/vllm/model_executor/layers/fused_moe/experts/xpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/xpu_moe.py @@ -171,6 +171,7 @@ class XPUExperts(mk.FusedMoEExpertsModular): num_experts=self.moe_config.num_local_experts, ep_rank=self.moe_config.ep_rank, ep_size=self.moe_config.ep_size, + expert_map=expert_map, gemm1_clamp_limit=self.gemm1_clamp_limit, ) assert self.fused_moe_impl is not None diff --git a/vllm/model_executor/layers/fused_moe/router/base_router.py b/vllm/model_executor/layers/fused_moe/router/base_router.py index 0509f02ead..b51dbf90dc 100644 --- a/vllm/model_executor/layers/fused_moe/router/base_router.py +++ b/vllm/model_executor/layers/fused_moe/router/base_router.py @@ -13,7 +13,7 @@ from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.v1.worker.ubatching import dbo_current_ubatch_id -if current_platform.is_cuda_alike(): +if current_platform.is_cuda_alike() or current_platform.is_xpu(): @triton.jit def _eplb_map_and_record_i32_kernel(