[XPU] Enable XPU eplb (#44987)

Signed-off-by: mayuyuace <qiming1.zhang@intel.com>
Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com>
This commit is contained in:
Qiming Zhang
2026-09-17 13:19:13 +08:00
committed by GitHub
parent 2743dc4f92
commit e6b1a5e5e3
16 changed files with 267 additions and 109 deletions
+1 -1
View File
@@ -935,7 +935,7 @@ def hf_runner():
def _default_block_size() -> int:
if torch.xpu.is_available():
if current_platform.is_xpu():
return 64
if current_platform.is_cpu():
return 128
+6 -2
View File
@@ -13,6 +13,7 @@ from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed.parallel_state import (
init_distributed_environment,
)
from vllm.platforms import current_platform
from vllm.utils.system_utils import update_environment_variables
mp.set_start_method("spawn", force=True)
@@ -73,13 +74,16 @@ def distributed_run(fn, world_size, *args):
def set_env_vars_and_device(env: dict[str, str]) -> None:
update_environment_variables(env)
local_rank = os.environ["LOCAL_RANK"]
device = torch.device(f"cuda:{local_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{local_rank}")
torch.accelerator.set_device_index(device)
backend = "xccl" if current_platform.is_xpu() else "nccl"
# Create a minimal vllm config for init_distributed_environment
vllm_config = VllmConfig()
with set_current_vllm_config(vllm_config):
init_distributed_environment()
init_distributed_environment(backend=backend)
atexit.register(_destroy_process_group_if_initialized)
# Ensure each worker process has the same random seed
random.seed(42)
+4
View File
@@ -10,6 +10,8 @@ from concurrent.futures import ThreadPoolExecutor
import pytest
import requests
from vllm.platforms import current_platform
from ..evals.gsm8k.gsm8k_eval import evaluate_gsm8k
from ..utils import RemoteOpenAIServer, multi_gpu_test
@@ -233,6 +235,8 @@ def test_elastic_ep_scaling(
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
if current_platform.is_xpu():
pytest.skip("Elastic EP with NIXL is not supported on XPU")
if all2all_backend == "nixl_ep" and not has_nixl_ep():
pytest.skip("NIXL EP is not installed")
+4 -1
View File
@@ -10,6 +10,7 @@ from vllm.distributed.eplb.eplb_state import (
compute_logical_maps,
)
from vllm.distributed.eplb.policy.default import DefaultEplbPolicy
from vllm.platforms import current_platform
def test_eplb_load_stats_reduce_across_ranks():
@@ -263,11 +264,13 @@ def test_global_load_balance_fallback():
assert torch.sum(logcnt) == num_replicas
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("device", ["cpu", "cuda", "xpu"])
def test_device_compatibility(device):
"""Test device compatibility."""
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA not available")
if device == "xpu" and not current_platform.is_xpu():
pytest.skip("XPU not available")
weight = torch.tensor([[10, 20, 30, 40]], device=device)
num_replicas = 6
+18 -11
View File
@@ -4,15 +4,17 @@
import threading
import time
import pytest
import torch
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
from vllm.platforms import current_platform
def test_wait_blocks_until_record():
event = CpuGpuEvent()
record_stream = torch.cuda.Stream()
wait_stream = torch.cuda.Stream()
record_stream = torch.Stream()
wait_stream = torch.Stream()
wait_returned = threading.Event()
def waiter():
@@ -33,8 +35,8 @@ def test_wait_blocks_until_record():
def test_reuse_across_multiple_cycles():
wrapper = CpuGpuEvent()
record_stream = torch.cuda.Stream()
wait_stream = torch.cuda.Stream()
record_stream = torch.Stream()
wait_stream = torch.Stream()
NUM_CYCLES = 8
completed_cycles = []
barriers = [threading.Barrier(2) for _ in range(NUM_CYCLES)]
@@ -56,13 +58,18 @@ def test_reuse_across_multiple_cycles():
assert len(completed_cycles) == NUM_CYCLES
def test_producer_consumer():
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared
GPU tensor on multiple CPU threads.
@pytest.mark.parametrize("device", ["cuda", "xpu"])
def test_producer_consumer(device):
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from
a shared GPU tensor on multiple CPU threads.
"""
worker_stream = torch.cuda.Stream()
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA not available")
if device == "xpu" and not current_platform.is_xpu():
pytest.skip("XPU not available")
worker_stream = torch.Stream()
# Create a single element counter that will be shared between two threads
buf = torch.zeros(1, device="cuda")
buf = torch.zeros(1, device=device)
NUM_ROUNDS = 5
ready_cpu = [threading.Event() for _ in range(NUM_ROUNDS)]
@@ -76,7 +83,7 @@ def test_producer_consumer():
if i > 0:
events[i - 1].wait(stream=worker_stream)
with torch.cuda.stream(worker_stream):
with device_stream(worker_stream):
buf.fill_(float(i + 1))
worker_stream.synchronize()
+78 -29
View File
@@ -23,6 +23,7 @@ from vllm.distributed.parallel_state import (
ensure_model_parallel_initialized,
get_tp_group,
)
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@@ -208,10 +209,14 @@ def verify_redundant_experts_have_same_weights(
ep_rank: int,
world_size: int,
num_local_experts: int,
cpu_group,
) -> bool:
"""Verify that all replicas of the same logical expert have the same weights."""
num_layers = len(expert_weights)
total_physical_experts = world_size * num_local_experts
# XPU: gloo does not support XPU tensors, so gather on CPU then move back.
# CUDA: gloo supports CUDA tensors directly, keep original device.
use_cpu_gather = current_platform.is_xpu()
ok = True
for layer in range(num_layers):
@@ -219,31 +224,35 @@ def verify_redundant_experts_have_same_weights(
all_weights: list[torch.Tensor] = []
for weight_idx, hidden_size in enumerate(hidden_sizes):
orig_device = expert_weights[layer][weight_idx].device
if use_cpu_gather:
local_weights = expert_weights[layer][weight_idx].cpu()
gather_device = "cpu"
else:
local_weights = expert_weights[layer][weight_idx]
gather_device = orig_device
# Create tensor to store all expert weights
# Shape: [total_physical_experts, hidden_size]
gathered_weights = torch.zeros(
total_physical_experts,
hidden_size,
device=expert_weights[layer][weight_idx].device,
dtype=expert_weights[layer][weight_idx].dtype,
device=gather_device,
dtype=local_weights.dtype,
)
# Use all_gather to collect expert weights from current node
# expert_weights[layer][weight_idx] shape:
# [num_local_experts, hidden_size]
local_weights = expert_weights[layer][
weight_idx
] # [num_local_experts, hidden_size]
# Split tensor along dim 0 into a list for all_gather
gathered_weights_list = torch.chunk(gathered_weights, world_size, dim=0)
torch.distributed.all_gather(
# Output list: each element corresponds to one rank's weights
list(gathered_weights_list),
local_weights, # Input: current rank's local weights
local_weights,
group=cpu_group,
)
if use_cpu_gather:
gathered_weights = gathered_weights.to(orig_device)
all_weights.append(gathered_weights)
# Verify that all replicas of the same logical expert have the same
@@ -294,9 +303,23 @@ def verify_redundant_experts_have_same_weights(
return ok
def assert_verification_synced(local_ok: bool, msg: str) -> None:
ok_tensor = torch.tensor([1 if local_ok else 0], device="cuda", dtype=torch.int32)
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
def assert_verification_synced(local_ok: bool, msg: str, cpu_group) -> None:
if current_platform.is_xpu():
# XPU: default group backend (gloo) does not support XPU tensors;
# use CPU tensor with cpu_group instead.
ok_tensor = torch.tensor(
[1 if local_ok else 0], device="cpu", dtype=torch.int32
)
torch.distributed.all_reduce(
ok_tensor, op=torch.distributed.ReduceOp.MIN, group=cpu_group
)
else:
# CUDA: nccl (default group) supports device tensors directly.
device = torch.accelerator.current_accelerator()
ok_tensor = torch.tensor(
[1 if local_ok else 0], device=device, dtype=torch.int32
)
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
assert bool(ok_tensor.item()), msg
@@ -336,8 +359,10 @@ def _test_async_transfer_layer_without_mtp_worker(
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
total_physical_experts = world_size * num_local_experts
hidden_sizes = [16, 32]
@@ -376,7 +401,7 @@ def _test_async_transfer_layer_without_mtp_worker(
new_indices_cpu = new_indices.cpu()
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
cuda_stream = torch.cuda.Stream(device=device)
stream = torch.Stream(device=device)
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
@@ -384,7 +409,7 @@ def _test_async_transfer_layer_without_mtp_worker(
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
communicator.set_stream(cuda_stream)
communicator.set_stream(stream)
for layer_idx in range(num_layers):
transfer_metadata = transfer_layer(
@@ -394,10 +419,10 @@ def _test_async_transfer_layer_without_mtp_worker(
expert_weights_buffer=expert_buffer,
ep_group=ep_group,
communicator=communicator,
cuda_stream=cuda_stream,
stream=stream,
layer_idx=layer_idx,
)
cuda_stream.synchronize()
stream.synchronize()
move_from_buffer(
expert_weights=expert_weights[layer_idx],
expert_weights_buffers=expert_buffer,
@@ -421,6 +446,7 @@ def _test_async_transfer_layer_without_mtp_worker(
ep_rank,
world_size,
num_local_experts,
cpu_group=cpu_group,
)
and local_ok
)
@@ -428,6 +454,7 @@ def _test_async_transfer_layer_without_mtp_worker(
local_ok,
"Async transfer verification failed on at least one rank. "
"See logs for details.",
cpu_group=cpu_group,
)
@@ -452,9 +479,11 @@ def _test_rearrange_expert_weights_with_redundancy(
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.cpu_group
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
# Test parameters
total_physical_experts = world_size * num_local_experts
@@ -523,12 +552,14 @@ def _test_rearrange_expert_weights_with_redundancy(
ep_rank,
world_size,
num_local_experts,
cpu_group=cpu_group,
)
and local_ok
)
assert_verification_synced(
local_ok,
"Rearrange verification failed on at least one rank. See logs for details.",
cpu_group=cpu_group,
)
@@ -556,7 +587,7 @@ def _test_rearrange_expert_weights_with_redundancy(
],
)
@pytest.mark.parametrize(
"eplb_communicator", ["torch_nccl", "torch_gloo", "pynccl", "nixl"]
"eplb_communicator", ["torch_nccl", "torch_gloo", "torch_xccl", "pynccl", "nixl"]
)
def test_rearrange_expert_weights_with_redundancy(
world_size,
@@ -568,6 +599,12 @@ def test_rearrange_expert_weights_with_redundancy(
"""Test the functionality of rearranging expert weights with redundancy."""
if eplb_communicator == "nixl" and not has_nixl():
pytest.skip("NIXL is not available")
if eplb_communicator == "nixl" and current_platform.is_xpu():
pytest.skip("NIXL does not support XPU")
if eplb_communicator in ("torch_nccl", "pynccl") and not torch.cuda.is_available():
pytest.skip(f"{eplb_communicator} requires CUDA")
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
pytest.skip("torch_xccl requires XPU")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
@@ -592,9 +629,11 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.cpu_group
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
num_layers = 2
num_local_experts = 2
@@ -623,9 +662,10 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
original_weights.append(layer_copy)
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
backend="torch_nccl",
backend=default_backend,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
@@ -657,6 +697,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
assert_verification_synced(
local_ok,
"No-change EPLB verification failed on at least one rank.",
cpu_group=cpu_group,
)
@@ -666,7 +707,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
(2, 2, 2, 3),
],
)
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "nixl"])
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "torch_xccl", "nixl"])
def test_async_transfer_layer_without_mtp(
world_size: int,
num_layers: int,
@@ -677,6 +718,10 @@ def test_async_transfer_layer_without_mtp(
"""Exercise async EPLB transfer path without MTP/spec decode."""
if eplb_communicator == "nixl" and not has_nixl():
pytest.skip("NIXL is not available")
if eplb_communicator == "nixl" and current_platform.is_xpu():
pytest.skip("NIXL does not support XPU")
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
pytest.skip("torch_xccl requires XPU")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
@@ -715,9 +760,11 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.cpu_group
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
num_layers = 1
num_local_experts = 2
@@ -753,9 +800,10 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
original_weights.append(layer_copy)
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
backend="torch_nccl",
backend=default_backend,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
@@ -788,6 +836,7 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
assert_verification_synced(
local_ok,
"Profile-mode EPLB verification failed on at least one rank.",
cpu_group=cpu_group,
)
@@ -17,6 +17,7 @@ from vllm.distributed.parallel_state import (
get_tp_group,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@@ -81,7 +82,7 @@ def make_fused_moe_layer(
)
re = fml.routed_experts
device = torch.device(f"cuda:{rank}")
device = torch.device(f"{torch.accelerator.current_accelerator().type}:{rank}")
from functools import partial
@@ -190,13 +191,14 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.parallel_config.enable_eplb = True
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
)
ep_group = get_tp_group().cpu_group
ep_group = get_tp_group().device_group
ep_rank = torch.distributed.get_rank()
fml_layers = [
@@ -216,9 +218,10 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend="torch_nccl",
backend=default_backend,
expert_weights=rank_expert_weights,
expert_buffer=expert_buffer,
)
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
ModelOptNvFp4FusedMoE,
)
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@@ -270,6 +271,8 @@ def test_eplb_fml(
):
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
if current_platform.is_xpu():
pytest.skip("NVFP4 quantization and flashinfer backends are CUDA-only")
num_local_experts = num_experts // world_size
num_topk = 4
+12 -5
View File
@@ -38,7 +38,9 @@ DistributedExecutorBackend = Literal["ray", "mp", "uni", "external_launcher"]
DataParallelBackend = Literal["ray", "mp"]
EPLBPolicyOption = Literal["default"]
DCPCommBackend = Literal["ag_rs", "a2a"]
EPLBCommunicatorBackend = Literal["torch_nccl", "torch_gloo", "nixl", "pynccl"]
EPLBCommunicatorBackend = Literal[
"torch_nccl", "torch_gloo", "torch_xccl", "nixl", "pynccl"
]
All2AllBackend = Literal[
"naive",
"pplx",
@@ -94,9 +96,11 @@ class EPLBConfig:
Backend for EPLB expert weight communication:
- "torch_nccl": Use torch.distributed on the device process group
- "torch_gloo": Use torch.distributed gloo with CPU staging
- "torch_xccl": Use torch.distributed XCCL device P2P on XPU
- "nixl": Use NIXL with staged send/recv buffers
- "pynccl": Use PyNccl send/recv
- None: Auto-select backend (prefers "nixl", falls back to "torch_gloo")
- None: Auto-select backend ("torch_xccl" on XPU, prefers "nixl"
on CUDA, falls back to "torch_gloo")
"""
@model_validator(mode="after")
@@ -525,10 +529,10 @@ class ParallelConfig:
)
if self.enable_eplb:
if not current_platform.is_cuda_alike():
if not current_platform.is_cuda_alike() and not current_platform.is_xpu():
raise ValueError(
"Expert parallelism load balancing is only supported on "
"CUDA devices or ROCm devices now."
"CUDA devices or ROCm devices or XPU devices now."
)
if not self.enable_expert_parallel:
raise ValueError("enable_expert_parallel must be True to use EPLB.")
@@ -1034,7 +1038,10 @@ class ParallelConfig:
# See https://github.com/pytorch/pytorch/issues/174288
from vllm.distributed.nixl_utils import is_nixl_available
if is_nixl_available():
if current_platform.is_xpu():
# On XPU, use the device-native XCCL P2P backend.
self.eplb_config.communicator = "torch_xccl"
elif is_nixl_available():
self.eplb_config.communicator = "nixl"
elif self.enable_elastic_ep:
self.eplb_config.communicator = "pynccl"
+14 -14
View File
@@ -10,7 +10,7 @@ import torch
from vllm.distributed.parallel_state import get_eplb_group
from vllm.logger import init_logger
from .eplb_utils import CpuGpuEvent
from .eplb_utils import CpuGpuEvent, device_stream
from .rebalance_execute import AsyncEplbLayerResult, transfer_layer
if TYPE_CHECKING:
@@ -24,17 +24,17 @@ def start_async_worker(
is_profile: bool = False,
) -> threading.Thread:
rank = get_eplb_group().device_group.rank()
device_index = state.cuda_device_index
device_index = state.device_index
assert state.is_async
def thread_target() -> None:
assert device_index is not None
torch.accelerator.set_device_index(device_index)
cuda_stream = torch.cuda.Stream(device=device_index)
stream = torch.Stream(device=torch.accelerator.current_accelerator())
try:
transfer_run_periodically(
state=state,
cuda_stream=cuda_stream,
stream=stream,
is_profile=is_profile,
)
except Exception as exc: # pragma: no cover - diagnostic path
@@ -49,13 +49,13 @@ def run_rebalance_experts(
model_state: "EplbModelState",
eplb_state: "EplbState",
physical_to_logical_map_cpu: torch.Tensor,
cuda_stream: torch.cuda.Stream,
stream: torch.Stream,
) -> torch.Tensor:
assert model_state.eplb_stats is not None
eplb_stats = model_state.eplb_stats
# Move the global expert load window to CPU for computation.
with torch.cuda.stream(cuda_stream):
with device_stream(stream):
global_expert_load_window = eplb_stats.global_expert_load_window.cpu()
# Compute new expert mappings for the model
new_physical_to_logical_map = eplb_state.policy.rebalance_experts(
@@ -73,11 +73,11 @@ def run_rebalance_experts(
def transfer_run_periodically(
state: "EplbState",
cuda_stream: torch.cuda.Stream,
stream: torch.Stream,
is_profile: bool = False,
) -> None:
while True:
state.rearrange_event.wait(stream=cuda_stream)
state.rearrange_event.wait(stream=stream)
eplb_group = get_eplb_group().device_group
eplb_cpu_group = get_eplb_group().cpu_group
@@ -87,16 +87,16 @@ def transfer_run_periodically(
for model_state in state.model_states.values():
layer_idx = 0
# Set the async worker's CUDA stream on the communicator
model_state.communicator.set_stream(cuda_stream)
model_state.communicator.set_stream(stream)
num_layers = model_state.model.num_moe_layers
# Snapshot the physical_to_logical_map (synchronized with
# rearrange_event) and copy it to CPU
with torch.cuda.stream(cuda_stream):
with device_stream(stream):
physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu()
new_physical_to_logical_map = run_rebalance_experts(
model_state, state, physical_to_logical_map_cpu, cuda_stream
model_state, state, physical_to_logical_map_cpu, stream
)
# Execute one EPLB layer transfer per model forward pass. Each iteration
@@ -132,13 +132,13 @@ def transfer_run_periodically(
communicator=model_state.communicator,
ep_group=eplb_group,
is_profile=is_profile,
cuda_stream=cuda_stream,
stream=stream,
layer_idx=layer_idx,
)
# Wait until all writes to expert_buffer have finished before making the
# AsyncEplbLayerResult visible to the main thread.
cuda_stream.synchronize()
stream.synchronize()
# This event guarantees that expert_buffer will not be overwritten by
# subsequent iterations of this loop until the main thread has consumed
@@ -155,6 +155,6 @@ def transfer_run_periodically(
# Block this thread until the main thread and main stream
# finish copying model_state.expert_buffer into
# model_state.model.expert_weights[layer_idx]
consumed_event.wait(stream=cuda_stream)
consumed_event.wait(stream=stream)
assert model_state.pending_result is None
layer_idx += 1
+78 -17
View File
@@ -22,6 +22,7 @@ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.device_communicators.pynccl_wrapper import (
ncclDataTypeEnum,
)
from vllm.distributed.eplb.eplb_utils import device_stream
from vllm.distributed.parallel_state import (
GroupCoordinator,
get_pp_group,
@@ -87,8 +88,8 @@ class EplbCommunicator(ABC):
communication buffers."""
return True
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
self._cuda_stream = cuda_stream
def set_stream(self, stream: torch.Stream | None) -> None:
self._stream = stream
def _log_initialized(self) -> None:
if is_local_first_rank():
@@ -101,10 +102,10 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator):
def __init__(
self,
ep_group: ProcessGroup,
cuda_stream: torch.cuda.Stream | None = None,
stream: torch.Stream | None = None,
) -> None:
self._ep_group = ep_group
self._cuda_stream = cuda_stream
self._stream = stream
self._p2p_ops: list[P2POp] = []
self._log_initialized()
@@ -144,7 +145,7 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator):
if not self._p2p_ops:
return
try:
with torch.cuda.stream(self._cuda_stream):
with device_stream(self._stream):
reqs = batch_isend_irecv(self._p2p_ops)
for req in reqs:
req.wait()
@@ -158,10 +159,10 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
def __init__(
self,
cpu_group: ProcessGroup,
cuda_stream: torch.cuda.Stream | None = None,
stream: torch.Stream | None = None,
) -> None:
self._cpu_group = cpu_group
self._cuda_stream = cuda_stream
self._stream = stream
self._ops: list[tuple[str, torch.Tensor, int]] = []
self._log_initialized()
@@ -217,7 +218,7 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
recv_staging.append((tensor, cpu_tensor))
try:
with torch.cuda.stream(self._cuda_stream):
with device_stream(self._stream):
build_ops()
finally:
self._ops.clear()
@@ -225,10 +226,10 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
# Wait for all D2H copies to finish
# before issuing gloo batch_isend_irecv operations.
with gpu_sync_allowed():
if self._cuda_stream is not None:
self._cuda_stream.synchronize()
if self._stream is not None:
self._stream.synchronize()
else:
torch.cuda.current_stream().synchronize()
torch.accelerator.current_stream().synchronize()
reqs = batch_isend_irecv(p2p_ops)
for req in reqs:
@@ -236,11 +237,69 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
if not recv_staging:
return
with torch.cuda.stream(self._cuda_stream):
with device_stream(self._stream):
for dst_tensor, cpu_tensor in recv_staging:
dst_tensor.copy_(cpu_tensor, non_blocking=True)
class TorchDistXCCLStagedEplbCommunicator(EplbCommunicator):
"""EPLB communicator using XCCL device-to-device P2P on XPU."""
def __init__(
self,
ep_group: ProcessGroup,
stream: torch.Stream | None = None,
) -> None:
self._ep_group = ep_group
self._stream = stream
self._ops: list[tuple[str, torch.Tensor, int]] = []
self._log_initialized()
def add_send(
self,
tensors: list[torch.Tensor],
dst_rank: int,
expert_id: int, # unused by this backend
) -> None:
for tensor in tensors:
self._ops.append(("send", tensor, dst_rank))
def add_recv(
self,
tensors: list[torch.Tensor],
src_rank: int,
expert_id: int, # unused by this backend
) -> None:
for tensor in tensors:
self._ops.append(("recv", tensor, src_rank))
def execute(self) -> None:
if not self._ops:
return
p2p_ops: list[P2POp] = []
try:
for op, tensor, peer_rank in self._ops:
send_or_recv = (
torch.distributed.isend if op == "send" else torch.distributed.irecv
)
p2p_ops.append(
P2POp(
send_or_recv,
tensor,
peer_rank,
self._ep_group,
)
)
finally:
self._ops.clear()
with device_stream(self._stream):
reqs = batch_isend_irecv(p2p_ops)
for req in reqs:
req.wait()
class NixlEplbCommunicator(EplbCommunicator):
"""EPLB communicator backed by NIXL READ transfers."""
@@ -343,7 +402,7 @@ class NixlEplbCommunicator(EplbCommunicator):
uid = uuid.uuid4().hex[:8]
return f"eplb-{self._rank}{pp_suffix}-{uid}"
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
def set_stream(self, stream: torch.Stream | None) -> None:
pass
def add_send(
@@ -602,10 +661,10 @@ class PyNcclEplbCommunicator(EplbCommunicator):
def __init__(
self,
pynccl_comm: PyNcclCommunicator,
cuda_stream: torch.cuda.Stream | None = None,
stream: torch.Stream | None = None,
) -> None:
self._pynccl_comm = pynccl_comm
self._cuda_stream = cuda_stream
self._stream = stream
self._group_started = False
self._log_initialized()
@@ -622,7 +681,7 @@ class PyNcclEplbCommunicator(EplbCommunicator):
) -> None:
self._ensure_group_started()
for tensor in tensors:
self._pynccl_comm.send(tensor, dst_rank, stream=self._cuda_stream)
self._pynccl_comm.send(tensor, dst_rank, stream=self._stream)
def add_recv(
self,
@@ -632,7 +691,7 @@ class PyNcclEplbCommunicator(EplbCommunicator):
) -> None:
self._ensure_group_started()
for tensor in tensors:
self._pynccl_comm.recv(tensor, src_rank, stream=self._cuda_stream)
self._pynccl_comm.recv(tensor, src_rank, stream=self._stream)
def execute(self) -> None:
if self._group_started:
@@ -750,6 +809,8 @@ def create_eplb_communicator(
return TorchDistGlooStagedEplbCommunicator(
cpu_group=group_coordinator.cpu_group,
)
elif backend == "torch_xccl":
return TorchDistXCCLStagedEplbCommunicator(ep_group=torch_group)
elif backend == "torch_nccl":
return TorchDistNcclEplbCommunicator(ep_group=torch_group)
elif backend == "pynccl":
+11 -11
View File
@@ -203,9 +203,9 @@ class EplbModelState:
"""
EPLB stats for the model.
"""
cuda_device_index: int | None
device_index: int | None
"""
CUDA device index for the async EPLB worker thread.
Device index for the async EPLB worker thread.
"""
communicator: EplbCommunicator
"""
@@ -286,14 +286,14 @@ class EplbState:
"""
Background thread handling async transfers.
"""
self.cuda_device_index: int | None = None
self.device_index: int | None = None
"""
CUDA device index for the async EPLB worker thread.
Device index for the async EPLB worker thread.
"""
if self.device.type == "cuda":
self.cuda_device_index = self.device.index
if self.cuda_device_index is None and torch.cuda.is_available():
self.cuda_device_index = torch.accelerator.current_device_index()
if self.device.type in ("cuda", "xpu"):
self.device_index = self.device.index
if self.device_index is None and torch.accelerator.is_available():
self.device_index = torch.accelerator.current_device_index()
@staticmethod
def build_initial_global_physical_to_logical_map(
@@ -504,7 +504,7 @@ class EplbState:
expert_buffer=expert_buffer,
rebalanced=False,
eplb_stats=None,
cuda_device_index=self.cuda_device_index,
device_index=self.device_index,
communicator=communicator,
num_unpadded_tokens_tensors=num_unpadded_tokens_tensors,
)
@@ -763,8 +763,8 @@ class EplbState:
is_main_rank = ep_rank == 0
if is_main_rank:
if not self.is_async or is_profile:
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
start_event = torch.Event(enable_timing=True)
end_event = torch.Event(enable_timing=True)
start_event.record()
logger.info(
"Rearranging experts %s %s...",
+21 -4
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions for EPLB (Expert Parallel Load Balancing)."""
import contextlib
import os
import threading
@@ -13,6 +14,22 @@ from vllm.logger import init_logger
logger = init_logger(__name__)
@contextlib.contextmanager
def device_stream(stream: torch.Stream | None):
"""Platform-agnostic context manager that activates *stream* as the
current accelerator stream for the duration of the ``with`` block.
A no-op when *stream* is ``None``."""
if stream is None:
yield
return
prev = torch.accelerator.current_stream()
torch.accelerator.set_stream(stream)
try:
yield
finally:
torch.accelerator.set_stream(prev)
class CpuGpuEvent:
"""Combines a CUDA event with a CPU threading event to enforce record->wait
ordering across two threads.
@@ -30,10 +47,10 @@ class CpuGpuEvent:
"""
def __init__(self):
self._event = torch.cuda.Event()
self._event = torch.Event()
self._recorded = threading.Event()
def wait(self, stream: torch.cuda.Stream | None = None):
def wait(self, stream: torch.Stream | None = None):
"""Blocks the calling thread until record finishes. Used to guarantee that the
record kernel is called before wait.
@@ -43,7 +60,7 @@ class CpuGpuEvent:
self._event.wait(stream)
self._recorded.clear()
def record(self, stream: torch.cuda.Stream | None = None):
def record(self, stream: torch.Stream | None = None):
"""Unblocks the waiting thread after calling event.record().
Should only be called by the main thread.
@@ -53,7 +70,7 @@ class CpuGpuEvent:
"CpuGpuEvent.record() called before the previous event was "
"consumed by wait()"
)
self._event = torch.cuda.Event()
self._event = torch.Event()
self._event.record(stream)
self._recorded.set()
+9 -10
View File
@@ -13,7 +13,7 @@ import torch
from torch.distributed import ProcessGroup, all_gather
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -173,7 +173,7 @@ def move_to_buffer(
new_indices: np.ndarray,
expert_weights: Sequence[torch.Tensor],
expert_weights_buffers: Sequence[torch.Tensor],
cuda_stream: torch.cuda.Stream | None,
stream: torch.Stream | None,
ep_rank: int,
communicator: EplbCommunicator,
layer_idx: int = 0,
@@ -183,12 +183,12 @@ def move_to_buffer(
Args:
num_local_experts: Number of local experts.
old_indices: (num_experts_total,) ndarray of current (old)
global-to-local expert assignments.
global-to-locals expert assignments.
new_indices: (num_experts_total,) ndarray of desired (new)
global-to-local assignments after rebalance.
expert_weights: Original expert weights for the layer.
expert_weights_buffers: Intermediate buffers (one per tensor).
cuda_stream: CUDA stream for async copies (can be None for sync mode).
stream: CUDA/XPU stream for async copies (can be None for sync mode).
ep_rank: Rank of this process in expert parallel group.
communicator: EplbCommunicator instance for P2P communication.
layer_idx: Index of the MoE layer being transferred.
@@ -261,10 +261,9 @@ def move_to_buffer(
expert = new_local_expert_ids[dst]
src_local = expert_to_src_map.get(expert, -1)
if src_local != -1:
with torch.cuda.stream(cuda_stream):
with device_stream(stream):
for w, b in zip(expert_weights, expert_weights_buffers):
b[dst].copy_(w[src_local], non_blocking=True)
communicator.set_transfer_context(old_indices, layer_idx)
# 2. Post sends
@@ -430,7 +429,7 @@ def transfer_layer(
ep_group: ProcessGroup,
communicator: EplbCommunicator,
is_profile: bool = False,
cuda_stream: torch.cuda.Stream | None = None,
stream: torch.Stream | None = None,
rank_mapping: dict[int, int] | None = None,
layer_idx: int = 0,
) -> TransferMetadata:
@@ -451,7 +450,7 @@ def transfer_layer(
is_profile (bool): If `True`, do not perform any actual weight copy.
This is used during profile run, where we only perform dummy
communications to reserve enough memory for the buffers.
cuda_stream: CUDA stream for async copies (can be None for sync mode).
stream: CUDA stream for async copies (can be None for sync mode).
rank_mapping: Optional rank mapping for elastic expert parallelism.
layer_idx: Index of the MoE layer being transferred.
@@ -499,7 +498,7 @@ def transfer_layer(
new_indices=new_layer_indices_np,
expert_weights=expert_weights,
expert_weights_buffers=expert_weights_buffer,
cuda_stream=cuda_stream,
stream=stream,
ep_rank=ep_group.rank(),
communicator=communicator,
layer_idx=layer_idx,
@@ -601,7 +600,7 @@ def rearrange_expert_weights_inplace(
new_indices=new_global_expert_indices_cpu[layer_idx],
expert_weights=expert_weights[layer_idx],
expert_weights_buffers=weights_buffer,
cuda_stream=None,
stream=None,
ep_rank=ep_rank,
communicator=communicator,
layer_idx=layer_idx,
@@ -171,6 +171,7 @@ class XPUExperts(mk.FusedMoEExpertsModular):
num_experts=self.moe_config.num_local_experts,
ep_rank=self.moe_config.ep_rank,
ep_size=self.moe_config.ep_size,
expert_map=expert_map,
gemm1_clamp_limit=self.gemm1_clamp_limit,
)
assert self.fused_moe_impl is not None
@@ -13,7 +13,7 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
if current_platform.is_cuda_alike():
if current_platform.is_cuda_alike() or current_platform.is_xpu():
@triton.jit
def _eplb_map_and_record_i32_kernel(