[XPU] Enable XPU eplb (#44987)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com> Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com>
This commit is contained in:
+1
-1
@@ -935,7 +935,7 @@ def hf_runner():
|
||||
|
||||
|
||||
def _default_block_size() -> int:
|
||||
if torch.xpu.is_available():
|
||||
if current_platform.is_xpu():
|
||||
return 64
|
||||
if current_platform.is_cpu():
|
||||
return 128
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
mp.set_start_method("spawn", force=True)
|
||||
@@ -73,13 +74,16 @@ def distributed_run(fn, world_size, *args):
|
||||
def set_env_vars_and_device(env: dict[str, str]) -> None:
|
||||
update_environment_variables(env)
|
||||
local_rank = os.environ["LOCAL_RANK"]
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
backend = "xccl" if current_platform.is_xpu() else "nccl"
|
||||
|
||||
# Create a minimal vllm config for init_distributed_environment
|
||||
vllm_config = VllmConfig()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
init_distributed_environment()
|
||||
init_distributed_environment(backend=backend)
|
||||
atexit.register(_destroy_process_group_if_initialized)
|
||||
# Ensure each worker process has the same random seed
|
||||
random.seed(42)
|
||||
|
||||
@@ -10,6 +10,8 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..evals.gsm8k.gsm8k_eval import evaluate_gsm8k
|
||||
from ..utils import RemoteOpenAIServer, multi_gpu_test
|
||||
|
||||
@@ -233,6 +235,8 @@ def test_elastic_ep_scaling(
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("Elastic EP with NIXL is not supported on XPU")
|
||||
if all2all_backend == "nixl_ep" and not has_nixl_ep():
|
||||
pytest.skip("NIXL EP is not installed")
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.distributed.eplb.eplb_state import (
|
||||
compute_logical_maps,
|
||||
)
|
||||
from vllm.distributed.eplb.policy.default import DefaultEplbPolicy
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_eplb_load_stats_reduce_across_ranks():
|
||||
@@ -263,11 +264,13 @@ def test_global_load_balance_fallback():
|
||||
assert torch.sum(logcnt) == num_replicas
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda", "xpu"])
|
||||
def test_device_compatibility(device):
|
||||
"""Test device compatibility."""
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
if device == "xpu" and not current_platform.is_xpu():
|
||||
pytest.skip("XPU not available")
|
||||
|
||||
weight = torch.tensor([[10, 20, 30, 40]], device=device)
|
||||
num_replicas = 6
|
||||
|
||||
@@ -4,15 +4,17 @@
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_wait_blocks_until_record():
|
||||
event = CpuGpuEvent()
|
||||
record_stream = torch.cuda.Stream()
|
||||
wait_stream = torch.cuda.Stream()
|
||||
record_stream = torch.Stream()
|
||||
wait_stream = torch.Stream()
|
||||
wait_returned = threading.Event()
|
||||
|
||||
def waiter():
|
||||
@@ -33,8 +35,8 @@ def test_wait_blocks_until_record():
|
||||
|
||||
def test_reuse_across_multiple_cycles():
|
||||
wrapper = CpuGpuEvent()
|
||||
record_stream = torch.cuda.Stream()
|
||||
wait_stream = torch.cuda.Stream()
|
||||
record_stream = torch.Stream()
|
||||
wait_stream = torch.Stream()
|
||||
NUM_CYCLES = 8
|
||||
completed_cycles = []
|
||||
barriers = [threading.Barrier(2) for _ in range(NUM_CYCLES)]
|
||||
@@ -56,13 +58,18 @@ def test_reuse_across_multiple_cycles():
|
||||
assert len(completed_cycles) == NUM_CYCLES
|
||||
|
||||
|
||||
def test_producer_consumer():
|
||||
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared
|
||||
GPU tensor on multiple CPU threads.
|
||||
@pytest.mark.parametrize("device", ["cuda", "xpu"])
|
||||
def test_producer_consumer(device):
|
||||
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from
|
||||
a shared GPU tensor on multiple CPU threads.
|
||||
"""
|
||||
worker_stream = torch.cuda.Stream()
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
if device == "xpu" and not current_platform.is_xpu():
|
||||
pytest.skip("XPU not available")
|
||||
worker_stream = torch.Stream()
|
||||
# Create a single element counter that will be shared between two threads
|
||||
buf = torch.zeros(1, device="cuda")
|
||||
buf = torch.zeros(1, device=device)
|
||||
NUM_ROUNDS = 5
|
||||
|
||||
ready_cpu = [threading.Event() for _ in range(NUM_ROUNDS)]
|
||||
@@ -76,7 +83,7 @@ def test_producer_consumer():
|
||||
if i > 0:
|
||||
events[i - 1].wait(stream=worker_stream)
|
||||
|
||||
with torch.cuda.stream(worker_stream):
|
||||
with device_stream(worker_stream):
|
||||
buf.fill_(float(i + 1))
|
||||
|
||||
worker_stream.synchronize()
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
@@ -208,10 +209,14 @@ def verify_redundant_experts_have_same_weights(
|
||||
ep_rank: int,
|
||||
world_size: int,
|
||||
num_local_experts: int,
|
||||
cpu_group,
|
||||
) -> bool:
|
||||
"""Verify that all replicas of the same logical expert have the same weights."""
|
||||
num_layers = len(expert_weights)
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
# XPU: gloo does not support XPU tensors, so gather on CPU then move back.
|
||||
# CUDA: gloo supports CUDA tensors directly, keep original device.
|
||||
use_cpu_gather = current_platform.is_xpu()
|
||||
|
||||
ok = True
|
||||
for layer in range(num_layers):
|
||||
@@ -219,31 +224,35 @@ def verify_redundant_experts_have_same_weights(
|
||||
all_weights: list[torch.Tensor] = []
|
||||
|
||||
for weight_idx, hidden_size in enumerate(hidden_sizes):
|
||||
orig_device = expert_weights[layer][weight_idx].device
|
||||
|
||||
if use_cpu_gather:
|
||||
local_weights = expert_weights[layer][weight_idx].cpu()
|
||||
gather_device = "cpu"
|
||||
else:
|
||||
local_weights = expert_weights[layer][weight_idx]
|
||||
gather_device = orig_device
|
||||
|
||||
# Create tensor to store all expert weights
|
||||
# Shape: [total_physical_experts, hidden_size]
|
||||
gathered_weights = torch.zeros(
|
||||
total_physical_experts,
|
||||
hidden_size,
|
||||
device=expert_weights[layer][weight_idx].device,
|
||||
dtype=expert_weights[layer][weight_idx].dtype,
|
||||
device=gather_device,
|
||||
dtype=local_weights.dtype,
|
||||
)
|
||||
|
||||
# Use all_gather to collect expert weights from current node
|
||||
# expert_weights[layer][weight_idx] shape:
|
||||
# [num_local_experts, hidden_size]
|
||||
local_weights = expert_weights[layer][
|
||||
weight_idx
|
||||
] # [num_local_experts, hidden_size]
|
||||
|
||||
# Split tensor along dim 0 into a list for all_gather
|
||||
gathered_weights_list = torch.chunk(gathered_weights, world_size, dim=0)
|
||||
|
||||
torch.distributed.all_gather(
|
||||
# Output list: each element corresponds to one rank's weights
|
||||
list(gathered_weights_list),
|
||||
local_weights, # Input: current rank's local weights
|
||||
local_weights,
|
||||
group=cpu_group,
|
||||
)
|
||||
|
||||
if use_cpu_gather:
|
||||
gathered_weights = gathered_weights.to(orig_device)
|
||||
all_weights.append(gathered_weights)
|
||||
|
||||
# Verify that all replicas of the same logical expert have the same
|
||||
@@ -294,9 +303,23 @@ def verify_redundant_experts_have_same_weights(
|
||||
return ok
|
||||
|
||||
|
||||
def assert_verification_synced(local_ok: bool, msg: str) -> None:
|
||||
ok_tensor = torch.tensor([1 if local_ok else 0], device="cuda", dtype=torch.int32)
|
||||
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
|
||||
def assert_verification_synced(local_ok: bool, msg: str, cpu_group) -> None:
|
||||
if current_platform.is_xpu():
|
||||
# XPU: default group backend (gloo) does not support XPU tensors;
|
||||
# use CPU tensor with cpu_group instead.
|
||||
ok_tensor = torch.tensor(
|
||||
[1 if local_ok else 0], device="cpu", dtype=torch.int32
|
||||
)
|
||||
torch.distributed.all_reduce(
|
||||
ok_tensor, op=torch.distributed.ReduceOp.MIN, group=cpu_group
|
||||
)
|
||||
else:
|
||||
# CUDA: nccl (default group) supports device tensors directly.
|
||||
device = torch.accelerator.current_accelerator()
|
||||
ok_tensor = torch.tensor(
|
||||
[1 if local_ok else 0], device=device, dtype=torch.int32
|
||||
)
|
||||
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
|
||||
assert bool(ok_tensor.item()), msg
|
||||
|
||||
|
||||
@@ -336,8 +359,10 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
hidden_sizes = [16, 32]
|
||||
@@ -376,7 +401,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
new_indices_cpu = new_indices.cpu()
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
cuda_stream = torch.cuda.Stream(device=device)
|
||||
stream = torch.Stream(device=device)
|
||||
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
@@ -384,7 +409,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
communicator.set_stream(cuda_stream)
|
||||
communicator.set_stream(stream)
|
||||
|
||||
for layer_idx in range(num_layers):
|
||||
transfer_metadata = transfer_layer(
|
||||
@@ -394,10 +419,10 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
expert_weights_buffer=expert_buffer,
|
||||
ep_group=ep_group,
|
||||
communicator=communicator,
|
||||
cuda_stream=cuda_stream,
|
||||
stream=stream,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
cuda_stream.synchronize()
|
||||
stream.synchronize()
|
||||
move_from_buffer(
|
||||
expert_weights=expert_weights[layer_idx],
|
||||
expert_weights_buffers=expert_buffer,
|
||||
@@ -421,6 +446,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
@@ -428,6 +454,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
local_ok,
|
||||
"Async transfer verification failed on at least one rank. "
|
||||
"See logs for details.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -452,9 +479,11 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
# Test parameters
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
@@ -523,12 +552,14 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Rearrange verification failed on at least one rank. See logs for details.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -556,7 +587,7 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"eplb_communicator", ["torch_nccl", "torch_gloo", "pynccl", "nixl"]
|
||||
"eplb_communicator", ["torch_nccl", "torch_gloo", "torch_xccl", "pynccl", "nixl"]
|
||||
)
|
||||
def test_rearrange_expert_weights_with_redundancy(
|
||||
world_size,
|
||||
@@ -568,6 +599,12 @@ def test_rearrange_expert_weights_with_redundancy(
|
||||
"""Test the functionality of rearranging expert weights with redundancy."""
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if eplb_communicator == "nixl" and current_platform.is_xpu():
|
||||
pytest.skip("NIXL does not support XPU")
|
||||
if eplb_communicator in ("torch_nccl", "pynccl") and not torch.cuda.is_available():
|
||||
pytest.skip(f"{eplb_communicator} requires CUDA")
|
||||
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
|
||||
pytest.skip("torch_xccl requires XPU")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
@@ -592,9 +629,11 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
num_layers = 2
|
||||
num_local_experts = 2
|
||||
@@ -623,9 +662,10 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend="torch_nccl",
|
||||
backend=default_backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
@@ -657,6 +697,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"No-change EPLB verification failed on at least one rank.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -666,7 +707,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
(2, 2, 2, 3),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "nixl"])
|
||||
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "torch_xccl", "nixl"])
|
||||
def test_async_transfer_layer_without_mtp(
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
@@ -677,6 +718,10 @@ def test_async_transfer_layer_without_mtp(
|
||||
"""Exercise async EPLB transfer path without MTP/spec decode."""
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if eplb_communicator == "nixl" and current_platform.is_xpu():
|
||||
pytest.skip("NIXL does not support XPU")
|
||||
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
|
||||
pytest.skip("torch_xccl requires XPU")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
|
||||
@@ -715,9 +760,11 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
num_layers = 1
|
||||
num_local_experts = 2
|
||||
@@ -753,9 +800,10 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend="torch_nccl",
|
||||
backend=default_backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
@@ -788,6 +836,7 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Profile-mode EPLB verification failed on at least one rank.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
@@ -81,7 +82,7 @@ def make_fused_moe_layer(
|
||||
)
|
||||
re = fml.routed_experts
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
device = torch.device(f"{torch.accelerator.current_accelerator().type}:{rank}")
|
||||
|
||||
from functools import partial
|
||||
|
||||
@@ -190,13 +191,14 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.parallel_config.enable_eplb = True
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group = get_tp_group().cpu_group
|
||||
ep_group = get_tp_group().device_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
|
||||
fml_layers = [
|
||||
@@ -216,9 +218,10 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend="torch_nccl",
|
||||
backend=default_backend,
|
||||
expert_weights=rank_expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
ModelOptNvFp4FusedMoE,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
@@ -270,6 +271,8 @@ def test_eplb_fml(
|
||||
):
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("NVFP4 quantization and flashinfer backends are CUDA-only")
|
||||
|
||||
num_local_experts = num_experts // world_size
|
||||
num_topk = 4
|
||||
|
||||
+12
-5
@@ -38,7 +38,9 @@ DistributedExecutorBackend = Literal["ray", "mp", "uni", "external_launcher"]
|
||||
DataParallelBackend = Literal["ray", "mp"]
|
||||
EPLBPolicyOption = Literal["default"]
|
||||
DCPCommBackend = Literal["ag_rs", "a2a"]
|
||||
EPLBCommunicatorBackend = Literal["torch_nccl", "torch_gloo", "nixl", "pynccl"]
|
||||
EPLBCommunicatorBackend = Literal[
|
||||
"torch_nccl", "torch_gloo", "torch_xccl", "nixl", "pynccl"
|
||||
]
|
||||
All2AllBackend = Literal[
|
||||
"naive",
|
||||
"pplx",
|
||||
@@ -94,9 +96,11 @@ class EPLBConfig:
|
||||
Backend for EPLB expert weight communication:
|
||||
- "torch_nccl": Use torch.distributed on the device process group
|
||||
- "torch_gloo": Use torch.distributed gloo with CPU staging
|
||||
- "torch_xccl": Use torch.distributed XCCL device P2P on XPU
|
||||
- "nixl": Use NIXL with staged send/recv buffers
|
||||
- "pynccl": Use PyNccl send/recv
|
||||
- None: Auto-select backend (prefers "nixl", falls back to "torch_gloo")
|
||||
- None: Auto-select backend ("torch_xccl" on XPU, prefers "nixl"
|
||||
on CUDA, falls back to "torch_gloo")
|
||||
"""
|
||||
|
||||
@model_validator(mode="after")
|
||||
@@ -525,10 +529,10 @@ class ParallelConfig:
|
||||
)
|
||||
|
||||
if self.enable_eplb:
|
||||
if not current_platform.is_cuda_alike():
|
||||
if not current_platform.is_cuda_alike() and not current_platform.is_xpu():
|
||||
raise ValueError(
|
||||
"Expert parallelism load balancing is only supported on "
|
||||
"CUDA devices or ROCm devices now."
|
||||
"CUDA devices or ROCm devices or XPU devices now."
|
||||
)
|
||||
if not self.enable_expert_parallel:
|
||||
raise ValueError("enable_expert_parallel must be True to use EPLB.")
|
||||
@@ -1034,7 +1038,10 @@ class ParallelConfig:
|
||||
# See https://github.com/pytorch/pytorch/issues/174288
|
||||
from vllm.distributed.nixl_utils import is_nixl_available
|
||||
|
||||
if is_nixl_available():
|
||||
if current_platform.is_xpu():
|
||||
# On XPU, use the device-native XCCL P2P backend.
|
||||
self.eplb_config.communicator = "torch_xccl"
|
||||
elif is_nixl_available():
|
||||
self.eplb_config.communicator = "nixl"
|
||||
elif self.enable_elastic_ep:
|
||||
self.eplb_config.communicator = "pynccl"
|
||||
|
||||
@@ -10,7 +10,7 @@ import torch
|
||||
from vllm.distributed.parallel_state import get_eplb_group
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .eplb_utils import CpuGpuEvent
|
||||
from .eplb_utils import CpuGpuEvent, device_stream
|
||||
from .rebalance_execute import AsyncEplbLayerResult, transfer_layer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -24,17 +24,17 @@ def start_async_worker(
|
||||
is_profile: bool = False,
|
||||
) -> threading.Thread:
|
||||
rank = get_eplb_group().device_group.rank()
|
||||
device_index = state.cuda_device_index
|
||||
device_index = state.device_index
|
||||
assert state.is_async
|
||||
|
||||
def thread_target() -> None:
|
||||
assert device_index is not None
|
||||
torch.accelerator.set_device_index(device_index)
|
||||
cuda_stream = torch.cuda.Stream(device=device_index)
|
||||
stream = torch.Stream(device=torch.accelerator.current_accelerator())
|
||||
try:
|
||||
transfer_run_periodically(
|
||||
state=state,
|
||||
cuda_stream=cuda_stream,
|
||||
stream=stream,
|
||||
is_profile=is_profile,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - diagnostic path
|
||||
@@ -49,13 +49,13 @@ def run_rebalance_experts(
|
||||
model_state: "EplbModelState",
|
||||
eplb_state: "EplbState",
|
||||
physical_to_logical_map_cpu: torch.Tensor,
|
||||
cuda_stream: torch.cuda.Stream,
|
||||
stream: torch.Stream,
|
||||
) -> torch.Tensor:
|
||||
assert model_state.eplb_stats is not None
|
||||
eplb_stats = model_state.eplb_stats
|
||||
|
||||
# Move the global expert load window to CPU for computation.
|
||||
with torch.cuda.stream(cuda_stream):
|
||||
with device_stream(stream):
|
||||
global_expert_load_window = eplb_stats.global_expert_load_window.cpu()
|
||||
# Compute new expert mappings for the model
|
||||
new_physical_to_logical_map = eplb_state.policy.rebalance_experts(
|
||||
@@ -73,11 +73,11 @@ def run_rebalance_experts(
|
||||
|
||||
def transfer_run_periodically(
|
||||
state: "EplbState",
|
||||
cuda_stream: torch.cuda.Stream,
|
||||
stream: torch.Stream,
|
||||
is_profile: bool = False,
|
||||
) -> None:
|
||||
while True:
|
||||
state.rearrange_event.wait(stream=cuda_stream)
|
||||
state.rearrange_event.wait(stream=stream)
|
||||
|
||||
eplb_group = get_eplb_group().device_group
|
||||
eplb_cpu_group = get_eplb_group().cpu_group
|
||||
@@ -87,16 +87,16 @@ def transfer_run_periodically(
|
||||
for model_state in state.model_states.values():
|
||||
layer_idx = 0
|
||||
# Set the async worker's CUDA stream on the communicator
|
||||
model_state.communicator.set_stream(cuda_stream)
|
||||
model_state.communicator.set_stream(stream)
|
||||
num_layers = model_state.model.num_moe_layers
|
||||
|
||||
# Snapshot the physical_to_logical_map (synchronized with
|
||||
# rearrange_event) and copy it to CPU
|
||||
with torch.cuda.stream(cuda_stream):
|
||||
with device_stream(stream):
|
||||
physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu()
|
||||
|
||||
new_physical_to_logical_map = run_rebalance_experts(
|
||||
model_state, state, physical_to_logical_map_cpu, cuda_stream
|
||||
model_state, state, physical_to_logical_map_cpu, stream
|
||||
)
|
||||
|
||||
# Execute one EPLB layer transfer per model forward pass. Each iteration
|
||||
@@ -132,13 +132,13 @@ def transfer_run_periodically(
|
||||
communicator=model_state.communicator,
|
||||
ep_group=eplb_group,
|
||||
is_profile=is_profile,
|
||||
cuda_stream=cuda_stream,
|
||||
stream=stream,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
|
||||
# Wait until all writes to expert_buffer have finished before making the
|
||||
# AsyncEplbLayerResult visible to the main thread.
|
||||
cuda_stream.synchronize()
|
||||
stream.synchronize()
|
||||
|
||||
# This event guarantees that expert_buffer will not be overwritten by
|
||||
# subsequent iterations of this loop until the main thread has consumed
|
||||
@@ -155,6 +155,6 @@ def transfer_run_periodically(
|
||||
# Block this thread until the main thread and main stream
|
||||
# finish copying model_state.expert_buffer into
|
||||
# model_state.model.expert_weights[layer_idx]
|
||||
consumed_event.wait(stream=cuda_stream)
|
||||
consumed_event.wait(stream=stream)
|
||||
assert model_state.pending_result is None
|
||||
layer_idx += 1
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
ncclDataTypeEnum,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_utils import device_stream
|
||||
from vllm.distributed.parallel_state import (
|
||||
GroupCoordinator,
|
||||
get_pp_group,
|
||||
@@ -87,8 +88,8 @@ class EplbCommunicator(ABC):
|
||||
communication buffers."""
|
||||
return True
|
||||
|
||||
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
|
||||
self._cuda_stream = cuda_stream
|
||||
def set_stream(self, stream: torch.Stream | None) -> None:
|
||||
self._stream = stream
|
||||
|
||||
def _log_initialized(self) -> None:
|
||||
if is_local_first_rank():
|
||||
@@ -101,10 +102,10 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator):
|
||||
def __init__(
|
||||
self,
|
||||
ep_group: ProcessGroup,
|
||||
cuda_stream: torch.cuda.Stream | None = None,
|
||||
stream: torch.Stream | None = None,
|
||||
) -> None:
|
||||
self._ep_group = ep_group
|
||||
self._cuda_stream = cuda_stream
|
||||
self._stream = stream
|
||||
self._p2p_ops: list[P2POp] = []
|
||||
self._log_initialized()
|
||||
|
||||
@@ -144,7 +145,7 @@ class TorchDistNcclEplbCommunicator(EplbCommunicator):
|
||||
if not self._p2p_ops:
|
||||
return
|
||||
try:
|
||||
with torch.cuda.stream(self._cuda_stream):
|
||||
with device_stream(self._stream):
|
||||
reqs = batch_isend_irecv(self._p2p_ops)
|
||||
for req in reqs:
|
||||
req.wait()
|
||||
@@ -158,10 +159,10 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
|
||||
def __init__(
|
||||
self,
|
||||
cpu_group: ProcessGroup,
|
||||
cuda_stream: torch.cuda.Stream | None = None,
|
||||
stream: torch.Stream | None = None,
|
||||
) -> None:
|
||||
self._cpu_group = cpu_group
|
||||
self._cuda_stream = cuda_stream
|
||||
self._stream = stream
|
||||
self._ops: list[tuple[str, torch.Tensor, int]] = []
|
||||
self._log_initialized()
|
||||
|
||||
@@ -217,7 +218,7 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
|
||||
recv_staging.append((tensor, cpu_tensor))
|
||||
|
||||
try:
|
||||
with torch.cuda.stream(self._cuda_stream):
|
||||
with device_stream(self._stream):
|
||||
build_ops()
|
||||
finally:
|
||||
self._ops.clear()
|
||||
@@ -225,10 +226,10 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
|
||||
# Wait for all D2H copies to finish
|
||||
# before issuing gloo batch_isend_irecv operations.
|
||||
with gpu_sync_allowed():
|
||||
if self._cuda_stream is not None:
|
||||
self._cuda_stream.synchronize()
|
||||
if self._stream is not None:
|
||||
self._stream.synchronize()
|
||||
else:
|
||||
torch.cuda.current_stream().synchronize()
|
||||
torch.accelerator.current_stream().synchronize()
|
||||
|
||||
reqs = batch_isend_irecv(p2p_ops)
|
||||
for req in reqs:
|
||||
@@ -236,11 +237,69 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
|
||||
|
||||
if not recv_staging:
|
||||
return
|
||||
with torch.cuda.stream(self._cuda_stream):
|
||||
with device_stream(self._stream):
|
||||
for dst_tensor, cpu_tensor in recv_staging:
|
||||
dst_tensor.copy_(cpu_tensor, non_blocking=True)
|
||||
|
||||
|
||||
class TorchDistXCCLStagedEplbCommunicator(EplbCommunicator):
|
||||
"""EPLB communicator using XCCL device-to-device P2P on XPU."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ep_group: ProcessGroup,
|
||||
stream: torch.Stream | None = None,
|
||||
) -> None:
|
||||
self._ep_group = ep_group
|
||||
self._stream = stream
|
||||
self._ops: list[tuple[str, torch.Tensor, int]] = []
|
||||
self._log_initialized()
|
||||
|
||||
def add_send(
|
||||
self,
|
||||
tensors: list[torch.Tensor],
|
||||
dst_rank: int,
|
||||
expert_id: int, # unused by this backend
|
||||
) -> None:
|
||||
for tensor in tensors:
|
||||
self._ops.append(("send", tensor, dst_rank))
|
||||
|
||||
def add_recv(
|
||||
self,
|
||||
tensors: list[torch.Tensor],
|
||||
src_rank: int,
|
||||
expert_id: int, # unused by this backend
|
||||
) -> None:
|
||||
for tensor in tensors:
|
||||
self._ops.append(("recv", tensor, src_rank))
|
||||
|
||||
def execute(self) -> None:
|
||||
if not self._ops:
|
||||
return
|
||||
|
||||
p2p_ops: list[P2POp] = []
|
||||
try:
|
||||
for op, tensor, peer_rank in self._ops:
|
||||
send_or_recv = (
|
||||
torch.distributed.isend if op == "send" else torch.distributed.irecv
|
||||
)
|
||||
p2p_ops.append(
|
||||
P2POp(
|
||||
send_or_recv,
|
||||
tensor,
|
||||
peer_rank,
|
||||
self._ep_group,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
self._ops.clear()
|
||||
|
||||
with device_stream(self._stream):
|
||||
reqs = batch_isend_irecv(p2p_ops)
|
||||
for req in reqs:
|
||||
req.wait()
|
||||
|
||||
|
||||
class NixlEplbCommunicator(EplbCommunicator):
|
||||
"""EPLB communicator backed by NIXL READ transfers."""
|
||||
|
||||
@@ -343,7 +402,7 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
uid = uuid.uuid4().hex[:8]
|
||||
return f"eplb-{self._rank}{pp_suffix}-{uid}"
|
||||
|
||||
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
|
||||
def set_stream(self, stream: torch.Stream | None) -> None:
|
||||
pass
|
||||
|
||||
def add_send(
|
||||
@@ -602,10 +661,10 @@ class PyNcclEplbCommunicator(EplbCommunicator):
|
||||
def __init__(
|
||||
self,
|
||||
pynccl_comm: PyNcclCommunicator,
|
||||
cuda_stream: torch.cuda.Stream | None = None,
|
||||
stream: torch.Stream | None = None,
|
||||
) -> None:
|
||||
self._pynccl_comm = pynccl_comm
|
||||
self._cuda_stream = cuda_stream
|
||||
self._stream = stream
|
||||
self._group_started = False
|
||||
self._log_initialized()
|
||||
|
||||
@@ -622,7 +681,7 @@ class PyNcclEplbCommunicator(EplbCommunicator):
|
||||
) -> None:
|
||||
self._ensure_group_started()
|
||||
for tensor in tensors:
|
||||
self._pynccl_comm.send(tensor, dst_rank, stream=self._cuda_stream)
|
||||
self._pynccl_comm.send(tensor, dst_rank, stream=self._stream)
|
||||
|
||||
def add_recv(
|
||||
self,
|
||||
@@ -632,7 +691,7 @@ class PyNcclEplbCommunicator(EplbCommunicator):
|
||||
) -> None:
|
||||
self._ensure_group_started()
|
||||
for tensor in tensors:
|
||||
self._pynccl_comm.recv(tensor, src_rank, stream=self._cuda_stream)
|
||||
self._pynccl_comm.recv(tensor, src_rank, stream=self._stream)
|
||||
|
||||
def execute(self) -> None:
|
||||
if self._group_started:
|
||||
@@ -750,6 +809,8 @@ def create_eplb_communicator(
|
||||
return TorchDistGlooStagedEplbCommunicator(
|
||||
cpu_group=group_coordinator.cpu_group,
|
||||
)
|
||||
elif backend == "torch_xccl":
|
||||
return TorchDistXCCLStagedEplbCommunicator(ep_group=torch_group)
|
||||
elif backend == "torch_nccl":
|
||||
return TorchDistNcclEplbCommunicator(ep_group=torch_group)
|
||||
elif backend == "pynccl":
|
||||
|
||||
@@ -203,9 +203,9 @@ class EplbModelState:
|
||||
"""
|
||||
EPLB stats for the model.
|
||||
"""
|
||||
cuda_device_index: int | None
|
||||
device_index: int | None
|
||||
"""
|
||||
CUDA device index for the async EPLB worker thread.
|
||||
Device index for the async EPLB worker thread.
|
||||
"""
|
||||
communicator: EplbCommunicator
|
||||
"""
|
||||
@@ -286,14 +286,14 @@ class EplbState:
|
||||
"""
|
||||
Background thread handling async transfers.
|
||||
"""
|
||||
self.cuda_device_index: int | None = None
|
||||
self.device_index: int | None = None
|
||||
"""
|
||||
CUDA device index for the async EPLB worker thread.
|
||||
Device index for the async EPLB worker thread.
|
||||
"""
|
||||
if self.device.type == "cuda":
|
||||
self.cuda_device_index = self.device.index
|
||||
if self.cuda_device_index is None and torch.cuda.is_available():
|
||||
self.cuda_device_index = torch.accelerator.current_device_index()
|
||||
if self.device.type in ("cuda", "xpu"):
|
||||
self.device_index = self.device.index
|
||||
if self.device_index is None and torch.accelerator.is_available():
|
||||
self.device_index = torch.accelerator.current_device_index()
|
||||
|
||||
@staticmethod
|
||||
def build_initial_global_physical_to_logical_map(
|
||||
@@ -504,7 +504,7 @@ class EplbState:
|
||||
expert_buffer=expert_buffer,
|
||||
rebalanced=False,
|
||||
eplb_stats=None,
|
||||
cuda_device_index=self.cuda_device_index,
|
||||
device_index=self.device_index,
|
||||
communicator=communicator,
|
||||
num_unpadded_tokens_tensors=num_unpadded_tokens_tensors,
|
||||
)
|
||||
@@ -763,8 +763,8 @@ class EplbState:
|
||||
is_main_rank = ep_rank == 0
|
||||
if is_main_rank:
|
||||
if not self.is_async or is_profile:
|
||||
start_event = torch.cuda.Event(enable_timing=True)
|
||||
end_event = torch.cuda.Event(enable_timing=True)
|
||||
start_event = torch.Event(enable_timing=True)
|
||||
end_event = torch.Event(enable_timing=True)
|
||||
start_event.record()
|
||||
logger.info(
|
||||
"Rearranging experts %s %s...",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility functions for EPLB (Expert Parallel Load Balancing)."""
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import threading
|
||||
|
||||
@@ -13,6 +14,22 @@ from vllm.logger import init_logger
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def device_stream(stream: torch.Stream | None):
|
||||
"""Platform-agnostic context manager that activates *stream* as the
|
||||
current accelerator stream for the duration of the ``with`` block.
|
||||
A no-op when *stream* is ``None``."""
|
||||
if stream is None:
|
||||
yield
|
||||
return
|
||||
prev = torch.accelerator.current_stream()
|
||||
torch.accelerator.set_stream(stream)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
torch.accelerator.set_stream(prev)
|
||||
|
||||
|
||||
class CpuGpuEvent:
|
||||
"""Combines a CUDA event with a CPU threading event to enforce record->wait
|
||||
ordering across two threads.
|
||||
@@ -30,10 +47,10 @@ class CpuGpuEvent:
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._event = torch.cuda.Event()
|
||||
self._event = torch.Event()
|
||||
self._recorded = threading.Event()
|
||||
|
||||
def wait(self, stream: torch.cuda.Stream | None = None):
|
||||
def wait(self, stream: torch.Stream | None = None):
|
||||
"""Blocks the calling thread until record finishes. Used to guarantee that the
|
||||
record kernel is called before wait.
|
||||
|
||||
@@ -43,7 +60,7 @@ class CpuGpuEvent:
|
||||
self._event.wait(stream)
|
||||
self._recorded.clear()
|
||||
|
||||
def record(self, stream: torch.cuda.Stream | None = None):
|
||||
def record(self, stream: torch.Stream | None = None):
|
||||
"""Unblocks the waiting thread after calling event.record().
|
||||
|
||||
Should only be called by the main thread.
|
||||
@@ -53,7 +70,7 @@ class CpuGpuEvent:
|
||||
"CpuGpuEvent.record() called before the previous event was "
|
||||
"consumed by wait()"
|
||||
)
|
||||
self._event = torch.cuda.Event()
|
||||
self._event = torch.Event()
|
||||
self._event.record(stream)
|
||||
self._recorded.set()
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import torch
|
||||
from torch.distributed import ProcessGroup, all_gather
|
||||
|
||||
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
@@ -173,7 +173,7 @@ def move_to_buffer(
|
||||
new_indices: np.ndarray,
|
||||
expert_weights: Sequence[torch.Tensor],
|
||||
expert_weights_buffers: Sequence[torch.Tensor],
|
||||
cuda_stream: torch.cuda.Stream | None,
|
||||
stream: torch.Stream | None,
|
||||
ep_rank: int,
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int = 0,
|
||||
@@ -183,12 +183,12 @@ def move_to_buffer(
|
||||
Args:
|
||||
num_local_experts: Number of local experts.
|
||||
old_indices: (num_experts_total,) ndarray of current (old)
|
||||
global-to-local expert assignments.
|
||||
global-to-locals expert assignments.
|
||||
new_indices: (num_experts_total,) ndarray of desired (new)
|
||||
global-to-local assignments after rebalance.
|
||||
expert_weights: Original expert weights for the layer.
|
||||
expert_weights_buffers: Intermediate buffers (one per tensor).
|
||||
cuda_stream: CUDA stream for async copies (can be None for sync mode).
|
||||
stream: CUDA/XPU stream for async copies (can be None for sync mode).
|
||||
ep_rank: Rank of this process in expert parallel group.
|
||||
communicator: EplbCommunicator instance for P2P communication.
|
||||
layer_idx: Index of the MoE layer being transferred.
|
||||
@@ -261,10 +261,9 @@ def move_to_buffer(
|
||||
expert = new_local_expert_ids[dst]
|
||||
src_local = expert_to_src_map.get(expert, -1)
|
||||
if src_local != -1:
|
||||
with torch.cuda.stream(cuda_stream):
|
||||
with device_stream(stream):
|
||||
for w, b in zip(expert_weights, expert_weights_buffers):
|
||||
b[dst].copy_(w[src_local], non_blocking=True)
|
||||
|
||||
communicator.set_transfer_context(old_indices, layer_idx)
|
||||
|
||||
# 2. Post sends
|
||||
@@ -430,7 +429,7 @@ def transfer_layer(
|
||||
ep_group: ProcessGroup,
|
||||
communicator: EplbCommunicator,
|
||||
is_profile: bool = False,
|
||||
cuda_stream: torch.cuda.Stream | None = None,
|
||||
stream: torch.Stream | None = None,
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
layer_idx: int = 0,
|
||||
) -> TransferMetadata:
|
||||
@@ -451,7 +450,7 @@ def transfer_layer(
|
||||
is_profile (bool): If `True`, do not perform any actual weight copy.
|
||||
This is used during profile run, where we only perform dummy
|
||||
communications to reserve enough memory for the buffers.
|
||||
cuda_stream: CUDA stream for async copies (can be None for sync mode).
|
||||
stream: CUDA stream for async copies (can be None for sync mode).
|
||||
rank_mapping: Optional rank mapping for elastic expert parallelism.
|
||||
layer_idx: Index of the MoE layer being transferred.
|
||||
|
||||
@@ -499,7 +498,7 @@ def transfer_layer(
|
||||
new_indices=new_layer_indices_np,
|
||||
expert_weights=expert_weights,
|
||||
expert_weights_buffers=expert_weights_buffer,
|
||||
cuda_stream=cuda_stream,
|
||||
stream=stream,
|
||||
ep_rank=ep_group.rank(),
|
||||
communicator=communicator,
|
||||
layer_idx=layer_idx,
|
||||
@@ -601,7 +600,7 @@ def rearrange_expert_weights_inplace(
|
||||
new_indices=new_global_expert_indices_cpu[layer_idx],
|
||||
expert_weights=expert_weights[layer_idx],
|
||||
expert_weights_buffers=weights_buffer,
|
||||
cuda_stream=None,
|
||||
stream=None,
|
||||
ep_rank=ep_rank,
|
||||
communicator=communicator,
|
||||
layer_idx=layer_idx,
|
||||
|
||||
@@ -171,6 +171,7 @@ class XPUExperts(mk.FusedMoEExpertsModular):
|
||||
num_experts=self.moe_config.num_local_experts,
|
||||
ep_rank=self.moe_config.ep_rank,
|
||||
ep_size=self.moe_config.ep_size,
|
||||
expert_map=expert_map,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
assert self.fused_moe_impl is not None
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
if current_platform.is_cuda_alike():
|
||||
if current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
|
||||
@triton.jit
|
||||
def _eplb_map_and_record_i32_kernel(
|
||||
|
||||
Reference in New Issue
Block a user