[MoE Refactor] Add sequence parallel tests to test_moe_layer.py (#41299)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-05-12 21:52:19 -04:00
committed by GitHub
parent 07534b8782
commit 18f6bf5a21
4 changed files with 189 additions and 72 deletions
+174 -63
View File
@@ -31,12 +31,13 @@ from vllm.config import (
VllmConfig, VllmConfig,
set_current_vllm_config, set_current_vllm_config,
) )
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator from vllm.distributed import (
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
from vllm.distributed.parallel_state import (
get_ep_group, get_ep_group,
get_eplb_group, get_eplb_group,
tensor_model_parallel_all_gather,
) )
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
from vllm.forward_context import set_forward_context from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoE, fused_experts from vllm.model_executor.layers.fused_moe import FusedMoE, fused_experts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.activation import MoEActivation
@@ -49,6 +50,7 @@ from vllm.model_executor.layers.quantization.modelopt import (
ModelOptFp8Config, ModelOptFp8Config,
ModelOptNvFp4Config, ModelOptNvFp4Config,
) )
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.flashinfer import ( from vllm.utils.flashinfer import (
has_flashinfer_nvlink_one_sided, has_flashinfer_nvlink_one_sided,
@@ -81,6 +83,9 @@ PARALLEL_COMBOS = [
[1, 4, False], [1, 4, False],
[2, 1, True], [2, 1, True],
[4, 1, True], [4, 1, True],
# This combination indicates sequence parallel.
# See ParallelConfig.use_sequence_parallel.
[2, 2, True],
] ]
# TODO: should this even be set manually? let oracles handle this # TODO: should this even be set manually? let oracles handle this
@@ -112,24 +117,24 @@ QUANT_METHODS = [
# Which quantization methods each backend supports. # Which quantization methods each backend supports.
# fmt: off # fmt: off
BACKEND_SUPPORTED_QUANTS: dict[str, set[str | None]] = { BACKEND_SUPPORTED_QUANTS: dict[str, set[str | None]] = {
"allgather_reducescatter": {None, "fp8", "modelopt_fp8", "modelopt_fp4"}, "allgather_reducescatter": {None, "fp8", "modelopt_fp8", "modelopt_fp4"}, # noqa: E501
"mori": {None, "fp8", "modelopt_fp8"}, "mori": {None, "fp8", "modelopt_fp8"},
"flashinfer_nvlink_two_sided": {None, "modelopt_fp8", "modelopt_fp4"}, "flashinfer_nvlink_two_sided": {None, "fp8_blocked", "modelopt_fp4"}, # noqa: E501
"flashinfer_nvlink_one_sided": {None, "modelopt_fp8", "modelopt_fp4"}, "flashinfer_nvlink_one_sided": {None, "modelopt_fp4"}, # noqa: E501
"deepep_low_latency": {None, "fp8_blocked", "modelopt_fp4"}, "deepep_low_latency": {None, "fp8_blocked", "modelopt_fp4"}, # noqa: E501
"deepep_high_throughput": {None, "fp8_blocked", "modelopt_fp8", "modelopt_fp4"}, # noqa: E501 "deepep_high_throughput": {None, "fp8_blocked", "modelopt_fp8", "modelopt_fp4"}, # noqa: E501
"nixl_ep": {None, "fp8", "modelopt_fp8"}, "nixl_ep": {None, "fp8_blocked", "modelopt_fp8"},
} }
# Map from backend -> (DP/EP support, DP support, TP support) # Map from backend -> (DP/EP support, DP support, TP support, SP support)
BACKEND_EP_DP_TP_SUPPORT: dict[str, tuple[bool, bool, bool]] = { BACKEND_EP_DP_TP_SUPPORT: dict[str, tuple[bool, bool, bool, bool]] = {
"allgather_reducescatter": (True, True, True), "allgather_reducescatter": (True, True, True, True),
"mori": (True, False, False), "mori": (True, False, False, True),
"flashinfer_nvlink_two_sided": (False, True, False), "flashinfer_nvlink_two_sided": (False, True, False, False),
"flashinfer_nvlink_one_sided": (False, True, False), "flashinfer_nvlink_one_sided": (False, True, False, False),
"deepep_low_latency": (True, False, False), "deepep_low_latency": (True, False, False, True),
"deepep_high_throughput": (True, False, False), "deepep_high_throughput": (True, False, False, True),
"nixl_ep": (True, False, False), "nixl_ep": (True, False, False, True),
} }
# fmt: on # fmt: on
@@ -163,6 +168,45 @@ def override_normalize_e4m3fn_to_e4m3fnuz():
vllm.model_executor.layers.quantization.utils.w8a8_utils.normalize_e4m3fn_to_e4m3fnuz = mock_normalize_e4m3fn_to_e4m3fnuz # noqa: E501 vllm.model_executor.layers.quantization.utils.w8a8_utils.normalize_e4m3fn_to_e4m3fnuz = mock_normalize_e4m3fn_to_e4m3fnuz # noqa: E501
def sp_wrapper(
fn: Callable | FusedMoE, is_sequence_parallel: bool | None = None
) -> Callable:
"""Wrapper to handle sequence parallelism chunking and gathering.
For SP with EP:
- The TP group is created with the original tensor_parallel_size (e.g., 2)
- get_tp_group() has the correct world_size for SP operations
- sequence_parallel_chunk() uses get_tensor_model_parallel_world_size()
- tensor_model_parallel_all_gather() uses get_tp_group()
- Both should work correctly even when EP is enabled
"""
if isinstance(fn, FusedMoE):
assert is_sequence_parallel is None
is_sequence_parallel = fn.is_sequence_parallel
else:
assert is_sequence_parallel is not None
if is_sequence_parallel:
def wrapper(
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
) -> torch.Tensor:
# Split sequence across TP ranks
# Both hidden_states and router_logits have [num_tokens, ...] shape
hidden_states = sequence_parallel_chunk(hidden_states)
router_logits = sequence_parallel_chunk(router_logits)
# Run MoE on local chunk
result = fn(hidden_states, router_logits)
# Gather results from all TP ranks
result = tensor_model_parallel_all_gather(result, 0)
# Remove any padding added by SP.
return result[: hidden_states.shape[0]]
return wrapper
return fn
def maybe_roundup_layer_hidden_size( def maybe_roundup_layer_hidden_size(
hidden_size: int, hidden_size: int,
act_dtype: torch.dtype, act_dtype: torch.dtype,
@@ -272,6 +316,15 @@ class MoETestConfig:
dp_size: int = 1 dp_size: int = 1
tp_size: int = 1 tp_size: int = 1
@property
def is_sequence_parallel(self) -> bool:
# Sequence parallelism: EP enabled + TP dimension used for sequence splitting
# In test config: ep_size represents total expert parallel size
# tp_size represents the original TP dimension (becomes sp_size in FusedMoE)
# dp_size represents data parallel size
# For SP: we need EP enabled (ep_size > 1) and sequence splitting (tp_size > 1)
return self.ep_size > 1 and self.tp_size > 1
# TODO: add more error messages # TODO: add more error messages
def id(self) -> str: def id(self) -> str:
def proc(s: str) -> str: def proc(s: str) -> str:
@@ -404,11 +457,6 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
"leads to large differences.", "leads to large differences.",
) )
# gate requires shared_experts (use_overlapped mode)
# TODO: also not sure this is true
if config.use_gate and not config.use_shared_experts:
return False, "gate requires shared_experts (use_overlapped mode)"
# Skip modelopt_fp4 if not on B100+ (compute capability 10.0+) # Skip modelopt_fp4 if not on B100+ (compute capability 10.0+)
if ( if (
config.quantization == "modelopt_fp4" config.quantization == "modelopt_fp4"
@@ -445,7 +493,7 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
) )
if config.backend == "nixl_ep": if config.backend == "nixl_ep":
from vllm.model_executor.layers.fused_moe.nixl_ep_prepare_finalize import ( # noqa: E501 from vllm.model_executor.layers.fused_moe.prepare_finalize.nixl_ep import ( # noqa: E501
NixlEPPrepareAndFinalize, NixlEPPrepareAndFinalize,
) )
@@ -456,11 +504,11 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
) )
if config.backend is not None: if config.backend is not None:
supports_ep_dp, supports_dp, supports_tp = BACKEND_EP_DP_TP_SUPPORT[ supports_ep_dp, supports_dp, supports_tp, supports_sp = (
config.backend BACKEND_EP_DP_TP_SUPPORT[config.backend]
] )
if config.tp_size > 1 and not supports_tp: if config.tp_size > 1 and not supports_tp and not config.is_sequence_parallel:
return False, f"{config.backend} does not support TP." return False, f"{config.backend} does not support TP."
if config.dp_size > 1 and config.ep_size == 1 and not supports_dp: if config.dp_size > 1 and config.ep_size == 1 and not supports_dp:
@@ -468,10 +516,34 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
if config.dp_size > 1 and config.ep_size > 1 and not supports_ep_dp: if config.dp_size > 1 and config.ep_size > 1 and not supports_ep_dp:
return False, f"{config.backend} does not support EP/DP." return False, f"{config.backend} does not support EP/DP."
if config.is_sequence_parallel and not supports_sp:
return False, f"{config.backend} does not support SP."
else: else:
if config.tp_size > 1 or config.ep_size > 1 or config.dp_size > 1: if config.tp_size > 1 or config.ep_size > 1 or config.dp_size > 1:
return False, "An all2all backend is required for parallelism." return False, "An all2all backend is required for parallelism."
# Sequence parallelism specific validations
if config.is_sequence_parallel:
if config.ep_size == 1:
return False, "Sequence parallelism requires EP to be enabled (ep_size > 1)"
if config.tp_size == 1:
return (
False,
"Sequence parallelism requires tp_size > 1 for sequence splitting",
)
# SP is essentially EP + sequence splitting
# Verify the relationship: ep_size should equal dp_size * tp_size
# (when pcp_size=1).
expected_ep_size = config.dp_size * config.tp_size
if config.ep_size != expected_ep_size:
return False, (
f"For sequence parallelism: ep_size ({config.ep_size}) should equal "
f"dp_size * tp_size ({expected_ep_size})"
)
if config.enable_eplb: if config.enable_eplb:
if config.ep_size == 1: if config.ep_size == 1:
return False, "EPLB requires EP." return False, "EPLB requires EP."
@@ -485,14 +557,6 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
if config.num_experts % config.dp_size != 0: if config.num_experts % config.dp_size != 0:
return False, "EPLB requires num_experts divisible by ep_size" return False, "EPLB requires num_experts divisible by ep_size"
# Disable fp4 tests until flashinfer is updated or the Dockerfile is
# modified to install cublasLt.h. See #39525.
if (
config.quantization == "modelopt_fp4"
and current_platform.is_device_capability_family(100)
):
return False, "Temporarily skip until #39525 is resolved"
return True, None return True, None
@@ -751,7 +815,8 @@ def create_shared_experts_from_config(
in_dtype: torch.dtype, in_dtype: torch.dtype,
tp_size: int = 1, tp_size: int = 1,
tp_rank: int = 0, tp_rank: int = 0,
device: torch.device | str | None = None, is_sequence_parallel: bool = False,
device: torch.device | str | None = "cuda",
) -> TestMLP | None: ) -> TestMLP | None:
"""Create TestMLP for shared experts from config. """Create TestMLP for shared experts from config.
@@ -772,7 +837,7 @@ def create_shared_experts_from_config(
s_w2 = shared_experts_config.w2 s_w2 = shared_experts_config.w2
# Apply TP chunking if needed # Apply TP chunking if needed
if tp_size > 1: if tp_size > 1 and not is_sequence_parallel:
s_w1 = tp_chunk_gate_up(s_w1, tp_rank, tp_size, dim=1, device=device) s_w1 = tp_chunk_gate_up(s_w1, tp_rank, tp_size, dim=1, device=device)
s_w2 = chunk_by_rank(s_w2, tp_rank, tp_size, dim=0, device=device) s_w2 = chunk_by_rank(s_w2, tp_rank, tp_size, dim=0, device=device)
else: else:
@@ -920,6 +985,7 @@ def make_fused_moe_layer(
routed_input_transform: torch.nn.Module | None = None, routed_input_transform: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None, routed_output_transform: torch.nn.Module | None = None,
pcp_size: int | None = 1, pcp_size: int | None = 1,
is_sequence_parallel: bool = False,
) -> FusedMoE: ) -> FusedMoE:
quant_config, qw = make_quant_config(quantization, w1, w2, global_num_experts) quant_config, qw = make_quant_config(quantization, w1, w2, global_num_experts)
@@ -959,6 +1025,7 @@ def make_fused_moe_layer(
enable_eplb=enable_eplb, enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts, num_redundant_experts=num_redundant_experts,
has_bias=has_bias, has_bias=has_bias,
is_sequence_parallel=is_sequence_parallel,
**kwargs, **kwargs,
) )
@@ -1014,6 +1081,7 @@ def make_fake_moe_layer(
tp_size: int = 1, tp_size: int = 1,
dp_size: int = 1, dp_size: int = 1,
ep_size: int = 1, ep_size: int = 1,
is_sequence_parallel: bool = False,
) -> Callable: ) -> Callable:
quant_dtype = None quant_dtype = None
activation = MoEActivation.from_str(activation) activation = MoEActivation.from_str(activation)
@@ -1043,7 +1111,8 @@ def make_fake_moe_layer(
w2_s = None w2_s = None
shared_experts = create_shared_experts_from_config( shared_experts = create_shared_experts_from_config(
shared_experts_config, in_dtype, 1, 0, "cuda" shared_experts_config,
in_dtype,
) )
quant_config = FusedMoEQuantConfig.make( quant_config = FusedMoEQuantConfig.make(
@@ -1114,7 +1183,7 @@ def make_fake_moe_layer(
def _test_body_regular( def _test_body_regular(
moe_layer: Callable, moe_layer: FusedMoE,
hidden_states: torch.Tensor, hidden_states: torch.Tensor,
router_logits: torch.Tensor, router_logits: torch.Tensor,
vllm_config: VllmConfig, vllm_config: VllmConfig,
@@ -1131,7 +1200,7 @@ def _test_body_regular(
num_tokens=num_tokens, num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp, num_tokens_across_dp=num_tokens_across_dp,
): ):
output = moe_layer(hidden_states, router_logits) output = sp_wrapper(moe_layer)(hidden_states, router_logits)
return baseline_output, output return baseline_output, output
@@ -1164,6 +1233,8 @@ def _test_body_eplb(
) -> tuple[torch.Tensor, torch.Tensor]: ) -> tuple[torch.Tensor, torch.Tensor]:
device = torch.accelerator.current_accelerator() device = torch.accelerator.current_accelerator()
is_sequence_parallel = moe_layer.is_sequence_parallel
"""EPLB test body: compare output before and after expert weight rearrangement.""" """EPLB test body: compare output before and after expert weight rearrangement."""
# Get "before" output with original weight arrangement # Get "before" output with original weight arrangement
with set_forward_context( with set_forward_context(
@@ -1172,7 +1243,7 @@ def _test_body_eplb(
num_tokens=num_tokens, num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp, num_tokens_across_dp=num_tokens_across_dp,
): ):
output_before = moe_layer(hidden_states, router_logits) output_before = sp_wrapper(moe_layer)(hidden_states, router_logits)
# Create a fresh FusedMoE layer with enable_eplb=True # Create a fresh FusedMoE layer with enable_eplb=True
# Delete the original layer's registration so the constructor can # Delete the original layer's registration so the constructor can
@@ -1203,6 +1274,7 @@ def _test_body_eplb(
gate=gate, gate=gate,
routed_input_transform=routed_input_transform, routed_input_transform=routed_input_transform,
routed_output_transform=routed_output_transform, routed_output_transform=routed_output_transform,
is_sequence_parallel=is_sequence_parallel,
) )
if eplb_moe_layer._expert_map is not None: if eplb_moe_layer._expert_map is not None:
@@ -1264,7 +1336,7 @@ def _test_body_eplb(
num_tokens=num_tokens, num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp, num_tokens_across_dp=num_tokens_across_dp,
): ):
output_after = eplb_moe_layer(hidden_states, router_logits) output_after = sp_wrapper(eplb_moe_layer)(hidden_states, router_logits)
return output_before, output_after return output_before, output_after
@@ -1272,11 +1344,12 @@ def _test_body_eplb(
# TODO: make this take a MoETestConfig # TODO: make this take a MoETestConfig
def _run_one_config( def _run_one_config(
vllm_config: VllmConfig, vllm_config: VllmConfig,
ep_size: int, ep_size: int, # Expert parallel size (total across all ranks)
dp_size: int, dp_size: int, # Data parallel size (number of DP groups)
tp_size: int, tp_size: int, # Tensor parallel size OR sequence parallel size (when use_ep=True)
dp_rank: int, dp_rank: int, # Current rank in data parallel dimension
tp_rank: int, tp_rank: int, # Current rank in tensor/sequence parallel dimension
is_sequence_parallel: bool, # Whether to use sequence parallelism
m: int, m: int,
n: int, n: int,
k: int, k: int,
@@ -1290,15 +1363,22 @@ def _run_one_config(
use_routed_input_transform: bool, use_routed_input_transform: bool,
**kwargs, **kwargs,
) -> None: ) -> None:
set_random_seed(7)
"""Generic test loop that sets up environment and delegates to test_body_fn. """Generic test loop that sets up environment and delegates to test_body_fn.
This function is called directly by test_moe_layer and test_moe_layer_eplb Parameter Interpretation:
via parallel_launch_with_config, passing either _test_body_regular or - When is_sequence_parallel=False (standard TP or EP):
_test_body_eplb as the test_body_fn parameter. * ep_size: Number of expert parallel ranks (or 1 if no EP)
* tp_size: Number of tensor parallel ranks (or 1 if no TP)
* Weights are chunked by ep_size (experts) and tp_size (tensors)
- When is_sequence_parallel=True (EP + sequence splitting):
* ep_size: Number of expert parallel ranks (equals dp_size * tp_size)
* tp_size: Number of ranks to split sequence across (becomes sp_size in FusedMoE)
* Weights are chunked by ep_size (experts) but NOT by tp_size
* Input sequences are chunked by tp_size (via sp_wrapper)
""" """
world_size = tp_size * dp_size set_random_seed(7)
use_ep = ep_size > 1 use_ep = ep_size > 1
assert vllm_config.parallel_config.enable_expert_parallel == use_ep assert vllm_config.parallel_config.enable_expert_parallel == use_ep
@@ -1334,6 +1414,8 @@ def _run_one_config(
routed_output_transform = test_data.routed_output_transform routed_output_transform = test_data.routed_output_transform
activation = "silu" activation = "silu"
# Create baseline layer with FULL weights (no EP chunking)
# Baseline represents the expected output using full model
baseline_layer = make_fake_moe_layer( baseline_layer = make_fake_moe_layer(
w1=w1, w1=w1,
w2=w2, w2=w2,
@@ -1351,26 +1433,45 @@ def _run_one_config(
ep_size=ep_size, ep_size=ep_size,
dp_size=dp_size, dp_size=dp_size,
activation=activation, activation=activation,
is_sequence_parallel=is_sequence_parallel,
) )
baseline_output = baseline_layer(hidden_states, router_logits) with set_current_vllm_config(vllm_config):
# Compute baseline output with SP wrapper if needed
# sp_wrapper handles sequence chunking/gathering for SP
baseline_output = sp_wrapper(baseline_layer, is_sequence_parallel)(
hidden_states, router_logits
)
del baseline_layer del baseline_layer
torch.accelerator.empty_cache() torch.accelerator.empty_cache()
with set_current_vllm_config(vllm_config): with set_current_vllm_config(vllm_config):
# Chunk weights for EP/TP (after baseline is created) # Chunk weights for EP BEFORE creating FusedMoE
# FusedMoE uses EP-chunked weights and handles reductions internally
if ep_size > 1: if ep_size > 1:
w1 = chunk_by_rank(w1, dp_rank, dp_size, dim=0, device=device) # Split experts across ranks (dimension 0 is the expert dimension)
w2 = chunk_by_rank(w2, dp_rank, dp_size, dim=0, device=device) # When EP is enabled, use EP group rank and ep_size for chunking
ep_rank = get_ep_group().rank_in_group
w1 = chunk_by_rank(w1, ep_rank, ep_size, dim=0, device=device)
w2 = chunk_by_rank(w2, ep_rank, ep_size, dim=0, device=device)
if tp_size > 1: # Chunk weights for TP (only if NOT doing sequence parallelism)
# Sequence parallelism splits tokens/sequences, not weight tensors
if tp_size > 1 and not is_sequence_parallel:
w1 = tp_chunk_gate_up(w1, tp_rank, tp_size, dim=1, device=device) w1 = tp_chunk_gate_up(w1, tp_rank, tp_size, dim=1, device=device)
w2 = chunk_by_rank(w2, tp_rank, tp_size, dim=2, device=device) w2 = chunk_by_rank(w2, tp_rank, tp_size, dim=2, device=device)
# Setup shared experts if needed # Setup shared experts if needed
# In SP mode, shared experts should NOT be TP-chunked (same as routed experts)
# tp_size is used for sequence splitting, not weight splitting
shared_experts = create_shared_experts_from_config( shared_experts = create_shared_experts_from_config(
shared_experts_config, in_dtype, tp_size, tp_rank, device shared_experts_config,
in_dtype,
tp_size,
tp_rank,
is_sequence_parallel,
device,
) )
# Determine hidden size for MoE layer # Determine hidden size for MoE layer
@@ -1396,14 +1497,17 @@ def _run_one_config(
routed_input_transform=routed_input_transform, routed_input_transform=routed_input_transform,
routed_output_transform=routed_output_transform, routed_output_transform=routed_output_transform,
activation=activation, activation=activation,
is_sequence_parallel=is_sequence_parallel,
) )
if moe_layer._expert_map is not None: if moe_layer._expert_map is not None:
moe_layer._expert_map = moe_layer._expert_map.to(device) moe_layer._expert_map = moe_layer._expert_map.to(device)
num_tokens = m num_tokens = m
# num_tokens_across_dp should have one entry per DP group, not per total rank
# When EP is enabled, dp_size represents the number of DP groups
num_tokens_across_dp = torch.tensor( num_tokens_across_dp = torch.tensor(
[num_tokens] * world_size, [num_tokens] * dp_size,
device=device, device=device,
dtype=torch.int, dtype=torch.int,
) )
@@ -1445,7 +1549,7 @@ def _run_one_config(
else: else:
atol, rtol = 3.5e-2, 3.5e-2 atol, rtol = 3.5e-2, 3.5e-2
elif quantization in ("fp8", "fp8_blocked", "modelopt_fp8"): elif quantization in ("fp8", "fp8_blocked", "modelopt_fp8"):
atol, rtol = 6e-2, 6e-2 atol, rtol = 6.5e-2, 6.5e-2
elif quantization == "modelopt_fp4": elif quantization == "modelopt_fp4":
if k >= 2048: if k >= 2048:
atol = rtol = 1e-1 + (k * 1e-4) atol = rtol = 1e-1 + (k * 1e-4)
@@ -1529,6 +1633,7 @@ def test_moe_layer_no_parallel(
test_config.tp_size, test_config.tp_size,
0, 0,
0, 0,
False,
test_config.m, test_config.m,
test_config.n, test_config.n,
test_config.k, test_config.k,
@@ -1589,6 +1694,7 @@ def _parallel_worker(
test_config.tp_size, test_config.tp_size,
dp_rank, dp_rank,
tp_rank, tp_rank,
test_config.is_sequence_parallel,
test_config.m, test_config.m,
test_config.n, test_config.n,
test_config.k, test_config.k,
@@ -1673,7 +1779,12 @@ def test_moe_layer(
""" """
num_gpus = current_platform.device_count() num_gpus = current_platform.device_count()
world_size = tp_size * dp_size world_size = tp_size * dp_size
ep_size = 1 if not use_ep else world_size # or dp_size? # When use_ep=True: FusedMoEParallelConfig flattens tp_size across dp ranks
# Result: ep_size = dp_size * pcp_size * tp_size
# Since pcp_size=1 in these tests: ep_size = dp_size * tp_size = world_size
# When use_ep=False: no expert parallelism, ep_size = 1
ep_size = 1 if not use_ep else world_size
assert world_size > 1 assert world_size > 1
# Check if enough GPUs available # Check if enough GPUs available
@@ -335,6 +335,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
_lock = threading.Lock() _lock = threading.Lock()
def __init__(self, cpu_group, tcp_store_group=None): def __init__(self, cpu_group, tcp_store_group=None):
assert tcp_store_group is not None
super().__init__(cpu_group, tcp_store_group) super().__init__(cpu_group, tcp_store_group)
self.max_num_ep_ranks = envs.VLLM_NIXL_EP_MAX_NUM_RANKS self.max_num_ep_ranks = envs.VLLM_NIXL_EP_MAX_NUM_RANKS
@@ -270,7 +270,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
input_tensor = input_.movedim(0, dim).contiguous() input_tensor = input_.movedim(0, dim).contiguous()
if sizes is not None: if sizes is not None:
assert len(sizes) == world_size assert len(sizes) == world_size, f"{len(sizes)} == {world_size}"
assert input_tensor.shape[0] == sum(sizes) assert input_tensor.shape[0] == sum(sizes)
chunk_size = sizes[self.rank_in_group] chunk_size = sizes[self.rank_in_group]
else: else:
@@ -185,16 +185,21 @@ def flashinfer_alltoall_dispatch(
ep_size, ep_size,
) )
x_sf = MnnvlMoe.mnnvl_moe_alltoallv( if x_sf is not None:
x_sf, x_sf = MnnvlMoe.mnnvl_moe_alltoallv(
alltoall_info, x_sf,
all2all_manager.workspace_tensor, # type: ignore[attr-defined] alltoall_info,
ep_rank, all2all_manager.workspace_tensor, # type: ignore[attr-defined]
ep_size, ep_rank,
) ep_size,
)
# Swizzle after the A2A if MoE kernel expects swizzled scales. # Swizzle after the A2A if MoE kernel expects swizzled scales.
if quant_config.quant_dtype == "nvfp4" and quant_config.is_scale_swizzled: if (
x_sf is not None
and quant_config.quant_dtype == "nvfp4"
and quant_config.is_scale_swizzled
):
if x_sf.element_size() == 1: if x_sf.element_size() == 1:
x_sf = x_sf.view(torch.uint8) x_sf = x_sf.view(torch.uint8)
x_sf = nvfp4_block_scale_interleave(x_sf) x_sf = nvfp4_block_scale_interleave(x_sf)