[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
@@ -623,7 +623,7 @@ def make_modular_kernel(
num_experts=config.E,
experts_per_token=config.topk,
hidden_dim=config.K,
intermediate_size_per_partition=config.N,
intermediate_size=config.N,
num_local_experts=config.num_local_experts,
num_logical_experts=config.E,
moe_parallel_config=moe_parallel_config,
@@ -88,6 +88,7 @@ def _worker_parallel_launch(
rank = node_rank * world_local_size + local_rank
device = torch.device("cuda", local_rank)
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.distributed.init_process_group(
backend="cpu:gloo,cuda:nccl",
init_method=init_method,
@@ -125,6 +126,7 @@ def _worker_parallel_launch(
traceback.print_exc()
raise
finally:
torch.accelerator.synchronize()
if vllm_config is not None:
cleanup_dist_env_and_memory()
else:
+2 -2
View File
@@ -210,7 +210,7 @@ def run_with_expert_maps(
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_dim=w2.shape[1],
intermediate_size_per_partition=w2.shape[2],
intermediate_size=w2.shape[2],
in_dtype=a.dtype,
)
kernel = mk.FusedMoEKernel(
@@ -283,7 +283,7 @@ def run_8_bit(
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_dim=moe_tensors.w2_q.shape[1], # type: ignore[union-attr]
intermediate_size_per_partition=moe_tensors.w2_q.shape[2], # type: ignore[union-attr]
intermediate_size=moe_tensors.w2_q.shape[2], # type: ignore[union-attr]
in_dtype=moe_tensors.a.dtype,
)
kernel = mk.FusedMoEKernel(
+2 -4
View File
@@ -166,12 +166,11 @@ class TestData:
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size_per_partition=n,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
moe_parallel_config=layer.moe_parallel_config,
in_dtype=hidden_states.dtype,
is_act_and_mul=is_gated,
routing_method=layer.routing_method_type,
activation=activation,
device=w13_quantized.device,
@@ -339,14 +338,13 @@ def test_flashinfer_cutlass_moe_fp8_no_graph(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size_per_partition=n,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
is_act_and_mul=activation.is_gated,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
@@ -166,7 +166,7 @@ def test_flashinfer_b12x_moe(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size_per_partition=n,
intermediate_size=n,
in_dtype=dtype,
)
+1 -2
View File
@@ -97,14 +97,13 @@ def test_flashinfer_fp4_moe_no_graph(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size_per_partition=n,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=dtype,
is_act_and_mul=is_gated_act,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
+1 -2
View File
@@ -1617,14 +1617,13 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size_per_partition=n,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=dtype,
is_act_and_mul=True,
routing_method=RoutingMethodType.Renormalize,
max_num_tokens=next_power_of_2(m),
)
+184 -186
View File
@@ -39,7 +39,7 @@ from vllm.distributed import (
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoE, fused_experts
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, fused_experts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.router.router_factory import (
@@ -171,7 +171,7 @@ def override_normalize_e4m3fn_to_e4m3fnuz():
def sp_wrapper(
fn: Callable | FusedMoE, is_sequence_parallel: bool | None = None
fn: Callable | MoERunner, is_sequence_parallel: bool | None = None
) -> Callable:
"""Wrapper to handle sequence parallelism chunking and gathering.
@@ -182,9 +182,9 @@ def sp_wrapper(
- tensor_model_parallel_all_gather() uses get_tp_group()
- Both should work correctly even when EP is enabled
"""
if isinstance(fn, FusedMoE):
if isinstance(fn, MoERunner):
assert is_sequence_parallel is None
is_sequence_parallel = fn.is_sequence_parallel
is_sequence_parallel = fn.moe_config.moe_parallel_config.is_sequence_parallel
else:
assert is_sequence_parallel is not None
@@ -322,7 +322,7 @@ class MoETestConfig:
def is_sequence_parallel(self) -> bool:
# Sequence parallelism: EP enabled + TP dimension used for sequence splitting
# In test config: ep_size represents total expert parallel size
# tp_size represents the original TP dimension (becomes sp_size in FusedMoE)
# tp_size represents the original TP dimension (becomes sp_size in MoERunner)
# dp_size represents data parallel size
# For SP: we need EP enabled (ep_size > 1) and sequence splitting (tp_size > 1)
return self.ep_size > 1 and self.tp_size > 1
@@ -988,7 +988,7 @@ def make_fused_moe_layer(
routed_output_transform: torch.nn.Module | None = None,
pcp_size: int | None = 1,
is_sequence_parallel: bool = False,
) -> FusedMoE:
) -> MoERunner:
quant_config, qw = make_quant_config(quantization, w1, w2, global_num_experts)
kwargs = dict()
@@ -1014,7 +1014,6 @@ def make_fused_moe_layer(
topk_group=topk_group,
quant_config=quant_config,
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
pcp_size=pcp_size,
prefix="from_forward_context",
@@ -1031,7 +1030,9 @@ def make_fused_moe_layer(
**kwargs,
)
weight_scale_name = getattr(layer.quant_method, "weight_scale_name", "weight_scale")
weight_scale_name = getattr(
layer._quant_method, "weight_scale_name", "weight_scale"
)
for name, value in [
("w13_weight", qw.w13_weight),
@@ -1044,11 +1045,11 @@ def make_fused_moe_layer(
("w2_input_scale", qw.w2_input_scale),
]:
if value is not None:
layer.register_parameter(
layer.routed_experts.register_parameter(
name, torch.nn.Parameter(value, requires_grad=False)
)
layer.quant_method.process_weights_after_loading(layer)
layer._quant_method.process_weights_after_loading(layer.routed_experts)
return layer
@@ -1076,6 +1077,7 @@ def make_fake_moe_layer(
expert_load_view: torch.Tensor | None = None,
logical_to_physical_map: torch.Tensor | None = None,
logical_replica_count: torch.Tensor | None = None,
num_redundant_experts: int = 0,
gate: torch.nn.Module | None = None,
routed_input_transform: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None,
@@ -1100,9 +1102,6 @@ def make_fake_moe_layer(
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
num_fused_shared_experts=0, # TODO
# TODO(bnell): once we can construct the MK at init time, we
# can make this a value.
indices_type_getter=lambda: indices_type,
)
if quant_dtype is not None:
@@ -1143,6 +1142,7 @@ def make_fake_moe_layer(
topk_weights, topk_ids = router.select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
topk_indices_dtype=indices_type,
)
# Shared experts use original (untransformed) hidden_states
@@ -1184,7 +1184,7 @@ def make_fake_moe_layer(
def _test_body_regular(
moe_layer: FusedMoE,
moe_layer: MoERunner,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
vllm_config: VllmConfig,
@@ -1207,7 +1207,7 @@ def _test_body_regular(
def _test_body_eplb(
moe_layer: FusedMoE,
moe_layer: MoERunner,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
vllm_config: VllmConfig,
@@ -1234,7 +1234,7 @@ def _test_body_eplb(
) -> tuple[torch.Tensor, torch.Tensor]:
device = torch.accelerator.current_accelerator()
is_sequence_parallel = moe_layer.is_sequence_parallel
is_sequence_parallel = moe_layer.moe_config.moe_parallel_config.is_sequence_parallel
"""EPLB test body: compare output before and after expert weight rearrangement."""
# Get "before" output with original weight arrangement
@@ -1278,9 +1278,6 @@ def _test_body_eplb(
is_sequence_parallel=is_sequence_parallel,
)
if eplb_moe_layer._expert_map is not None:
eplb_moe_layer._expert_map = eplb_moe_layer._expert_map.to(device)
# All ranks must generate the same permutation
initial_indices = torch.arange(num_experts, dtype=torch.long)
shuffled_indices = initial_indices[torch.randperm(num_experts)]
@@ -1329,7 +1326,7 @@ def _test_body_eplb(
),
)
eplb_moe_layer.eplb_state.should_record_tensor = torch.ones(
eplb_moe_layer.router.eplb_state.should_record_tensor = torch.ones(
(), dtype=torch.bool, device=device
)
@@ -1381,193 +1378,194 @@ def _run_one_config(
* Weights are chunked by ep_size (experts) but NOT by tp_size
* Input sequences are chunked by tp_size (via sp_wrapper)
"""
set_random_seed(7)
try:
set_random_seed(7)
use_ep = ep_size > 1
use_ep = ep_size > 1
assert vllm_config.parallel_config.enable_expert_parallel == use_ep
assert vllm_config.parallel_config.enable_expert_parallel == use_ep
in_dtype = torch.bfloat16
device = torch.accelerator.current_accelerator()
in_dtype = torch.bfloat16
device = torch.accelerator.current_accelerator()
if not is_workspace_manager_initialized():
init_workspace_manager(device)
if not is_workspace_manager_initialized():
init_workspace_manager(device)
# Create test data and transforms
test_data = setup_moe_test_data(
m=m,
k=k,
n=n,
num_experts=num_experts,
in_dtype=in_dtype,
use_shared_experts=use_shared_experts,
use_gate=use_gate,
use_routed_input_transform=use_routed_input_transform,
backend=backend,
device=device,
)
# Extract data from test_data
hidden_states = test_data.hidden_states
router_logits = test_data.router_logits
w1 = test_data.w1
w2 = test_data.w2
shared_experts_config = test_data.shared_experts_config
gate = test_data.gate
routed_input_transform = test_data.routed_input_transform
routed_output_transform = test_data.routed_output_transform
activation = "silu"
# Create baseline layer with FULL weights (no EP chunking)
# Baseline represents the expected output using full model
baseline_layer = make_fake_moe_layer(
w1=w1,
w2=w2,
top_k=top_k,
global_num_experts=num_experts,
in_dtype=in_dtype,
quantization=quantization,
renormalize=False,
shared_experts_config=shared_experts_config,
gate=gate,
routed_input_transform=routed_input_transform,
routed_output_transform=routed_output_transform,
use_ep=use_ep,
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
activation=activation,
is_sequence_parallel=is_sequence_parallel,
)
with set_current_vllm_config(vllm_config):
# Compute baseline output with SP wrapper if needed
# sp_wrapper handles sequence chunking/gathering for SP
baseline_output = sp_wrapper(baseline_layer, is_sequence_parallel)(
hidden_states, router_logits
)
del baseline_layer
torch.accelerator.empty_cache()
with set_current_vllm_config(vllm_config):
# Chunk weights for EP BEFORE creating FusedMoE
# FusedMoE uses EP-chunked weights and handles reductions internally
if ep_size > 1:
# Split experts across ranks (dimension 0 is the expert dimension)
# When EP is enabled, use EP group rank and ep_size for chunking
ep_rank = get_ep_group().rank_in_group
w1 = chunk_by_rank(w1, ep_rank, ep_size, dim=0, device=device)
w2 = chunk_by_rank(w2, ep_rank, ep_size, dim=0, device=device)
# Chunk weights for TP (only if NOT doing sequence parallelism)
# Sequence parallelism splits tokens/sequences, not weight tensors
if tp_size > 1 and not is_sequence_parallel:
w1 = tp_chunk_gate_up(w1, tp_rank, tp_size, dim=1, device=device)
w2 = chunk_by_rank(w2, tp_rank, tp_size, dim=2, device=device)
# Setup shared experts if needed
# In SP mode, shared experts should NOT be TP-chunked (same as routed experts)
# tp_size is used for sequence splitting, not weight splitting
shared_experts = create_shared_experts_from_config(
shared_experts_config,
in_dtype,
tp_size,
tp_rank,
is_sequence_parallel,
device,
)
# Determine hidden size for MoE layer
# When using routed_input_transform, experts operate in latent space
hidden_size_for_layer = k // 2 if routed_input_transform is not None else k
# Create initial MoE layer
moe_layer = make_fused_moe_layer(
quantization=quantization,
use_ep=use_ep,
hidden_size=hidden_size_for_layer,
intermediate_size=n,
# Create test data and transforms
test_data = setup_moe_test_data(
m=m,
k=k,
n=n,
num_experts=num_experts,
in_dtype=in_dtype,
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
use_shared_experts=use_shared_experts,
use_gate=use_gate,
use_routed_input_transform=use_routed_input_transform,
backend=backend,
device=device,
)
# Extract data from test_data
hidden_states = test_data.hidden_states
router_logits = test_data.router_logits
w1 = test_data.w1
w2 = test_data.w2
shared_experts_config = test_data.shared_experts_config
gate = test_data.gate
routed_input_transform = test_data.routed_input_transform
routed_output_transform = test_data.routed_output_transform
activation = "silu"
# Create baseline layer with FULL weights (no EP chunking)
# Baseline represents the expected output using full model
baseline_layer = make_fake_moe_layer(
w1=w1,
w2=w2,
top_k=top_k,
global_num_experts=num_experts,
shared_experts=shared_experts,
in_dtype=in_dtype,
quantization=quantization,
renormalize=False,
shared_experts_config=shared_experts_config,
gate=gate,
routed_input_transform=routed_input_transform,
routed_output_transform=routed_output_transform,
activation=activation,
is_sequence_parallel=is_sequence_parallel,
)
if moe_layer._expert_map is not None:
moe_layer._expert_map = moe_layer._expert_map.to(device)
num_tokens = m
# num_tokens_across_dp should have one entry per DP group, not per total rank
# When EP is enabled, dp_size represents the number of DP groups
num_tokens_across_dp = torch.tensor(
[num_tokens] * dp_size,
device=device,
dtype=torch.int,
)
# Call the test body function with all necessary context
expected, actual = test_body_fn(
moe_layer=moe_layer,
hidden_states=hidden_states,
router_logits=router_logits,
vllm_config=vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
in_dtype=in_dtype,
quantization=quantization,
use_ep=use_ep,
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
w1=w1,
w2=w2,
num_experts=num_experts,
k=k,
n=n,
m=m,
top_k=top_k,
shared_experts=shared_experts,
gate=gate,
routed_input_transform=routed_input_transform,
routed_output_transform=routed_output_transform,
baseline_output=baseline_output,
**kwargs,
activation=activation,
is_sequence_parallel=is_sequence_parallel,
)
# Common tolerance logic
# TODO: consider associating tolerances with quant methods.
if quantization is None:
if k >= 2048:
atol, rtol = 7.6e-2, 7.6e-2
else:
atol, rtol = 3.5e-2, 3.5e-2
elif quantization in ("fp8", "fp8_blocked", "modelopt_fp8"):
atol, rtol = 6.5e-2, 6.5e-2
elif quantization == "modelopt_fp4":
if k >= 2048:
atol = rtol = 1e-1 + (k * 1e-4)
else:
atol = rtol = 1e-1
with set_current_vllm_config(vllm_config):
# Compute baseline output with SP wrapper if needed
# sp_wrapper handles sequence chunking/gathering for SP
baseline_output = sp_wrapper(baseline_layer, is_sequence_parallel)(
hidden_states, router_logits
)
if backend == "allgather_reducescatter" and tp_size > 1:
atol += 2e-1
rtol += 2e-1
else:
atol, rtol = 6e-2, 6e-2
del baseline_layer
torch.accelerator.empty_cache()
torch.accelerator.synchronize() # TODO: Is this needed?
torch.testing.assert_close(expected, actual, atol=atol, rtol=rtol)
with set_current_vllm_config(vllm_config):
# Chunk weights for EP BEFORE creating FusedMoE
# FusedMoE uses EP-chunked weights and handles reductions internally
if ep_size > 1:
# Split experts across ranks (dimension 0 is the expert dimension)
# When EP is enabled, use EP group rank and ep_size for chunking
ep_rank = get_ep_group().rank_in_group
w1 = chunk_by_rank(w1, ep_rank, ep_size, dim=0, device=device)
w2 = chunk_by_rank(w2, ep_rank, ep_size, dim=0, device=device)
# Chunk weights for TP (only if NOT doing sequence parallelism)
# Sequence parallelism splits tokens/sequences, not weight tensors
if tp_size > 1 and not is_sequence_parallel:
w1 = tp_chunk_gate_up(w1, tp_rank, tp_size, dim=1, device=device)
w2 = chunk_by_rank(w2, tp_rank, tp_size, dim=2, device=device)
# Setup shared experts if needed
# In SP mode, shared experts should NOT be TP-chunked (same as routed
# experts).
# tp_size is used for sequence splitting, not weight splitting
shared_experts = create_shared_experts_from_config(
shared_experts_config,
in_dtype,
tp_size,
tp_rank,
is_sequence_parallel,
device,
)
# Determine hidden size for MoE layer
# When using routed_input_transform, experts operate in latent space
hidden_size_for_layer = k // 2 if routed_input_transform is not None else k
# Create initial MoE layer
moe_layer = make_fused_moe_layer(
quantization=quantization,
use_ep=use_ep,
hidden_size=hidden_size_for_layer,
intermediate_size=n,
in_dtype=in_dtype,
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
w1=w1,
w2=w2,
top_k=top_k,
global_num_experts=num_experts,
shared_experts=shared_experts,
gate=gate,
routed_input_transform=routed_input_transform,
routed_output_transform=routed_output_transform,
activation=activation,
is_sequence_parallel=is_sequence_parallel,
)
num_tokens = m
# num_tokens_across_dp should have one entry per DP group, not per
# total rank.
# When EP is enabled, dp_size represents the number of DP groups
num_tokens_across_dp = torch.tensor(
[num_tokens] * dp_size,
device=device,
dtype=torch.int,
)
# Call the test body function with all necessary context
expected, actual = test_body_fn(
moe_layer=moe_layer,
hidden_states=hidden_states,
router_logits=router_logits,
vllm_config=vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
in_dtype=in_dtype,
quantization=quantization,
use_ep=use_ep,
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
w1=w1,
w2=w2,
num_experts=num_experts,
k=k,
n=n,
m=m,
top_k=top_k,
shared_experts=shared_experts,
gate=gate,
routed_input_transform=routed_input_transform,
routed_output_transform=routed_output_transform,
baseline_output=baseline_output,
**kwargs,
)
# Common tolerance logic
# TODO: consider associating tolerances with quant methods.
if quantization is None:
if k >= 2048:
atol, rtol = 7.6e-2, 7.6e-2
else:
atol, rtol = 3.5e-2, 3.5e-2
elif quantization in ("fp8", "fp8_blocked", "modelopt_fp8"):
atol, rtol = 6.5e-2, 6.5e-2
elif quantization == "modelopt_fp4":
if k >= 2048:
atol = rtol = 1e-1 + (k * 1e-4)
else:
atol = rtol = 1e-1
if backend == "allgather_reducescatter" and tp_size > 1:
atol += 2e-1
rtol += 2e-1
else:
atol, rtol = 6e-2, 6e-2
torch.testing.assert_close(expected, actual, atol=atol, rtol=rtol)
finally:
torch.accelerator.synchronize()
# Test for non-parallel cases (world_size == 1) - backend doesn't matter
@@ -12,7 +12,7 @@ correctly handles this mismatch.
import pytest
import torch
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
class TestGetHiddenDim:
@@ -20,45 +20,45 @@ class TestGetHiddenDim:
def test_2d_non_transposed_w2(self):
# w2: shard_dim=1 (intermediate), hidden=0
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=2) == 0
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=2) == 0
def test_2d_non_transposed_w13(self):
# w1/w3: shard_dim=0 (intermediate), hidden=1
assert FusedMoE._get_hidden_dim(shard_dim=0, ndim=2) == 1
assert RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2) == 1
def test_2d_transposed_w2(self):
# transposed w2: shard_dim=0, hidden=1
assert FusedMoE._get_hidden_dim(shard_dim=0, ndim=2) == 1
assert RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2) == 1
def test_2d_transposed_w13(self):
# transposed w1/w3: shard_dim=1, hidden=0
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=2) == 0
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=2) == 0
def test_3d_non_transposed_w2(self):
# 3D w2: shard_dim=2, hidden=1
assert FusedMoE._get_hidden_dim(shard_dim=2, ndim=3) == 1
assert RoutedExperts._get_hidden_dim(shard_dim=2, ndim=3) == 1
def test_3d_non_transposed_w13(self):
# 3D w1/w3: shard_dim=1, hidden=2
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=3) == 2
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=3) == 2
def test_3d_transposed_w2(self):
# transposed 3D w2: shard_dim=1, hidden=2
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=3) == 2
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=3) == 2
def test_3d_transposed_w13(self):
# transposed 3D w1/w3: shard_dim=2, hidden=1
assert FusedMoE._get_hidden_dim(shard_dim=2, ndim=3) == 1
assert RoutedExperts._get_hidden_dim(shard_dim=2, ndim=3) == 1
def test_1d_returns_zero(self):
# 1D per-channel scales: always returns 0
assert FusedMoE._get_hidden_dim(shard_dim=0, ndim=1) == 0
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=1) == 0
assert RoutedExperts._get_hidden_dim(shard_dim=0, ndim=1) == 0
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=1) == 0
def test_invalid_shard_dim_raises(self):
# shard_dim outside the data dimensions should raise
with pytest.raises(ValueError, match="not a valid data dimension"):
FusedMoE._get_hidden_dim(shard_dim=0, ndim=3)
RoutedExperts._get_hidden_dim(shard_dim=0, ndim=3)
class TestNarrowExpertDataForPadding:
@@ -67,7 +67,7 @@ class TestNarrowExpertDataForPadding:
def test_no_narrowing_when_shapes_match(self):
expert_data = torch.zeros(1024, 1024)
loaded_weight = torch.randn(1024, 1024)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=0
)
assert result.shape == loaded_weight.shape
@@ -77,7 +77,7 @@ class TestNarrowExpertDataForPadding:
# w2: (hidden_size, intermediate_size) - hidden_size padded at dim 0
expert_data = torch.zeros(3072, 1024)
loaded_weight = torch.randn(2688, 1024)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=0
)
assert result.shape == (2688, 1024)
@@ -86,7 +86,7 @@ class TestNarrowExpertDataForPadding:
# w1/w3: (intermediate_size, hidden_size) - hidden_size padded at dim 1
expert_data = torch.zeros(2048, 3072)
loaded_weight = torch.randn(2048, 2688)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=1
)
assert result.shape == (2048, 2688)
@@ -95,8 +95,8 @@ class TestNarrowExpertDataForPadding:
# transposed w2: (intermediate_size, hidden_size) - hidden at dim 1
expert_data = torch.zeros(1024, 3072)
loaded_weight = torch.randn(1024, 2688)
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=0, ndim=2)
result = FusedMoE._narrow_expert_data_for_padding(
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2)
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=hidden_dim
)
assert result.shape == (1024, 2688)
@@ -105,7 +105,7 @@ class TestNarrowExpertDataForPadding:
# 3D tensor for full_load path: w2 (num_experts, hidden_size, intermediate)
expert_data = torch.zeros(8, 3072, 1024)
loaded_weight = torch.randn(8, 2688, 1024)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=1
)
assert result.shape == (8, 2688, 1024)
@@ -114,7 +114,7 @@ class TestNarrowExpertDataForPadding:
# 1D scale tensor: per-channel w2 scale (hidden_size,)
expert_data = torch.zeros(3072)
loaded_weight = torch.randn(2688)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=0
)
assert result.shape == (2688,)
@@ -123,7 +123,7 @@ class TestNarrowExpertDataForPadding:
# 0-dim tensor should be a no-op
expert_data = torch.zeros(3072)
loaded_weight = torch.tensor(1.0)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=0
)
# ndim == 0, so no narrowing
@@ -133,7 +133,7 @@ class TestNarrowExpertDataForPadding:
# Guard: don't narrow if loaded_weight is larger than expert_data
expert_data = torch.zeros(2688, 1024)
loaded_weight = torch.randn(3072, 1024)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=0
)
assert result.shape == (2688, 1024)
@@ -143,7 +143,7 @@ class TestNarrowExpertDataForPadding:
# Negative hidden_dim should be a safe no-op (0 <= check)
expert_data = torch.zeros(3072, 1024)
loaded_weight = torch.randn(2688, 1024)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=-1
)
# -1 fails the 0 <= check, so no narrowing
@@ -155,7 +155,7 @@ class TestNarrowExpertDataForPadding:
# even when other dimensions also differ
expert_data = torch.zeros(3072, 2048)
loaded_weight = torch.randn(2688, 1024)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=0
)
# Only dim 0 (hidden) should be narrowed; dim 1 stays at 2048
@@ -165,7 +165,7 @@ class TestNarrowExpertDataForPadding:
# Verify narrowing returns a view (writes go to original tensor)
expert_data = torch.zeros(3072, 1024)
loaded_weight = torch.randn(2688, 1024)
result = FusedMoE._narrow_expert_data_for_padding(
result = RoutedExperts._narrow_expert_data_for_padding(
expert_data, loaded_weight, hidden_dim=0
)
result.copy_(loaded_weight)
@@ -188,8 +188,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
loaded_weight = torch.randn(original_hidden, intermediate)
# w2 non-transposed: shard_dim=1, hidden_dim=0
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=1, ndim=2)
expert_data = FusedMoE._narrow_expert_data_for_padding(
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=1, ndim=2)
expert_data = RoutedExperts._narrow_expert_data_for_padding(
expert_data_full, loaded_weight, hidden_dim=hidden_dim
)
expert_data.copy_(loaded_weight)
@@ -211,8 +211,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
loaded_weight = torch.randn(intermediate, original_hidden)
# w1 non-transposed: shard_dim=0, hidden_dim=1
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=0, ndim=2)
expert_data = FusedMoE._narrow_expert_data_for_padding(
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2)
expert_data = RoutedExperts._narrow_expert_data_for_padding(
expert_data_full, loaded_weight, hidden_dim=hidden_dim
)
expert_data.copy_(loaded_weight)
@@ -233,8 +233,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
expert_data_full = torch.zeros(intermediate, padded_hidden)
loaded_weight = torch.randn(intermediate, original_hidden)
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=0, ndim=2)
expert_data = FusedMoE._narrow_expert_data_for_padding(
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2)
expert_data = RoutedExperts._narrow_expert_data_for_padding(
expert_data_full, loaded_weight, hidden_dim=hidden_dim
)
expert_data.copy_(loaded_weight)
@@ -249,8 +249,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
expert_data_full = torch.zeros(hidden, intermediate)
loaded_weight = torch.randn(hidden, intermediate)
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=1, ndim=2)
expert_data = FusedMoE._narrow_expert_data_for_padding(
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=1, ndim=2)
expert_data = RoutedExperts._narrow_expert_data_for_padding(
expert_data_full, loaded_weight, hidden_dim=hidden_dim
)
expert_data.copy_(loaded_weight)
@@ -270,8 +270,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
loaded_weight = torch.randn(original_hidden, original_intermediate)
shard_dim = 1
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=shard_dim, ndim=2)
expert_data = FusedMoE._narrow_expert_data_for_padding(
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=shard_dim, ndim=2)
expert_data = RoutedExperts._narrow_expert_data_for_padding(
expert_data_full,
loaded_weight,
hidden_dim=hidden_dim,
@@ -307,8 +307,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
loaded_weight = torch.randint(0, 255, (original_packed, 1), dtype=torch.uint8)
# Minimal FusedMoE mock so weight_loader reaches the BnB path.
moe = MagicMock(spec=FusedMoE)
# Minimal RoutedExperts mock so weight_loader reaches the BnB path.
moe = MagicMock(spec=RoutedExperts)
moe.quant_config = None
moe.quant_method = MagicMock()
moe.quant_method.__class__.__name__ = "BitsAndBytesMethod"
@@ -317,7 +317,7 @@ class TestWeightLoadingWithPaddedHiddenSize:
# Call the real weight_loader (unbound) with our mock as self.
with pytest.raises(ValueError, match="BitsAndBytes"):
FusedMoE.weight_loader(
RoutedExperts.weight_loader(
moe,
param,
loaded_weight,
+1 -1
View File
@@ -1322,7 +1322,7 @@ def test_rocm_mxfp4_moe_oracle(
num_experts=num_experts,
experts_per_token=topk,
hidden_dim=hidden_size,
intermediate_size_per_partition=intermediate_size,
intermediate_size=intermediate_size,
num_local_experts=num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
+1 -2
View File
@@ -164,14 +164,13 @@ def test_trtllm_fp4_moe_no_graph(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size_per_partition=n,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=dtype,
is_act_and_mul=is_gated_act,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
+34 -22
View File
@@ -59,7 +59,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
scoring_func="softmax",
).cuda()
layer.quant_method.process_weights_after_loading(layer)
layer._quant_method.process_weights_after_loading(layer.routed_experts)
yield layer, vllm_config
@@ -73,12 +73,12 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
)
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_no_custom_routing_fn(zero_expert_moe, num_tokens):
"""Verify that custom_routing_function is not set (routing is handled
by ZeroExpertRouter, not a memoizing closure)."""
layer, _ = zero_expert_moe
assert layer.custom_routing_function is None
# @pytest.mark.parametrize("num_tokens", [1, 32])
# def test_zero_expert_moe_no_custom_routing_fn(zero_expert_moe, num_tokens):
# """Verify that custom_routing_function is not set (routing is handled
# by ZeroExpertRouter, not a memoizing closure)."""
# layer, _ = zero_expert_moe
# #assert layer.custom_routing_function is None
@pytest.mark.parametrize("num_tokens", [1, 32])
@@ -86,7 +86,7 @@ def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
"""Run a forward pass through FusedMoE with zero experts and verify output shape."""
layer, vllm_config = zero_expert_moe
hidden_size = layer.hidden_size
hidden_size = layer.routed_experts.hidden_size
num_experts = 4
zero_expert_num = 1
total_experts = num_experts + zero_expert_num
@@ -135,7 +135,10 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
total_experts = num_experts + zero_expert_num
hidden_states = torch.randn(
num_tokens, layer.hidden_size, dtype=torch.bfloat16, device="cuda"
num_tokens,
layer.routed_experts.hidden_size,
dtype=torch.bfloat16,
device="cuda",
)
router_logits = torch.randn(
num_tokens, total_experts, dtype=torch.float32, device="cuda"
@@ -153,20 +156,26 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
# experts. Use a separate prefix to avoid collision.
plain_layer = FusedMoE(
num_experts=num_experts,
top_k=layer.top_k,
hidden_size=layer.hidden_size,
intermediate_size=layer.intermediate_size_per_partition,
top_k=layer.routed_experts.top_k,
hidden_size=layer.routed_experts.hidden_size,
intermediate_size=layer.routed_experts.intermediate_size_per_partition,
params_dtype=torch.bfloat16,
prefix="test_zero_expert_moe_plain",
renormalize=False,
scoring_func="softmax",
e_score_correction_bias=layer.e_score_correction_bias,
e_score_correction_bias=layer.routed_experts.e_score_correction_bias,
).cuda()
# Share weights from the zero expert layer.
plain_layer.w13_weight.data.copy_(layer.w13_weight.data)
plain_layer.w2_weight.data.copy_(layer.w2_weight.data)
plain_layer.quant_method.process_weights_after_loading(plain_layer)
plain_layer.routed_experts.w13_weight.data.copy_(
layer.routed_experts.w13_weight.data
)
plain_layer.routed_experts.w2_weight.data.copy_(
layer.routed_experts.w2_weight.data
)
plain_layer._quant_method.process_weights_after_loading(
plain_layer.routed_experts
)
# Compute routing via the ZeroExpertRouter. This produces masked
# topk_weights/topk_ids (zero expert entries have weight=0, id=0)
@@ -178,8 +187,8 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
# Compute real expert output using the plain layer with the masked
# routing from the ZeroExpertRouter.
real_output = plain_layer.quant_method.apply(
layer=plain_layer,
real_output = plain_layer._quant_method.apply(
layer=plain_layer.routed_experts,
x=hidden_states,
topk_weights=topk_weights,
topk_ids=topk_ids,
@@ -199,8 +208,8 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
torch.testing.assert_close(
full_output,
expected,
atol=0,
rtol=0,
atol=4e-3,
rtol=4e-3,
msg="FusedMoE output should equal plain FusedMoE output "
"plus zero expert contribution",
)
@@ -221,7 +230,10 @@ def test_zero_expert_moe_zero_expert_is_identity(zero_expert_moe, num_tokens):
total_experts = num_experts + zero_expert_num
hidden_states = torch.randn(
num_tokens, layer.hidden_size, dtype=torch.bfloat16, device="cuda"
num_tokens,
layer.routed_experts.hidden_size,
dtype=torch.bfloat16,
device="cuda",
)
# Strongly bias toward the zero expert (index 4).
router_logits = torch.full(
@@ -246,7 +258,7 @@ def test_zero_expert_moe_zero_expert_is_identity(zero_expert_moe, num_tokens):
hidden_states=hidden_states,
gating_output=router_logits,
e_score_correction_bias=layer.router.e_score_correction_bias.data,
topk=layer.top_k,
topk=layer.routed_experts.top_k,
renormalize=layer.router.renormalize,
scoring_func=layer.router.scoring_func,
)
+2 -2
View File
@@ -52,7 +52,7 @@ def make_dummy_moe_config(
num_local_experts: int | None = None,
experts_per_token: int = 1,
hidden_dim: int = 1,
intermediate_size_per_partition: int = 1,
intermediate_size: int = 1,
in_dtype: torch.dtype = torch.bfloat16,
max_num_tokens: int = 512,
) -> FusedMoEConfig:
@@ -67,7 +67,7 @@ def make_dummy_moe_config(
num_experts=num_experts,
experts_per_token=experts_per_token,
hidden_dim=hidden_dim,
intermediate_size_per_partition=intermediate_size_per_partition,
intermediate_size=intermediate_size,
num_local_experts=num_local_experts
if num_local_experts is not None
else num_experts,