[MoE] Generalize masked activation for padded layouts (#51217)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Misha Goin
2026-09-01 11:34:13 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent a232e29e9d
commit 82b7d49a6e
15 changed files with 1217 additions and 196 deletions
+208
View File
@@ -22,7 +22,13 @@ from vllm.model_executor.layers.activation import (
SwigluStepAndMul,
swiglustep_and_mul_triton,
)
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -34,6 +40,100 @@ CUDA_DEVICES = [
]
def test_masked_moe_activation_rejects_unsupported_activation() -> None:
input = torch.empty(1, 1, 2)
output = torch.empty(1, 1, 1)
valid_token_counts = torch.ones(1, dtype=torch.int32)
with pytest.raises(NotImplementedError, match="relu2"):
apply_moe_activation(
MoEActivation.RELU2,
output,
input,
valid_token_counts=valid_token_counts,
)
def test_moe_silu_clamp_uses_native_xpu_fallback(
default_vllm_config, monkeypatch
) -> None:
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
clamp_limit = 3.0
input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]])
output = torch.empty(2, 2)
apply_moe_activation(
MoEActivation.SILU,
output,
input,
activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit),
)
expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native(
input
)
torch.testing.assert_close(output, expected)
def _assert_masked_moe_activation(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
*,
dtype: torch.dtype,
mask_layout: str,
d: int,
max_num_tokens: int,
) -> None:
device = CUDA_DEVICES[0]
num_experts = 4
input_dim = 2 * d if activation.is_gated else d
if mask_layout == "flat":
input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device)
valid_token_counts = torch.tensor(
[max_num_tokens // 2], dtype=torch.int32, device=device
)
output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device)
else:
input = torch.randn(
num_experts, max_num_tokens, input_dim, dtype=dtype, device=device
)
valid_token_counts = torch.tensor(
[0, 1, max_num_tokens // 2, max_num_tokens],
dtype=torch.int32,
device=device,
)
output = torch.full(
(num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device
)
apply_moe_activation(
activation,
output,
input,
activation_config=activation_config,
valid_token_counts=valid_token_counts,
)
batched_input = input.view(-1, max_num_tokens, input_dim)
batched_output = output.view(-1, max_num_tokens, d)
for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()):
if num_tokens:
expected = torch.empty((num_tokens, d), dtype=dtype, device=device)
apply_moe_activation(
activation,
expected,
batched_input[expert, :num_tokens].clone(),
activation_config=activation_config,
)
torch.testing.assert_close(
batched_output[expert, :num_tokens],
expected,
atol=get_default_atol(output),
rtol=get_default_rtol(output),
)
assert torch.all(batched_output[expert, num_tokens:] == 42.0)
@pytest.mark.parametrize(
"activation",
[
@@ -252,6 +352,114 @@ def test_masked_situ_and_mul(
)
@pytest.mark.parametrize(
("activation", "activation_config"),
[
(MoEActivation.SILU, ApplyMoEActivationConfig()),
(
MoEActivation.SILU,
ApplyMoEActivationConfig(clamp_limit=3.0),
),
(MoEActivation.GELU, ApplyMoEActivationConfig()),
(MoEActivation.GELU_TANH, ApplyMoEActivationConfig()),
(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
),
(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig()),
(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
),
(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig()),
(MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.GELU_TANH_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig()),
],
ids=[
"silu",
"silu_clamp",
"gelu",
"gelu_tanh",
"situ",
"swigluoai",
"swigluoai_uninterleave",
"swiglustep",
"silu_no_mul",
"gelu_no_mul",
"gelu_tanh_no_mul",
"relu2_no_mul",
],
)
@torch.inference_mode()
def test_masked_moe_activation_dispatch(
default_vllm_config,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
) -> None:
_assert_masked_moe_activation(
activation,
activation_config,
dtype=torch.bfloat16,
mask_layout="batched_experts",
d=513,
max_num_tokens=7,
)
@pytest.mark.parametrize(
("activation", "activation_config", "mask_layout"),
[
pytest.param(
MoEActivation.SILU,
ApplyMoEActivationConfig(),
"flat",
id="flat",
),
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
"batched_experts",
id="batched-experts",
),
],
)
@torch.inference_mode()
def test_masked_moe_activation_grid_stride(
default_vllm_config,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
mask_layout: str,
) -> None:
_assert_masked_moe_activation(
activation,
activation_config,
dtype=torch.half,
mask_layout=mask_layout,
d=513,
max_num_tokens=67,
)
@torch.inference_mode()
def test_masked_moe_activation_opcheck(default_vllm_config) -> None:
device = CUDA_DEVICES[0]
input = torch.randn(2, 3, 64, dtype=torch.half, device=device)
output = torch.empty(2, 3, 32, dtype=torch.half, device=device)
valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device)
opcheck(
torch.ops._C.masked_moe_activation,
(output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0),
)
@pytest.mark.parametrize(
"activation",
[