[MoE] Generalize masked activation for padded layouts (#51217)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a232e29e9d
commit
82b7d49a6e
@@ -22,7 +22,13 @@ from vllm.model_executor.layers.activation import (
|
||||
SwigluStepAndMul,
|
||||
swiglustep_and_mul_triton,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.half, torch.bfloat16, torch.float]
|
||||
@@ -34,6 +40,100 @@ CUDA_DEVICES = [
|
||||
]
|
||||
|
||||
|
||||
def test_masked_moe_activation_rejects_unsupported_activation() -> None:
|
||||
input = torch.empty(1, 1, 2)
|
||||
output = torch.empty(1, 1, 1)
|
||||
valid_token_counts = torch.ones(1, dtype=torch.int32)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="relu2"):
|
||||
apply_moe_activation(
|
||||
MoEActivation.RELU2,
|
||||
output,
|
||||
input,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
|
||||
def test_moe_silu_clamp_uses_native_xpu_fallback(
|
||||
default_vllm_config, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
clamp_limit = 3.0
|
||||
input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]])
|
||||
output = torch.empty(2, 2)
|
||||
|
||||
apply_moe_activation(
|
||||
MoEActivation.SILU,
|
||||
output,
|
||||
input,
|
||||
activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit),
|
||||
)
|
||||
|
||||
expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native(
|
||||
input
|
||||
)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _assert_masked_moe_activation(
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
mask_layout: str,
|
||||
d: int,
|
||||
max_num_tokens: int,
|
||||
) -> None:
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts = 4
|
||||
input_dim = 2 * d if activation.is_gated else d
|
||||
if mask_layout == "flat":
|
||||
input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device)
|
||||
valid_token_counts = torch.tensor(
|
||||
[max_num_tokens // 2], dtype=torch.int32, device=device
|
||||
)
|
||||
output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device)
|
||||
else:
|
||||
input = torch.randn(
|
||||
num_experts, max_num_tokens, input_dim, dtype=dtype, device=device
|
||||
)
|
||||
valid_token_counts = torch.tensor(
|
||||
[0, 1, max_num_tokens // 2, max_num_tokens],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
output = torch.full(
|
||||
(num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
activation_config=activation_config,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
batched_input = input.view(-1, max_num_tokens, input_dim)
|
||||
batched_output = output.view(-1, max_num_tokens, d)
|
||||
for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()):
|
||||
if num_tokens:
|
||||
expected = torch.empty((num_tokens, d), dtype=dtype, device=device)
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
expected,
|
||||
batched_input[expert, :num_tokens].clone(),
|
||||
activation_config=activation_config,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
batched_output[expert, :num_tokens],
|
||||
expected,
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.all(batched_output[expert, num_tokens:] == 42.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
@@ -252,6 +352,114 @@ def test_masked_situ_and_mul(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config"),
|
||||
[
|
||||
(MoEActivation.SILU, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SILU,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0),
|
||||
),
|
||||
(MoEActivation.GELU, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_TANH, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
),
|
||||
(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
|
||||
),
|
||||
(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_TANH_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig()),
|
||||
],
|
||||
ids=[
|
||||
"silu",
|
||||
"silu_clamp",
|
||||
"gelu",
|
||||
"gelu_tanh",
|
||||
"situ",
|
||||
"swigluoai",
|
||||
"swigluoai_uninterleave",
|
||||
"swiglustep",
|
||||
"silu_no_mul",
|
||||
"gelu_no_mul",
|
||||
"gelu_tanh_no_mul",
|
||||
"relu2_no_mul",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_dispatch(
|
||||
default_vllm_config,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
) -> None:
|
||||
_assert_masked_moe_activation(
|
||||
activation,
|
||||
activation_config,
|
||||
dtype=torch.bfloat16,
|
||||
mask_layout="batched_experts",
|
||||
d=513,
|
||||
max_num_tokens=7,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config", "mask_layout"),
|
||||
[
|
||||
pytest.param(
|
||||
MoEActivation.SILU,
|
||||
ApplyMoEActivationConfig(),
|
||||
"flat",
|
||||
id="flat",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
"batched_experts",
|
||||
id="batched-experts",
|
||||
),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_grid_stride(
|
||||
default_vllm_config,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
mask_layout: str,
|
||||
) -> None:
|
||||
_assert_masked_moe_activation(
|
||||
activation,
|
||||
activation_config,
|
||||
dtype=torch.half,
|
||||
mask_layout=mask_layout,
|
||||
d=513,
|
||||
max_num_tokens=67,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_opcheck(default_vllm_config) -> None:
|
||||
device = CUDA_DEVICES[0]
|
||||
input = torch.randn(2, 3, 64, dtype=torch.half, device=device)
|
||||
output = torch.empty(2, 3, 32, dtype=torch.half, device=device)
|
||||
valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device)
|
||||
opcheck(
|
||||
torch.ops._C.masked_moe_activation,
|
||||
(output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user