[MoE] Generalize masked activation for padded layouts (#51217)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Misha Goin
2026-09-01 11:34:13 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent a232e29e9d
commit 82b7d49a6e
15 changed files with 1217 additions and 196 deletions
@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -134,11 +136,31 @@ _APPLY_MOE_ACTIVATIONS = frozenset(
)
_MASKED_MOE_ACTIVATION_NAMES: dict[MoEActivation, str] = {
MoEActivation.SILU: "silu",
MoEActivation.GELU: "gelu",
MoEActivation.GELU_TANH: "gelu_tanh",
MoEActivation.SITU: "situ",
MoEActivation.SWIGLUOAI: "swigluoai",
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "swigluoai_uninterleave",
MoEActivation.SWIGLUSTEP: "swiglustep",
MoEActivation.SILU_NO_MUL: "silu_no_mul",
MoEActivation.GELU_NO_MUL: "gelu_no_mul",
MoEActivation.GELU_TANH_NO_MUL: "gelu_tanh_no_mul",
MoEActivation.RELU2_NO_MUL: "relu2_no_mul",
}
def apply_moe_activation_supported(activation: MoEActivation) -> bool:
"""Whether ``apply_moe_activation`` supports an activation."""
return activation in _APPLY_MOE_ACTIVATIONS
def apply_moe_activation_masked_supported(activation: MoEActivation) -> bool:
"""Whether the masked ``apply_moe_activation`` path supports an activation."""
return activation in _MASKED_MOE_ACTIVATION_NAMES
@dataclass(frozen=True)
class ApplyMoEActivationConfig:
"""Configuration forwarded to ``apply_moe_activation``."""
@@ -177,6 +199,96 @@ class ApplyMoEActivationConfig:
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG = ApplyMoEActivationConfig()
def _validate_moe_activation_shapes(
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
expected_dim: int,
) -> None:
assert input.dim() == expected_dim, f"Input must be {expected_dim}D"
assert output.dim() == expected_dim, f"Output must be {expected_dim}D"
assert input.shape[:-1] == output.shape[:-1], (
f"Input/output leading shapes must match: {input.shape} vs {output.shape}"
)
if activation.is_gated:
assert output.size(-1) * 2 == input.size(-1), (
f"{activation.value} expects 2x ratio: "
f"{output.size(-1) * 2} vs {input.size(-1)}"
)
else:
assert output.size(-1) == input.size(-1), (
f"{activation.value} expects equal sizes: "
f"{output.size(-1)} vs {input.size(-1)}"
)
def _apply_moe_activation_masked(
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
valid_token_counts: torch.Tensor,
config: ApplyMoEActivationConfig,
) -> torch.Tensor:
masked_activation = _MASKED_MOE_ACTIVATION_NAMES.get(activation)
if masked_activation is None:
raise NotImplementedError(
f"Masked MoE activation is not implemented for {activation.value}"
)
assert input.dim() in (2, 3), "Masked input must be 2D or 3D"
_validate_moe_activation_shapes(activation, output, input, expected_dim=input.dim())
assert input.dtype == output.dtype, "Input and output dtypes must match"
assert input.device == output.device, "Input and output devices must match"
assert input.is_contiguous(), "Input must be contiguous"
assert output.is_contiguous(), "Output must be contiguous"
assert valid_token_counts.dtype == torch.int32, (
"valid_token_counts must use torch.int32"
)
assert valid_token_counts.dim() == 1, "valid_token_counts must be 1D"
assert valid_token_counts.device == input.device, (
"valid_token_counts must be on the input device"
)
assert valid_token_counts.is_contiguous(), "valid_token_counts must be contiguous"
expected_counts = input.size(0) if input.dim() == 3 else 1
assert valid_token_counts.size(0) == expected_counts, (
f"valid_token_counts must have {expected_counts} element(s) for "
f"{input.dim()}D input"
)
masked_clamp_limit = 0.0 if config.clamp_limit is None else config.clamp_limit
masked_alpha = config.alpha
if activation == MoEActivation.SILU and config.clamp_limit is not None:
masked_activation = "silu_with_clamp"
elif activation == MoEActivation.SITU:
assert config.activation_situ_beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
elif activation == MoEActivation.SWIGLUOAI:
masked_clamp_limit = 7.0
masked_alpha = 1.702
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
assert config.clamp_limit is not None, (
"SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
)
elif activation == MoEActivation.SWIGLUSTEP:
masked_clamp_limit = 7.0
torch.ops._C.masked_moe_activation(
output,
input,
valid_token_counts,
masked_activation,
masked_clamp_limit,
masked_alpha,
config.beta,
1.0 if config.activation_situ_beta is None else config.activation_situ_beta,
-1.0
if config.activation_situ_linear_beta is None
else config.activation_situ_linear_beta,
)
return output
def silu_and_mul_with_clamp(
output: torch.Tensor,
input: torch.Tensor,
@@ -184,7 +296,12 @@ def silu_and_mul_with_clamp(
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
) -> None:
if topk_ids is not None and expert_map is not None:
if current_platform.is_xpu():
d = input.shape[-1] // 2
gate = torch.clamp(input[..., :d], max=clamp_limit)
up = torch.clamp(input[..., d:], min=-clamp_limit, max=clamp_limit)
output.copy_(gate * torch.sigmoid(gate) * up)
elif topk_ids is not None and expert_map is not None:
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
swiglu_limit_func(output, input, clamp_limit, topk_ids, expert_map)
@@ -201,12 +318,15 @@ def apply_moe_activation(
activation_config: ApplyMoEActivationConfig | None = None,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
valid_token_counts: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply MoE activation function.
The configuration drives specialized activation behavior. Routing tensors
remain per-call inputs because they depend on the current token assignment.
and valid token counts remain per-call inputs because they depend on the
current token assignment. A single token count masks a flat ``[T, D]``
buffer; one count per expert masks each prefix in a padded ``[E, T, D]``
buffer.
"""
config = (
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG
@@ -214,19 +334,17 @@ def apply_moe_activation(
else activation_config
)
assert input.dim() == 2, "Input must be 2D"
assert output.dim() == 2, "Output must be 2D"
if activation.is_gated:
assert output.size(-1) * 2 == input.size(-1), (
f"{activation.value} expects 2x ratio: "
f"{output.size(-1) * 2} vs {input.size(-1)}"
)
else:
assert output.size(-1) == input.size(-1), (
f"{activation.value} expects equal sizes: "
f"{output.size(-1)} vs {input.size(-1)}"
if valid_token_counts is not None:
return _apply_moe_activation_masked(
activation,
output,
input,
valid_token_counts,
config,
)
_validate_moe_activation_shapes(activation, output, input, expected_dim=2)
# Activations with gated multiplication (gate × activation(up))
if activation == MoEActivation.SILU:
if config.clamp_limit is not None:
@@ -257,7 +375,6 @@ def apply_moe_activation(
-1.0
if config.activation_situ_linear_beta is None
else config.activation_situ_linear_beta,
valid_rows,
)
elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input)
@@ -298,7 +415,7 @@ def situ_and_mul_quant(
beta: float,
linear_beta: float | None,
group_size: int = 0,
valid_rows: torch.Tensor | None = None,
num_valid_tokens: torch.Tensor | None = None,
topk: int = 1,
) -> None:
"""Fused Kimi SITU activation + dynamic FP8 quantization.
@@ -314,11 +431,11 @@ def situ_and_mul_quant(
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
``SituAndMul(linear_beta=None)``. ``num_valid_tokens`` (int32 scalar tensor)
is the DeepEP v2 contiguous-layout token count (e.g. ``psum[-1:]``); the
kernel multiplies it by ``topk`` to bound rows. Padding rows are skipped and
receive a benign scale of 1.0. Kept in this module so that every
``torch.ops._C.situ*`` call lives in one file.
"""
torch.ops._C.situ_and_mul_quant(
output,
@@ -327,6 +444,6 @@ def situ_and_mul_quant(
beta,
-1.0 if linear_beta is None else linear_beta,
group_size,
valid_rows,
num_valid_tokens,
topk,
)
@@ -5,7 +5,10 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
apply_moe_activation_masked_supported,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
@@ -19,7 +22,6 @@ from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
moe_kernel_quantize_input,
normalize_batched_scales_shape,
swiglu_limit_func,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -839,16 +841,18 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
if current_platform.is_xpu():
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
return apply_moe_activation_masked_supported(activation)
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
@@ -858,20 +862,6 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
# Let PrepareAndFinalize::finalize() decide the impl.
return TopKWeightAndReduceDelegate()
def activation(
self,
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
**kwargs,
) -> None:
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
return
super().activation(activation, output, input)
def workspace_shapes(
self,
M: int,
@@ -998,11 +988,19 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
intermediate_cache2.fill_(0)
# TODO (bnell): use triton utility from batched deep gemm.
self.activation(
activation,
intermediate_cache2.view(-1, activation_out_dim),
intermediate_cache1.view(-1, N),
)
if current_platform.is_xpu():
self.activation(
activation,
intermediate_cache2.view(-1, activation_out_dim),
intermediate_cache1.view(-1, N),
)
else:
self.activation(
activation,
intermediate_cache2,
intermediate_cache1,
valid_token_counts=expert_num_tokens,
)
qintermediate_cache2, a2q_scale = batched_moe_kernel_quantize_input(
intermediate_cache2,
@@ -35,10 +35,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceDelegate,
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
swiglu_limit_func,
)
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
from vllm.model_executor.layers.quantization.utils.quant_utils import (
INT4_DTYPE,
INT8_DTYPE,
@@ -103,17 +100,18 @@ def _is_supported_wna16_weight_key(weight_key: QuantKey | None) -> bool:
)
def get_humming_moe_gemm_type() -> str:
def get_humming_moe_gemm_type(moe_config: FusedMoEConfig | None = None) -> str:
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
gemm_type = "indexed"
if env_gemm_type is not None:
if env_gemm_type is not None and env_gemm_type.lower() != "auto":
env_gemm_type = env_gemm_type.lower()
if env_gemm_type == "indexed":
gemm_type = env_gemm_type
elif env_gemm_type in ["grouped_contiguous", "grouped"]:
if env_gemm_type == "grouped":
gemm_type = "grouped_contiguous"
else:
gemm_type = "indexed"
gemm_type = env_gemm_type
elif moe_config is not None and moe_config.moe_parallel_config.use_ep:
gemm_type = "grouped_contiguous"
else:
gemm_type = "indexed"
logger.info_once(f"Using {gemm_type} gemm for humming moe") # noqa
return gemm_type
@@ -178,7 +176,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
max_num_tokens=max_num_tokens,
num_dispatchers=num_dispatchers,
)
self._permute_scratch: MoEPermuteScratch | None = None
self._permute_scratch: dict[int, MoEPermuteScratch] = {}
def init_humming_moe(self):
from vllm.utils.humming import get_heuristics_config
@@ -257,26 +255,36 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
**kwargs,
)
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if self._permute_scratch is None and moe_permute_unpermute_supported():
self._permute_scratch = MoEPermuteScratch(
max_num_tokens=self.moe_config.max_num_tokens,
topk=self.moe_config.experts_per_token,
def _get_permute_scratch(self, topk: int) -> MoEPermuteScratch | None:
if not moe_permute_unpermute_supported():
return None
scratch = self._permute_scratch.get(topk)
if scratch is None:
max_expanded_rows = (
self.moe_config.max_num_tokens
* self.moe_config.dp_size
* self.moe_config.experts_per_token
)
scratch = MoEPermuteScratch(
max_num_tokens=math.ceil(max_expanded_rows / topk),
topk=topk,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
hidden_size=self.moe_config.hidden_dim,
hidden_dtype=self.moe_config.in_dtype,
)
return self._permute_scratch
self._permute_scratch[topk] = scratch
return scratch
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
num_tokens = topk_ids.size(0)
ctx = get_forward_context()
if ctx.dp_metadata is not None:
num_tokens = ctx.dp_metadata.num_tokens_across_dp_cpu.sum().item()
return num_tokens * self.moe_config.experts_per_token
return num_tokens * topk_ids.size(1)
return topk_ids.size(0) * topk_ids.size(1)
def estimate_local_valid_shape_m(self, topk_ids: torch.Tensor):
# estimate shape_m for kernel tuning
@@ -645,7 +653,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
if supported:
assert hasattr(cls, "humming_gemm_type")
gemm_type = cls.humming_gemm_type().value.lower()
preferred_gemm_type = get_humming_moe_gemm_type()
preferred_gemm_type = get_humming_moe_gemm_type(moe_config)
supported = preferred_gemm_type.lower() == gemm_type
if not supported:
reason = (
@@ -660,25 +668,16 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
valid_rows: torch.Tensor | None = None,
valid_token_counts: torch.Tensor | None = None,
) -> None:
activation_config = self.activation_config
if (
activation == MoEActivation.SILU
and activation_config.clamp_limit is not None
):
swiglu_limit_func(
output=output,
input=input,
swiglu_limit=activation_config.clamp_limit,
)
else:
self.activation(
activation=activation,
input=input,
output=output,
valid_rows=valid_rows,
)
activation_kwargs: dict[str, Any] = dict(
activation=activation,
input=input,
output=output,
)
if valid_token_counts is not None:
activation_kwargs["valid_token_counts"] = valid_token_counts
self.activation(**activation_kwargs)
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
"""Whether the SITU activation + w2 quant can be fused into one kernel.
@@ -718,7 +717,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
self,
gate_up_output: torch.Tensor,
quanted_down_input: torch.Tensor,
valid_rows: torch.Tensor | None,
num_valid_tokens: torch.Tensor | None,
topk: int = 1,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the fused SITU activation + FP8 quant for the w2 input.
@@ -750,7 +749,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
beta=cfg.activation_situ_beta,
linear_beta=cfg.activation_situ_linear_beta,
group_size=group_size,
valid_rows=valid_rows,
num_valid_tokens=num_valid_tokens,
topk=topk,
)
return quanted_down_input, input_scale
@@ -878,8 +877,8 @@ class HummingIndexedExperts(HummingExpertsBase):
)
# psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
# situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds
# on tokens, so both read this pointer directly -- no host cast/multiply.
# The fused path and mul_sum consume tokens directly. The unfused
# activation consumes the expanded row count (tokens * topk).
valid_tokens = None
topk = topk_ids.size(1)
if (
@@ -894,21 +893,18 @@ class HummingIndexedExperts(HummingExpertsBase):
inputs, input_scale = self.fused_situ_quant(
gate_up_output=buffers["gate_up_output"],
quanted_down_input=buffers["quanted_down_input"],
valid_rows=valid_tokens,
num_valid_tokens=valid_tokens,
topk=topk,
)
else:
# Fallback situ_and_mul takes int64 row counts (tokens*topk).
valid_rows = (
valid_tokens.to(torch.int64) * topk
if valid_tokens is not None
else None
valid_token_counts = (
valid_tokens * topk if valid_tokens is not None else None
)
self.apply_activation(
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
valid_rows=valid_rows,
valid_token_counts=valid_token_counts,
)
inputs, input_scale = self.quantize_input(
@@ -1002,7 +998,7 @@ class HummingGroupedExperts(HummingExpertsBase):
n_expert=global_num_experts,
n_local_expert=self.num_experts,
expert_map=expert_map,
scratch=self._get_permute_scratch(),
scratch=self._get_permute_scratch(topk_ids.size(1)),
)
inputs, input_scale = self.quantize_input(
@@ -1028,6 +1024,11 @@ class HummingGroupedExperts(HummingExpertsBase):
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
valid_token_counts=(
expert_first_token_offset[-1:].to(torch.int32)
if expert_tokens_meta is None
else None
),
)
inputs, input_scale = self.quantize_input(
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
apply_moe_activation_masked_supported,
apply_moe_activation_supported,
)
from vllm.model_executor.layers.fused_moe.config import (
@@ -955,6 +956,10 @@ class BatchedMarlinExperts(MarlinExpertsBase):
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.BatchedExperts
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return apply_moe_activation_masked_supported(activation)
def workspace_shapes(
self,
M: int,
@@ -1000,26 +1005,18 @@ class BatchedMarlinExperts(MarlinExpertsBase):
act: MoEActivation,
act_output: torch.Tensor,
act_input: torch.Tensor,
**kwargs,
*,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
) -> None:
if act != MoEActivation.SITU:
self.activation(
act,
act_output,
act_input,
**kwargs,
)
return
num_experts, max_num_tokens = hidden_states.shape[:2]
beta = self.activation_config.activation_situ_beta
linear_beta = self.activation_config.activation_situ_linear_beta
torch.ops._C.masked_situ_and_mul(
self.activation(
act,
act_output.view(num_experts, max_num_tokens, -1),
act_input.view(num_experts, max_num_tokens, -1),
expert_tokens_meta.expert_num_tokens,
1.0 if beta is None else beta,
-1.0 if linear_beta is None else linear_beta,
topk_ids=topk_ids,
expert_map=expert_map,
valid_token_counts=expert_tokens_meta.expert_num_tokens,
)
return batched_fused_marlin_moe(
@@ -902,7 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
*,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
valid_token_counts: torch.Tensor | None = None,
) -> None:
apply_moe_activation(
activation,
@@ -911,7 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
activation_config=self.activation_config,
topk_ids=topk_ids,
expert_map=expert_map,
valid_rows=valid_rows,
valid_token_counts=valid_token_counts,
)
@abstractmethod