Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+5 -8
View File
@@ -479,7 +479,6 @@ class SwigluOAIAndMul(CustomOp):
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
"""PyTorch-native implementation equivalent to forward()."""
gate, up = x[..., ::2], x[..., 1::2]
gate = gate.clamp(min=None, max=self.limit)
up = up.clamp(min=-self.limit, max=self.limit)
@@ -647,9 +646,7 @@ class QuickGELU(CustomOp):
# --8<-- [start:relu2]
@CustomOp.register("relu2")
class ReLUSquaredActivation(CustomOp):
"""
Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
"""
"""Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2."""
# --8<-- [end:relu2]
@@ -676,8 +673,7 @@ class ReLUSquaredActivation(CustomOp):
# --8<-- [start:xielu]
@CustomOp.register("xielu")
class XIELU(CustomOp):
"""
Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010
"""Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010
If the user has installed the nickjbrowning/XIELU, we import xIELU CUDA
Otherwise, we emit a single warning and use xIELU Python
"""
@@ -747,7 +743,7 @@ class XIELU(CustomOp):
)
def _xielu_cuda(self, x: torch.Tensor) -> torch.Tensor:
"""Firewall function to prevent torch.compile from seeing .item()"""
"""Firewall function to prevent torch.compile from seeing .item()."""
assert self._xielu_cuda_obj is not None, "XIELU CUDA object must not be None"
original_shape = x.shape
# CUDA kernel expects 3D tensors, reshape if needed
@@ -808,7 +804,8 @@ _ACTIVATION_REGISTRY = LazyDict(
def _get_gelu_pytorch_tanh() -> nn.Module:
"""Get PyTorch GELU with tanh approximation, with ROCm fallback
and fast GELU for ARM."""
and fast GELU for ARM.
"""
if current_platform.is_rocm():
# TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile
logger.warning_once(
@@ -166,8 +166,8 @@ def _init_kv_cache_quant(
layer: The attention layer instance to initialize.
quant_config: Optional quantization configuration.
prefix: Layer name prefix for quantization method lookup.
"""
"""
# Note [Register q/k/v/prob scales in state dict]
# When calling model.to(device), only parameters/buffers in state dict are
# moved. If not registering q/k/v/prob scales in state dict, there would
@@ -254,8 +254,7 @@ class Attention(nn.Module, AttentionLayerBase):
head_size_v: int | None = None,
**extra_impl_args,
) -> None:
"""
The KV cache is stored inside this class and is accessed via
"""The KV cache is stored inside this class and is accessed via
`self.kv_cache`.
"""
super().__init__()
@@ -493,8 +492,7 @@ class Attention(nn.Module, AttentionLayerBase):
output_shape: torch.Size | None = None,
output_dtype: torch.dtype | None = None,
) -> torch.Tensor:
"""
The KV cache is stored inside this class and is accessed via
"""The KV cache is stored inside this class and is accessed via
`self.kv_cache`.
Attention metadata (`attn_metadata`) is set using a context manager in
@@ -691,6 +689,7 @@ def get_attention_context(
Note: attn_metadata may be None, but attn_layer and kv_cache are always
extracted from the forward context.
"""
forward_context: ForwardContext = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
@@ -718,8 +717,7 @@ def unified_kv_cache_update(
value: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
"""
Returns a dummy that is passed to unified_attention to signal a side effect and
"""Returns a dummy that is passed to unified_attention to signal a side effect and
the data dependency between them to ensure torch.compile preserves ordering.
"""
layer_name = _resolve_layer_name(layer_name)
@@ -34,7 +34,6 @@ def _get_cross_slot_mapping(
device: torch.device,
) -> torch.Tensor:
"""Get cross-attention slot mappings."""
block_size = kv_cache_spec.block_size
slot_mappings = []
@@ -187,8 +186,7 @@ def create_cross_attention_backend(
class CrossAttention(Attention):
"""
Cross-attention for encoder-decoder models.
"""Cross-attention for encoder-decoder models.
Handles attention between decoder queries and encoder keys/values.
"""
@@ -49,9 +49,7 @@ def create_encoder_only_attention_backend(
class EncoderOnlyAttention(Attention):
"""
Encoder attention is a special case that doesn't need a KV Cache.
"""
"""Encoder attention is a special case that doesn't need a KV Cache."""
def __init__(
self,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
# MLA Common Components
"""# MLA Common Components
This file implements common components for MLA implementations.
@@ -1415,8 +1414,7 @@ def unified_mla_kv_cache_update(
kv_cache_dtype: str,
k_scale: torch.Tensor,
) -> torch.Tensor:
"""
Returns a dummy that is passed to unified_attention to signal a side effect and
"""Returns a dummy that is passed to unified_attention to signal a side effect and
the data dependency between them to ensure torch.compile preserves ordering.
"""
layer_name = _resolve_layer_name(layer_name)
@@ -1539,8 +1537,7 @@ def dynamic_per_batched_tensor_quant(
dynamic_arg_dims={"decode_ql_nope": 0, "decode_q_pe": 0},
)
class _DecodeConcatQuantFP8(QuantFP8):
"""
QuantFP8 variant that concatenates decode_ql_nope and decode_q_pe before
"""QuantFP8 variant that concatenates decode_ql_nope and decode_q_pe before
quantization. When disabled, forward_native is compiled via torch.compile,
fusing cat/reshape/quant/view together.
"""
@@ -1571,7 +1568,8 @@ class MLACommonBackend(AttentionBackend):
@classmethod
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""Per-token-head modes pack an inline fp32 scale pair after the
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
latent data (single-sided: ``head_size_v == 0`` for MLA).
"""
spec = super().customize_spec(spec)
mode = spec.kv_quant_mode
if spec.state_content_bytes is not None or not mode.is_per_token_head:
@@ -2030,6 +2028,7 @@ def build_mla_chunked_context_metadata(
Returns:
The chunked-context metadata, or None when no prefill has any context.
"""
# NOTE: it is recommended you read the `Chunked Prefill` section in the
# comment at the top of the file before trying to understand this code.
@@ -2230,8 +2229,7 @@ def build_mla_chunked_context_metadata(
class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
"""
NOTE: Please read the comment at the top of the file before trying to
"""NOTE: Please read the comment at the top of the file before trying to
understand this class
"""
@@ -2316,8 +2314,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
vllm_config: VllmConfig,
model_dtype: torch.dtype,
) -> torch.dtype:
"""
Determine the query data type for prefill queries.
"""Determine the query data type for prefill queries.
Return FP8 dtype if cache is FP8 and prefill query quantization
is enabled, else model dtype.
"""
@@ -2484,8 +2481,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
def build_for_cudagraph_capture(
self, common_attn_metadata: CommonAttentionMetadata
) -> M:
"""
This method builds the metadata for full cudagraph capture.
"""This method builds the metadata for full cudagraph capture.
Currently, only decode is supported for full cudagraphs with MLA.
"""
m = common_attn_metadata
@@ -2671,14 +2667,16 @@ def reorg_kvcache(
max_seq_len: int,
toks: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
reorg and unpad kvcache after cp local gather to tp layout for attn kernel.
"""Reorg and unpad kvcache after cp local gather to tp layout for attn kernel.
e.g.
allgatered_kv_c_normed = [T0_0, T0_1, T0_2, T0_3, T1_0, T1_1, ...,
T0_4, T0_5, pad, pad, T1_2, pad, ...]
-> reorganized_kv_c_normed = [T0_0, T0_1, T0_2, T0_3, T0_4, T0_5,
T1_0, T1_1, T1_2, ...]
Args:
allgatered_kv_c_normed: all-gathered, padded latent KV cache.
allgatered_k_pe: all-gathered, padded RoPE key cache.
padded_local_chunk_seq_lens_lst: local chunk context lengths
under current CP rank.
local_context_lens_allranks: local context lengths on each CP rank.
@@ -2687,6 +2685,7 @@ def reorg_kvcache(
sum_seq_len: the sum of cp_chunk_seq_lens_lst.
max_seq_len: the max value of cp_chunk_seq_lens_lst.
toks: the number of tokens for local gather cache.
"""
kv_c_segments = []
k_pe_segments = []
@@ -2789,10 +2788,16 @@ def accumulate_mla_context_chunk(
remaining token range is initialized.
Args:
chunk: The context chunk being folded in.
attn_output: The chunk's attention output.
attn_softmax_lse: The chunk's log-sum-exp values.
output: Running context partial, updated in place.
output_lse: Running log-sum-exp, updated in place.
output_written: The chunk's attention output already landed in
``output[chunk.token_slice]`` because the backend was handed it as
``out``, leaving only the lse to fold. Invalid for a continuation
chunk, whose leading tokens must be merged rather than overwritten.
"""
token_start = chunk.token_slice.start
token_end = chunk.token_slice.stop
@@ -2819,8 +2824,7 @@ def accumulate_mla_context_chunk(
class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
"""
Shared MLA base providing dense-MHA prefill (via the selected
"""Shared MLA base providing dense-MHA prefill (via the selected
MLAPrefillBackend) for both dense and sparse impls; subclasses add decode
(``forward_mqa``).
"""
@@ -2856,8 +2860,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
def _concat_k_nope_k_pe(
self, k_nope: torch.Tensor, k_pe: torch.Tensor
) -> torch.Tensor:
"""
Efficiently concatenate k_nope and k_pe tensors along the last dimension.
"""Efficiently concatenate k_nope and k_pe tensors along the last dimension.
This function avoids the performance penalty of torch.cat with expanded
non-contiguous tensors by pre-allocating the output and using direct copies.
@@ -2869,6 +2872,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
Returns:
Tensor of shape [..., nope_dim + pe_dim]
"""
if k_pe.shape[-1] == 0:
# NoPE MLA: nothing to append, so no copy either.
@@ -3224,8 +3228,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
"""
NOTE: Please read the comment at the top of the file before trying to
"""NOTE: Please read the comment at the top of the file before trying to
understand this class
"""
@@ -322,14 +322,14 @@ class MMEncoderAttention(CustomOp):
num_kv_heads: int | None = None,
prefix: str = "",
) -> None:
"""
Args:
num_heads: number of attention heads per partition.
head_size: hidden_size per attention head.
scale: scale factor.
num_kv_heads: number of kv heads.
prefix: This has no effect, it is only here to make it easier to
swap between Attention and MultiHeadAttention
"""Args:
num_heads: number of attention heads per partition.
head_size: hidden_size per attention head.
scale: scale factor.
num_kv_heads: number of kv heads.
prefix: This has no effect, it is only here to make it easier to
swap between Attention and MultiHeadAttention
"""
super().__init__()
@@ -517,8 +517,7 @@ class MMEncoderAttention(CustomOp):
q_len: int,
kv_len: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Reshape query, key, value to 4D tensors:
"""Reshape query, key, value to 4D tensors:
(batch_size, seq_len, num_heads, head_size)
"""
query = query.view(bsz, q_len, self.num_heads, self.head_size)
@@ -114,9 +114,7 @@ def create_static_sink_attention_backend(
@CustomOp.register("static_sink_attention")
class StaticSinkAttention(Attention, CustomOp):
"""
Attention with static sink tokens
"""
"""Attention with static sink tokens."""
def __init__(
self,
@@ -12,8 +12,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpec
class AttentionLayerBase(ABC):
"""
Base class for attention-like layers (Attention, Mamba, etc.)
"""Base class for attention-like layers (Attention, Mamba, etc.)
that support the v1 engine.
This provides a common interface for getting attention backends
@@ -38,8 +37,7 @@ class AttentionLayerBase(ABC):
@abstractmethod
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
"""
Get the KV cache spec for this layer.
"""Get the KV cache spec for this layer.
May be None if the layer does not need KV cache.
"""
pass
+2 -2
View File
@@ -143,7 +143,7 @@ class Conv2dLayer(ConvLayerBase):
return x
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
"""Expected input shape: (batch_size, in_channels, height, width)"""
"""Expected input shape: (batch_size, in_channels, height, width)."""
assert x.dim() == 4
if self.enable_linear:
return self._forward_mulmat(x)
@@ -193,7 +193,7 @@ class Conv3dLayer(ConvLayerBase):
return x
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
"""Expected input shape: (batch_size, in_channels, time, height, width)"""
"""Expected input shape: (batch_size, in_channels, time, height, width)."""
if self.enable_linear:
return self._forward_mulmat(x)
else:
@@ -55,7 +55,8 @@ _FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
"""Workspace token budget for flashinfer fused all-reduce, or None if the
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``.
"""
from vllm.config.compilation import PassConfig
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
@@ -170,7 +170,8 @@ def _fused_embed_eh_norm_kernel(
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
cat. Requires the full table on-rank (replicated embedding)."""
cat. Requires the full table on-rank (replicated embedding).
"""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
@@ -56,7 +56,8 @@ class MoEActivation(Enum):
@property
def custom_op_name(self) -> str:
"""Maps to the CustomOp name of activations
in vllm/model_executor/layers/activation.py."""
in vllm/model_executor/layers/activation.py.
"""
return _CUSTOM_OP_NAMES[self]
def without_mul(self) -> "MoEActivation":
@@ -115,6 +116,7 @@ def activation_without_mul(activation: str) -> str:
Returns:
The non-gated activation name (e.g., "silu_no_mul", "gelu_no_mul")
"""
return MoEActivation.from_str(activation).without_mul().value
@@ -122,8 +122,7 @@ def maybe_roundup_layer_hidden_size(
act_dtype: torch.dtype,
moe_parallel_config: FusedMoEParallelConfig,
) -> int:
"""
Given layer hidden size and MoE configurations, round up hidden_size
"""Given layer hidden size and MoE configurations, round up hidden_size
if necessary.
Args:
@@ -135,6 +134,7 @@ def maybe_roundup_layer_hidden_size(
Rounded up hidden_size if rounding up is required based on the configs
and all2all backend.
Original hidden size otherwise.
"""
if moe_parallel_config.use_deepep_ht_kernels:
hidden_size = DeepEPHTPrepareAndFinalize.maybe_roundup_layer_hidden_size(
+26 -62
View File
@@ -41,8 +41,7 @@ def _get_config_dtype_str(
use_int4_w4a16: bool = False,
ocp_mx_scheme: str | None = None,
) -> str | None:
"""
Return a string used to construct the filename that contains the
"""Return a string used to construct the filename that contains the
tuning info for a particular quantization scheme. See
try_get_optimal_moe_config in fused_moe.py.
"""
@@ -72,9 +71,7 @@ def _quant_flags_to_group_shape(
per_out_ch_quant: bool,
block_shape: list[int] | None,
) -> tuple[GroupShape | None, GroupShape | None]:
"""
Convert MoE quantization flags into more generic GroupShapes.
"""
"""Convert MoE quantization flags into more generic GroupShapes."""
a_shape: GroupShape | None
w_shape: GroupShape | None
if block_shape is not None:
@@ -173,8 +170,7 @@ def get_routing_method_type(
@dataclass
class FusedMoEQuantDesc:
"""
A quantization descriptor for fused MoE ops. This class can describe
"""A quantization descriptor for fused MoE ops. This class can describe
either activations or weights.
"""
@@ -212,8 +208,7 @@ class FusedMoEQuantDesc:
# e.g. for specific arguments bias, precision, etc.
@dataclass
class FusedMoEQuantConfig:
"""
The FusedMoEQuantConfig contains all the quantization parameters for
"""The FusedMoEQuantConfig contains all the quantization parameters for
a single FusedMoEMethodBase operation. It consists of four
FusedMoEQuantDescs, one for each activation and set of weights.
@@ -435,8 +430,7 @@ class FusedMoEQuantConfig:
return self._a1.dtype == "fp8" and self._w1.dtype == "mxfp4"
def config_name(self, dtype: torch.dtype) -> str | None:
"""
Return a string used to construct the filename that contains the
"""Return a string used to construct the filename that contains the
tuning info for a particular quantization scheme. See
try_get_optimal_moe_config in fused_moe.py.
"""
@@ -454,8 +448,7 @@ class FusedMoEQuantConfig:
max_tokens: int,
hidden_dim: int,
) -> tuple[int, int] | None:
"""
Construct the proper activation scale shape for this
"""Construct the proper activation scale shape for this
config.
"""
if self.is_quantized:
@@ -477,8 +470,7 @@ class FusedMoEQuantConfig:
max_tokens: int,
hidden_dim: int,
) -> tuple[int, int, int] | None:
"""
Construct the proper activation batched scale shape for this
"""Construct the proper activation batched scale shape for this
config, e.g. (num experts, *scale_shape).
"""
if self.is_quantized:
@@ -512,8 +504,7 @@ class FusedMoEQuantConfig:
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> "FusedMoEQuantConfig":
"""
General builder function for a FusedMoEQuantConfig.
"""General builder function for a FusedMoEQuantConfig.
- quant_dtype: Optional quantization type. None if activations are
unquantized or quantized prior to calling. Note: "nvfp4", "mxfp4",
"mxfp6_e3m2", "mxfp6_e2m3" are the only valid string values
@@ -607,9 +598,7 @@ def fp8_w8a8_moe_quant_config(
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for fp8 activations and fp8 weights.
"""
"""Construct a quant config for fp8 activations and fp8 weights."""
return FusedMoEQuantConfig.make(
current_platform.fp8_dtype(),
w1_scale=w1_scale,
@@ -640,9 +629,7 @@ def int8_w8a8_moe_quant_config(
w2_bias: torch.Tensor | None = None,
per_act_token_quant: bool = False,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for int8 activations and int8 weights.
"""
"""Construct a quant config for int8 activations and int8 weights."""
return FusedMoEQuantConfig.make(
torch.int8,
w1_scale=w1_scale,
@@ -667,9 +654,7 @@ def gptq_marlin_moe_quant_config(
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
):
"""
Construct a quant config for gptq marlin quantization.
"""
"""Construct a quant config for gptq marlin quantization."""
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
w_shape = None if group_size == -1 else GroupShape(row=1, col=group_size)
@@ -702,9 +687,7 @@ def mxfp4_w4a16_moe_quant_config(
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for unquantized activations and mxfp4 weights.
"""
"""Construct a quant config for unquantized activations and mxfp4 weights."""
return FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(),
_a2=FusedMoEQuantDesc(),
@@ -730,9 +713,7 @@ def mxfp4_mxfp8_moe_quant_config(
mx_alignment: int = 0,
is_scale_swizzled: bool = True,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for mxfp4 activations and mxfp4 weights.
"""
"""Construct a quant config for mxfp4 activations and mxfp4 weights."""
return FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc("mxfp8"),
_a2=FusedMoEQuantDesc("mxfp8"),
@@ -756,9 +737,7 @@ def mxfp4_w4a8_moe_quant_config(
block_shape: list[int] | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for fp8 activations and mxfp4 weights.
"""
"""Construct a quant config for fp8 activations and mxfp4 weights."""
return FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc("fp8", None, a1_scale, None, None, None),
_a2=FusedMoEQuantDesc("fp8", None, a2_scale, None, None, None),
@@ -782,9 +761,7 @@ def ocp_mx_moe_quant_config(
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for mxfp4 activations and mxfp4 weights.
"""
"""Construct a quant config for mxfp4 activations and mxfp4 weights."""
assert quant_dtype in OCP_MX_DTYPES
return FusedMoEQuantConfig.make(
quant_dtype=quant_dtype,
@@ -818,9 +795,7 @@ def nvfp4_moe_quant_config(
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for mxfp4 activations and nvp4 weights.
"""
"""Construct a quant config for mxfp4 activations and nvp4 weights."""
return FusedMoEQuantConfig.make(
"nvfp4",
w1_scale=w1_scale,
@@ -845,8 +820,7 @@ def mxfp4_moe_quant_config(
w1_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for MXFP4 x MXFP4 MoE.
"""Construct a quant config for MXFP4 x MXFP4 MoE.
MXFP4 uses block scaling only (E8M0 scales, 32-element groups), with no
separate alphas / global activation scales in this config.
"""
@@ -869,9 +843,7 @@ def nvfp4_w4a16_moe_quant_config(
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for 16-but activations and nvp4 weights.
"""
"""Construct a quant config for 16-but activations and nvp4 weights."""
return FusedMoEQuantConfig.make(
quant_dtype=None,
w1_scale=w1_scale,
@@ -899,9 +871,7 @@ def int4_w4a16_moe_quant_config(
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for 16-bit float activations and int4 weights.
"""
"""Construct a quant config for 16-bit float activations and int4 weights."""
group_shape = GroupShape(*block_shape) if block_shape is not None else None
return FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(shape=group_shape, alpha_or_gscale=a1_gscale),
@@ -924,9 +894,7 @@ def fp8_w8a16_moe_quant_config(
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for 16-bit float activations and fp8 weights.
"""
"""Construct a quant config for 16-bit float activations and fp8 weights."""
group_shape = GroupShape(*block_shape) if block_shape is not None else None
fp8_dtype = current_platform.fp8_dtype()
return FusedMoEQuantConfig(
@@ -968,9 +936,7 @@ def int8_w8a16_moe_quant_config(
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for 16-bit float activations and int8 weights.
"""
"""Construct a quant config for 16-bit float activations and int8 weights."""
group_shape = GroupShape(*block_shape) if block_shape is not None else None
return FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(shape=group_shape, alpha_or_gscale=a1_gscale),
@@ -992,9 +958,7 @@ def int4_w4afp8_moe_quant_config(
per_out_ch_quant: bool = False,
block_shape: list[int] | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for fp8 activations and int4 weights.
"""
"""Construct a quant config for fp8 activations and int4 weights."""
return FusedMoEQuantConfig.make(
torch.float8_e4m3fn, # quant dtype for activations
w1_scale=w1_scale,
@@ -1015,8 +979,7 @@ def biased_moe_quant_config(
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for unquantized activations with biases.
"""Construct a quant config for unquantized activations with biases.
gemm1_alpha/gemm1_beta/gemm1_clamp_limit carry the SwiGLU gate params
through to the fused activation kernel (e.g. swigluoai_uninterleave).
@@ -1136,8 +1099,7 @@ class FusedMoEParallelConfig:
sp_size_: int,
vllm_parallel_config: ParallelConfig,
) -> "FusedMoEParallelConfig":
"""
Determine MoE parallel configuration. Based on the input `tp_size_`,
"""Determine MoE parallel configuration. Based on the input `tp_size_`,
`dp_size_` and vllm's parallel config, determine what
level's of parallelism to use in the fused moe layer.
@@ -1145,6 +1107,7 @@ class FusedMoEParallelConfig:
tp_size_ (int): `tp_size` passed into the FusedMoEFactory constructor.
pcp_size_ (int): `pcp_size` passed into the FusedMoEFactory constructor.
dp_size_ (int): `dp_size` passed into the FusedMoEFactory constructor.
sp_size_ (int): `sp_size` passed into the FusedMoEFactory constructor.
vllm_parallel_config (ParallelConfig): vLLM's parallel config
object which contains the `enable_expert_parallel` flag.
@@ -1208,6 +1171,7 @@ class FusedMoEParallelConfig:
- device 3: TP = {1, 0} DP = {2, 1} EP = {4, 3}
- Comment: There are 2 engine instances and the experts are split
between the 4 devices.
"""
use_ep = (
dp_size_ * pcp_size_ * tp_size_ > 1
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Taken from https://github.com/ModelTC/LightLLM/blob/8ed97c74c18f11505b048b1ba00ba5c0cef8bff6/lightllm/common/fused_moe/deepep_scatter_gather.py
"""Taken from https://github.com/ModelTC/LightLLM/blob/8ed97c74c18f11505b048b1ba00ba5c0cef8bff6/lightllm/common/fused_moe/deepep_scatter_gather.py
and updated to fit vllm needs and terminology.
"""
@@ -27,8 +27,7 @@ def determine_expert_map(
num_fused_shared_experts: int = 0,
return_expert_mask: bool = False,
) -> tuple[int, torch.Tensor | None, torch.Tensor | None]:
"""
Calculates how many experts should be assigned to each rank for EP and
"""Calculates how many experts should be assigned to each rank for EP and
creates a mapping from global to local expert index. Experts are
distributed evenly across ranks. Any remaining are assigned to the
last rank.
@@ -56,6 +55,7 @@ def determine_expert_map(
and 0 for sentinel.
Returns None if ep_size is 1.
Used only when AITER MOE is enabled.
"""
from typing import get_args
@@ -150,8 +150,7 @@ def determine_expert_placement_strategy(
class ExpertMapManager:
"""
Manages expert ID mappings and placement for Expert Parallelism.
"""Manages expert ID mappings and placement for Expert Parallelism.
Responsibilities:
- Calculate local vs global expert counts
@@ -193,16 +192,22 @@ class ExpertMapManager:
num_fused_shared_experts: int = 0,
rocm_aiter_enabled: bool = False,
):
"""
Initialize expert map manager.
"""Initialize expert map manager.
Args:
max_num_batched_tokens: Maximum number of batched tokens per step
top_k: Number of experts each token is routed to
global_num_experts: Total number of experts across all ranks
num_redundant_experts: Number of redundant expert replicas (EPLB)
num_expert_group: Number of expert groups, when grouped routing is
used
moe_parallel_config: MoE parallel configuration (contains ep_size,
ep_rank, backend flags)
placement_strategy: Strategy for placing experts ('linear' or 'round_robin')
num_fused_shared_experts: Number of fused shared experts (for AITER)
rocm_aiter_enabled: Whether ROCm AITER fusion is enabled
enable_eplb: Whether expert-parallel load balancing is enabled
"""
self.global_num_experts = global_num_experts
self.moe_parallel_config = moe_parallel_config
@@ -295,8 +300,7 @@ class ExpertMapManager:
@property
def expert_map(self) -> torch.Tensor | None:
"""
Mapping from global expert ID to local expert ID.
"""Mapping from global expert ID to local expert ID.
Returns tensor of shape (global_num_experts,) where:
- expert_map[global_id] = local_id if expert is on this rank
@@ -308,8 +312,7 @@ class ExpertMapManager:
@property
def expert_mask(self) -> torch.Tensor | None:
"""
Expert mask for AITER fusion (ROCm-specific).
"""Expert mask for AITER fusion (ROCm-specific).
Returns tensor of shape (global_num_experts + num_fused_shared + 1,)
where 1 indicates expert is on this rank, 0 otherwise.
@@ -325,8 +328,7 @@ class ExpertMapManager:
def routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
"""
Routing tables for round-robin placement.
"""Routing tables for round-robin placement.
Returns (global_to_physical, physical_to_global, local_to_global)
or None if not using round-robin or tables not needed.
@@ -334,8 +336,7 @@ class ExpertMapManager:
return self._routing_tables
def map_global_to_local(self, global_id: int) -> int:
"""
Map global expert ID to local expert ID.
"""Map global expert ID to local expert ID.
Args:
global_id: Global expert ID (0 to global_num_experts - 1)
@@ -345,6 +346,7 @@ class ExpertMapManager:
Raises:
ValueError: If expert is not on this rank
"""
if self._expert_map is None:
return global_id
@@ -369,8 +371,7 @@ class ExpertMapManager:
moe_parallel_config: FusedMoEParallelConfig,
global_num_experts: int,
) -> None:
"""
Update expert mappings for new EP configuration.
"""Update expert mappings for new EP configuration.
Used during dynamic reconfiguration (e.g., elastic scaling).
@@ -378,6 +379,7 @@ class ExpertMapManager:
global_num_experts: New total number of experts across all ranks
moe_parallel_config: New MoE parallel configuration (contains ep_size,
ep_rank, backend flags)
"""
self.moe_parallel_config = moe_parallel_config
self.global_num_experts = global_num_experts
@@ -397,8 +399,7 @@ class ExpertMapManager:
self._init_aiter_shared_experts_topK_buffer()
def get_compressed_map_string(self) -> str:
"""
Get compressed string representation of expert map for logging.
"""Get compressed string representation of expert map for logging.
Returns string mapping local to global expert IDs.
"""
@@ -457,8 +458,7 @@ class ExpertMapManager:
def _init_routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
"""
Ensure routing tables are initialized if needed for round-robin.
"""Ensure routing tables are initialized if needed for round-robin.
This is a public method that can be called to explicitly initialize
routing tables. It's safe to call multiple times (idempotent).
@@ -56,9 +56,7 @@ def _aiter_w4a16_silu_via_a8w4(
unpadded_N_w2,
unpadded_K_w2,
) -> torch.Tensor:
"""
MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation.
"""
"""MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation."""
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
from aiter.ops.triton.quant import dynamic_mxfp8_quant
@@ -29,8 +29,7 @@ __all__ = [
# _get_padding_mask and patch_gating_output after
# https://github.com/ROCm/aiter/pull/4530 is released and consumed by vLLM.
def _get_padding_mask() -> torch.Tensor | None:
"""
Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
"""Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
`slot_mapping < 0` comes from:
@@ -247,8 +246,7 @@ def triton_kernel_fused_mxfp4_w4a8_experts(
class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
"""
Monolithic MXFP4 W4A8 expert using AITER triton kernels.
"""Monolithic MXFP4 W4A8 expert using AITER triton kernels.
This backend uses:
- aiter.ops.triton.moe_routing.routing for routing
@@ -36,7 +36,8 @@ def is_aiter_mxfp8_moe_available() -> bool:
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
closed (returns False -> triton dot_scaled), which is always safe."""
closed (returns False -> triton dot_scaled), which is always safe.
"""
if not (current_platform.is_rocm() and current_platform.supports_mx()):
return False
try:
@@ -307,8 +307,7 @@ class BatchedDeepGemmExperts(mk.FusedMoEExpertsModular):
max_num_tokens: int,
num_dispatchers: int,
):
"""
max_num_tokens: Maximum number of tokens from a DP Rank
"""max_num_tokens: Maximum number of tokens from a DP Rank
num_dispatchers: The number of DP dispatchers.
quant_config: Quantization configuration
"""
@@ -350,8 +349,7 @@ class BatchedDeepGemmExperts(mk.FusedMoEExpertsModular):
return True
def supports_packed_ue8m0_act_scales(self) -> bool:
"""
DeepGemm supports packed ue8m0 activation scales on Blackwell-family
"""DeepGemm supports packed ue8m0 activation scales on Blackwell-family
GPUs (SM100 datacenter and SM120 consumer).
"""
return is_deep_gemm_e8m0_used() and (
@@ -172,6 +172,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
0 = SwiGLU_Gu (SiLU(g)*u)
1 = SwiGLU_Ug (SiLU(u)*g)
2 = SiLU
"""
if activation == MoEActivation.SWIGLUSTEP:
return 0
@@ -217,6 +218,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
Returns:
Output tensor after MoE computation
"""
from vllm.model_executor.layers.fused_moe.router.cpu_router import (
select_experts,
@@ -192,7 +192,8 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsModular):
def _pad_moe_intermediate(self, layer: torch.nn.Module) -> None:
"""Zero-pad the per-partition MoE intermediate dim of both weights and
the expert bias, see `_padded_intermediate_size`."""
the expert bias, see `_padded_intermediate_size`.
"""
intermediate_size = self.moe_config.intermediate_size_per_partition
padded_size = self._padded_intermediate_size(self.moe_config)
if padded_size == intermediate_size:
@@ -745,6 +746,7 @@ def prepare_int4_moe_layer_for_cpu(
Returns:
(blocked_w13, blocked_w2, blocked_s13, blocked_s2, blocked_z13, blocked_z2)
"""
E = w13_packed.size(0)
@@ -951,7 +953,8 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular):
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
modular MoE experts."""
modular MoE experts.
"""
def __init__(
self,
@@ -1055,7 +1058,6 @@ class CPUExpertsInt8(mk.FusedMoEExpertsModular):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
w13 = torch.ops._C.convert_weight_packed(layer.w13_weight)
w2 = torch.ops._C.convert_weight_packed(layer.w2_weight)
replace_parameter(layer, "w13_weight", w13)
@@ -1285,7 +1287,8 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular):
class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
quantization, dispatched through zentorch."""
quantization, dispatched through zentorch.
"""
def __init__(
self,
@@ -518,8 +518,7 @@ def run_cutlass_moe_fp4(
*,
activation_config: ApplyMoEActivationConfig | None = None,
) -> None:
"""
MoE implementation for FP4 Inputs
"""MoE implementation for FP4 Inputs.
# Gemm 1
a: Input tensor: [m, k] (half/bfloat16)
@@ -58,8 +58,7 @@ def _valid_deep_gemm_shape(M: int, N: int, K: int) -> bool:
def _valid_deep_gemm(
hidden_states: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor
) -> bool:
"""
Check if the given problem size is supported by the DeepGemm grouped
"""Check if the given problem size is supported by the DeepGemm grouped
gemm kernel. All of M, N, K and the quantization block_shape must be
aligned by `dg.get_m_alignment_for_contiguous_layout()`.
"""
@@ -30,8 +30,7 @@ class FallbackExperts(mk.FusedMoEExpertsModular, ABC):
type[mk.FusedMoEExpertsModular],
type[mk.FusedMoEExpertsModular],
]:
"""
Get the cls for the experts and fallback experts.
"""Get the cls for the experts and fallback experts.
Subclasses should implement this method, so that
we have a consistent way to call the _supports_*
@@ -107,8 +107,7 @@ class FlashInferCuteDSLBatchedExperts(mk.FusedMoEExpertsModular):
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
"""
Compute the shapes for the temporary and final outputs of the two gemms
"""Compute the shapes for the temporary and final outputs of the two gemms
and activation in the fused expert function. Since the gemms are
independent, the workspace for the first gemm can be shared with the
workspace for the last gemm.
@@ -123,7 +122,6 @@ class FlashInferCuteDSLBatchedExperts(mk.FusedMoEExpertsModular):
- Note: in order for activation chunking to work, the first dimension
of each tuple must be the number of tokens.
"""
# We use global_num_experts due to how moe_align_block_size handles
# expert_maps.
K_dim = K * 2 if self.use_deep_ep_ll_nvfp4_dispatch else K
@@ -212,8 +210,7 @@ def flashinfer_cutedsl_moe_masked(
workspace: torch.Tensor,
out: torch.Tensor,
):
"""
Perform masked Mixture-of-Experts computation with FlashInfer's CuteDSL
"""Perform masked Mixture-of-Experts computation with FlashInfer's CuteDSL
kernels.
Args:
@@ -229,13 +226,14 @@ def flashinfer_cutedsl_moe_masked(
a2_global_scale (torch.Tensor): (l,)
w2_blockscale (torch.Tensor): blockscale factors, e4m3,
w2_alpha (torch.Tensor): (l,)
out (torch.Tensor): output tensor written in place, [l, m, k], bf16
masked_m (torch.Tensor): Masked dimension indices
workspace (torch.Tensor): For gateup_output
Notes:
- Assumes max(masked_m) <= m.
"""
"""
# === Assertions on dtypes ===
assert w1.dtype == torch.uint8, f"w1 must be uint8, got {w1.dtype}"
assert w1_blockscale.dtype == torch.float8_e4m3fn, (
@@ -29,8 +29,7 @@ from vllm.utils.flashinfer import (
class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
"""
CuteDSL NvFP4 MoE experts using the FlashInfer functional API.
"""CuteDSL NvFP4 MoE experts using the FlashInfer functional API.
Uses Standard activation format (non-batched). The kernel handles
routing, expert computation, and reduction internally.
@@ -38,8 +38,7 @@ logger = init_logger(__name__)
def is_valid_flashinfer_cutlass_fused_moe(
hidden_states: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor
) -> bool:
"""
Check if the given problem size is supported by the FlashInfer CUTLASS MoE
"""Check if the given problem size is supported by the FlashInfer CUTLASS MoE
kernel.
"""
if not has_flashinfer_cutlass_fused_moe():
@@ -219,8 +218,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# We use global_num_experts due to how moe_align_block_size handles
# expert_maps.
"""
Compute the shapes for the temporary and final outputs of the two gemms
"""Compute the shapes for the temporary and final outputs of the two gemms
and activation in the fused expert function. Since the gemms are
independent, the workspace for the first gemm can be shared with the
workspace for the last gemm.
@@ -551,8 +551,7 @@ def invoke_moe_batched_triton_kernel(
class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
"""
A reference MoE expert class that operates on expert batched format,
"""A reference MoE expert class that operates on expert batched format,
i.e. E x max_num_tokens x K. This is the format that the batched
dispatch/combine kernels use.
"""
@@ -775,8 +774,7 @@ def batched_moe_kernel_quantize_input(
class BatchedTritonExperts(mk.FusedMoEExpertsModular):
"""
A Triton based MoE expert class that operates on expert batched format,
"""A Triton based MoE expert class that operates on expert batched format,
i.e. E x max_num_tokens x K. This is the format that the batched
dispatch/combine kernels use.
"""
@@ -352,8 +352,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
)
def _prequantizes_dispatch_activation(self) -> bool:
"""
Whether the prepare/finalize step should quantize activations before
"""Whether the prepare/finalize step should quantize activations before
the (EP all-to-all) dispatch instead of leaving it to Humming.
This is enabled only for block-FP8 (group-128) activations: quantizing
@@ -373,8 +372,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
@property
def expects_unquantized_inputs(self) -> bool:
"""
Whether the prepare/finalize step should defer input quantization to
"""Whether the prepare/finalize step should defer input quantization to
the experts (by setting defer_input_quant=True and passing unquantized
inputs).
@@ -832,8 +830,7 @@ class HummingIndexedExperts(HummingExpertsBase):
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
"""
Standard apply implementation for Humming indexed experts.
"""Standard apply implementation for Humming indexed experts.
Note: Humming kernels handle weights internally through the layer
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
@@ -967,8 +964,7 @@ class HummingGroupedExperts(HummingExpertsBase):
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
"""
Standard apply implementation for Humming grouped experts.
"""Standard apply implementation for Humming grouped experts.
Note: Humming kernels handle weights internally through the layer
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
@@ -1090,8 +1086,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
"""
Standard apply implementation for Humming batched grouped experts.
"""Standard apply implementation for Humming batched grouped experts.
Note: Humming kernels handle weights internally through the layer
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
@@ -502,8 +502,7 @@ def pack_bitmatrix(
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
):
"""
Packs topk_ids into a bitmatrix.
"""Packs topk_ids into a bitmatrix.
code reference:
https://github.com/triton-lang/triton/blob/dd1bbc52b34d202dfe5ffea1e04fb16166c5c04e/python/triton_kernels/bench/distributed.py#L264
"""
@@ -949,8 +948,7 @@ class BaseOAITritonExperts(mk.FusedMoEExpertsModular):
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
"""
Extract the MoE problem size from the given tensor arguments:
"""Extract the MoE problem size from the given tensor arguments:
- a: The hidden states, input to the MoE layer.
- w1: The first set of expert weights.
- w2: The second set of expert weights.
@@ -1075,8 +1073,7 @@ class OAITritonExperts(BaseOAITritonExperts):
class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
"""
A Triton based MoE expert class that operates on expert standard
"""A Triton based MoE expert class that operates on expert standard
format and explicitly keeps the activation and reduction (moe_sum) steps
unfused from the matmul_ogs kernel. This exposes injection points
for activation and moe_sum.
@@ -10,8 +10,7 @@ from vllm.lora.punica_wrapper.punica_base import PunicaWrapperBase
@dataclass
class MoELoRAContext:
"""
Carries all LoRA state for one MoE forward pass.
"""Carries all LoRA state for one MoE forward pass.
Built by FusedMoEWithLoRA.forward() and propagated explicitly through the
modular kernel path (FusedMoEKernel -> FusedMoEExpertsModular.apply) so
@@ -7,8 +7,7 @@ from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRACon
class LoRAExpertsMixin:
"""
Mixin for FusedMoEExpertsModular subclasses that natively handle
"""Mixin for FusedMoEExpertsModular subclasses that natively handle
MoELoRAContext inside their apply() implementation.
Mixing this class in:
@@ -251,26 +251,56 @@ def fused_marlin_moe(
input_dtype: torch.dtype | None = None,
activation_config: ApplyMoEActivationConfig | None = None,
) -> torch.Tensor:
"""
This function computes a Mixture of Experts (MoE) layer using two sets of
"""This function computes a Mixture of Experts (MoE) layer using two sets of
weights, w1 and w2, and top-k gating mechanism.
Parameters:
- hidden_states (torch.Tensor): The input tensor to the MoE layer.
- w1 (torch.Tensor): The first set of expert weights.
- w2 (torch.Tensor): The second set of expert weights.
- w1_scale (torch.Tensor): Scale to be used for w1.
- w2_scale (torch.Tensor): Scale to be used for w2.
- topk_weights (torch.Tensor): Top-k weights.
- topk_ids (torch.Tensor): Indices of topk-k elements.
- w1_zeros (torch.Tensor|None): Optional zero points to be used for w1.
- w2_zeros (torch.Tensor|None): Optional zero points to be used for w2.
- num_bits (bool): The number of bits in expert weights quantization.
Args:
hidden_states (torch.Tensor): The input tensor to the MoE layer.
w1 (torch.Tensor): The first set of expert weights.
w2 (torch.Tensor): The second set of expert weights.
bias1 (torch.Tensor|None): Optional bias for the first GEMM.
bias2 (torch.Tensor|None): Optional bias for the second GEMM.
w1_scale (torch.Tensor): Scale to be used for w1.
w2_scale (torch.Tensor): Scale to be used for w2.
topk_weights (torch.Tensor): Top-k weights.
topk_ids (torch.Tensor): Indices of topk-k elements.
quant_type_id (int): Id of the `ScalarType` the expert weights are
quantized to. Determines the 4- or 8-bit Marlin kernel path.
apply_router_weight_on_input (bool): Apply the topk weights to the
input rather than the output. Only valid when topk is 1.
global_num_experts (int): Total number of experts across all expert
parallel shards. -1 means the local expert count.
activation (MoEActivation): Activation applied between the two GEMMs.
activation_func (Callable|None): Override for the activation
implementation.
moe_sum (Callable|None): Override for the final reduction over the
topk dimension. Defaults to `ops.moe_sum` / `torch.sum`.
expert_map (torch.Tensor|None): Maps global expert indices to the
local expert space of this expert parallel shard.
input_global_scale1 (torch.Tensor|None): NVFP4 global input scale for
the first GEMM.
input_global_scale2 (torch.Tensor|None): NVFP4 global input scale for
the second GEMM.
global_scale1 (torch.Tensor|None): NVFP4 global weight scale for w1.
global_scale2 (torch.Tensor|None): NVFP4 global weight scale for w2.
w1_zeros (torch.Tensor|None): Optional zero points to be used for w1.
w2_zeros (torch.Tensor|None): Optional zero points to be used for w2.
workspace (torch.Tensor|None): Scratch buffer for the Marlin kernels.
intermediate_cache13 (torch.Tensor|None): Scratch buffer for the
first and third intermediate activations.
intermediate_cache2 (torch.Tensor|None): Scratch buffer for the
second intermediate activation.
output (torch.Tensor|None): Tensor to write the result into. A new
tensor is allocated when omitted.
input_dtype (torch.dtype|None): dtype the activations are quantized
to. A 1-byte dtype raises the M block size to at least 16.
activation_config (ApplyMoEActivationConfig|None): Extra
configuration forwarded to the activation.
Returns:
- torch.Tensor: The output tensor after applying the MoE layer.
"""
torch.Tensor: The output tensor after applying the MoE layer.
"""
quant_type = ScalarType.from_id(quant_type_id)
assert quant_type in [
scalar_types.uint4,
@@ -399,8 +429,7 @@ def batched_fused_marlin_moe(
activation_func: Callable[..., None] | None = None,
activation_config: ApplyMoEActivationConfig | None = None,
) -> torch.Tensor:
"""
This function massages the inputs so the batched hidden_states can be
"""This function massages the inputs so the batched hidden_states can be
presented as a 2D contiguous tensor that could be used with
_fused_marlin_moe.
@@ -423,7 +452,6 @@ def batched_fused_marlin_moe(
expert_ids tensors, so only relevant/valid rows of A (hidden_states)
are accessed and are processed with the correct expert_ids.
"""
assert hidden_states.ndim == 3, (
f"hidden states must be batched. e.g. [B, MAX_TOKENS, K]."
f"But got {hidden_states.size()}"
@@ -34,7 +34,8 @@ from vllm.triton_utils import tl, triton
def _select_cfg(M, N, K, block_m):
"""Pick the launch config from host constants only (M=num_valid_tokens, N, K,
block_m) — graph-capture safe (no GPU-scalar branch)."""
block_m) — graph-capture safe (no GPU-scalar branch).
"""
# Per-regime winners (measured, isolated cuda-event A/B on gfx950, GPU 3):
# BLOCK_K=256 (fewer K-iters + bigger MX scale-load coalesced with the dot),
# num_stages=2 (software-pipeline overlaps the E8M0 scale-load with the scaled
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
NVFP4 quantization emulation for MoE.
"""NVFP4 quantization emulation for MoE.
This file implements NVFP4 emulation for NVFP4 MOE in case the hardware used does not
natively support NVFP4 MOE.
@@ -86,8 +85,7 @@ def fused_moe_nvfp4_emulation_kernel(
compute_type: tl.constexpr,
group_size: tl.constexpr,
):
"""
Fused MoE kernel for emulated NVFP4 weight-only dequantization + GEMM.
"""Fused MoE kernel for emulated NVFP4 weight-only dequantization + GEMM.
Activations A are BF16 (already QDQ'd externally).
Weights B are packed uint8 NVFP4 [E, N, K//2] — two FP4 values per byte
@@ -334,8 +332,7 @@ def invoke_fused_moe_nvfp4_emulation_kernel(
class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
"""
Extension of TritonExperts to support emulated NVFP4 MoE experts.
"""Extension of TritonExperts to support emulated NVFP4 MoE experts.
It may be used for NVFP4 models when the device does not have
native support for this dtype.
@@ -427,8 +424,7 @@ class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
"""
Apply emulated quantized MoE computation.
"""Apply emulated quantized MoE computation.
This dequantizes the weights on the fly and calls fused_experts_impl
with activation quantization support.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
OCP MX quantization emulation for MoE.
"""OCP MX quantization emulation for MoE.
This file implements OCP MX (MXFP4/MXFP6) emulation for MoE in case the
hardware used does not natively support OCP MX MoE.
@@ -50,6 +49,7 @@ def activation_quant_dtype(
Raises:
NotImplementedError: If the scheme has no emulated activation dtype.
"""
if ocp_mx_scheme in {
OCP_MX_Scheme.w_mxfp4,
@@ -82,8 +82,7 @@ def activation_quant_dtype(
class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
"""
Extension of TritonExperts to support emulated OCP MX MoE experts.
"""Extension of TritonExperts to support emulated OCP MX MoE experts.
It may be used for OCP MX (MXFP4/MXFP6) models when the device does not
have native support for these dtypes.
@@ -190,8 +189,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
"""
Apply emulated quantized MoE computation.
"""Apply emulated quantized MoE computation.
This dequantizes the weights on the fly and calls TritonExperts.apply
with activation quantization support.
@@ -32,8 +32,7 @@ from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
class TrtLlmBf16ExpertsBase:
"""
BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and
"""BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and
monolithic interfaces.
"""
@@ -124,9 +123,7 @@ class TrtLlmBf16ExpertsBase:
class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular):
"""
BF16 unquantized TRTLLM-Gen MoE kernels. Supports modular interface.
"""
"""BF16 unquantized TRTLLM-Gen MoE kernels. Supports modular interface."""
@staticmethod
def _supports_parallel_config(
@@ -224,9 +221,7 @@ class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular)
class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonolithic):
"""
BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface.
"""
"""BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface."""
@staticmethod
def _supports_parallel_config(
@@ -58,8 +58,7 @@ def prepare_deepseek_fp8_x_sf(x: torch.Tensor, x_sf: torch.Tensor) -> torch.Tens
class TrtLlmFp8ExpertsBase:
"""
Fp8 TRTLLM-Gen MoE kernels. Shared base for modular and monolithic
"""Fp8 TRTLLM-Gen MoE kernels. Shared base for modular and monolithic
interfaces.
"""
@@ -194,9 +193,7 @@ class TrtLlmFp8ExpertsBase:
class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
"""
Fp8 TRTLLM-Gen MoE kernels. Supports modular interface.
"""
"""Fp8 TRTLLM-Gen MoE kernels. Supports modular interface."""
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
@@ -330,9 +327,7 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolithic):
"""
Fp8 TRTLLM-Gen MoE kernels. Supports monolithic interface.
"""
"""Fp8 TRTLLM-Gen MoE kernels. Supports monolithic interface."""
def supports_routing_replay_capture(self) -> bool:
return True
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
LoRA-aware FlashInfer TRT-LLM MoE experts (BF16).
"""LoRA-aware FlashInfer TRT-LLM MoE experts (BF16).
Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153:
@@ -109,7 +108,7 @@ def _finalize_lora_kernel(
class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
"""LoRA-aware trtllm MoE experts"""
"""LoRA-aware trtllm MoE experts."""
def __init__(
self,
@@ -35,9 +35,7 @@ from vllm.utils.flashinfer import has_flashinfer
class TrtLlmMxfp4ExpertsBase:
"""
MXFP4 TRTLLM-Gen MoE kernels. Shared base for modular and monolithic.
"""
"""MXFP4 TRTLLM-Gen MoE kernels. Shared base for modular and monolithic."""
def __init__(
self,
@@ -164,8 +162,7 @@ class TrtLlmMxfp4ExpertsBase:
class TrtLlmMxfp4ExpertsMonolithic(
TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsMonolithic
):
"""
Monolithic version of the MXFP4 TRTLLM kernel (router + experts).
"""Monolithic version of the MXFP4 TRTLLM kernel (router + experts).
Wraps flashinfer.trtllm_fp4_block_scale_moe().
"""
@@ -287,8 +284,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModular):
"""
Modular version of the MXFP4 TRTLLM kernel (just the experts).
"""Modular version of the MXFP4 TRTLLM kernel (just the experts).
Wraps flashinfer.trtllm_fp4_block_scale_routed_moe().
Moved from trtllm_moe.py.
"""
@@ -19,8 +19,7 @@ from vllm.platforms import current_platform
class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
"""
FlashInfer TRT-LLM MxInt4 MoE kernel. Monolithic interface
"""FlashInfer TRT-LLM MxInt4 MoE kernel. Monolithic interface
(fused router + experts).
Wraps flashinfer_trtllm_mxint4_moe().
@@ -42,9 +42,7 @@ _PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
class TrtLlmNvFp4ExpertsBase:
"""
NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface.
"""
"""NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface."""
def __init__(
self,
@@ -300,9 +298,7 @@ class TrtLlmNvFp4ExpertsBase:
class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModular):
"""
Modular version of the implementation (just the experts).
"""
"""Modular version of the implementation (just the experts)."""
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
@@ -467,9 +463,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
class TrtLlmNvFp4ExpertsMonolithic(
TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsMonolithic
):
"""
Monolithic version of the kernel (router + experts).
"""
"""Monolithic version of the kernel (router + experts)."""
def supports_routing_replay_capture(self) -> bool:
return True
@@ -110,8 +110,7 @@ def fused_moe_kernel_gptq_awq(
use_int4_w4a16: tl.constexpr,
use_int8_w8a16: tl.constexpr,
):
"""
Implements the fused computation for a Mixture of Experts (MOE) using
"""Implements the fused computation for a Mixture of Experts (MOE) using
token and expert matrices.
Key Parameters:
@@ -353,8 +352,7 @@ def fused_moe_kernel(
# Tensor-descriptor path for the A gather and B load in the K-loop.
USE_TD: tl.constexpr = False,
):
"""
Implements the fused computation for a Mixture of Experts (MOE) using
"""Implements the fused computation for a Mixture of Experts (MOE) using
token and expert matrices.
Key Parameters:
@@ -1115,15 +1113,13 @@ def get_moe_configs(
block_n: int | None = None,
block_k: int | None = None,
) -> dict[int, Any] | None:
"""
Return optimized configurations for the fused MoE kernel.
"""Return optimized configurations for the fused MoE kernel.
The return value will be a dictionary that maps an irregular grid of
batch sizes to configurations of the fused_moe kernel. To evaluate the
kernel on a given batch size bs, the closest batch size in the grid should
be picked and the associated configuration chosen to invoke the kernel.
"""
# Avoid optimizing for the batch invariant case. Use default config
if envs.VLLM_BATCH_INVARIANT:
return None
@@ -1187,6 +1183,7 @@ def _ensure_block_size_k_divisible(
Returns:
A valid BLOCK_SIZE_K that divides size_k and is divisible by group_size.
"""
# Fast path: already valid
if size_k % block_size_k == 0 and block_size_k % group_size == 0:
@@ -1638,8 +1635,7 @@ def _get_config_quant_dtype(
use_fp8_w8a8: bool,
use_int8_w8a8: bool,
) -> None | torch.dtype | str:
"""
Get the quantization type based on the quantization strategy flags.
"""Get the quantization type based on the quantization strategy flags.
We don't have a quant_config at this point so we need to work backwards.
A return type of None means no quantization is required because the
input is unquantized or has been quantized prior to calling
@@ -59,8 +59,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
raise NotImplementedError
def uses_weight_scale_2_pattern(self) -> bool:
"""
Returns True if this quantization method uses 'weight_scale_2' pattern
"""Returns True if this quantization method uses 'weight_scale_2' pattern
for per-tensor weight scales (e.g., FP4 variants), False otherwise.
This method should be overridden by subclasses that use the
@@ -75,8 +74,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
act_dtype: torch.dtype,
moe_parallel_config: FusedMoEParallelConfig,
) -> tuple[int, int]:
"""
Given layer hidden size and intermediate size per partition and MoE
"""Given layer hidden size and intermediate size per partition and MoE
configurations, round up hidden_size and intermediate_size_per_partition
if necessary.
@@ -93,6 +91,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
- rounded_hidden_size is the possibly rounded up hidden size.
- rounded_intermediate_size_per_partition is the possibly rounded
up intermediate size per partition.
"""
from .all2all_utils import maybe_roundup_layer_hidden_size
@@ -119,8 +118,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
@property
def has_unpadded_output(self) -> bool:
"""
Indicates that the hidden_states output might be the unpadded
"""Indicates that the hidden_states output might be the unpadded
hidden_states shape rather than the full padded shape.
"""
return False
@@ -151,8 +149,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
shared_experts: "SharedExperts | None",
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
"""
Apply the MoE operation using modular kernels.
"""Apply the MoE operation using modular kernels.
Args:
layer: RoutedExperts instance containing weight parameters
@@ -160,9 +157,11 @@ class FusedMoEMethodBase(QuantizeMethodBase):
topk_weights: Expert weights from router
topk_ids: Selected expert IDs from router
shared_experts_input: Input for shared experts (if any)
shared_experts: The shared experts module, if any
Returns:
Output tensor from routed experts.
"""
raise NotImplementedError
@@ -173,15 +172,16 @@ class FusedMoEMethodBase(QuantizeMethodBase):
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""
Apply the MoE operation using monolithic kernels.
"""Apply the MoE operation using monolithic kernels.
Args:
layer: RoutedExperts instance containing weight parameters
x: Input tensor
router_logits: Router logits (routing done internally)
input_ids: Token ids, for routers that condition on them
Returns:
Finalized routed states or a deferred-finalize output.
"""
raise NotImplementedError
@@ -122,8 +122,7 @@ class HPCExperts(mk.FusedMoEExpertsModular):
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# We use global_num_experts due to how moe_align_block_size handles
# expert_maps.
"""
Compute the shapes for the temporary and final outputs of the two gemms
"""Compute the shapes for the temporary and final outputs of the two gemms
and activation in the fused expert function. Since the gemms are
independent, the workspace for the first gemm can be shared with the
workspace for the last gemm.
@@ -152,6 +152,11 @@ def FusedMoEFactory(
copy that naming convention here and handle any remapping in the
load_weights function in each model implementation.
Args:
intermediate_pad: Padding added to the intermediate size, if any.
swiglu_alpha: Optional alpha parameter for the SwiGLU activation.
swiglu_beta: Optional beta parameter for the SwiGLU activation.
Args:
num_experts: Number of experts in the model (global count)
top_k: Number of experts selected for each token
@@ -211,6 +216,7 @@ def FusedMoEFactory(
Returns:
MoERunner: Configured MoE execution pipeline ready for forward passes
"""
vllm_config = get_current_vllm_config()
@@ -84,9 +84,7 @@ logger = init_logger(__name__)
class FusedMoEActivationFormat(Enum):
"""
The standard activation format (num_tokens, hidden dim).
"""
"""The standard activation format (num_tokens, hidden dim)."""
Standard = ("standard",)
"""
@@ -97,9 +95,7 @@ class FusedMoEActivationFormat(Enum):
@dataclass
class ExpertTokensMetadata:
"""
Metadata regarding expert-token routing.
"""
"""Metadata regarding expert-token routing."""
expert_num_tokens: torch.Tensor | None
expert_num_tokens_cpu: torch.Tensor | None
@@ -122,9 +118,7 @@ class ExpertTokensMetadata:
class TopKWeightAndReduce(ABC):
"""
An abstract base class for weight application and reduction implementations.
"""
"""An abstract base class for weight application and reduction implementations."""
@abstractmethod
def apply(
@@ -135,8 +129,7 @@ class TopKWeightAndReduce(ABC):
topk_ids: torch.Tensor,
apply_router_weight_on_input: bool,
) -> torch.Tensor:
"""
Apply topk_weights to the fused_experts_outputs and/or reduce.
"""Apply topk_weights to the fused_experts_outputs and/or reduce.
If an output tensor is not passed, it will be created in the
function.
"""
@@ -185,8 +178,7 @@ ReceiverType = Callable[[], PrepareResultType]
class FusedMoEPrepareAndFinalize(ABC):
"""
An abstract base class for the [Quantize-Prepare] and [Finalize] steps
"""An abstract base class for the [Quantize-Prepare] and [Finalize] steps
described above.
There are two variants of this class:
@@ -195,8 +187,7 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
def post_init_setup(self, fused_experts: "FusedMoEExperts"):
"""
Initialize FusedMoEPrepareAndFinalizeModular settings that depend on
"""Initialize FusedMoEPrepareAndFinalizeModular settings that depend on
FusedMoEExpertsModular experts object.
The FusedMoEPrepareAndFinalizeModular implementations that have such
dependencies may choose to override this function.
@@ -206,16 +197,14 @@ class FusedMoEPrepareAndFinalize(ABC):
@property
@abstractmethod
def activation_format(self) -> FusedMoEActivationFormat:
"""
A property indicating the output format of the activations for the
"""A property indicating the output format of the activations for the
'prepare' method.
"""
raise NotImplementedError
@abstractmethod
def topk_indices_dtype(self) -> torch.dtype | None:
"""
The PrepareFinalize All2All implementations generally constrain the
"""The PrepareFinalize All2All implementations generally constrain the
dtype of the topk_ids they support. This function returns the
required topk indices dtype so it can be respected.
Return None if there are no such restrictions.
@@ -224,8 +213,7 @@ class FusedMoEPrepareAndFinalize(ABC):
@abstractmethod
def max_num_tokens_per_rank(self) -> int | None:
"""
Some PrepareFinalize All2All implementations are batched. Meaning,
"""Some PrepareFinalize All2All implementations are batched. Meaning,
they can process only as set of tokens at a time. This
function returns the batch size i.e the maximum number of tokens
the implementation can process at a time.
@@ -239,15 +227,13 @@ class FusedMoEPrepareAndFinalize(ABC):
@abstractmethod
def output_is_reduced(self) -> bool:
"""
Indicates whether or not the output of finalize is reduced across all
"""Indicates whether or not the output of finalize is reduced across all
ranks.
"""
raise NotImplementedError
def supports_async(self) -> bool:
"""
Indicates whether or not this class implements prepare_async and
"""Indicates whether or not this class implements prepare_async and
finalize_async.
"""
return False
@@ -255,8 +241,7 @@ class FusedMoEPrepareAndFinalize(ABC):
# TODO: pass FusedMoEParallelConfig in as ctor parameter?
class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
"""
An abstract base class for the [Quantize-Prepare] and [Finalize] steps
"""An abstract base class for the [Quantize-Prepare] and [Finalize] steps
described above for the Modular case.
"""
@@ -272,8 +257,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool,
) -> PrepareResultType:
"""
Perform any quantization (and/or) dispatching needed for this kernel.
"""Perform any quantization (and/or) dispatching needed for this kernel.
- a1: The (unquantized) input to the MoE layer.
- topk_ids: The topk ids.
- topk_weights: The topk weights.
@@ -309,8 +293,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool,
) -> tuple[Callable, ReceiverType] | ReceiverType:
"""
Perform any quantization (and/or) dispatching needed for this kernel
"""Perform any quantization (and/or) dispatching needed for this kernel
but do not wait for results from other workers.
- a1: The (unquantized) input to the MoE layer.
- a1_scale: Optional scales for a1
@@ -360,8 +343,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
apply_router_weight_on_input: bool,
weight_and_reduce_impl: TopKWeightAndReduce,
) -> None:
"""
Perform any combine plus apply weights and perform a reduction on the
"""Perform any combine plus apply weights and perform a reduction on the
fused experts output.
- output: The output tensor, written in place. Must be (M, K) shape.
- fused_expert_output: The unweighted, unreduced output of the fused
@@ -384,8 +366,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
apply_router_weight_on_input: bool,
weight_and_reduce_impl: TopKWeightAndReduce,
) -> tuple[Callable, Callable] | Callable:
"""
Perform any combine plus apply weights and perform a reduction on the
"""Perform any combine plus apply weights and perform a reduction on the
fused experts output but do not wait for results from other workers.
- output: The output tensor, written in place. Must be (M, K) shape.
- fused_expert_output: The unweighted, unreduced output of the fused
@@ -419,8 +400,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
"""
An abstract base class for the [Quantize-Prepare] and [Finalize] steps
"""An abstract base class for the [Quantize-Prepare] and [Finalize] steps
described above for the monolithic case.
"""
@@ -435,8 +415,7 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> PrepareMonolithicResultType:
"""
Optional method for subclasses compatible with monolithic
"""Optional method for subclasses compatible with monolithic
FusedMoEExpertsModular kernels.
Perform any quantization (and/or) dispatching needed for this kernel.
@@ -453,8 +432,7 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
@abstractmethod
def finalize(self, fused_expert_output: torch.Tensor) -> torch.Tensor:
"""
Optional method for subclasses compatible with monolithic
"""Optional method for subclasses compatible with monolithic
FusedMoEExpertsModular kernels.
Perform any combine plus apply weights and perform a reduction on the
@@ -485,8 +463,7 @@ class FusedMoEExperts(ABC):
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
"""
moe_config: MoE layer configuration.
"""moe_config: MoE layer configuration.
quant_config: Quantization parameters for this experts instance.
"""
if self.activation_format() == FusedMoEActivationFormat.Standard and (
@@ -521,8 +498,7 @@ class FusedMoEExperts(ABC):
@property
def expects_unquantized_inputs(self) -> bool:
"""
Whether or not the PrepareFinalize should defer input quantization
"""Whether or not the PrepareFinalize should defer input quantization
in the prepare step. If True, then the Experts kernel will
execute the input quantization itself.
@@ -533,8 +509,7 @@ class FusedMoEExperts(ABC):
@staticmethod
@abstractmethod
def activation_format() -> FusedMoEActivationFormat:
"""
A property which is a tuple of the input and output activation formats
"""A property which is a tuple of the input and output activation formats
for the 'apply' method.
"""
raise NotImplementedError
@@ -595,8 +570,7 @@ class FusedMoEExperts(ABC):
@staticmethod
@abstractmethod
def _supports_current_device() -> bool:
"""
Whether the kernel supports the current device type
"""Whether the kernel supports the current device type
(compute cability and current platform).
"""
raise NotImplementedError
@@ -604,8 +578,7 @@ class FusedMoEExperts(ABC):
@staticmethod
@abstractmethod
def _supports_no_act_and_mul() -> bool:
"""
Whether the kernel supports act_and_mul=False, i.e.
"""Whether the kernel supports act_and_mul=False, i.e.
non-gated MoE models like Nemotron-Nano.
"""
raise NotImplementedError
@@ -621,16 +594,13 @@ class FusedMoEExperts(ABC):
@staticmethod
@abstractmethod
def _supports_activation(activation: MoEActivation) -> bool:
"""
Whether the kernel supports a particular act function.
"""
"""Whether the kernel supports a particular act function."""
raise NotImplementedError
@staticmethod
@abstractmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
"""
Whether the kernel supports deployment in particular parallel config.
"""Whether the kernel supports deployment in particular parallel config.
Can be overridden if a kernel does not support EP, SP or some other
configuration.
@@ -643,8 +613,7 @@ class FusedMoEExperts(ABC):
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
"""
Whether the kernel supports a routing method (e.g. GroupedTopK).
"""Whether the kernel supports a routing method (e.g. GroupedTopK).
Can be overridden by monolithic kernels that execute the router
in addition to the experts if certain routers are not supported.
@@ -656,8 +625,7 @@ class FusedMoEExperts(ABC):
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
"""
Whether a kernel supports a particular dtype for router logits input.
"""Whether a kernel supports a particular dtype for router logits input.
Can be overridden by monolithic kernels that execute the router
in addition to the experts if certain dtypes are not supported.
@@ -666,16 +634,14 @@ class FusedMoEExperts(ABC):
@staticmethod
def _supports_shape(hidden_dim: int) -> bool:
"""
Whether a kernel supports a particular shape. Can be overridden if a kernel
"""Whether a kernel supports a particular shape. Can be overridden if a kernel
has specific shape requirements.
"""
return True
@staticmethod
def _supports_batch_invariance() -> bool:
"""
Whether the kernel supports batch invariance, i.e. the output does not
"""Whether the kernel supports batch invariance, i.e. the output does not
depend on the order of the tokens in the input batch. This is useful
for determining if the kernel can used with VLLM_BATCH_INVARIANT=1.
"""
@@ -764,17 +730,15 @@ class FusedMoEExperts(ABC):
return False
def supports_packed_ue8m0_act_scales(self) -> bool:
"""
A flag indicating whether or not this class can process packed ue8m0
"""A flag indicating whether or not this class can process packed ue8m0
activation scales.
"""
return False
class FusedMoEExpertsModular(FusedMoEExperts):
"""
An abstract base class for the [Permute-Experts-Unpermute] step described
above.
"""An abstract base class for the [Permute-Experts-Unpermute] step described
above.
"""
@staticmethod
@@ -788,8 +752,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
"""
Extract the MoE problem size from the given tensor arguments:
"""Extract the MoE problem size from the given tensor arguments:
- a: The hidden states, input to the MoE layer.
- w1: The first set of expert weights.
- w2: The second set of expert weights.
@@ -825,9 +788,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
return E, M, N, K, topk
def workspace_dtype(self, act_dtype: torch.dtype) -> torch.dtype:
"""
Workspace type: The dtype to use for the workspace tensors.
"""
"""Workspace type: The dtype to use for the workspace tensors."""
return act_dtype
@abstractmethod
@@ -842,8 +803,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
expert_tokens_meta: ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
"""
Compute the shapes for the temporary and final outputs of the two gemms
"""Compute the shapes for the temporary and final outputs of the two gemms
and activation in the fused expert function. Since the gemms are
independent, the workspace for the first gemm can be shared with the
workspace for the last gemm.
@@ -873,8 +833,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
@staticmethod
def adjust_N_for_activation(N: int, activation: MoEActivation) -> int:
"""
Calculate the output dimension for the activation function.
"""Calculate the output dimension for the activation function.
For *_no_mul activations (e.g. relu2_no_mul),
there's no gate/up split, so output size equals input size (N).
@@ -888,6 +847,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
Returns:
The output dimension after activation.
"""
return N if not activation.is_gated else N // 2
@@ -934,49 +894,50 @@ class FusedMoEExpertsModular(FusedMoEExperts):
expert_tokens_meta: ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
"""
This function computes the intermediate result of a Mixture of Experts
"""This function computes the intermediate result of a Mixture of Experts
(MoE) layer using two sets of weights, w1 and w2.
Parameters:
- output: (torch.Tensor): The unweighted, unreduced output tensor.
- hidden_states: (torch.Tensor): The (quantized) input tensor to the MoE
layer.
- w1 (torch.Tensor): The first set of expert weights.
- w2 (torch.Tensor): The second set of expert weights.
- topk_weights: A map of row to expert weights. Some implementations
choose to do weight application.
- topk_ids (torch.Tensor): A map of row to expert id.
- activation (str): The activation function to apply after the first
MoE layer.
- global_num_experts (int): The total number of experts in the global
expert space.
- expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
from the global expert space to the local expert space of the expert
parallel shard.
- a1q_scale (Optional[torch.Tensor]): Optional quantized scale to be
used for a1. Result of quantization from prepare/finalize and not
from the FusedMoEQuantConfig.
- workspace13 (torch.Tensor): A scratch tensor used for gemm outputs
must be large enough to hold output of either MoE gemm.
- workspace2 (torch.Tensor): A scratch tensor used for the activation
function.
- expert_tokens_meta (Optional[ExpertTokensMetadata]) - An optional
ExpertTokensMetadata object containing gpu/cpu tensors
as big as the number of local experts with the information about the
number of tokens assigned to each local expert.
- apply_router_weight_on_input: True if router weights are already
applied on the input. This is relevant if the implementation
chooses to do weight application.
Args:
output: (torch.Tensor): The unweighted, unreduced output tensor.
hidden_states: (torch.Tensor): The (quantized) input tensor to the MoE
layer.
w1 (torch.Tensor): The first set of expert weights.
w2 (torch.Tensor): The second set of expert weights.
topk_weights: A map of row to expert weights. Some implementations
choose to do weight application.
topk_ids (torch.Tensor): A map of row to expert id.
activation (str): The activation function to apply after the first
MoE layer.
global_num_experts (int): The total number of experts in the global
expert space.
expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
from the global expert space to the local expert space of the
expert parallel shard.
a1q_scale (Optional[torch.Tensor]): Optional quantized scale to be
used for a1. Result of quantization from prepare/finalize and not
from the FusedMoEQuantConfig.
a2_scale (Optional[torch.Tensor]): Optional quantized scale to be
used for the second gemm's activations.
workspace13 (torch.Tensor): A scratch tensor used for gemm outputs
must be large enough to hold output of either MoE gemm.
workspace2 (torch.Tensor): A scratch tensor used for the activation
function.
expert_tokens_meta (Optional[ExpertTokensMetadata]): An optional
ExpertTokensMetadata object containing gpu/cpu tensors
as big as the number of local experts with the information about
the number of tokens assigned to each local expert.
apply_router_weight_on_input: True if router weights are already
applied on the input. This is relevant if the implementation
chooses to do weight application.
"""
raise NotImplementedError
class FusedMoEExpertsMonolithic(FusedMoEExperts):
"""
An abstract base class for the [Permute-Experts-Unpermute] step described
above, but with the monolithic interface (accepts router logits
rather than topk ids and weights).
"""An abstract base class for the [Permute-Experts-Unpermute] step described
above, but with the monolithic interface (accepts router logits
rather than topk ids and weights).
"""
@staticmethod
@@ -985,8 +946,7 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
"""
Whether the kernel supports a routing method (e.g. GroupedTopK).
"""Whether the kernel supports a routing method (e.g. GroupedTopK).
Monolithic kernels should explicitly opt-in to support.
"""
@@ -997,8 +957,7 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
"""
Whether the kernel supports a dtype for router logits.
"""Whether the kernel supports a dtype for router logits.
Modular kernels should opt-in to support.
"""
@@ -1083,8 +1042,7 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""
Same as apply(), except uses router_logits as opposed
"""Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed
to the topk_ids and topk_weights. This is useful for kernels
with fused router and fused_experts (e.g. FLASHINFER_TRTLLM).
"""
@@ -1128,8 +1086,7 @@ class FusedMoEKernelModularImpl:
expert_tokens_meta: ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Allocate temporary and output buffers for the fused experts op.
"""Allocate temporary and output buffers for the fused experts op.
Inputs:
- out_dtype: output type of workspace and output tensors.
- device: the device of the workspace and output tensors.
@@ -1223,11 +1180,9 @@ class FusedMoEKernelModularImpl:
torch.Tensor,
torch.Tensor,
]:
"""
The _prepare method is a wrapper around self.prepare_finalize.prepare
"""The _prepare method is a wrapper around self.prepare_finalize.prepare
that handles DBO and async.
"""
if not self.prepare_finalize.supports_async():
# We shouldn't be running an a2a kernel that doesn't
# support async prepare/finalize
@@ -1392,17 +1347,24 @@ class FusedMoEKernelModularImpl:
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
"""
The _finalize method is a wrapper around self.prepare_finalize.finalize
"""The _finalize method is a wrapper around self.prepare_finalize.finalize
that handles DBO, async and shared expert overlap.
Args:
output: Tensor the finalized result is written into.
fused_out: The unweighted, unreduced output of the fused experts.
hidden_states: The input tensor to the MoE layer.
topk_weights: A map of row to expert weights.
topk_ids: A map of row to expert id.
apply_router_weight_on_input: True if the router weights were
already applied on the input.
shared_experts: SharedExperts | None. The shared experts if any.
shared_experts_input: Optional separate input for shared experts.
When latent MoE is used, hidden_states is the latent-projected
tensor (smaller dimension) used by routed experts, while
shared_experts_input is the original hidden_states (full
dimension) needed by the shared expert MLP.
"""
if not self.prepare_finalize.supports_async():
assert not dbo_enabled()
@@ -1463,33 +1425,34 @@ class FusedMoEKernelModularImpl:
shared_experts: SharedExperts | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
"""
This function computes a Mixture of Experts (MoE) layer using two sets
"""This function computes a Mixture of Experts (MoE) layer using two sets
of weights, w1 and w2, and top-k gating mechanism.
Parameters:
- hidden_states: (torch.Tensor): The input tensor to the MoE layer.
- w1 (torch.Tensor): The first set of expert weights.
- w2 (torch.Tensor): The second set of expert weights.
- topk_weights (torch.Tensor): The topk weights applied at the end of the layer.
- topk_ids (torch.Tensor): A map of row to expert id.
- activation (MoEActivation): The activation function to apply after the first
MoE layer.
- global_num_experts (int): The total number of experts in the global
expert space.
- expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
from the global expert space to the local expert space of the expert
parallel shard.
- apply_router_weight_on_input (bool): When true, the topk weights are
applied directly on the inputs. This is only applicable when topk is
1.
- shared_experts: SharedExperts | None. The shared experts if any.
- shared_experts_input (Optional[torch.Tensor]): Optional separate
input for shared experts. For latent MoE, this is the original
hidden_states before latent projection.
Args:
hidden_states: (torch.Tensor): The input tensor to the MoE layer.
w1 (torch.Tensor): The first set of expert weights.
w2 (torch.Tensor): The second set of expert weights.
topk_ids (torch.Tensor): A map of row to expert id.
topk_weights (torch.Tensor): The topk weights applied at the end of
the layer.
activation (MoEActivation): The activation function to apply after
the first MoE layer.
global_num_experts (int): The total number of experts in the global
expert space.
expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
from the global expert space to the local expert space of the
expert parallel shard.
apply_router_weight_on_input (bool): When true, the topk weights are
applied directly on the inputs. This is only applicable when
topk is 1.
shared_experts: SharedExperts | None. The shared experts if any.
shared_experts_input (Optional[torch.Tensor]): Optional separate
input for shared experts. For latent MoE, this is the original
hidden_states before latent projection.
Returns:
- torch.Tensor: The output tensor after applying the MoE layer.
torch.Tensor: The output tensor after applying the MoE layer.
"""
output = torch.empty_like(hidden_states)
@@ -1571,12 +1534,10 @@ class FusedMoEKernelMonolithicImpl:
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""
Same as forward(), except uses router_logits as opposed
"""Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
"""
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
hidden_states,
router_logits=router_logits,
@@ -1674,8 +1635,7 @@ class FusedMoEKernel:
return self.fused_experts.supports_lora()
def _post_init_setup(self):
"""
Resolve any leftover setup dependencies between self.prepare_finalize
"""Resolve any leftover setup dependencies between self.prepare_finalize
and self.fused_experts here.
"""
self.prepare_finalize.post_init_setup(self.impl.fused_experts)
@@ -1685,8 +1645,7 @@ class FusedMoEKernel:
)
def output_is_reduced(self) -> bool:
"""
Indicates whether or not the output of fused MoE kernel
"""Indicates whether or not the output of fused MoE kernel
is reduced across all ranks.
"""
return self.prepare_finalize.output_is_reduced()
@@ -16,8 +16,7 @@ def moe_align_block_size(
pad_sorted_ids: bool = False,
ignore_invalid_experts: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Aligns the token distribution across experts to be compatible with block
"""Aligns the token distribution across experts to be compatible with block
size for matrix multiplication.
Note: In the case of expert_parallel, moe_align_block_size initially
@@ -26,23 +25,23 @@ def moe_align_block_size(
the current GPU rank as -1 so the MoE matmuls could skip those blocks.
This requires the num_experts input arg to be the num global experts.
Parameters:
- topk_ids: A tensor of shape [total_tokens, top_k] representing the
top-k expert indices for each token.
- block_size: The block size used in block matrix multiplication.
- num_experts: The total number of experts.
- expert_map: A tensor of shape [num_experts] that maps the expert index
from the global space to the local index space of the current
expert parallel shard. If the expert is not in the current expert
parallel shard, the mapping is set to -1.
- pad_sorted_ids: A flag indicating whether the sorted_token_ids length
should be padded to a multiple of block_size,
- ignore_invalid_experts: A flag indicating whether to ignore invalid
experts. When False, all expert_ids in topk_ids will participate in
counting and ranking, but invalid experts in expert_ids will be marked
as -1. When True, all invalid expert_ids in topk_ids will be ignored
and will not participate in counting or ranking, and there will be no
-1 in expert_ids.
Args:
topk_ids: A tensor of shape [total_tokens, top_k] representing the
top-k expert indices for each token.
block_size: The block size used in block matrix multiplication.
num_experts: The total number of experts.
expert_map: A tensor of shape [num_experts] that maps the expert index
from the global space to the local index space of the current
expert parallel shard. If the expert is not in the current expert
parallel shard, the mapping is set to -1.
pad_sorted_ids: A flag indicating whether the sorted_token_ids length
should be padded to a multiple of block_size,
ignore_invalid_experts: A flag indicating whether to ignore invalid
experts. When False, all expert_ids in topk_ids will participate in
counting and ranking, but invalid experts in expert_ids will be marked
as -1. When True, all invalid expert_ids in topk_ids will be ignored
and will not participate in counting or ranking, and there will be no
-1 in expert_ids.
Returns:
- sorted_token_ids: A tensor containing the sorted token indices according
@@ -70,6 +69,7 @@ def moe_align_block_size(
the subsequent matrix multiplication.
- The padding ensures that the total number of tokens is now divisible
by block_size for proper block matrix operations.
"""
max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
if pad_sorted_ids:
@@ -106,8 +106,7 @@ def moe_align_block_size(
def batched_moe_align_block_size(
max_tokens_per_batch: int, block_size: int, expert_num_tokens: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Given num_batches, max_tokens_per_batch, block_size and the number of
"""Given num_batches, max_tokens_per_batch, block_size and the number of
valid-tokens in each batch, prepare sorted_token_ids, expert_ids and
num_tokens_post_pad. sorted_token_ids, expert_ids and num_tokens_post_pad
have the same semantics as in moe_align_block_size.
@@ -115,12 +114,12 @@ def batched_moe_align_block_size(
This function is intended to be a drop in replacement for
moe_align_batch_size for the batched case.
Parameters:
- max_tokens_per_batch (int): Number of tokens in each batch (both
valid and invalid).
- block_size (int): block_size to align the data to.
- expert_num_tokens (torch.Tensor): expert_num_tokens[i], indicates
the number of valid tokens in batch i.
Args:
max_tokens_per_batch (int): Number of tokens in each batch (both
valid and invalid).
block_size (int): block_size to align the data to.
expert_num_tokens (torch.Tensor): expert_num_tokens[i], indicates
the number of valid tokens in batch i.
Returns:
- sorted_token_ids (torch.Tensor): Torch tensor of size
@@ -132,6 +131,7 @@ def batched_moe_align_block_size(
- num_tokens_post_pad (torch.Tensor): Torch tensor of size 1
indicating the number of valid blocks with actual data to
process. This is represented in terms of num tokens.
Example:
Let num_batches=5, max_tokens_per_batch=8, block_size=4, and
expert_num_tokens=[2, 3, 0, 6, 8]. This expert_num_tokens tensor
@@ -166,8 +166,8 @@ def batched_moe_align_block_size(
num_tokens_post_pad will be 24 as sorted_token_ids has valid entries
until 24.
"""
"""
B = expert_num_tokens.size(0)
device = expert_num_tokens.device
@@ -92,8 +92,7 @@ def moe_fused_mul_sum(
expert_map: torch.Tensor | None = None,
num_valid_tokens: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Fused kernel for MoE (Mixture of Experts) to perform weighted summation
"""Fused kernel for MoE (Mixture of Experts) to perform weighted summation
of expert outputs.
Args:
@@ -122,6 +121,7 @@ def moe_fused_mul_sum(
Returns:
The fused weighted sum of expert outputs.
Shape: (num_tokens, hidden_size).
"""
assert inputs.ndim == 3
assert topk_weights.ndim == 2
@@ -57,8 +57,8 @@ def convert_flashinfer_moe_output(
Raises:
ValueError: If FlashInfer returns an unexpected layout.
"""
"""
if num_tokens < 0:
raise ValueError("num_tokens must be non-negative.")
if top_k <= 0:
@@ -112,20 +112,24 @@ def moe_permute(
permuted_hidden_states: torch.Tensor | None = None,
scratch: MoEPermuteScratch | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
This function expands and permutes activation to gather uncontinuous tokens
"""This function expands and permutes activation to gather uncontinuous tokens
for each expert.
Parameters:
- hidden_states (torch.Tensor): The input tensor to the MoE layer.
- a1q_scale (Optional[torch.Tensor]): quant scale for hidden_states
- topk_ids (torch.Tensor): topk expert route id for each token.
- n_expert (int): The number of expert.
- n_local_expert (int): The number of expert in current EP rank.
- expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
from the global expert space to the local expert space of the expert
parallel shard.
- permuted_hidden_states (Optional[torch.Tensor]): Optional output tensor.
If None, the output tensor will be created in this function.
Args:
hidden_states (torch.Tensor): The input tensor to the MoE layer.
a1q_scale (Optional[torch.Tensor]): quant scale for hidden_states
topk_ids (torch.Tensor): topk expert route id for each token.
n_expert (int): The number of expert.
n_local_expert (int): The number of expert in current EP rank.
expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
from the global expert space to the local expert space of the expert
parallel shard.
permuted_hidden_states (Optional[torch.Tensor]): Optional output tensor.
If None, the output tensor will be created in this function.
scratch (Optional[MoEPermuteScratch]): Optional preallocated scratch
buffers. Validated against hidden_states and topk_ids when given,
otherwise the buffers are allocated in this function.
Returns:
- permuted_hidden_states (torch.Tensor): permuted activation.
- a1q_scale (Optional[torch.Tensor]): permuted quant scale for hidden_states
@@ -134,6 +138,7 @@ def moe_permute(
of each expert for standard grouped gemm.
- inv_permuted_idx (torch.Tensor): idx map for moe_unpermute.
- permuted_idx (torch.Tensor): idx map from hidden to permuted_hidden.
"""
n_token, n_hidden = hidden_states.size()
topk = topk_ids.size(1)
@@ -249,19 +254,21 @@ def moe_unpermute(
inv_permuted_idx: torch.Tensor,
expert_first_token_offset: torch.Tensor | None = None,
) -> None:
"""
This function expands and permutes activation to gathering uncontinuous
"""This function expands and permutes activation to gathering uncontinuous
tokens for each expert.
Parameters:
- out (torch.Tensor): output tensor
- permuted_hidden_states (torch.Tensor): permuted activation.
- topk_weights (torch.Tensor): topk expert route weight for each token.
- inv_permuted_idx (torch.Tensor): row idx map for moe_unpermute.
- expert_first_token_offset (Optional[torch.Tensor]): offset of the first
token of each expert for grouped gemm.
Args:
out (torch.Tensor): output tensor
permuted_hidden_states (torch.Tensor): permuted activation.
topk_weights (torch.Tensor): topk expert route weight for each token.
inv_permuted_idx (torch.Tensor): row idx map for moe_unpermute.
expert_first_token_offset (Optional[torch.Tensor]): offset of the first
token of each expert for grouped gemm.
Returns:
- hidden_states (torch.Tensor): The reduced and unpermuted activation
tensor.
"""
topk = topk_weights.size(1)
n_hidden = permuted_hidden_states.size(-1)
@@ -55,24 +55,28 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
@abstractmethod
def backend_enum_cls(self) -> type[BackendT]:
"""Return the concrete `Enum` class enumerating this oracle's
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`)."""
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`).
"""
@abstractmethod
def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]:
"""Return platform-appropriate backends in priority order for
this `moe_config`."""
this `moe_config`.
"""
@abstractmethod
def backend_to_kernel_cls(
self, backend: BackendT
) -> list[type[mk.FusedMoEExperts]]:
"""Map a backend enum value to its concrete `FusedMoEExperts`
subclasses, in selection priority order."""
subclasses, in selection priority order.
"""
@abstractmethod
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
"""Map a user-facing `MoEBackend` (from the runner config) to
this oracle's enum."""
this oracle's enum.
"""
@abstractmethod
def select_backend(
@@ -104,7 +108,8 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
"""Construct the `FusedMoEKernel` (Prepare/Finalize + Experts
combinator) for the chosen backend."""
combinator) for the chosen backend.
"""
def convert_to_kernel_format(
self,
@@ -72,12 +72,10 @@ def _get_priority_backends(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> list[Fp8MoeBackend]:
"""
Get available backends in priority order based on platform and config.
"""Get available backends in priority order based on platform and config.
This function can be extended to become more complex as needed.
"""
_AVAILABLE_BACKENDS = [
Fp8MoeBackend.AITER,
Fp8MoeBackend.FLASHINFER_TRTLLM,
@@ -273,8 +271,7 @@ def refine_fp8_moe_block_shape(
config: FusedMoEConfig,
weight_block_size: list[int],
) -> list[int] | None:
"""
Compute a refined block shape for block-quantized FP8 MoE weights whose
"""Compute a refined block shape for block-quantized FP8 MoE weights whose
checkpoint blocks cannot be sharded exactly across TP ranks.
TP shards the intermediate dim of the expert weights, so a per-shard size
@@ -308,11 +305,9 @@ def select_fp8_moe_backend(
activation_key: QuantKey | None,
allow_vllm_cutlass: bool = False,
) -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts] | None]:
"""
Select the primary FP8 MoE backend
"""Select the primary FP8 MoE backend
Note: Shape-specific fallbacks may still occur at runtime.
"""
# NOTE: the kernels are selected in the following order.
AVAILABLE_BACKENDS = _get_priority_backends(config, weight_key, activation_key)
@@ -456,7 +451,8 @@ def _humming_fp8_weight_schema(
layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
"""Build the humming weight schema from the canonical on-device fp8/mxfp8
tensors (scale dtype/shape, block size), not the producing quant method."""
tensors (scale dtype/shape, block size), not the producing quant method.
"""
# mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
# no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
@@ -614,8 +610,7 @@ def make_fp8_moe_quant_config(
gemm1_beta: float | None = None,
layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig:
"""
Create FusedMoEQuantConfig for the specified FP8 Backend.
"""Create FusedMoEQuantConfig for the specified FP8 Backend.
The FusedMoEQuantConfig holds the scales that are used
at runtime by the Modular Kernel abstraction.
@@ -626,7 +621,6 @@ def make_fp8_moe_quant_config(
In a future PR, we will have this function should be
a method of the modular kernel itself.
"""
# MARLIN and CPU are mixed precision W8A16 config.
if fp8_backend == Fp8MoeBackend.MARLIN or fp8_backend == Fp8MoeBackend.CPU:
return fp8_w8a16_moe_quant_config(
@@ -37,9 +37,7 @@ class Int8MoeBackend(Enum):
def _get_priority_backends(
moe_config: FusedMoEConfig,
) -> list[Int8MoeBackend]:
"""
Get available backends in priority order based on platform and config.
"""
"""Get available backends in priority order based on platform and config."""
return [
Int8MoeBackend.TRITON,
Int8MoeBackend.HUMMING,
@@ -100,11 +98,9 @@ def select_int8_moe_backend(
weight_key: QuantKey | None = kInt8StaticChannelSym,
activation_key: QuantKey | None = kInt8DynamicTokenSym,
) -> tuple[Int8MoeBackend, type[mk.FusedMoEExperts]]:
"""
Select the primary Int8 MoE backend.
"""Select the primary Int8 MoE backend.
Note: Shape-specific fallbacks may still occur at runtime.
"""
AVAILABLE_BACKENDS = _get_priority_backends(config)
activation_format = (
@@ -228,7 +224,8 @@ def _humming_int8_weight_schema(
weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
"""Build the humming compressed-tensors int8 schema from the canonical
on-device tensors; humming does the signed-int8 -> native conversion."""
on-device tensors; humming does the signed-int8 -> native conversion.
"""
config: dict[str, Any] = {
"quant_method": "compressed-tensors",
"format": "int-quantized",
@@ -115,9 +115,7 @@ def backend_to_kernel_cls(
def _get_priority_backends() -> list[WNA16MoEBackend]:
"""
Get available backends in priority order based on platform and config.
"""
"""Get available backends in priority order based on platform and config."""
if current_platform.is_cpu():
return [WNA16MoEBackend.CPU]
if current_platform.is_xpu():
@@ -241,11 +239,13 @@ def select_wna16_moe_backend(
quant_config: Quantization structure and checkpoint format description.
may_have_zp: Whether the integration can provide weight zero points.
may_have_bias: Whether the integration can provide expert bias.
allow_tile_padding: Whether backends that require padding the weights
up to a tile boundary may be selected.
Returns:
A tuple of (``WNA16MoEBackend``, experts class or ``None``).
"""
"""
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
if config.moe_parallel_config.use_batched_activation_format
@@ -512,7 +512,8 @@ def _process_weights_flashinfer(
def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor:
"""Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)``
tensor along its last dim, from ``unit`` to ``padded_unit`` columns."""
tensor along its last dim, from ``unit`` to ``padded_unit`` columns.
"""
if padded_unit == unit:
return x
e, rows, _ = x.shape
@@ -1080,7 +1081,8 @@ def _humming_wna16_weight_schema(
quant_config: QuantizationConfig | QuantizationArgs | None,
) -> dict[str, Any]:
"""Humming weight schema for a WNA16 checkpoint, derived from the quant
config rather than the running kernel."""
config rather than the running kernel.
"""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
@@ -1194,6 +1196,7 @@ def _unpack_and_dequant_int4_gptq(
Returns:
Dequantized weight tensor in the requested layout.
"""
E, K_packed, N = w_int32.shape
K = K_packed * 8
@@ -1256,6 +1259,7 @@ def _unpack_and_dequant_int4_awq(
Returns:
Dequantized weight tensor in the requested layout.
"""
E, K, N_packed = w_int32.shape
N = N_packed * 8
@@ -1442,6 +1446,15 @@ def convert_to_wna16_moe_kernel_format(
layer: the ``MoERunner`` layer whose parameters are being prepared.
quant_config: the ``QuantizationConfig`` for this layer.
input_dtype: optional activation dtype, usually should be 16 bit.
w13: fused gate/up expert weights.
w2: down-projection expert weights.
w13_scale: quantization scales for ``w13``.
w2_scale: quantization scales for ``w2``.
w13_qzeros: optional zero points for ``w13``.
w2_qzeros: optional zero points for ``w2``.
w13_bias: optional bias for ``w13``.
w2_bias: optional bias for ``w2``.
"""
if backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
@@ -331,7 +331,8 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
"""Available backends in priority order, BF16-act variant before
activation-quantized variant within each vendor family."""
activation-quantized variant within each vendor family.
"""
_AVAILABLE_BACKENDS = [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
@@ -355,8 +356,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
"""
Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
"""Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
backend-level ``is_supported_config`` check filters by device capability).
"""
@@ -411,7 +411,8 @@ def _resolve_activation_key(
model_activation_key: QuantKey | None,
) -> QuantKey | None:
"""Combine the model-supplied activation key with the user override.
Raises on conflict (both set and disagreeing)."""
Raises on conflict (both set and disagreeing).
"""
user_override = _user_moe_activation_override()
if user_override is None:
return model_activation_key
@@ -461,7 +462,8 @@ def _filter_by_activation(
) -> list[Mxfp4MoeBackend]:
"""Pick variants matching ``requested_activation_key``; without one,
prefer BF16 if the list has any, else keep the list as-is so explicit
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
non-BF16 picks (e.g. the ``_afp8`` aliases) still land.
"""
if requested_activation_key is not None:
return [
b
@@ -526,8 +528,7 @@ def select_mxfp4_moe_backend(
config: FusedMoEConfig,
activation_key: QuantKey | None = None,
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts] | None]:
"""
Select the primary MXFP4 MoE backend.
"""Select the primary MXFP4 MoE backend.
Args:
config: MoE configuration
@@ -536,6 +537,7 @@ def select_mxfp4_moe_backend(
Use kFp8StaticTensorSym for W4A8 scheme.
Note: Shape-specific fallbacks may still occur at runtime.
"""
runner_backend = config.moe_backend
requested_activation_key = _resolve_activation_key(activation_key)
@@ -648,8 +650,7 @@ def select_mxfp4_moe_backend(
def select_deepseek_v4_mxfp4_moe_backend(
config: FusedMoEConfig,
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts] | None]:
"""
Select the MXFP4 MoE backend with MXFP8 activation as top priority.
"""Select the MXFP4 MoE backend with MXFP8 activation as top priority.
Falls back through BF16 and other backends.
"""
activation_format = (
@@ -799,7 +800,6 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
torch.Tensor | None,
]:
"""Convert loaded weights into backend-specific kernel format."""
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
w13_weight,
@@ -111,8 +111,8 @@ def select_mxfp8_moe_backend(
Returns:
A tuple of (fp8_backend, experts_cls).
"""
"""
runner_backend = config.moe_backend
if runner_backend != "auto":
backend = _BACKEND_NAME_MAP.get(runner_backend)
@@ -184,11 +184,9 @@ def select_nvfp4_moe_backend(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> tuple[NvFp4MoeBackend, type[mk.FusedMoEExperts]]:
"""
Select the primary NvFP4 MoE backend
"""Select the primary NvFP4 MoE backend
Note: Shape-specific fallbacks may still occur at runtime.
"""
# NOTE: the kernels are selected in the following order.
# FLASHINFER_B12X is intentionally excluded from auto-selection until
# the upstream CUTLASS SM121 MMA op guard is resolved; use
@@ -46,8 +46,7 @@ class UnquantizedMoeBackend(Enum):
def _get_priority_backends(moe_config: FusedMoEConfig) -> list[UnquantizedMoeBackend]:
"""
Get available backends in priority order based on platform and config.
"""Get available backends in priority order based on platform and config.
This function can be extended to become more complex as needed.
"""
@@ -208,11 +207,9 @@ def _trtllm_bf16_lora_supported(moe_config: FusedMoEConfig) -> bool:
def select_unquantized_moe_backend(
moe_config: FusedMoEConfig,
) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]:
"""
Select the primary Unquantized MoE backend.
"""Select the primary Unquantized MoE backend.
Note: Shape-specific fallbacks may still occur at runtime.
"""
if current_platform.is_tpu():
return UnquantizedMoeBackend.TPU, None
@@ -32,8 +32,7 @@ class W4A8Int8MoeBackend(Enum):
def _get_priority_backends(
moe_config: FusedMoEConfig,
) -> list[W4A8Int8MoeBackend]:
"""
Get available backends in priority order based on platform and config.
"""Get available backends in priority order based on platform and config.
Currently only CPU INT4 backend is available for W4A8 INT8 MoE.
"""
@@ -75,8 +74,7 @@ def select_w4a8_int8_moe_backend(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> tuple[W4A8Int8MoeBackend, type[mk.FusedMoEExperts]]:
"""
Select the primary W4A8 Int8 MoE backend.
"""Select the primary W4A8 Int8 MoE backend.
Args:
config: MoE configuration
@@ -85,8 +83,8 @@ def select_w4a8_int8_moe_backend(
Returns:
Tuple of (backend, kernel_class)
"""
"""
AVAILABLE_BACKENDS = _get_priority_backends(config)
if not AVAILABLE_BACKENDS:
@@ -160,8 +158,7 @@ def select_w4a8_int8_moe_backend(
def make_w4a8_int8_moe_quant_config(
block_shape: tuple[int, int] | None = None,
) -> FusedMoEQuantConfig:
"""
Create FusedMoEQuantConfig for W4A8 Int8 MoE.
"""Create FusedMoEQuantConfig for W4A8 Int8 MoE.
Args:
block_shape: Quantization block shape (row, col).
@@ -170,6 +167,7 @@ def make_w4a8_int8_moe_quant_config(
Returns:
FusedMoEQuantConfig with appropriate settings for W4A8 Int8
"""
# W4A8 Int8 uses static weight quantization, dynamic activation quantization
# Weights are 4-bit (stored as int8, packed to uint8),
@@ -196,8 +194,7 @@ def pack_int4_weights_for_kleidi(
out_features: int,
group_size: int,
) -> torch.Tensor:
"""
Pack INT4 weights (stored as int8 in [-8,7]) to KleidiAI format.
"""Pack INT4 weights (stored as int8 in [-8,7]) to KleidiAI format.
Args:
int4_as_int8: [out, in] int8 tensor with values in [-8, 7]
@@ -209,6 +206,7 @@ def pack_int4_weights_for_kleidi(
Returns:
Packed weight tensor in KleidiAI format
"""
# Shift to unsigned nibble [0, 15]
tmp = int4_as_int8.add(8)
@@ -250,8 +248,7 @@ def convert_to_w4a8_int8_moe_format(
torch.Tensor | None,
torch.Tensor | None,
]:
"""
Pack INT4 MoE weights to KleidiAI format.
"""Pack INT4 MoE weights to KleidiAI format.
This function packs the INT4 weights (stored as int8 values) into
the format expected by the KleidiAI dynamic_4bit_int_moe kernel.
@@ -267,6 +264,7 @@ def convert_to_w4a8_int8_moe_format(
Returns:
Tuple of (w13_packed, w2_packed) tensors
"""
# Derive dimensions from tensor shapes
E = w13_weight.shape[0] # num_experts
@@ -315,8 +313,7 @@ def make_w4a8_int8_moe_kernel(
experts_cls: type[mk.FusedMoEExperts],
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
"""
Create FusedMoEKernel for W4A8 Int8 MoE.
"""Create FusedMoEKernel for W4A8 Int8 MoE.
Args:
moe_quant_config: Quantization configuration
@@ -326,6 +323,7 @@ def make_w4a8_int8_moe_kernel(
Returns:
Configured FusedMoEKernel instance
"""
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -16,8 +16,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""
A reference prepare/finalize class that reorganizes the tokens into
"""A reference prepare/finalize class that reorganizes the tokens into
expert batched format, i.e. E x max_num_tokens x K. This is the format
that the batched dispatch/combine kernels use.
"""
@@ -27,9 +27,7 @@ from vllm.v1.worker.ubatching import (
class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""
Prepare/Finalize using DeepEP High-Throughput kernels.
"""
"""Prepare/Finalize using DeepEP High-Throughput kernels."""
@staticmethod
def maybe_roundup_layer_hidden_size(hidden_size: int, dtype: torch.dtype) -> int:
@@ -34,9 +34,7 @@ logger = init_logger(__name__)
def dequant_fp8(
expert_x_fp8: torch.Tensor, expert_x_scales: torch.Tensor
) -> torch.Tensor:
"""
Return dequantized tensor in fp32
"""
"""Return dequantized tensor in fp32."""
# TODO (varun) : Optimize leverage num_tokens_per_expert counts
assert expert_x_fp8.is_contiguous()
expert_x_scales = expert_x_scales.contiguous()
@@ -50,9 +48,7 @@ def dequant_fp8(
class DeepEPLLPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""
Prepare/Finalize using DeepEP low-latency kernels.
"""
"""Prepare/Finalize using DeepEP low-latency kernels."""
# DeepEP low-latency kernels are compiled only for certain
# specific hidden sizes.
@@ -28,8 +28,7 @@ from vllm.v1.worker.ubatching import (
def _quantize_before_dispatch(
quant_config: FusedMoEQuantConfig, defer_input_quant: bool
) -> bool:
"""
Do quantized dispatch for blockfp8 and mxfp8, unless the
"""Do quantized dispatch for blockfp8 and mxfp8, unless the
subsequent moe kernel requires bf16 inputs.
"""
if defer_input_quant:
@@ -69,8 +68,7 @@ def _unpack_mxfp8_scale(
class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""
Prepare/Finalize using DeepEP v2 ElasticBuffer (unified API).
"""Prepare/Finalize using DeepEP v2 ElasticBuffer (unified API).
Supports two modes controlled by the `use_cudagraph` constructor arg:
@@ -13,9 +13,7 @@ logger = init_logger(__name__)
class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""
Prepare/Finalize using MoRI kernels.
"""
"""Prepare/Finalize using MoRI kernels."""
def __init__(
self,
@@ -60,8 +58,7 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> mk.PrepareResultType:
"""
Returns a tuple of:
"""Returns a tuple of:
- quantized + dispatched a.
- Optional quantized + dispatched a1_scales.
- Optional ExpertTokensMetadata containing gpu/cpu tensors
@@ -69,8 +69,7 @@ def _unwrap_scale_and_prepare_for_moe(
class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular):
"""
Naive Prepare/Finalize for Dp/Ep case for Modular Kernels.
"""Naive Prepare/Finalize for Dp/Ep case for Modular Kernels.
Uses Torch AR/RS or AR for dispatch/combine operations, applied
to the topk weights and ids.
@@ -121,7 +120,6 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
defer_input_quant: bool = False,
) -> mk.PrepareResultType:
"""Quantize and Dispatch Topk Weights and Topk Ids."""
if apply_router_weight_on_input:
topk = topk_ids.size(1)
assert topk == 1, (
@@ -210,8 +208,7 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):
"""
Naive Prepare/Finalize for Dp/Ep case for Modular Kernels.
"""Naive Prepare/Finalize for Dp/Ep case for Modular Kernels.
Uses Torch AR/RS or AR for dispatch/combine operations, applied
to the router logits (the MoE kernel runs the router internally).
@@ -250,7 +247,6 @@ class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMono
defer_input_quant: bool = False,
) -> mk.PrepareMonolithicResultType:
"""Quantize and Dispatch Router Logits."""
a1q, scales, a1q_scale_orig = _quantize_and_setup_dispatch(
a1, quant_config, defer_input_quant
)
@@ -33,9 +33,7 @@ assert isinstance(NIXL_EP_TOPK_INDICES_DTYPE, torch.dtype)
def dequant_fp8(
expert_x_fp8: torch.Tensor, expert_x_scales: torch.Tensor
) -> torch.Tensor:
"""
Return dequantized tensor in fp32
"""
"""Return dequantized tensor in fp32."""
assert expert_x_fp8.is_contiguous()
expert_x_scales = expert_x_scales.contiguous()
num_experts = expert_x_fp8.size(0)
@@ -48,9 +46,7 @@ def dequant_fp8(
class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""
Prepare/Finalize using NIXL EP kernels.
"""
"""Prepare/Finalize using NIXL EP kernels."""
# NIXL EP kernels are compiled only for certain specific hidden sizes.
# NOTE: Keep this list sorted, maybe_roundup_layer_hidden_size depends
@@ -44,8 +44,7 @@ class FusedMoeWeightScaleSupported(Enum):
@PluggableLayer.register("routed_experts")
class RoutedExperts(PluggableLayer):
"""
Container for routed expert weights and execution logic.
"""Container for routed expert weights and execution logic.
This module owns the expert weight parameters (w13_weight, w2_weight, scales, etc.)
and handles:
@@ -188,8 +187,7 @@ class RoutedExperts(PluggableLayer):
quant_config: QuantizationConfig | None,
moe_config: FusedMoEConfig,
) -> FusedMoEMethodBase:
"""
Helper method to ensure quant_method is never None and
"""Helper method to ensure quant_method is never None and
of the proper type.
"""
quant_method = None
@@ -318,8 +316,7 @@ class RoutedExperts(PluggableLayer):
param: torch.Tensor,
tp_rank: int,
):
"""
Load w13 weight scales assuming that w1 weight scales and w3 weight
"""Load w13 weight scales assuming that w1 weight scales and w3 weight
scales are stored in the same loaded_weight tensor.
"""
shard_size = param.shape[shard_dim]
@@ -336,8 +333,7 @@ class RoutedExperts(PluggableLayer):
loaded_weight: torch.Tensor,
tp_rank: int,
):
"""
Load grouped weight scales for group quantization or model weights
"""Load grouped weight scales for group quantization or model weights
Args:
shard_dim: dimension to shard
@@ -345,6 +341,7 @@ class RoutedExperts(PluggableLayer):
shard_id: either w1, w2, or w3
loaded_weight: checkpoint weight to load into the param
tp_rank: tensor parallel rank
"""
if shard_id == "w2":
self._load_w2(
@@ -444,6 +441,7 @@ class RoutedExperts(PluggableLayer):
Must be non-negative.
shard_dim: The dimension index corresponding to the shard
(intermediate) dimension. Defaults to `None`.
"""
dims = (hidden_dim,) if shard_dim is None else (hidden_dim, shard_dim)
if loaded_weight.ndim > 0:
@@ -990,8 +988,7 @@ class RoutedExperts(PluggableLayer):
lora_base_layer_prefix_on_param_name: str = "",
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
"""
Create expert parameter mapping for weight loading with redundant experts.
"""Create expert parameter mapping for weight loading with redundant experts.
This mapping handles the physical-to-logical expert ID conversion needed
when loading weights with EPLB redundant experts.
@@ -1011,6 +1008,7 @@ class RoutedExperts(PluggableLayer):
``make_expert_params_mapping`` indexes the model-wide
``params_dict`` (prefix included).
include_fused: Prepend the fused pre-fused-checkpoint entries
routed_experts_prefix: Prefix of the routed experts submodule
Returns:
List of tuples (param_name, weight_name, expert_id, shard_id)
@@ -1019,6 +1017,7 @@ class RoutedExperts(PluggableLayer):
- weight_name: Weight name in checkpoint
- expert_id: Physical expert ID
- shard_id: Shard identifier (w1, w2, w3)
"""
num_physical_experts = num_experts + num_redundant_experts
@@ -1097,8 +1096,7 @@ class RoutedExperts(PluggableLayer):
def _maybe_make_contiguous(
name: str, p: torch.nn.Parameter
) -> torch.nn.Parameter:
"""
In some cases, the last 2 dimensions (the non-expert dimensions)
"""In some cases, the last 2 dimensions (the non-expert dimensions)
of the weight scale tensor are transposed. This function
transforms the tensor (view update) so the tensor is contiguous().
Example: A non-contiguous scale tensor,
@@ -1178,8 +1176,7 @@ class RoutedExperts(PluggableLayer):
shared_experts: "SharedExperts | None" = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Execute routed experts using the quantization method's apply function.
"""Execute routed experts using the quantization method's apply function.
This is called by the runner after router selection (for modular kernels)
quant_method.apply() which accesses the weights on this RoutedExperts
@@ -1194,6 +1191,7 @@ class RoutedExperts(PluggableLayer):
Returns:
Output tensor from routed experts.
"""
assert not self.quant_method.is_monolithic
@@ -1213,8 +1211,7 @@ class RoutedExperts(PluggableLayer):
router_logits: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""
Execute routed experts using the quantization method's apply function.
"""Execute routed experts using the quantization method's apply function.
This is called by the runner after router selection (for modular kernels)
or with router logits (for monolithic kernels). It delegates to
@@ -1228,6 +1225,7 @@ class RoutedExperts(PluggableLayer):
Returns:
Finalized routed states or a deferred-finalize output.
"""
assert self.quant_method.is_monolithic
@@ -136,8 +136,8 @@ class RoutedExpertsCapturer:
Args:
layer_id: The layer index.
topk_ids: Tensor of shape (batch_size, num_routed_experts).
"""
"""
ctx = get_forward_context()
if ctx.dp_metadata is None: # single dp
start_loc = 0
@@ -241,6 +241,7 @@ class RoutedExpertsCapturer:
slot_mappings: Per-KV-cache-group slot mappings for this step,
shape ``(num_kv_cache_groups, max_num_batched_tokens)``.
num_tokens: Total number of tokens scheduled in this step.
"""
return RoutedExpertsTensors(
routing_data=self.device_buffer[:num_tokens].clone(),
@@ -417,6 +418,7 @@ class RoutedExpertsManager:
Returns:
Array of shape (num_tokens - token_start, num_layers,
num_experts_per_tok).
"""
bs = self.block_size
block_ids_array = np.array(block_ids, dtype=np.int32)
@@ -16,8 +16,7 @@ from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
class AiterSharedRoutedFusedMoERouter(BaseRouter):
"""
ROCm AITER router for models with fused shared experts (e.g. Qwen3-MoE).
"""ROCm AITER router for models with fused shared experts (e.g. Qwen3-MoE).
When the AITER topk_softmax kernel supports sigmoid fusion, the routing
softmax and shared-expert sigmoid are computed in a single kernel launch.
@@ -157,8 +157,7 @@ else:
class BaseRouter(FusedMoERouter):
"""
Base router class that provides common functionality for all router implementations.
"""Base class providing common functionality for all router implementations.
This class implements the template method pattern where select_experts() handles
common pre-processing and post-processing, delegating the actual routing logic
@@ -171,11 +170,11 @@ class BaseRouter(FusedMoERouter):
global_num_experts: int,
eplb_state: EplbLayerState | None = None,
):
"""
Args:
top_k: Number of experts to select per token
global_num_experts: Total number of experts
eplb_state: Optional EPLBLayerState for load balancing
"""Args:
top_k: Number of experts to select per token
global_num_experts: Total number of experts
eplb_state: Optional EPLBLayerState for load balancing
"""
super().__init__(eplb_state=eplb_state)
self.top_k = top_k
@@ -241,8 +240,7 @@ class BaseRouter(FusedMoERouter):
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Compute the actual routing logic.
"""Compute the actual routing logic.
This method must be implemented by subclasses to provide the specific
routing algorithm (e.g., grouped_topk, fused_topk, custom routing, etc.).
@@ -251,9 +249,11 @@ class BaseRouter(FusedMoERouter):
hidden_states: Input hidden states
router_logits: Router logits for expert selection
indices_type: Desired dtype for expert indices (may be None)
input_ids: Token ids, for routers that condition on them
Returns:
tuple of (topk_weights, topk_ids)
"""
raise NotImplementedError
@@ -265,8 +265,7 @@ class BaseRouter(FusedMoERouter):
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Route the input hidden states to the top-k experts based on the
"""Route the input hidden states to the top-k experts based on the
router logits.
This method implements the template method pattern:
@@ -283,6 +282,7 @@ class BaseRouter(FusedMoERouter):
**Compatibility**: When EPLB is not enabled, the returned ids are
equivalent to global logical ids, so should be compatible with
plain MoE implementations without redundant experts.
"""
# Step 1: Validate EPLB state
self._validate_eplb_state()
@@ -10,8 +10,7 @@ from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
class FusedMoERouter(ABC):
"""
FusedMoERouter is an abstract class that provides a 'select_experts'
"""FusedMoERouter is an abstract class that provides a 'select_experts'
method that is used for routing hidden states based on router logits.
"""
@@ -54,8 +53,7 @@ class FusedMoERouter(ABC):
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Route the input hidden states to the top-k experts based on the
"""Route the input hidden states to the top-k experts based on the
router logits.
Returns:
@@ -66,8 +64,8 @@ class FusedMoERouter(ABC):
**Compatibility**: When EPLB is not enabled, the returned ids are
equivalent to global logical ids, so should be compatible with
plain MoE implementations without redundant experts.
"""
"""
topk_weights, topk_ids = self._select_experts(
hidden_states,
router_logits,
@@ -217,8 +217,7 @@ def fp32_router_gemm_dispatch_impl(
weight: torch.Tensor,
allow_bf16x3_router_gemm: bool,
) -> torch.Tensor:
"""
Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
otherwise optionally run the experimental BF16x3 kernel for medium/large
SM100 router batches, then fall back to F.linear.
This must be wrapped in a custom op because our torch.compile integration
@@ -66,8 +66,7 @@ def create_fused_moe_router(
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
) -> FusedMoERouter:
"""
Factory function to create the appropriate FusedMoERouter subclass based on
"""Factory function to create the appropriate FusedMoERouter subclass based on
the provided parameters.
The selection logic follows this priority order:
@@ -114,8 +113,8 @@ def create_fused_moe_router(
Returns:
An instance of the appropriate FusedMoERouter subclass
"""
"""
routing_strategy = envs.VLLM_MOE_ROUTING_SIMULATION_STRATEGY
if routing_strategy != "":
return RoutingSimulatorRouter(
@@ -25,8 +25,7 @@ class RoutingStrategy(ABC):
top_k: int,
indices_type: torch.dtype | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Route tokens to experts.
"""Route tokens to experts.
Args:
hidden_states: Input hidden states [num_tokens, hidden_size]
@@ -36,13 +35,13 @@ class RoutingStrategy(ABC):
Returns:
tuple of (topk_weights, topk_ids)
"""
pass
class DistributionBasedRouting(RoutingStrategy):
"""
Distribution-based random routing strategy with configurable distributions.
"""Distribution-based random routing strategy with configurable distributions.
This routing strategy randomly selects experts for each token based on
different probability distributions. Currently supports uniform and normal
@@ -50,8 +49,7 @@ class DistributionBasedRouting(RoutingStrategy):
"""
def __init__(self, distribution: str = "uniform", **distribution_params: Any):
"""
Initialize distribution-based routing.
"""Initialize distribution-based routing.
Args:
distribution: Type of distribution to use for sampling
@@ -61,6 +59,7 @@ class DistributionBasedRouting(RoutingStrategy):
chosen distribution
For "uniform": No additional parameters needed
For "normal": mean (default: 0.0), std (default: 1.0)
"""
self.distribution = distribution.lower()
self.distribution_params = distribution_params
@@ -90,8 +89,7 @@ class DistributionBasedRouting(RoutingStrategy):
top_k: int,
indices_type: torch.dtype | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Randomly select experts for each token using the specified distribution.
"""Randomly select experts for each token using the specified distribution.
Args:
hidden_states: Input hidden states [num_tokens, hidden_size]
@@ -103,6 +101,7 @@ class DistributionBasedRouting(RoutingStrategy):
tuple of (topk_weights, topk_ids) where:
- topk_weights: Weights based on distribution sampling
- topk_ids: Expert indices sampled from the distribution
"""
num_tokens = hidden_states.shape[0]
num_experts = router_logits.shape[-1]
@@ -129,7 +128,6 @@ class DistributionBasedRouting(RoutingStrategy):
indices_type: torch.dtype,
) -> torch.Tensor:
"""Sample expert IDs based on the specified distribution."""
if self.distribution == "uniform":
# Generate random scores, and take the top-k to avoid duplicate topk_ids
scores = torch.rand(num_tokens, num_experts, device=device)
@@ -218,8 +216,7 @@ class DistributionBasedRouting(RoutingStrategy):
class RoutingSimulator:
"""
Token-to-Expert Routing Simulator.
"""Token-to-Expert Routing Simulator.
This class provides a framework for testing and comparing different
routing strategies for MoE models. It can simulate routing behavior
@@ -239,22 +236,22 @@ class RoutingSimulator:
@classmethod
def register_strategy(cls, name: str, strategy: RoutingStrategy):
"""
Register a custom routing strategy.
"""Register a custom routing strategy.
Args:
name: Name of the strategy
strategy: RoutingStrategy instance
"""
cls._routing_strategies[name] = strategy
@classmethod
def get_available_strategies(cls) -> list[str]:
"""
Get list of available routing strategy names.
"""Get list of available routing strategy names.
Returns:
List of available strategy names
"""
return list(cls._routing_strategies.keys())
@@ -266,8 +263,7 @@ class RoutingSimulator:
top_k: int,
indices_type: torch.dtype | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Simulate token-to-expert routing using the specified strategy.
"""Simulate token-to-expert routing using the specified strategy.
Args:
hidden_states: Input hidden states [num_tokens, hidden_size]
@@ -278,6 +274,7 @@ class RoutingSimulator:
Returns:
tuple of (topk_weights, topk_ids)
"""
if strategy_name not in RoutingSimulator._routing_strategies:
raise ValueError(
@@ -71,7 +71,8 @@ class ZeroExpertRouter(BaseRouter):
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compute routing with full bias, compute zero expert output,
mask zero expert IDs."""
mask zero expert IDs.
"""
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=router_logits,
@@ -225,8 +225,7 @@ def _unpack(
class MoERunner(MoERunnerInterface):
"""
Standard MoE runner implementation for executing Mixture of Experts layers.
"""Standard MoE runner implementation for executing Mixture of Experts layers.
This is the primary concrete implementation of MoE execution logic, providing
comprehensive support for standard MoE operations. It handles:
@@ -704,7 +703,6 @@ class MoERunner(MoERunnerInterface):
1. pytorch cannot handle union types in custom op signatures so
_moe_forward and _moe_forward_shared must be split.
"""
# Apply transform for routed experts (e.g., latent projection for
# latent MoE). When the caller pre-applies the routed input transform
# outside the runner (e.g. to overlap it on a separate stream), it
@@ -1019,8 +1017,7 @@ class MoERunner(MoERunnerInterface):
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
"""
Register the EPLB state in this layer.
"""Register the EPLB state in this layer.
This is used later in forward pass, where we get the expert mapping
and record the load metrics in `expert_load_view`.
@@ -17,8 +17,7 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
class MoERunnerInterface(PluggableLayer, ABC):
"""
Abstract base class for Mixture of Experts (MoE) runners.
"""Abstract base class for Mixture of Experts (MoE) runners.
This class defines the interface that all MoE runner implementations must follow.
MoE runners are responsible for executing the forward pass of MoE layers, handling
@@ -9,8 +9,7 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
class TopKWeightAndReduceDelegate(mk.TopKWeightAndReduce):
"""
Useful in the case when some FusedMoEExpertsModular
"""Useful in the case when some FusedMoEExpertsModular
implementation does not perform weight application and reduction
but cannot address the needs of all the compatible PrepareAndFinalize
implementations.
@@ -42,8 +41,7 @@ class TopKWeightAndReduceDelegate(mk.TopKWeightAndReduce):
class TopKWeightAndReduceNoOP(mk.TopKWeightAndReduce):
"""
The fused_experts outputs have already been weight applied and reduced.
"""The fused_experts outputs have already been weight applied and reduced.
This implementation is a no-op.
"""
@@ -78,8 +76,7 @@ class TopKWeightAndReduceNoOP(mk.TopKWeightAndReduce):
class TopKWeightAndReduceContiguous(mk.TopKWeightAndReduce):
"""
TopKWeightAndReduce implementation for a fused_experts output
"""TopKWeightAndReduce implementation for a fused_experts output
of shape (m, topk, K)
"""
@@ -122,8 +119,7 @@ class TopKWeightAndReduceContiguous(mk.TopKWeightAndReduce):
class TopKWeightAndReduceNaiveBatched(mk.TopKWeightAndReduce):
"""
TopKWeightAndReduce implementation for a fused_experts output
"""TopKWeightAndReduce implementation for a fused_experts output
of shape (num_experts, batch_size, K)
"""
+12 -16
View File
@@ -69,6 +69,7 @@ def resolve_layer_fused_shared_expert(
Raises:
ValueError: If requested shared-expert fusion is quantization-incompatible.
"""
# NOTE: is_fusion_moe_shared_experts_enabled is decorated with @if_aiter_supported
# that returns None if AITER is not available.
@@ -164,20 +165,19 @@ def _count_expert_num_tokens(
def count_expert_num_tokens(
topk_ids: torch.Tensor, num_local_experts: int, expert_map: torch.Tensor | None
) -> torch.Tensor:
"""
Count the number to tokens assigned to each expert.
"""Count the number to tokens assigned to each expert.
Parameters:
- topk_ids (torch.Tensor): Tensor mapping each token to its
list of experts.
- num_local_experts (int): Number of experts in this rank.
- expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
from the global expert space to the local expert space of the expert
parallel shard.
Args:
topk_ids (torch.Tensor): Tensor mapping each token to its list of experts.
num_local_experts (int): Number of experts in this rank.
expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
from the global expert space to the local expert space of the expert
parallel shard.
Returns:
A tensor of size num_local_experts, where tensor[i] holds the number
of tokens assigned to the ith expert.
"""
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
expert_num_tokens = torch.empty(
@@ -202,8 +202,7 @@ def count_expert_num_tokens(
def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
"""
Shrink the given tensor and apply the given view to it. This is
"""Shrink the given tensor and apply the given view to it. This is
used to resize the intermediate fused_moe caches.
"""
assert prod(v) <= x.numel(), (
@@ -226,8 +225,7 @@ def _fp8_quantize(
per_act_token: bool,
block_shape: list[int] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Perform fp8 quantization on the inputs. If a block_shape
"""Perform fp8 quantization on the inputs. If a block_shape
is provided, the output will be blocked.
"""
if block_shape is None:
@@ -252,11 +250,9 @@ def _int8_quantize(
per_act_token: bool,
block_shape: list[int] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Perform int8 quantization on the inputs. If a block_shape
"""Perform int8 quantization on the inputs. If a block_shape
is provided, the output will be blocked.
"""
# If weights are per-channel (per_channel_quant=True), then
# activations apply per-token quantization. Otherwise, assume
# activation tensor-wise fp8/int8 quantization, dynamic or static
@@ -170,6 +170,7 @@ def fused_qk_rmsnorm_rope_gate(
Returns:
(q_out, k_out, gate_out) -- all contiguous (n_tokens, heads * head_dim).
``gate_out`` is the raw (pre-sigmoid) gate.
"""
if rotary_dim <= 0 or rotary_dim > head_dim or rotary_dim % 2 != 0:
raise ValueError(
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Producer side of the QuantizedActivation contract for activation layers.
"""Producer side of the QuantizedActivation contract for activation layers.
Given an activation module and the downstream linear it feeds, fuse the
activation with that linear's input quantization into a single kernel when the
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
A QuantizedActivation is a pre-quantized activation produced by a fused kernel
"""A QuantizedActivation is a pre-quantized activation produced by a fused kernel
and consumed directly by a linear layer, letting the layer skip its own input
quantization. A linear advertises the key its kernel can consume via
expose_input_quant_key; the kernel validates and reads the activation via
@@ -107,6 +107,7 @@ def hpc_gated_mla_gemm(
Returns:
``[m, n]`` bfloat16.
"""
from hpc.gemm import gated_mla_gemm
@@ -122,6 +122,7 @@ class HpcIHCPre(HpcModule):
When given, its weight/eps are folded into the kernel so a single
launch covers pre + RMSNorm (the caller then skips the separate
layernorm). None keeps the original pre-only behaviour.
"""
def __init__(
+7 -3
View File
@@ -87,6 +87,7 @@ def deep_select_topk(
Returns:
The (num_rows, topk) indices tensor.
"""
assert input.dim() == 2 and input.stride(1) == 1
assert (
@@ -162,7 +163,8 @@ class SparseIndexerTopk(torch.nn.Module):
) -> str:
"""Resolve the decode top-k implementation from the configured
backend ("auto" = the pre-existing chain, or a validated explicit
value)."""
value).
"""
if self._backend == "auto":
return self._resolve_auto(logits, topk_tokens, num_rows)
@@ -209,7 +211,8 @@ class SparseIndexerTopk(torch.nn.Module):
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""The pre-existing priority chain: cooperative -> persistent ->
per_row. deep_select/flashinfer/torch are opt-in only."""
per_row. deep_select/flashinfer/torch are opt-in only.
"""
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
return "cooperative"
if self._is_cuda and topk_tokens in (512, 1024, 2048):
@@ -271,7 +274,8 @@ class SparseIndexerTopk(torch.nn.Module):
max_seq_len: int,
) -> None:
"""Run the resolved decode top-k implementation, writing into
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
"""
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
if backend == "deep_select":
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
+2 -3
View File
@@ -231,6 +231,7 @@ class RMSNormGated(CustomOp):
device: Device to create parameters on
dtype: Data type for parameters
activation: Activation function name for gating
"""
factory_kwargs = {"device": device, "dtype": dtype}
super().__init__()
@@ -335,9 +336,7 @@ class RMSNormGated(CustomOp):
class LayerNorm(nn.Module):
"""
Layer Normalization.
"""
"""Layer Normalization."""
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
+5 -6
View File
@@ -541,8 +541,7 @@ def lightning_attention(
block_size: int = 256,
kv_history: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Apply lightning attention algorithm
"""Apply lightning attention algorithm
to compute attention efficiently.
Args:
@@ -556,6 +555,7 @@ def lightning_attention(
Returns:
output: Attention output
kv: Updated key-value history
"""
d = q.shape[-1]
e = v.shape[-1]
@@ -610,8 +610,7 @@ def _linear_attn_decode_kernel(
pad_slot_id: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""
Kernel for linear attention decoding with KV cache.
"""Kernel for linear attention decoding with KV cache.
This kernel computes attention for a single token using the KV cache.
"""
@@ -683,8 +682,7 @@ def linear_decode_forward_triton(
slot_idx: torch.Tensor,
BLOCK_SIZE: int = 32,
) -> torch.Tensor:
"""
Perform linear attention decoding using Triton kernels.
"""Perform linear attention decoding using Triton kernels.
Args:
q: Query tensor of shape [B, H, 1, D]
@@ -697,6 +695,7 @@ def linear_decode_forward_triton(
Returns:
output: Attention output tensor
"""
B, H, _, D = q.shape
assert k.shape == (B, H, 1, D)
+12 -6
View File
@@ -146,6 +146,7 @@ class LinearMethodBase(QuantizeMethodBase):
input_size: Size of the input dim of the weight across all ranks.
output_size: Size of the output dim of the weight across all ranks.
params_dtype: Datatype of the parameters.
"""
raise NotImplementedError
@@ -157,7 +158,8 @@ class LinearMethodBase(QuantizeMethodBase):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply the weights in layer to the input tensor.
Expects create_weights to have been called before on the layer."""
Expects create_weights to have been called before on the layer.
"""
raise NotImplementedError
@@ -250,6 +252,7 @@ class LinearBase(PluggableLayer):
prefix: Prefix for parameter names.
return_bias: If true, return bias together with outputs in forward pass.
disable_tp: If true, tensor parallelism will be disabled for this layer.
"""
requires_unquantized_input = False
@@ -335,6 +338,7 @@ class ReplicatedLinear(LinearBase):
(e.g. model.layers.0.qkv_proj)
return_bias: If true, return bias together with outputs in forward pass.
disable_tp: Take no effect for replicated linear layers.
"""
# --8<-- [end:replicated_linear]
@@ -454,6 +458,7 @@ class ColumnParallelLinear(LinearBase):
shard per rank (see ``DCPGroupColumnParallelLinear``).
tp_size: Override the tensor-parallel world size used for sharding.
Defaults to the global TP world size.
"""
# --8<-- [end:column_parallel_linear]
@@ -690,6 +695,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
return_bias: If true, return bias together with outputs in forward pass.
disable_tp: If true, all weights matrix won't be sharded, this layer
will be treated as a "Replicated" MergedLinear.
"""
def __init__(
@@ -867,8 +873,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
loaded_weight: torch.Tensor,
output_sizes: list[int] | None = None,
):
"""
Handle special case for models where MLP layers are already
"""Handle special case for models where MLP layers are already
fused on disk. In this case, we have no shard id. This function
determines the shard id by splitting these layers and then calls
the weight loader using the shard id.
@@ -876,7 +881,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
An example of a model with these fused layers:
https://huggingface.co/microsoft/Phi-3-mini-4k-instruct
"""
current_shard_offset = 0
shard_offsets: list[tuple[int, int, int]] = []
output_sizes = output_sizes or self.output_sizes
@@ -1018,6 +1022,7 @@ class QKVParallelLinear(ColumnParallelLinear):
(e.g. model.layers.0.qkv_proj)
return_bias: If true, return bias together with outputs in forward pass.
disable_tp: If true, weights matrix won't be sharded through tp rank.
"""
def __init__(
@@ -1102,8 +1107,7 @@ class QKVParallelLinear(ColumnParallelLinear):
def _load_fused_module_from_checkpoint(
self, param: BasevLLMParameter, loaded_weight: torch.Tensor
):
"""
Handle special case for models where QKV layers are already
"""Handle special case for models where QKV layers are already
fused on disk. In this case, we have no shard id. This function
determines the shard id by splitting these layers and then calls
the weight loader using the shard id.
@@ -1632,6 +1636,7 @@ class RowParallelLinear(LinearBase):
| . |
| A_p |
- -
Arguments:
input_size: first dimension of matrix A.
output_size: second dimension of matrix A.
@@ -1651,6 +1656,7 @@ class RowParallelLinear(LinearBase):
(e.g. model.layers.0.down_proj)
return_bias: If true, return bias together with outputs in forward pass.
disable_tp: If true, weights matrix won't be sharded through tp rank.
"""
# --8<-- [end:row_parallel_linear]
@@ -74,9 +74,9 @@ class LogitsProcessor(PluggableLayer):
logits_as_input: bool = False,
soft_cap: float | None = None,
) -> None:
"""
Args:
scale: A scaling factor to apply to the logits.
"""Args:
scale: A scaling factor to apply to the logits.
"""
super().__init__()
self.scale = scale
+2 -4
View File
@@ -16,8 +16,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
class MambaBase(AttentionLayerBase):
"""
Base class for Mamba-like layers which support the v1 engine.
"""Base class for Mamba-like layers which support the v1 engine.
Inherit from this class if you implement a custom layer.
"""
@@ -44,8 +43,7 @@ class MambaBase(AttentionLayerBase):
@abstractmethod
def get_state_shape(self) -> Iterable[tuple[int, ...]]:
"""
Defines the shape of the state.
"""Defines the shape of the state.
For mamba layers this is usually a (conv_state, ssm_state) tuple.
In this case, returns (conv_state_shape, ssm_state_shape).
"""
@@ -45,8 +45,7 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
@PluggableLayer.register("olmo_hybrid_gated_delta_net_attention")
class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
"""
Gated DeltaNet linear attention layer for OLMo Hybrid.
"""Gated DeltaNet linear attention layer for OLMo Hybrid.
This implements the linear attention mechanism that replaces sliding window
attention in the hybrid architecture.
@@ -294,9 +293,7 @@ class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
a: torch.Tensor,
core_attn_out: torch.Tensor,
):
"""
Core attention computation (called by custom op).
"""
"""Core attention computation (called by custom op)."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
@@ -639,9 +639,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_qkvz: torch.Tensor,
mixed_ba: torch.Tensor,
):
"""
Derives `query`, `key` and `value` tensors from `mixed_qkvzba`.
"""
"""Derives `query`, `key` and `value` tensors from `mixed_qkvzba`."""
new_tensor_shape_qkvz = mixed_qkvz.size()[:-1] + (
self.num_k_heads // self.tp_size,
(
@@ -692,8 +690,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_ba: torch.Tensor,
num_tokens: int,
):
"""
Derives mixed_qkv, z, b, a from projected qkvz/ba for the GDN custom op.
"""Derives mixed_qkv, z, b, a from projected qkvz/ba for the GDN custom op.
For gqa_interleaved_layout (Qwen3-Next): unpack the interleaved
[ng, (hk + hk + np/ng*hv + np/ng*hv)] layout into contiguous qkv.
@@ -864,7 +861,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
hidden_states: torch.Tensor,
) -> torch.Tensor:
"""ROCm forward using AITER Triton fused projection+attention when
available, otherwise falling back to the generic CUDA path."""
available, otherwise falling back to the generic CUDA path.
"""
if GDN_AITER_TRITON_AVAILABLE:
num_tokens = hidden_states.size(0)
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
@@ -899,8 +897,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
"""
Forward pass with three parts:
"""Forward pass with three parts:
1. Input projection
2. Core attention (custom op)
3. Output projection
@@ -977,8 +974,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
"""
Forward pass with three parts:
"""Forward pass with three parts:
1. Input projection
2. Core attention (custom op)
3. Output projection
@@ -1221,6 +1217,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
z_out: **output** buffer for z (num_tokens, num_heads,
head_dim); mutated in-place.
core_attn_out: Pre-allocated output buffer for attention results.
"""
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
@@ -1279,6 +1276,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
b: beta gating vector (num_tokens, num_heads)
a: alpha gating vector (num_tokens, num_heads)
core_attn_out: Pre-allocated output buffer for attention results.
"""
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
@@ -1658,9 +1656,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
core_attn_out: torch.Tensor,
attn_metadata: GDNAttentionMetadata,
):
"""
Core attention computation with a packed non-spec decode fast path.
"""
"""Core attention computation with a packed non-spec decode fast path."""
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor # noqa: E501
self_kv_cache = self.kv_cache
# conv_state must be (..., dim, width-1) for the conv kernels.
@@ -2030,8 +2026,7 @@ def fused_gdn_gating(
beta: float = 1.0,
threshold: float = 20.0,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Fused computation of g and beta for Gated Delta Net.
"""Fused computation of g and beta for Gated Delta Net.
g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
beta_output = b.sigmoid()
TODO maybe use torch.compile to replace this triton kernel
@@ -523,7 +523,7 @@ class BailingMoELinearAttention(LinearAttention):
output: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""Forward method called by torch.ops.vllm.linear_attention"""
"""Forward method called by torch.ops.vllm.linear_attention."""
torch.ops.vllm.linear_attention(
hidden_states,
output,
@@ -49,8 +49,7 @@ from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
# --8<-- [start:mamba_mixer]
@PluggableLayer.register("mamba_mixer")
class MambaMixer(MambaBase, PluggableLayer):
"""
Compute ∆, A, B, C, and D the state space parameters and compute
"""Compute ∆, A, B, C, and D the state space parameters and compute
the `contextualized_states`. A, D are input independent
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
for why A isn't selective) ∆, B, C are input-dependent
@@ -237,8 +236,7 @@ class MambaMixer(MambaBase, PluggableLayer):
)
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
"""
Run the Mamba-1 SSM pipeline.
"""Run the Mamba-1 SSM pipeline.
Steps
-----
@@ -258,7 +256,6 @@ class MambaMixer(MambaBase, PluggableLayer):
decode tokens), both sets of kernels are executed independently
and their outputs are concatenated before the final output projection.
"""
forward_context: ForwardContext = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
@@ -252,8 +252,7 @@ def mamba_v2_sharded_weight_loader(
# --8<-- [start:mamba_mixer2]
@PluggableLayer.register("mamba_mixer2")
class MambaMixer2(MambaBase, PluggableLayer):
"""
Compute ∆, A, B, C, and D the state space parameters and compute
"""Compute ∆, A, B, C, and D the state space parameters and compute
the `contextualized_states`. A, D are input independent
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
for why A isn't selective) ∆, B, C are input-dependent
@@ -1221,7 +1220,6 @@ def share_replayssm_ring_trackers(
The final local layer in each group advances its cursors after every layer
in that group has consumed the previous values.
"""
replayssm_mixers: dict[str, MambaMixer2] = {}
for layer_name in ordered_layer_names:
layer = forward_context[layer_name]
@@ -261,8 +261,8 @@ class MambaStateShapeCalculator:
@classmethod
def extra_groups_for_head_shards(cls, ngroups: int, tp_size: int):
"""Compute the increase in group numbers to account for
replication in order to accompany the head shards."""
replication in order to accompany the head shards.
"""
# in the case ngoups % tp_size == 0, this will be zero
if ngroups % tp_size == 0:
return 0
@@ -344,13 +344,13 @@ class MambaStateShapeCalculator:
@dataclass
class MambaCopySpec:
"""
Data class specifying the memory-copy parameters for Mamba states used for
"""Data class specifying the memory-copy parameters for Mamba states used for
prefix caching in align mode.
Attributes:
start_addr (int): Starting address for the memory copy operation.
num_elements (int): Number of elements to copy from the starting address.
"""
start_addr: int
@@ -497,7 +497,7 @@ def causal_conv1d_fn(
metadata=None,
validate_data=False,
):
"""support varlen + continuous batching when x is 2D tensor
"""Support varlen + continuous batching when x is 2D tensor.
x: (dim,cu_seq_len)
cu_seq_len = total tokens of all seqs in that batch
@@ -1109,8 +1109,7 @@ def causal_conv1d_update(
validate_data=False,
out: torch.Tensor | None = None,
):
"""
x: Input tensor which can take the following shapes:
"""x: Input tensor which can take the following shapes:
- `[batch, dim]` - single token prediction
- `[batch, dim, seqlen]` - single or multiple tokens prediction
@@ -123,8 +123,7 @@ def causal_conv1d_update_torch(
bias: torch.Tensor | None = None,
activation: str | None = None,
) -> torch.Tensor:
"""
Pure PyTorch fallback for causal_conv1d_update.
"""Pure PyTorch fallback for causal_conv1d_update.
Currently used as a fallback for Arm (aarch64) to leverage
oneDNN/ACL F.conv1d kernels for batched decoding.
"""

Some files were not shown because too many files have changed in this diff Show More