Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -479,7 +479,6 @@ class SwigluOAIAndMul(CustomOp):
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""PyTorch-native implementation equivalent to forward()."""
|
||||
|
||||
gate, up = x[..., ::2], x[..., 1::2]
|
||||
gate = gate.clamp(min=None, max=self.limit)
|
||||
up = up.clamp(min=-self.limit, max=self.limit)
|
||||
@@ -647,9 +646,7 @@ class QuickGELU(CustomOp):
|
||||
# --8<-- [start:relu2]
|
||||
@CustomOp.register("relu2")
|
||||
class ReLUSquaredActivation(CustomOp):
|
||||
"""
|
||||
Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
|
||||
"""
|
||||
"""Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2."""
|
||||
|
||||
# --8<-- [end:relu2]
|
||||
|
||||
@@ -676,8 +673,7 @@ class ReLUSquaredActivation(CustomOp):
|
||||
# --8<-- [start:xielu]
|
||||
@CustomOp.register("xielu")
|
||||
class XIELU(CustomOp):
|
||||
"""
|
||||
Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010
|
||||
"""Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010
|
||||
If the user has installed the nickjbrowning/XIELU, we import xIELU CUDA
|
||||
Otherwise, we emit a single warning and use xIELU Python
|
||||
"""
|
||||
@@ -747,7 +743,7 @@ class XIELU(CustomOp):
|
||||
)
|
||||
|
||||
def _xielu_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Firewall function to prevent torch.compile from seeing .item()"""
|
||||
"""Firewall function to prevent torch.compile from seeing .item()."""
|
||||
assert self._xielu_cuda_obj is not None, "XIELU CUDA object must not be None"
|
||||
original_shape = x.shape
|
||||
# CUDA kernel expects 3D tensors, reshape if needed
|
||||
@@ -808,7 +804,8 @@ _ACTIVATION_REGISTRY = LazyDict(
|
||||
|
||||
def _get_gelu_pytorch_tanh() -> nn.Module:
|
||||
"""Get PyTorch GELU with tanh approximation, with ROCm fallback
|
||||
and fast GELU for ARM."""
|
||||
and fast GELU for ARM.
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
# TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile
|
||||
logger.warning_once(
|
||||
|
||||
@@ -166,8 +166,8 @@ def _init_kv_cache_quant(
|
||||
layer: The attention layer instance to initialize.
|
||||
quant_config: Optional quantization configuration.
|
||||
prefix: Layer name prefix for quantization method lookup.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Note [Register q/k/v/prob scales in state dict]
|
||||
# When calling model.to(device), only parameters/buffers in state dict are
|
||||
# moved. If not registering q/k/v/prob scales in state dict, there would
|
||||
@@ -254,8 +254,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
head_size_v: int | None = None,
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
"""
|
||||
The KV cache is stored inside this class and is accessed via
|
||||
"""The KV cache is stored inside this class and is accessed via
|
||||
`self.kv_cache`.
|
||||
"""
|
||||
super().__init__()
|
||||
@@ -493,8 +492,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
output_shape: torch.Size | None = None,
|
||||
output_dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
The KV cache is stored inside this class and is accessed via
|
||||
"""The KV cache is stored inside this class and is accessed via
|
||||
`self.kv_cache`.
|
||||
|
||||
Attention metadata (`attn_metadata`) is set using a context manager in
|
||||
@@ -691,6 +689,7 @@ def get_attention_context(
|
||||
|
||||
Note: attn_metadata may be None, but attn_layer and kv_cache are always
|
||||
extracted from the forward context.
|
||||
|
||||
"""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
@@ -718,8 +717,7 @@ def unified_kv_cache_update(
|
||||
value: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Returns a dummy that is passed to unified_attention to signal a side effect and
|
||||
"""Returns a dummy that is passed to unified_attention to signal a side effect and
|
||||
the data dependency between them to ensure torch.compile preserves ordering.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
|
||||
@@ -34,7 +34,6 @@ def _get_cross_slot_mapping(
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
"""Get cross-attention slot mappings."""
|
||||
|
||||
block_size = kv_cache_spec.block_size
|
||||
slot_mappings = []
|
||||
|
||||
@@ -187,8 +186,7 @@ def create_cross_attention_backend(
|
||||
|
||||
|
||||
class CrossAttention(Attention):
|
||||
"""
|
||||
Cross-attention for encoder-decoder models.
|
||||
"""Cross-attention for encoder-decoder models.
|
||||
Handles attention between decoder queries and encoder keys/values.
|
||||
"""
|
||||
|
||||
|
||||
@@ -49,9 +49,7 @@ def create_encoder_only_attention_backend(
|
||||
|
||||
|
||||
class EncoderOnlyAttention(Attention):
|
||||
"""
|
||||
Encoder attention is a special case that doesn't need a KV Cache.
|
||||
"""
|
||||
"""Encoder attention is a special case that doesn't need a KV Cache."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
# MLA Common Components
|
||||
"""# MLA Common Components
|
||||
|
||||
This file implements common components for MLA implementations.
|
||||
|
||||
@@ -1415,8 +1414,7 @@ def unified_mla_kv_cache_update(
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Returns a dummy that is passed to unified_attention to signal a side effect and
|
||||
"""Returns a dummy that is passed to unified_attention to signal a side effect and
|
||||
the data dependency between them to ensure torch.compile preserves ordering.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
@@ -1539,8 +1537,7 @@ def dynamic_per_batched_tensor_quant(
|
||||
dynamic_arg_dims={"decode_ql_nope": 0, "decode_q_pe": 0},
|
||||
)
|
||||
class _DecodeConcatQuantFP8(QuantFP8):
|
||||
"""
|
||||
QuantFP8 variant that concatenates decode_ql_nope and decode_q_pe before
|
||||
"""QuantFP8 variant that concatenates decode_ql_nope and decode_q_pe before
|
||||
quantization. When disabled, forward_native is compiled via torch.compile,
|
||||
fusing cat/reshape/quant/view together.
|
||||
"""
|
||||
@@ -1571,7 +1568,8 @@ class MLACommonBackend(AttentionBackend):
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Per-token-head modes pack an inline fp32 scale pair after the
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA).
|
||||
"""
|
||||
spec = super().customize_spec(spec)
|
||||
mode = spec.kv_quant_mode
|
||||
if spec.state_content_bytes is not None or not mode.is_per_token_head:
|
||||
@@ -2030,6 +2028,7 @@ def build_mla_chunked_context_metadata(
|
||||
|
||||
Returns:
|
||||
The chunked-context metadata, or None when no prefill has any context.
|
||||
|
||||
"""
|
||||
# NOTE: it is recommended you read the `Chunked Prefill` section in the
|
||||
# comment at the top of the file before trying to understand this code.
|
||||
@@ -2230,8 +2229,7 @@ def build_mla_chunked_context_metadata(
|
||||
|
||||
|
||||
class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
"""
|
||||
NOTE: Please read the comment at the top of the file before trying to
|
||||
"""NOTE: Please read the comment at the top of the file before trying to
|
||||
understand this class
|
||||
"""
|
||||
|
||||
@@ -2316,8 +2314,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
vllm_config: VllmConfig,
|
||||
model_dtype: torch.dtype,
|
||||
) -> torch.dtype:
|
||||
"""
|
||||
Determine the query data type for prefill queries.
|
||||
"""Determine the query data type for prefill queries.
|
||||
Return FP8 dtype if cache is FP8 and prefill query quantization
|
||||
is enabled, else model dtype.
|
||||
"""
|
||||
@@ -2484,8 +2481,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
def build_for_cudagraph_capture(
|
||||
self, common_attn_metadata: CommonAttentionMetadata
|
||||
) -> M:
|
||||
"""
|
||||
This method builds the metadata for full cudagraph capture.
|
||||
"""This method builds the metadata for full cudagraph capture.
|
||||
Currently, only decode is supported for full cudagraphs with MLA.
|
||||
"""
|
||||
m = common_attn_metadata
|
||||
@@ -2671,14 +2667,16 @@ def reorg_kvcache(
|
||||
max_seq_len: int,
|
||||
toks: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
reorg and unpad kvcache after cp local gather to tp layout for attn kernel.
|
||||
"""Reorg and unpad kvcache after cp local gather to tp layout for attn kernel.
|
||||
e.g.
|
||||
allgatered_kv_c_normed = [T0_0, T0_1, T0_2, T0_3, T1_0, T1_1, ...,
|
||||
T0_4, T0_5, pad, pad, T1_2, pad, ...]
|
||||
-> reorganized_kv_c_normed = [T0_0, T0_1, T0_2, T0_3, T0_4, T0_5,
|
||||
T1_0, T1_1, T1_2, ...]
|
||||
|
||||
Args:
|
||||
allgatered_kv_c_normed: all-gathered, padded latent KV cache.
|
||||
allgatered_k_pe: all-gathered, padded RoPE key cache.
|
||||
padded_local_chunk_seq_lens_lst: local chunk context lengths
|
||||
under current CP rank.
|
||||
local_context_lens_allranks: local context lengths on each CP rank.
|
||||
@@ -2687,6 +2685,7 @@ def reorg_kvcache(
|
||||
sum_seq_len: the sum of cp_chunk_seq_lens_lst.
|
||||
max_seq_len: the max value of cp_chunk_seq_lens_lst.
|
||||
toks: the number of tokens for local gather cache.
|
||||
|
||||
"""
|
||||
kv_c_segments = []
|
||||
k_pe_segments = []
|
||||
@@ -2789,10 +2788,16 @@ def accumulate_mla_context_chunk(
|
||||
remaining token range is initialized.
|
||||
|
||||
Args:
|
||||
chunk: The context chunk being folded in.
|
||||
attn_output: The chunk's attention output.
|
||||
attn_softmax_lse: The chunk's log-sum-exp values.
|
||||
output: Running context partial, updated in place.
|
||||
output_lse: Running log-sum-exp, updated in place.
|
||||
output_written: The chunk's attention output already landed in
|
||||
``output[chunk.token_slice]`` because the backend was handed it as
|
||||
``out``, leaving only the lse to fold. Invalid for a continuation
|
||||
chunk, whose leading tokens must be merged rather than overwritten.
|
||||
|
||||
"""
|
||||
token_start = chunk.token_slice.start
|
||||
token_end = chunk.token_slice.stop
|
||||
@@ -2819,8 +2824,7 @@ def accumulate_mla_context_chunk(
|
||||
|
||||
|
||||
class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
"""
|
||||
Shared MLA base providing dense-MHA prefill (via the selected
|
||||
"""Shared MLA base providing dense-MHA prefill (via the selected
|
||||
MLAPrefillBackend) for both dense and sparse impls; subclasses add decode
|
||||
(``forward_mqa``).
|
||||
"""
|
||||
@@ -2856,8 +2860,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
def _concat_k_nope_k_pe(
|
||||
self, k_nope: torch.Tensor, k_pe: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Efficiently concatenate k_nope and k_pe tensors along the last dimension.
|
||||
"""Efficiently concatenate k_nope and k_pe tensors along the last dimension.
|
||||
|
||||
This function avoids the performance penalty of torch.cat with expanded
|
||||
non-contiguous tensors by pre-allocating the output and using direct copies.
|
||||
@@ -2869,6 +2872,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
|
||||
Returns:
|
||||
Tensor of shape [..., nope_dim + pe_dim]
|
||||
|
||||
"""
|
||||
if k_pe.shape[-1] == 0:
|
||||
# NoPE MLA: nothing to append, so no copy either.
|
||||
@@ -3224,8 +3228,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
|
||||
|
||||
class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
|
||||
"""
|
||||
NOTE: Please read the comment at the top of the file before trying to
|
||||
"""NOTE: Please read the comment at the top of the file before trying to
|
||||
understand this class
|
||||
"""
|
||||
|
||||
|
||||
@@ -322,14 +322,14 @@ class MMEncoderAttention(CustomOp):
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
num_heads: number of attention heads per partition.
|
||||
head_size: hidden_size per attention head.
|
||||
scale: scale factor.
|
||||
num_kv_heads: number of kv heads.
|
||||
prefix: This has no effect, it is only here to make it easier to
|
||||
swap between Attention and MultiHeadAttention
|
||||
"""Args:
|
||||
num_heads: number of attention heads per partition.
|
||||
head_size: hidden_size per attention head.
|
||||
scale: scale factor.
|
||||
num_kv_heads: number of kv heads.
|
||||
prefix: This has no effect, it is only here to make it easier to
|
||||
swap between Attention and MultiHeadAttention
|
||||
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
@@ -517,8 +517,7 @@ class MMEncoderAttention(CustomOp):
|
||||
q_len: int,
|
||||
kv_len: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Reshape query, key, value to 4D tensors:
|
||||
"""Reshape query, key, value to 4D tensors:
|
||||
(batch_size, seq_len, num_heads, head_size)
|
||||
"""
|
||||
query = query.view(bsz, q_len, self.num_heads, self.head_size)
|
||||
|
||||
@@ -114,9 +114,7 @@ def create_static_sink_attention_backend(
|
||||
|
||||
@CustomOp.register("static_sink_attention")
|
||||
class StaticSinkAttention(Attention, CustomOp):
|
||||
"""
|
||||
Attention with static sink tokens
|
||||
"""
|
||||
"""Attention with static sink tokens."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -12,8 +12,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
|
||||
class AttentionLayerBase(ABC):
|
||||
"""
|
||||
Base class for attention-like layers (Attention, Mamba, etc.)
|
||||
"""Base class for attention-like layers (Attention, Mamba, etc.)
|
||||
that support the v1 engine.
|
||||
|
||||
This provides a common interface for getting attention backends
|
||||
@@ -38,8 +37,7 @@ class AttentionLayerBase(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
"""
|
||||
Get the KV cache spec for this layer.
|
||||
"""Get the KV cache spec for this layer.
|
||||
May be None if the layer does not need KV cache.
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -143,7 +143,7 @@ class Conv2dLayer(ConvLayerBase):
|
||||
return x
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Expected input shape: (batch_size, in_channels, height, width)"""
|
||||
"""Expected input shape: (batch_size, in_channels, height, width)."""
|
||||
assert x.dim() == 4
|
||||
if self.enable_linear:
|
||||
return self._forward_mulmat(x)
|
||||
@@ -193,7 +193,7 @@ class Conv3dLayer(ConvLayerBase):
|
||||
return x
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Expected input shape: (batch_size, in_channels, time, height, width)"""
|
||||
"""Expected input shape: (batch_size, in_channels, time, height, width)."""
|
||||
if self.enable_linear:
|
||||
return self._forward_mulmat(x)
|
||||
else:
|
||||
|
||||
@@ -55,7 +55,8 @@ _FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
|
||||
"""Workspace token budget for flashinfer fused all-reduce, or None if the
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``.
|
||||
"""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
|
||||
|
||||
@@ -170,7 +170,8 @@ def _fused_embed_eh_norm_kernel(
|
||||
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
|
||||
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
|
||||
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
|
||||
cat. Requires the full table on-rank (replicated embedding)."""
|
||||
cat. Requires the full table on-rank (replicated embedding).
|
||||
"""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
@@ -56,7 +56,8 @@ class MoEActivation(Enum):
|
||||
@property
|
||||
def custom_op_name(self) -> str:
|
||||
"""Maps to the CustomOp name of activations
|
||||
in vllm/model_executor/layers/activation.py."""
|
||||
in vllm/model_executor/layers/activation.py.
|
||||
"""
|
||||
return _CUSTOM_OP_NAMES[self]
|
||||
|
||||
def without_mul(self) -> "MoEActivation":
|
||||
@@ -115,6 +116,7 @@ def activation_without_mul(activation: str) -> str:
|
||||
|
||||
Returns:
|
||||
The non-gated activation name (e.g., "silu_no_mul", "gelu_no_mul")
|
||||
|
||||
"""
|
||||
return MoEActivation.from_str(activation).without_mul().value
|
||||
|
||||
|
||||
@@ -122,8 +122,7 @@ def maybe_roundup_layer_hidden_size(
|
||||
act_dtype: torch.dtype,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> int:
|
||||
"""
|
||||
Given layer hidden size and MoE configurations, round up hidden_size
|
||||
"""Given layer hidden size and MoE configurations, round up hidden_size
|
||||
if necessary.
|
||||
|
||||
Args:
|
||||
@@ -135,6 +134,7 @@ def maybe_roundup_layer_hidden_size(
|
||||
Rounded up hidden_size if rounding up is required based on the configs
|
||||
and all2all backend.
|
||||
Original hidden size otherwise.
|
||||
|
||||
"""
|
||||
if moe_parallel_config.use_deepep_ht_kernels:
|
||||
hidden_size = DeepEPHTPrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
||||
|
||||
@@ -41,8 +41,7 @@ def _get_config_dtype_str(
|
||||
use_int4_w4a16: bool = False,
|
||||
ocp_mx_scheme: str | None = None,
|
||||
) -> str | None:
|
||||
"""
|
||||
Return a string used to construct the filename that contains the
|
||||
"""Return a string used to construct the filename that contains the
|
||||
tuning info for a particular quantization scheme. See
|
||||
try_get_optimal_moe_config in fused_moe.py.
|
||||
"""
|
||||
@@ -72,9 +71,7 @@ def _quant_flags_to_group_shape(
|
||||
per_out_ch_quant: bool,
|
||||
block_shape: list[int] | None,
|
||||
) -> tuple[GroupShape | None, GroupShape | None]:
|
||||
"""
|
||||
Convert MoE quantization flags into more generic GroupShapes.
|
||||
"""
|
||||
"""Convert MoE quantization flags into more generic GroupShapes."""
|
||||
a_shape: GroupShape | None
|
||||
w_shape: GroupShape | None
|
||||
if block_shape is not None:
|
||||
@@ -173,8 +170,7 @@ def get_routing_method_type(
|
||||
|
||||
@dataclass
|
||||
class FusedMoEQuantDesc:
|
||||
"""
|
||||
A quantization descriptor for fused MoE ops. This class can describe
|
||||
"""A quantization descriptor for fused MoE ops. This class can describe
|
||||
either activations or weights.
|
||||
"""
|
||||
|
||||
@@ -212,8 +208,7 @@ class FusedMoEQuantDesc:
|
||||
# e.g. for specific arguments bias, precision, etc.
|
||||
@dataclass
|
||||
class FusedMoEQuantConfig:
|
||||
"""
|
||||
The FusedMoEQuantConfig contains all the quantization parameters for
|
||||
"""The FusedMoEQuantConfig contains all the quantization parameters for
|
||||
a single FusedMoEMethodBase operation. It consists of four
|
||||
FusedMoEQuantDescs, one for each activation and set of weights.
|
||||
|
||||
@@ -435,8 +430,7 @@ class FusedMoEQuantConfig:
|
||||
return self._a1.dtype == "fp8" and self._w1.dtype == "mxfp4"
|
||||
|
||||
def config_name(self, dtype: torch.dtype) -> str | None:
|
||||
"""
|
||||
Return a string used to construct the filename that contains the
|
||||
"""Return a string used to construct the filename that contains the
|
||||
tuning info for a particular quantization scheme. See
|
||||
try_get_optimal_moe_config in fused_moe.py.
|
||||
"""
|
||||
@@ -454,8 +448,7 @@ class FusedMoEQuantConfig:
|
||||
max_tokens: int,
|
||||
hidden_dim: int,
|
||||
) -> tuple[int, int] | None:
|
||||
"""
|
||||
Construct the proper activation scale shape for this
|
||||
"""Construct the proper activation scale shape for this
|
||||
config.
|
||||
"""
|
||||
if self.is_quantized:
|
||||
@@ -477,8 +470,7 @@ class FusedMoEQuantConfig:
|
||||
max_tokens: int,
|
||||
hidden_dim: int,
|
||||
) -> tuple[int, int, int] | None:
|
||||
"""
|
||||
Construct the proper activation batched scale shape for this
|
||||
"""Construct the proper activation batched scale shape for this
|
||||
config, e.g. (num experts, *scale_shape).
|
||||
"""
|
||||
if self.is_quantized:
|
||||
@@ -512,8 +504,7 @@ class FusedMoEQuantConfig:
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> "FusedMoEQuantConfig":
|
||||
"""
|
||||
General builder function for a FusedMoEQuantConfig.
|
||||
"""General builder function for a FusedMoEQuantConfig.
|
||||
- quant_dtype: Optional quantization type. None if activations are
|
||||
unquantized or quantized prior to calling. Note: "nvfp4", "mxfp4",
|
||||
"mxfp6_e3m2", "mxfp6_e2m3" are the only valid string values
|
||||
@@ -607,9 +598,7 @@ def fp8_w8a8_moe_quant_config(
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for fp8 activations and fp8 weights.
|
||||
"""
|
||||
"""Construct a quant config for fp8 activations and fp8 weights."""
|
||||
return FusedMoEQuantConfig.make(
|
||||
current_platform.fp8_dtype(),
|
||||
w1_scale=w1_scale,
|
||||
@@ -640,9 +629,7 @@ def int8_w8a8_moe_quant_config(
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for int8 activations and int8 weights.
|
||||
"""
|
||||
"""Construct a quant config for int8 activations and int8 weights."""
|
||||
return FusedMoEQuantConfig.make(
|
||||
torch.int8,
|
||||
w1_scale=w1_scale,
|
||||
@@ -667,9 +654,7 @@ def gptq_marlin_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
Construct a quant config for gptq marlin quantization.
|
||||
"""
|
||||
"""Construct a quant config for gptq marlin quantization."""
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
|
||||
w_shape = None if group_size == -1 else GroupShape(row=1, col=group_size)
|
||||
@@ -702,9 +687,7 @@ def mxfp4_w4a16_moe_quant_config(
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for unquantized activations and mxfp4 weights.
|
||||
"""
|
||||
"""Construct a quant config for unquantized activations and mxfp4 weights."""
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(),
|
||||
_a2=FusedMoEQuantDesc(),
|
||||
@@ -730,9 +713,7 @@ def mxfp4_mxfp8_moe_quant_config(
|
||||
mx_alignment: int = 0,
|
||||
is_scale_swizzled: bool = True,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for mxfp4 activations and mxfp4 weights.
|
||||
"""
|
||||
"""Construct a quant config for mxfp4 activations and mxfp4 weights."""
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc("mxfp8"),
|
||||
_a2=FusedMoEQuantDesc("mxfp8"),
|
||||
@@ -756,9 +737,7 @@ def mxfp4_w4a8_moe_quant_config(
|
||||
block_shape: list[int] | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for fp8 activations and mxfp4 weights.
|
||||
"""
|
||||
"""Construct a quant config for fp8 activations and mxfp4 weights."""
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc("fp8", None, a1_scale, None, None, None),
|
||||
_a2=FusedMoEQuantDesc("fp8", None, a2_scale, None, None, None),
|
||||
@@ -782,9 +761,7 @@ def ocp_mx_moe_quant_config(
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for mxfp4 activations and mxfp4 weights.
|
||||
"""
|
||||
"""Construct a quant config for mxfp4 activations and mxfp4 weights."""
|
||||
assert quant_dtype in OCP_MX_DTYPES
|
||||
return FusedMoEQuantConfig.make(
|
||||
quant_dtype=quant_dtype,
|
||||
@@ -818,9 +795,7 @@ def nvfp4_moe_quant_config(
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for mxfp4 activations and nvp4 weights.
|
||||
"""
|
||||
"""Construct a quant config for mxfp4 activations and nvp4 weights."""
|
||||
return FusedMoEQuantConfig.make(
|
||||
"nvfp4",
|
||||
w1_scale=w1_scale,
|
||||
@@ -845,8 +820,7 @@ def mxfp4_moe_quant_config(
|
||||
w1_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for MXFP4 x MXFP4 MoE.
|
||||
"""Construct a quant config for MXFP4 x MXFP4 MoE.
|
||||
MXFP4 uses block scaling only (E8M0 scales, 32-element groups), with no
|
||||
separate alphas / global activation scales in this config.
|
||||
"""
|
||||
@@ -869,9 +843,7 @@ def nvfp4_w4a16_moe_quant_config(
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for 16-but activations and nvp4 weights.
|
||||
"""
|
||||
"""Construct a quant config for 16-but activations and nvp4 weights."""
|
||||
return FusedMoEQuantConfig.make(
|
||||
quant_dtype=None,
|
||||
w1_scale=w1_scale,
|
||||
@@ -899,9 +871,7 @@ def int4_w4a16_moe_quant_config(
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for 16-bit float activations and int4 weights.
|
||||
"""
|
||||
"""Construct a quant config for 16-bit float activations and int4 weights."""
|
||||
group_shape = GroupShape(*block_shape) if block_shape is not None else None
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(shape=group_shape, alpha_or_gscale=a1_gscale),
|
||||
@@ -924,9 +894,7 @@ def fp8_w8a16_moe_quant_config(
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for 16-bit float activations and fp8 weights.
|
||||
"""
|
||||
"""Construct a quant config for 16-bit float activations and fp8 weights."""
|
||||
group_shape = GroupShape(*block_shape) if block_shape is not None else None
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
return FusedMoEQuantConfig(
|
||||
@@ -968,9 +936,7 @@ def int8_w8a16_moe_quant_config(
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for 16-bit float activations and int8 weights.
|
||||
"""
|
||||
"""Construct a quant config for 16-bit float activations and int8 weights."""
|
||||
group_shape = GroupShape(*block_shape) if block_shape is not None else None
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(shape=group_shape, alpha_or_gscale=a1_gscale),
|
||||
@@ -992,9 +958,7 @@ def int4_w4afp8_moe_quant_config(
|
||||
per_out_ch_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for fp8 activations and int4 weights.
|
||||
"""
|
||||
"""Construct a quant config for fp8 activations and int4 weights."""
|
||||
return FusedMoEQuantConfig.make(
|
||||
torch.float8_e4m3fn, # quant dtype for activations
|
||||
w1_scale=w1_scale,
|
||||
@@ -1015,8 +979,7 @@ def biased_moe_quant_config(
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for unquantized activations with biases.
|
||||
"""Construct a quant config for unquantized activations with biases.
|
||||
|
||||
gemm1_alpha/gemm1_beta/gemm1_clamp_limit carry the SwiGLU gate params
|
||||
through to the fused activation kernel (e.g. swigluoai_uninterleave).
|
||||
@@ -1136,8 +1099,7 @@ class FusedMoEParallelConfig:
|
||||
sp_size_: int,
|
||||
vllm_parallel_config: ParallelConfig,
|
||||
) -> "FusedMoEParallelConfig":
|
||||
"""
|
||||
Determine MoE parallel configuration. Based on the input `tp_size_`,
|
||||
"""Determine MoE parallel configuration. Based on the input `tp_size_`,
|
||||
`dp_size_` and vllm's parallel config, determine what
|
||||
level's of parallelism to use in the fused moe layer.
|
||||
|
||||
@@ -1145,6 +1107,7 @@ class FusedMoEParallelConfig:
|
||||
tp_size_ (int): `tp_size` passed into the FusedMoEFactory constructor.
|
||||
pcp_size_ (int): `pcp_size` passed into the FusedMoEFactory constructor.
|
||||
dp_size_ (int): `dp_size` passed into the FusedMoEFactory constructor.
|
||||
sp_size_ (int): `sp_size` passed into the FusedMoEFactory constructor.
|
||||
vllm_parallel_config (ParallelConfig): vLLM's parallel config
|
||||
object which contains the `enable_expert_parallel` flag.
|
||||
|
||||
@@ -1208,6 +1171,7 @@ class FusedMoEParallelConfig:
|
||||
- device 3: TP = {1, 0} DP = {2, 1} EP = {4, 3}
|
||||
- Comment: There are 2 engine instances and the experts are split
|
||||
between the 4 devices.
|
||||
|
||||
"""
|
||||
use_ep = (
|
||||
dp_size_ * pcp_size_ * tp_size_ > 1
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Taken from https://github.com/ModelTC/LightLLM/blob/8ed97c74c18f11505b048b1ba00ba5c0cef8bff6/lightllm/common/fused_moe/deepep_scatter_gather.py
|
||||
"""Taken from https://github.com/ModelTC/LightLLM/blob/8ed97c74c18f11505b048b1ba00ba5c0cef8bff6/lightllm/common/fused_moe/deepep_scatter_gather.py
|
||||
and updated to fit vllm needs and terminology.
|
||||
"""
|
||||
|
||||
|
||||
@@ -27,8 +27,7 @@ def determine_expert_map(
|
||||
num_fused_shared_experts: int = 0,
|
||||
return_expert_mask: bool = False,
|
||||
) -> tuple[int, torch.Tensor | None, torch.Tensor | None]:
|
||||
"""
|
||||
Calculates how many experts should be assigned to each rank for EP and
|
||||
"""Calculates how many experts should be assigned to each rank for EP and
|
||||
creates a mapping from global to local expert index. Experts are
|
||||
distributed evenly across ranks. Any remaining are assigned to the
|
||||
last rank.
|
||||
@@ -56,6 +55,7 @@ def determine_expert_map(
|
||||
and 0 for sentinel.
|
||||
Returns None if ep_size is 1.
|
||||
Used only when AITER MOE is enabled.
|
||||
|
||||
"""
|
||||
from typing import get_args
|
||||
|
||||
@@ -150,8 +150,7 @@ def determine_expert_placement_strategy(
|
||||
|
||||
|
||||
class ExpertMapManager:
|
||||
"""
|
||||
Manages expert ID mappings and placement for Expert Parallelism.
|
||||
"""Manages expert ID mappings and placement for Expert Parallelism.
|
||||
|
||||
Responsibilities:
|
||||
- Calculate local vs global expert counts
|
||||
@@ -193,16 +192,22 @@ class ExpertMapManager:
|
||||
num_fused_shared_experts: int = 0,
|
||||
rocm_aiter_enabled: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize expert map manager.
|
||||
"""Initialize expert map manager.
|
||||
|
||||
Args:
|
||||
max_num_batched_tokens: Maximum number of batched tokens per step
|
||||
top_k: Number of experts each token is routed to
|
||||
global_num_experts: Total number of experts across all ranks
|
||||
num_redundant_experts: Number of redundant expert replicas (EPLB)
|
||||
num_expert_group: Number of expert groups, when grouped routing is
|
||||
used
|
||||
moe_parallel_config: MoE parallel configuration (contains ep_size,
|
||||
ep_rank, backend flags)
|
||||
placement_strategy: Strategy for placing experts ('linear' or 'round_robin')
|
||||
num_fused_shared_experts: Number of fused shared experts (for AITER)
|
||||
rocm_aiter_enabled: Whether ROCm AITER fusion is enabled
|
||||
enable_eplb: Whether expert-parallel load balancing is enabled
|
||||
|
||||
"""
|
||||
self.global_num_experts = global_num_experts
|
||||
self.moe_parallel_config = moe_parallel_config
|
||||
@@ -295,8 +300,7 @@ class ExpertMapManager:
|
||||
|
||||
@property
|
||||
def expert_map(self) -> torch.Tensor | None:
|
||||
"""
|
||||
Mapping from global expert ID to local expert ID.
|
||||
"""Mapping from global expert ID to local expert ID.
|
||||
|
||||
Returns tensor of shape (global_num_experts,) where:
|
||||
- expert_map[global_id] = local_id if expert is on this rank
|
||||
@@ -308,8 +312,7 @@ class ExpertMapManager:
|
||||
|
||||
@property
|
||||
def expert_mask(self) -> torch.Tensor | None:
|
||||
"""
|
||||
Expert mask for AITER fusion (ROCm-specific).
|
||||
"""Expert mask for AITER fusion (ROCm-specific).
|
||||
|
||||
Returns tensor of shape (global_num_experts + num_fused_shared + 1,)
|
||||
where 1 indicates expert is on this rank, 0 otherwise.
|
||||
@@ -325,8 +328,7 @@ class ExpertMapManager:
|
||||
def routing_tables(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
|
||||
"""
|
||||
Routing tables for round-robin placement.
|
||||
"""Routing tables for round-robin placement.
|
||||
|
||||
Returns (global_to_physical, physical_to_global, local_to_global)
|
||||
or None if not using round-robin or tables not needed.
|
||||
@@ -334,8 +336,7 @@ class ExpertMapManager:
|
||||
return self._routing_tables
|
||||
|
||||
def map_global_to_local(self, global_id: int) -> int:
|
||||
"""
|
||||
Map global expert ID to local expert ID.
|
||||
"""Map global expert ID to local expert ID.
|
||||
|
||||
Args:
|
||||
global_id: Global expert ID (0 to global_num_experts - 1)
|
||||
@@ -345,6 +346,7 @@ class ExpertMapManager:
|
||||
|
||||
Raises:
|
||||
ValueError: If expert is not on this rank
|
||||
|
||||
"""
|
||||
if self._expert_map is None:
|
||||
return global_id
|
||||
@@ -369,8 +371,7 @@ class ExpertMapManager:
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
global_num_experts: int,
|
||||
) -> None:
|
||||
"""
|
||||
Update expert mappings for new EP configuration.
|
||||
"""Update expert mappings for new EP configuration.
|
||||
|
||||
Used during dynamic reconfiguration (e.g., elastic scaling).
|
||||
|
||||
@@ -378,6 +379,7 @@ class ExpertMapManager:
|
||||
global_num_experts: New total number of experts across all ranks
|
||||
moe_parallel_config: New MoE parallel configuration (contains ep_size,
|
||||
ep_rank, backend flags)
|
||||
|
||||
"""
|
||||
self.moe_parallel_config = moe_parallel_config
|
||||
self.global_num_experts = global_num_experts
|
||||
@@ -397,8 +399,7 @@ class ExpertMapManager:
|
||||
self._init_aiter_shared_experts_topK_buffer()
|
||||
|
||||
def get_compressed_map_string(self) -> str:
|
||||
"""
|
||||
Get compressed string representation of expert map for logging.
|
||||
"""Get compressed string representation of expert map for logging.
|
||||
|
||||
Returns string mapping local to global expert IDs.
|
||||
"""
|
||||
@@ -457,8 +458,7 @@ class ExpertMapManager:
|
||||
def _init_routing_tables(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
|
||||
"""
|
||||
Ensure routing tables are initialized if needed for round-robin.
|
||||
"""Ensure routing tables are initialized if needed for round-robin.
|
||||
|
||||
This is a public method that can be called to explicitly initialize
|
||||
routing tables. It's safe to call multiple times (idempotent).
|
||||
|
||||
@@ -56,9 +56,7 @@ def _aiter_w4a16_silu_via_a8w4(
|
||||
unpadded_N_w2,
|
||||
unpadded_K_w2,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation.
|
||||
"""
|
||||
"""MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation."""
|
||||
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
|
||||
from aiter.ops.triton.quant import dynamic_mxfp8_quant
|
||||
|
||||
|
||||
@@ -29,8 +29,7 @@ __all__ = [
|
||||
# _get_padding_mask and patch_gating_output after
|
||||
# https://github.com/ROCm/aiter/pull/4530 is released and consumed by vLLM.
|
||||
def _get_padding_mask() -> torch.Tensor | None:
|
||||
"""
|
||||
Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
|
||||
"""Retrieves a boolean mask with non-padding (0) and padding (1) tokens.
|
||||
|
||||
`slot_mapping < 0` comes from:
|
||||
|
||||
@@ -247,8 +246,7 @@ def triton_kernel_fused_mxfp4_w4a8_experts(
|
||||
|
||||
|
||||
class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
"""
|
||||
Monolithic MXFP4 W4A8 expert using AITER triton kernels.
|
||||
"""Monolithic MXFP4 W4A8 expert using AITER triton kernels.
|
||||
|
||||
This backend uses:
|
||||
- aiter.ops.triton.moe_routing.routing for routing
|
||||
|
||||
@@ -36,7 +36,8 @@ def is_aiter_mxfp8_moe_available() -> bool:
|
||||
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
|
||||
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
|
||||
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
|
||||
closed (returns False -> triton dot_scaled), which is always safe."""
|
||||
closed (returns False -> triton dot_scaled), which is always safe.
|
||||
"""
|
||||
if not (current_platform.is_rocm() and current_platform.supports_mx()):
|
||||
return False
|
||||
try:
|
||||
|
||||
@@ -307,8 +307,7 @@ class BatchedDeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
max_num_tokens: int,
|
||||
num_dispatchers: int,
|
||||
):
|
||||
"""
|
||||
max_num_tokens: Maximum number of tokens from a DP Rank
|
||||
"""max_num_tokens: Maximum number of tokens from a DP Rank
|
||||
num_dispatchers: The number of DP dispatchers.
|
||||
quant_config: Quantization configuration
|
||||
"""
|
||||
@@ -350,8 +349,7 @@ class BatchedDeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
return True
|
||||
|
||||
def supports_packed_ue8m0_act_scales(self) -> bool:
|
||||
"""
|
||||
DeepGemm supports packed ue8m0 activation scales on Blackwell-family
|
||||
"""DeepGemm supports packed ue8m0 activation scales on Blackwell-family
|
||||
GPUs (SM100 datacenter and SM120 consumer).
|
||||
"""
|
||||
return is_deep_gemm_e8m0_used() and (
|
||||
|
||||
@@ -172,6 +172,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
0 = SwiGLU_Gu (SiLU(g)*u)
|
||||
1 = SwiGLU_Ug (SiLU(u)*g)
|
||||
2 = SiLU
|
||||
|
||||
"""
|
||||
if activation == MoEActivation.SWIGLUSTEP:
|
||||
return 0
|
||||
@@ -217,6 +218,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
|
||||
Returns:
|
||||
Output tensor after MoE computation
|
||||
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import (
|
||||
select_experts,
|
||||
|
||||
@@ -192,7 +192,8 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
def _pad_moe_intermediate(self, layer: torch.nn.Module) -> None:
|
||||
"""Zero-pad the per-partition MoE intermediate dim of both weights and
|
||||
the expert bias, see `_padded_intermediate_size`."""
|
||||
the expert bias, see `_padded_intermediate_size`.
|
||||
"""
|
||||
intermediate_size = self.moe_config.intermediate_size_per_partition
|
||||
padded_size = self._padded_intermediate_size(self.moe_config)
|
||||
if padded_size == intermediate_size:
|
||||
@@ -745,6 +746,7 @@ def prepare_int4_moe_layer_for_cpu(
|
||||
|
||||
Returns:
|
||||
(blocked_w13, blocked_w2, blocked_s13, blocked_s2, blocked_z13, blocked_z2)
|
||||
|
||||
"""
|
||||
E = w13_packed.size(0)
|
||||
|
||||
@@ -951,7 +953,8 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular):
|
||||
|
||||
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
|
||||
modular MoE experts."""
|
||||
modular MoE experts.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1055,7 +1058,6 @@ class CPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
|
||||
|
||||
w13 = torch.ops._C.convert_weight_packed(layer.w13_weight)
|
||||
w2 = torch.ops._C.convert_weight_packed(layer.w2_weight)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -1285,7 +1287,8 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
|
||||
class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
|
||||
quantization, dispatched through zentorch."""
|
||||
quantization, dispatched through zentorch.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -518,8 +518,7 @@ def run_cutlass_moe_fp4(
|
||||
*,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
MoE implementation for FP4 Inputs
|
||||
"""MoE implementation for FP4 Inputs.
|
||||
|
||||
# Gemm 1
|
||||
a: Input tensor: [m, k] (half/bfloat16)
|
||||
|
||||
@@ -58,8 +58,7 @@ def _valid_deep_gemm_shape(M: int, N: int, K: int) -> bool:
|
||||
def _valid_deep_gemm(
|
||||
hidden_states: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the given problem size is supported by the DeepGemm grouped
|
||||
"""Check if the given problem size is supported by the DeepGemm grouped
|
||||
gemm kernel. All of M, N, K and the quantization block_shape must be
|
||||
aligned by `dg.get_m_alignment_for_contiguous_layout()`.
|
||||
"""
|
||||
|
||||
@@ -30,8 +30,7 @@ class FallbackExperts(mk.FusedMoEExpertsModular, ABC):
|
||||
type[mk.FusedMoEExpertsModular],
|
||||
type[mk.FusedMoEExpertsModular],
|
||||
]:
|
||||
"""
|
||||
Get the cls for the experts and fallback experts.
|
||||
"""Get the cls for the experts and fallback experts.
|
||||
|
||||
Subclasses should implement this method, so that
|
||||
we have a consistent way to call the _supports_*
|
||||
|
||||
@@ -107,8 +107,7 @@ class FlashInferCuteDSLBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
"""
|
||||
Compute the shapes for the temporary and final outputs of the two gemms
|
||||
"""Compute the shapes for the temporary and final outputs of the two gemms
|
||||
and activation in the fused expert function. Since the gemms are
|
||||
independent, the workspace for the first gemm can be shared with the
|
||||
workspace for the last gemm.
|
||||
@@ -123,7 +122,6 @@ class FlashInferCuteDSLBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
- Note: in order for activation chunking to work, the first dimension
|
||||
of each tuple must be the number of tokens.
|
||||
"""
|
||||
|
||||
# We use global_num_experts due to how moe_align_block_size handles
|
||||
# expert_maps.
|
||||
K_dim = K * 2 if self.use_deep_ep_ll_nvfp4_dispatch else K
|
||||
@@ -212,8 +210,7 @@ def flashinfer_cutedsl_moe_masked(
|
||||
workspace: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
Perform masked Mixture-of-Experts computation with FlashInfer's CuteDSL
|
||||
"""Perform masked Mixture-of-Experts computation with FlashInfer's CuteDSL
|
||||
kernels.
|
||||
|
||||
Args:
|
||||
@@ -229,13 +226,14 @@ def flashinfer_cutedsl_moe_masked(
|
||||
a2_global_scale (torch.Tensor): (l,)
|
||||
w2_blockscale (torch.Tensor): blockscale factors, e4m3,
|
||||
w2_alpha (torch.Tensor): (l,)
|
||||
out (torch.Tensor): output tensor written in place, [l, m, k], bf16
|
||||
masked_m (torch.Tensor): Masked dimension indices
|
||||
workspace (torch.Tensor): For gateup_output
|
||||
|
||||
Notes:
|
||||
- Assumes max(masked_m) <= m.
|
||||
"""
|
||||
|
||||
"""
|
||||
# === Assertions on dtypes ===
|
||||
assert w1.dtype == torch.uint8, f"w1 must be uint8, got {w1.dtype}"
|
||||
assert w1_blockscale.dtype == torch.float8_e4m3fn, (
|
||||
|
||||
@@ -29,8 +29,7 @@ from vllm.utils.flashinfer import (
|
||||
|
||||
|
||||
class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
CuteDSL NvFP4 MoE experts using the FlashInfer functional API.
|
||||
"""CuteDSL NvFP4 MoE experts using the FlashInfer functional API.
|
||||
|
||||
Uses Standard activation format (non-batched). The kernel handles
|
||||
routing, expert computation, and reduction internally.
|
||||
|
||||
@@ -38,8 +38,7 @@ logger = init_logger(__name__)
|
||||
def is_valid_flashinfer_cutlass_fused_moe(
|
||||
hidden_states: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the given problem size is supported by the FlashInfer CUTLASS MoE
|
||||
"""Check if the given problem size is supported by the FlashInfer CUTLASS MoE
|
||||
kernel.
|
||||
"""
|
||||
if not has_flashinfer_cutlass_fused_moe():
|
||||
@@ -219,8 +218,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# We use global_num_experts due to how moe_align_block_size handles
|
||||
# expert_maps.
|
||||
"""
|
||||
Compute the shapes for the temporary and final outputs of the two gemms
|
||||
"""Compute the shapes for the temporary and final outputs of the two gemms
|
||||
and activation in the fused expert function. Since the gemms are
|
||||
independent, the workspace for the first gemm can be shared with the
|
||||
workspace for the last gemm.
|
||||
|
||||
@@ -551,8 +551,7 @@ def invoke_moe_batched_triton_kernel(
|
||||
|
||||
|
||||
class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
A reference MoE expert class that operates on expert batched format,
|
||||
"""A reference MoE expert class that operates on expert batched format,
|
||||
i.e. E x max_num_tokens x K. This is the format that the batched
|
||||
dispatch/combine kernels use.
|
||||
"""
|
||||
@@ -775,8 +774,7 @@ def batched_moe_kernel_quantize_input(
|
||||
|
||||
|
||||
class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
A Triton based MoE expert class that operates on expert batched format,
|
||||
"""A Triton based MoE expert class that operates on expert batched format,
|
||||
i.e. E x max_num_tokens x K. This is the format that the batched
|
||||
dispatch/combine kernels use.
|
||||
"""
|
||||
|
||||
@@ -352,8 +352,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
)
|
||||
|
||||
def _prequantizes_dispatch_activation(self) -> bool:
|
||||
"""
|
||||
Whether the prepare/finalize step should quantize activations before
|
||||
"""Whether the prepare/finalize step should quantize activations before
|
||||
the (EP all-to-all) dispatch instead of leaving it to Humming.
|
||||
|
||||
This is enabled only for block-FP8 (group-128) activations: quantizing
|
||||
@@ -373,8 +372,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
"""
|
||||
Whether the prepare/finalize step should defer input quantization to
|
||||
"""Whether the prepare/finalize step should defer input quantization to
|
||||
the experts (by setting defer_input_quant=True and passing unquantized
|
||||
inputs).
|
||||
|
||||
@@ -832,8 +830,7 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Standard apply implementation for Humming indexed experts.
|
||||
"""Standard apply implementation for Humming indexed experts.
|
||||
|
||||
Note: Humming kernels handle weights internally through the layer
|
||||
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||
@@ -967,8 +964,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Standard apply implementation for Humming grouped experts.
|
||||
"""Standard apply implementation for Humming grouped experts.
|
||||
|
||||
Note: Humming kernels handle weights internally through the layer
|
||||
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||
@@ -1090,8 +1086,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Standard apply implementation for Humming batched grouped experts.
|
||||
"""Standard apply implementation for Humming batched grouped experts.
|
||||
|
||||
Note: Humming kernels handle weights internally through the layer
|
||||
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||
|
||||
@@ -502,8 +502,7 @@ def pack_bitmatrix(
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
BLOCK_SIZE_K: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Packs topk_ids into a bitmatrix.
|
||||
"""Packs topk_ids into a bitmatrix.
|
||||
code reference:
|
||||
https://github.com/triton-lang/triton/blob/dd1bbc52b34d202dfe5ffea1e04fb16166c5c04e/python/triton_kernels/bench/distributed.py#L264
|
||||
"""
|
||||
@@ -949,8 +948,7 @@ class BaseOAITritonExperts(mk.FusedMoEExpertsModular):
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
"""
|
||||
Extract the MoE problem size from the given tensor arguments:
|
||||
"""Extract the MoE problem size from the given tensor arguments:
|
||||
- a: The hidden states, input to the MoE layer.
|
||||
- w1: The first set of expert weights.
|
||||
- w2: The second set of expert weights.
|
||||
@@ -1075,8 +1073,7 @@ class OAITritonExperts(BaseOAITritonExperts):
|
||||
|
||||
|
||||
class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
|
||||
"""
|
||||
A Triton based MoE expert class that operates on expert standard
|
||||
"""A Triton based MoE expert class that operates on expert standard
|
||||
format and explicitly keeps the activation and reduction (moe_sum) steps
|
||||
unfused from the matmul_ogs kernel. This exposes injection points
|
||||
for activation and moe_sum.
|
||||
|
||||
@@ -10,8 +10,7 @@ from vllm.lora.punica_wrapper.punica_base import PunicaWrapperBase
|
||||
|
||||
@dataclass
|
||||
class MoELoRAContext:
|
||||
"""
|
||||
Carries all LoRA state for one MoE forward pass.
|
||||
"""Carries all LoRA state for one MoE forward pass.
|
||||
|
||||
Built by FusedMoEWithLoRA.forward() and propagated explicitly through the
|
||||
modular kernel path (FusedMoEKernel -> FusedMoEExpertsModular.apply) so
|
||||
|
||||
@@ -7,8 +7,7 @@ from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRACon
|
||||
|
||||
|
||||
class LoRAExpertsMixin:
|
||||
"""
|
||||
Mixin for FusedMoEExpertsModular subclasses that natively handle
|
||||
"""Mixin for FusedMoEExpertsModular subclasses that natively handle
|
||||
MoELoRAContext inside their apply() implementation.
|
||||
|
||||
Mixing this class in:
|
||||
|
||||
@@ -251,26 +251,56 @@ def fused_marlin_moe(
|
||||
input_dtype: torch.dtype | None = None,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a Mixture of Experts (MoE) layer using two sets of
|
||||
"""This function computes a Mixture of Experts (MoE) layer using two sets of
|
||||
weights, w1 and w2, and top-k gating mechanism.
|
||||
|
||||
Parameters:
|
||||
- hidden_states (torch.Tensor): The input tensor to the MoE layer.
|
||||
- w1 (torch.Tensor): The first set of expert weights.
|
||||
- w2 (torch.Tensor): The second set of expert weights.
|
||||
- w1_scale (torch.Tensor): Scale to be used for w1.
|
||||
- w2_scale (torch.Tensor): Scale to be used for w2.
|
||||
- topk_weights (torch.Tensor): Top-k weights.
|
||||
- topk_ids (torch.Tensor): Indices of topk-k elements.
|
||||
- w1_zeros (torch.Tensor|None): Optional zero points to be used for w1.
|
||||
- w2_zeros (torch.Tensor|None): Optional zero points to be used for w2.
|
||||
- num_bits (bool): The number of bits in expert weights quantization.
|
||||
Args:
|
||||
hidden_states (torch.Tensor): The input tensor to the MoE layer.
|
||||
w1 (torch.Tensor): The first set of expert weights.
|
||||
w2 (torch.Tensor): The second set of expert weights.
|
||||
bias1 (torch.Tensor|None): Optional bias for the first GEMM.
|
||||
bias2 (torch.Tensor|None): Optional bias for the second GEMM.
|
||||
w1_scale (torch.Tensor): Scale to be used for w1.
|
||||
w2_scale (torch.Tensor): Scale to be used for w2.
|
||||
topk_weights (torch.Tensor): Top-k weights.
|
||||
topk_ids (torch.Tensor): Indices of topk-k elements.
|
||||
quant_type_id (int): Id of the `ScalarType` the expert weights are
|
||||
quantized to. Determines the 4- or 8-bit Marlin kernel path.
|
||||
apply_router_weight_on_input (bool): Apply the topk weights to the
|
||||
input rather than the output. Only valid when topk is 1.
|
||||
global_num_experts (int): Total number of experts across all expert
|
||||
parallel shards. -1 means the local expert count.
|
||||
activation (MoEActivation): Activation applied between the two GEMMs.
|
||||
activation_func (Callable|None): Override for the activation
|
||||
implementation.
|
||||
moe_sum (Callable|None): Override for the final reduction over the
|
||||
topk dimension. Defaults to `ops.moe_sum` / `torch.sum`.
|
||||
expert_map (torch.Tensor|None): Maps global expert indices to the
|
||||
local expert space of this expert parallel shard.
|
||||
input_global_scale1 (torch.Tensor|None): NVFP4 global input scale for
|
||||
the first GEMM.
|
||||
input_global_scale2 (torch.Tensor|None): NVFP4 global input scale for
|
||||
the second GEMM.
|
||||
global_scale1 (torch.Tensor|None): NVFP4 global weight scale for w1.
|
||||
global_scale2 (torch.Tensor|None): NVFP4 global weight scale for w2.
|
||||
w1_zeros (torch.Tensor|None): Optional zero points to be used for w1.
|
||||
w2_zeros (torch.Tensor|None): Optional zero points to be used for w2.
|
||||
workspace (torch.Tensor|None): Scratch buffer for the Marlin kernels.
|
||||
intermediate_cache13 (torch.Tensor|None): Scratch buffer for the
|
||||
first and third intermediate activations.
|
||||
intermediate_cache2 (torch.Tensor|None): Scratch buffer for the
|
||||
second intermediate activation.
|
||||
output (torch.Tensor|None): Tensor to write the result into. A new
|
||||
tensor is allocated when omitted.
|
||||
input_dtype (torch.dtype|None): dtype the activations are quantized
|
||||
to. A 1-byte dtype raises the M block size to at least 16.
|
||||
activation_config (ApplyMoEActivationConfig|None): Extra
|
||||
configuration forwarded to the activation.
|
||||
|
||||
Returns:
|
||||
- torch.Tensor: The output tensor after applying the MoE layer.
|
||||
"""
|
||||
torch.Tensor: The output tensor after applying the MoE layer.
|
||||
|
||||
"""
|
||||
quant_type = ScalarType.from_id(quant_type_id)
|
||||
assert quant_type in [
|
||||
scalar_types.uint4,
|
||||
@@ -399,8 +429,7 @@ def batched_fused_marlin_moe(
|
||||
activation_func: Callable[..., None] | None = None,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function massages the inputs so the batched hidden_states can be
|
||||
"""This function massages the inputs so the batched hidden_states can be
|
||||
presented as a 2D contiguous tensor that could be used with
|
||||
_fused_marlin_moe.
|
||||
|
||||
@@ -423,7 +452,6 @@ def batched_fused_marlin_moe(
|
||||
expert_ids tensors, so only relevant/valid rows of A (hidden_states)
|
||||
are accessed and are processed with the correct expert_ids.
|
||||
"""
|
||||
|
||||
assert hidden_states.ndim == 3, (
|
||||
f"hidden states must be batched. e.g. [B, MAX_TOKENS, K]."
|
||||
f"But got {hidden_states.size()}"
|
||||
|
||||
@@ -34,7 +34,8 @@ from vllm.triton_utils import tl, triton
|
||||
|
||||
def _select_cfg(M, N, K, block_m):
|
||||
"""Pick the launch config from host constants only (M=num_valid_tokens, N, K,
|
||||
block_m) — graph-capture safe (no GPU-scalar branch)."""
|
||||
block_m) — graph-capture safe (no GPU-scalar branch).
|
||||
"""
|
||||
# Per-regime winners (measured, isolated cuda-event A/B on gfx950, GPU 3):
|
||||
# BLOCK_K=256 (fewer K-iters + bigger MX scale-load coalesced with the dot),
|
||||
# num_stages=2 (software-pipeline overlaps the E8M0 scale-load with the scaled
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
NVFP4 quantization emulation for MoE.
|
||||
"""NVFP4 quantization emulation for MoE.
|
||||
|
||||
This file implements NVFP4 emulation for NVFP4 MOE in case the hardware used does not
|
||||
natively support NVFP4 MOE.
|
||||
@@ -86,8 +85,7 @@ def fused_moe_nvfp4_emulation_kernel(
|
||||
compute_type: tl.constexpr,
|
||||
group_size: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fused MoE kernel for emulated NVFP4 weight-only dequantization + GEMM.
|
||||
"""Fused MoE kernel for emulated NVFP4 weight-only dequantization + GEMM.
|
||||
|
||||
Activations A are BF16 (already QDQ'd externally).
|
||||
Weights B are packed uint8 NVFP4 [E, N, K//2] — two FP4 values per byte
|
||||
@@ -334,8 +332,7 @@ def invoke_fused_moe_nvfp4_emulation_kernel(
|
||||
|
||||
|
||||
class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
|
||||
"""
|
||||
Extension of TritonExperts to support emulated NVFP4 MoE experts.
|
||||
"""Extension of TritonExperts to support emulated NVFP4 MoE experts.
|
||||
|
||||
It may be used for NVFP4 models when the device does not have
|
||||
native support for this dtype.
|
||||
@@ -427,8 +424,7 @@ class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
"""
|
||||
Apply emulated quantized MoE computation.
|
||||
"""Apply emulated quantized MoE computation.
|
||||
|
||||
This dequantizes the weights on the fly and calls fused_experts_impl
|
||||
with activation quantization support.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
OCP MX quantization emulation for MoE.
|
||||
"""OCP MX quantization emulation for MoE.
|
||||
|
||||
This file implements OCP MX (MXFP4/MXFP6) emulation for MoE in case the
|
||||
hardware used does not natively support OCP MX MoE.
|
||||
@@ -50,6 +49,7 @@ def activation_quant_dtype(
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If the scheme has no emulated activation dtype.
|
||||
|
||||
"""
|
||||
if ocp_mx_scheme in {
|
||||
OCP_MX_Scheme.w_mxfp4,
|
||||
@@ -82,8 +82,7 @@ def activation_quant_dtype(
|
||||
|
||||
|
||||
class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
|
||||
"""
|
||||
Extension of TritonExperts to support emulated OCP MX MoE experts.
|
||||
"""Extension of TritonExperts to support emulated OCP MX MoE experts.
|
||||
|
||||
It may be used for OCP MX (MXFP4/MXFP6) models when the device does not
|
||||
have native support for these dtypes.
|
||||
@@ -190,8 +189,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
"""
|
||||
Apply emulated quantized MoE computation.
|
||||
"""Apply emulated quantized MoE computation.
|
||||
|
||||
This dequantizes the weights on the fly and calls TritonExperts.apply
|
||||
with activation quantization support.
|
||||
|
||||
@@ -32,8 +32,7 @@ from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
|
||||
|
||||
class TrtLlmBf16ExpertsBase:
|
||||
"""
|
||||
BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and
|
||||
"""BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and
|
||||
monolithic interfaces.
|
||||
"""
|
||||
|
||||
@@ -124,9 +123,7 @@ class TrtLlmBf16ExpertsBase:
|
||||
|
||||
|
||||
class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
BF16 unquantized TRTLLM-Gen MoE kernels. Supports modular interface.
|
||||
"""
|
||||
"""BF16 unquantized TRTLLM-Gen MoE kernels. Supports modular interface."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
@@ -224,9 +221,7 @@ class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular)
|
||||
|
||||
|
||||
class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonolithic):
|
||||
"""
|
||||
BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface.
|
||||
"""
|
||||
"""BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
|
||||
@@ -58,8 +58,7 @@ def prepare_deepseek_fp8_x_sf(x: torch.Tensor, x_sf: torch.Tensor) -> torch.Tens
|
||||
|
||||
|
||||
class TrtLlmFp8ExpertsBase:
|
||||
"""
|
||||
Fp8 TRTLLM-Gen MoE kernels. Shared base for modular and monolithic
|
||||
"""Fp8 TRTLLM-Gen MoE kernels. Shared base for modular and monolithic
|
||||
interfaces.
|
||||
"""
|
||||
|
||||
@@ -194,9 +193,7 @@ class TrtLlmFp8ExpertsBase:
|
||||
|
||||
|
||||
class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
Fp8 TRTLLM-Gen MoE kernels. Supports modular interface.
|
||||
"""
|
||||
"""Fp8 TRTLLM-Gen MoE kernels. Supports modular interface."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
@@ -330,9 +327,7 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
|
||||
|
||||
class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolithic):
|
||||
"""
|
||||
Fp8 TRTLLM-Gen MoE kernels. Supports monolithic interface.
|
||||
"""
|
||||
"""Fp8 TRTLLM-Gen MoE kernels. Supports monolithic interface."""
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return True
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
LoRA-aware FlashInfer TRT-LLM MoE experts (BF16).
|
||||
"""LoRA-aware FlashInfer TRT-LLM MoE experts (BF16).
|
||||
|
||||
Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153:
|
||||
|
||||
@@ -109,7 +108,7 @@ def _finalize_lora_kernel(
|
||||
|
||||
|
||||
class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
"""LoRA-aware trtllm MoE experts"""
|
||||
"""LoRA-aware trtllm MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -35,9 +35,7 @@ from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
|
||||
class TrtLlmMxfp4ExpertsBase:
|
||||
"""
|
||||
MXFP4 TRTLLM-Gen MoE kernels. Shared base for modular and monolithic.
|
||||
"""
|
||||
"""MXFP4 TRTLLM-Gen MoE kernels. Shared base for modular and monolithic."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -164,8 +162,7 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
class TrtLlmMxfp4ExpertsMonolithic(
|
||||
TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsMonolithic
|
||||
):
|
||||
"""
|
||||
Monolithic version of the MXFP4 TRTLLM kernel (router + experts).
|
||||
"""Monolithic version of the MXFP4 TRTLLM kernel (router + experts).
|
||||
Wraps flashinfer.trtllm_fp4_block_scale_moe().
|
||||
"""
|
||||
|
||||
@@ -287,8 +284,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
|
||||
|
||||
class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
Modular version of the MXFP4 TRTLLM kernel (just the experts).
|
||||
"""Modular version of the MXFP4 TRTLLM kernel (just the experts).
|
||||
Wraps flashinfer.trtllm_fp4_block_scale_routed_moe().
|
||||
Moved from trtllm_moe.py.
|
||||
"""
|
||||
|
||||
@@ -19,8 +19,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
"""
|
||||
FlashInfer TRT-LLM MxInt4 MoE kernel. Monolithic interface
|
||||
"""FlashInfer TRT-LLM MxInt4 MoE kernel. Monolithic interface
|
||||
(fused router + experts).
|
||||
|
||||
Wraps flashinfer_trtllm_mxint4_moe().
|
||||
|
||||
@@ -42,9 +42,7 @@ _PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
|
||||
|
||||
|
||||
class TrtLlmNvFp4ExpertsBase:
|
||||
"""
|
||||
NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface.
|
||||
"""
|
||||
"""NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -300,9 +298,7 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
|
||||
|
||||
class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
Modular version of the implementation (just the experts).
|
||||
"""
|
||||
"""Modular version of the implementation (just the experts)."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
@@ -467,9 +463,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
class TrtLlmNvFp4ExpertsMonolithic(
|
||||
TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsMonolithic
|
||||
):
|
||||
"""
|
||||
Monolithic version of the kernel (router + experts).
|
||||
"""
|
||||
"""Monolithic version of the kernel (router + experts)."""
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return True
|
||||
|
||||
@@ -110,8 +110,7 @@ def fused_moe_kernel_gptq_awq(
|
||||
use_int4_w4a16: tl.constexpr,
|
||||
use_int8_w8a16: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
"""Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
token and expert matrices.
|
||||
|
||||
Key Parameters:
|
||||
@@ -353,8 +352,7 @@ def fused_moe_kernel(
|
||||
# Tensor-descriptor path for the A gather and B load in the K-loop.
|
||||
USE_TD: tl.constexpr = False,
|
||||
):
|
||||
"""
|
||||
Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
"""Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
token and expert matrices.
|
||||
|
||||
Key Parameters:
|
||||
@@ -1115,15 +1113,13 @@ def get_moe_configs(
|
||||
block_n: int | None = None,
|
||||
block_k: int | None = None,
|
||||
) -> dict[int, Any] | None:
|
||||
"""
|
||||
Return optimized configurations for the fused MoE kernel.
|
||||
"""Return optimized configurations for the fused MoE kernel.
|
||||
|
||||
The return value will be a dictionary that maps an irregular grid of
|
||||
batch sizes to configurations of the fused_moe kernel. To evaluate the
|
||||
kernel on a given batch size bs, the closest batch size in the grid should
|
||||
be picked and the associated configuration chosen to invoke the kernel.
|
||||
"""
|
||||
|
||||
# Avoid optimizing for the batch invariant case. Use default config
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
return None
|
||||
@@ -1187,6 +1183,7 @@ def _ensure_block_size_k_divisible(
|
||||
|
||||
Returns:
|
||||
A valid BLOCK_SIZE_K that divides size_k and is divisible by group_size.
|
||||
|
||||
"""
|
||||
# Fast path: already valid
|
||||
if size_k % block_size_k == 0 and block_size_k % group_size == 0:
|
||||
@@ -1638,8 +1635,7 @@ def _get_config_quant_dtype(
|
||||
use_fp8_w8a8: bool,
|
||||
use_int8_w8a8: bool,
|
||||
) -> None | torch.dtype | str:
|
||||
"""
|
||||
Get the quantization type based on the quantization strategy flags.
|
||||
"""Get the quantization type based on the quantization strategy flags.
|
||||
We don't have a quant_config at this point so we need to work backwards.
|
||||
A return type of None means no quantization is required because the
|
||||
input is unquantized or has been quantized prior to calling
|
||||
|
||||
@@ -59,8 +59,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
raise NotImplementedError
|
||||
|
||||
def uses_weight_scale_2_pattern(self) -> bool:
|
||||
"""
|
||||
Returns True if this quantization method uses 'weight_scale_2' pattern
|
||||
"""Returns True if this quantization method uses 'weight_scale_2' pattern
|
||||
for per-tensor weight scales (e.g., FP4 variants), False otherwise.
|
||||
|
||||
This method should be overridden by subclasses that use the
|
||||
@@ -75,8 +74,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
act_dtype: torch.dtype,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Given layer hidden size and intermediate size per partition and MoE
|
||||
"""Given layer hidden size and intermediate size per partition and MoE
|
||||
configurations, round up hidden_size and intermediate_size_per_partition
|
||||
if necessary.
|
||||
|
||||
@@ -93,6 +91,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
- rounded_hidden_size is the possibly rounded up hidden size.
|
||||
- rounded_intermediate_size_per_partition is the possibly rounded
|
||||
up intermediate size per partition.
|
||||
|
||||
"""
|
||||
from .all2all_utils import maybe_roundup_layer_hidden_size
|
||||
|
||||
@@ -119,8 +118,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
|
||||
@property
|
||||
def has_unpadded_output(self) -> bool:
|
||||
"""
|
||||
Indicates that the hidden_states output might be the unpadded
|
||||
"""Indicates that the hidden_states output might be the unpadded
|
||||
hidden_states shape rather than the full padded shape.
|
||||
"""
|
||||
return False
|
||||
@@ -151,8 +149,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
shared_experts: "SharedExperts | None",
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply the MoE operation using modular kernels.
|
||||
"""Apply the MoE operation using modular kernels.
|
||||
|
||||
Args:
|
||||
layer: RoutedExperts instance containing weight parameters
|
||||
@@ -160,9 +157,11 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
topk_weights: Expert weights from router
|
||||
topk_ids: Selected expert IDs from router
|
||||
shared_experts_input: Input for shared experts (if any)
|
||||
shared_experts: The shared experts module, if any
|
||||
|
||||
Returns:
|
||||
Output tensor from routed experts.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -173,15 +172,16 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""
|
||||
Apply the MoE operation using monolithic kernels.
|
||||
"""Apply the MoE operation using monolithic kernels.
|
||||
|
||||
Args:
|
||||
layer: RoutedExperts instance containing weight parameters
|
||||
x: Input tensor
|
||||
router_logits: Router logits (routing done internally)
|
||||
input_ids: Token ids, for routers that condition on them
|
||||
|
||||
Returns:
|
||||
Finalized routed states or a deferred-finalize output.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -122,8 +122,7 @@ class HPCExperts(mk.FusedMoEExpertsModular):
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# We use global_num_experts due to how moe_align_block_size handles
|
||||
# expert_maps.
|
||||
"""
|
||||
Compute the shapes for the temporary and final outputs of the two gemms
|
||||
"""Compute the shapes for the temporary and final outputs of the two gemms
|
||||
and activation in the fused expert function. Since the gemms are
|
||||
independent, the workspace for the first gemm can be shared with the
|
||||
workspace for the last gemm.
|
||||
|
||||
@@ -152,6 +152,11 @@ def FusedMoEFactory(
|
||||
copy that naming convention here and handle any remapping in the
|
||||
load_weights function in each model implementation.
|
||||
|
||||
Args:
|
||||
intermediate_pad: Padding added to the intermediate size, if any.
|
||||
swiglu_alpha: Optional alpha parameter for the SwiGLU activation.
|
||||
swiglu_beta: Optional beta parameter for the SwiGLU activation.
|
||||
|
||||
Args:
|
||||
num_experts: Number of experts in the model (global count)
|
||||
top_k: Number of experts selected for each token
|
||||
@@ -211,6 +216,7 @@ def FusedMoEFactory(
|
||||
|
||||
Returns:
|
||||
MoERunner: Configured MoE execution pipeline ready for forward passes
|
||||
|
||||
"""
|
||||
vllm_config = get_current_vllm_config()
|
||||
|
||||
|
||||
@@ -84,9 +84,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FusedMoEActivationFormat(Enum):
|
||||
"""
|
||||
The standard activation format (num_tokens, hidden dim).
|
||||
"""
|
||||
"""The standard activation format (num_tokens, hidden dim)."""
|
||||
|
||||
Standard = ("standard",)
|
||||
"""
|
||||
@@ -97,9 +95,7 @@ class FusedMoEActivationFormat(Enum):
|
||||
|
||||
@dataclass
|
||||
class ExpertTokensMetadata:
|
||||
"""
|
||||
Metadata regarding expert-token routing.
|
||||
"""
|
||||
"""Metadata regarding expert-token routing."""
|
||||
|
||||
expert_num_tokens: torch.Tensor | None
|
||||
expert_num_tokens_cpu: torch.Tensor | None
|
||||
@@ -122,9 +118,7 @@ class ExpertTokensMetadata:
|
||||
|
||||
|
||||
class TopKWeightAndReduce(ABC):
|
||||
"""
|
||||
An abstract base class for weight application and reduction implementations.
|
||||
"""
|
||||
"""An abstract base class for weight application and reduction implementations."""
|
||||
|
||||
@abstractmethod
|
||||
def apply(
|
||||
@@ -135,8 +129,7 @@ class TopKWeightAndReduce(ABC):
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply topk_weights to the fused_experts_outputs and/or reduce.
|
||||
"""Apply topk_weights to the fused_experts_outputs and/or reduce.
|
||||
If an output tensor is not passed, it will be created in the
|
||||
function.
|
||||
"""
|
||||
@@ -185,8 +178,7 @@ ReceiverType = Callable[[], PrepareResultType]
|
||||
|
||||
|
||||
class FusedMoEPrepareAndFinalize(ABC):
|
||||
"""
|
||||
An abstract base class for the [Quantize-Prepare] and [Finalize] steps
|
||||
"""An abstract base class for the [Quantize-Prepare] and [Finalize] steps
|
||||
described above.
|
||||
|
||||
There are two variants of this class:
|
||||
@@ -195,8 +187,7 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
"""
|
||||
|
||||
def post_init_setup(self, fused_experts: "FusedMoEExperts"):
|
||||
"""
|
||||
Initialize FusedMoEPrepareAndFinalizeModular settings that depend on
|
||||
"""Initialize FusedMoEPrepareAndFinalizeModular settings that depend on
|
||||
FusedMoEExpertsModular experts object.
|
||||
The FusedMoEPrepareAndFinalizeModular implementations that have such
|
||||
dependencies may choose to override this function.
|
||||
@@ -206,16 +197,14 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
def activation_format(self) -> FusedMoEActivationFormat:
|
||||
"""
|
||||
A property indicating the output format of the activations for the
|
||||
"""A property indicating the output format of the activations for the
|
||||
'prepare' method.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def topk_indices_dtype(self) -> torch.dtype | None:
|
||||
"""
|
||||
The PrepareFinalize All2All implementations generally constrain the
|
||||
"""The PrepareFinalize All2All implementations generally constrain the
|
||||
dtype of the topk_ids they support. This function returns the
|
||||
required topk indices dtype so it can be respected.
|
||||
Return None if there are no such restrictions.
|
||||
@@ -224,8 +213,7 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def max_num_tokens_per_rank(self) -> int | None:
|
||||
"""
|
||||
Some PrepareFinalize All2All implementations are batched. Meaning,
|
||||
"""Some PrepareFinalize All2All implementations are batched. Meaning,
|
||||
they can process only as set of tokens at a time. This
|
||||
function returns the batch size i.e the maximum number of tokens
|
||||
the implementation can process at a time.
|
||||
@@ -239,15 +227,13 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def output_is_reduced(self) -> bool:
|
||||
"""
|
||||
Indicates whether or not the output of finalize is reduced across all
|
||||
"""Indicates whether or not the output of finalize is reduced across all
|
||||
ranks.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def supports_async(self) -> bool:
|
||||
"""
|
||||
Indicates whether or not this class implements prepare_async and
|
||||
"""Indicates whether or not this class implements prepare_async and
|
||||
finalize_async.
|
||||
"""
|
||||
return False
|
||||
@@ -255,8 +241,7 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
|
||||
# TODO: pass FusedMoEParallelConfig in as ctor parameter?
|
||||
class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
|
||||
"""
|
||||
An abstract base class for the [Quantize-Prepare] and [Finalize] steps
|
||||
"""An abstract base class for the [Quantize-Prepare] and [Finalize] steps
|
||||
described above for the Modular case.
|
||||
"""
|
||||
|
||||
@@ -272,8 +257,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool,
|
||||
) -> PrepareResultType:
|
||||
"""
|
||||
Perform any quantization (and/or) dispatching needed for this kernel.
|
||||
"""Perform any quantization (and/or) dispatching needed for this kernel.
|
||||
- a1: The (unquantized) input to the MoE layer.
|
||||
- topk_ids: The topk ids.
|
||||
- topk_weights: The topk weights.
|
||||
@@ -309,8 +293,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool,
|
||||
) -> tuple[Callable, ReceiverType] | ReceiverType:
|
||||
"""
|
||||
Perform any quantization (and/or) dispatching needed for this kernel
|
||||
"""Perform any quantization (and/or) dispatching needed for this kernel
|
||||
but do not wait for results from other workers.
|
||||
- a1: The (unquantized) input to the MoE layer.
|
||||
- a1_scale: Optional scales for a1
|
||||
@@ -360,8 +343,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: TopKWeightAndReduce,
|
||||
) -> None:
|
||||
"""
|
||||
Perform any combine plus apply weights and perform a reduction on the
|
||||
"""Perform any combine plus apply weights and perform a reduction on the
|
||||
fused experts output.
|
||||
- output: The output tensor, written in place. Must be (M, K) shape.
|
||||
- fused_expert_output: The unweighted, unreduced output of the fused
|
||||
@@ -384,8 +366,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: TopKWeightAndReduce,
|
||||
) -> tuple[Callable, Callable] | Callable:
|
||||
"""
|
||||
Perform any combine plus apply weights and perform a reduction on the
|
||||
"""Perform any combine plus apply weights and perform a reduction on the
|
||||
fused experts output but do not wait for results from other workers.
|
||||
- output: The output tensor, written in place. Must be (M, K) shape.
|
||||
- fused_expert_output: The unweighted, unreduced output of the fused
|
||||
@@ -419,8 +400,7 @@ class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
|
||||
|
||||
|
||||
class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
|
||||
"""
|
||||
An abstract base class for the [Quantize-Prepare] and [Finalize] steps
|
||||
"""An abstract base class for the [Quantize-Prepare] and [Finalize] steps
|
||||
described above for the monolithic case.
|
||||
"""
|
||||
|
||||
@@ -435,8 +415,7 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> PrepareMonolithicResultType:
|
||||
"""
|
||||
Optional method for subclasses compatible with monolithic
|
||||
"""Optional method for subclasses compatible with monolithic
|
||||
FusedMoEExpertsModular kernels.
|
||||
|
||||
Perform any quantization (and/or) dispatching needed for this kernel.
|
||||
@@ -453,8 +432,7 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
|
||||
|
||||
@abstractmethod
|
||||
def finalize(self, fused_expert_output: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Optional method for subclasses compatible with monolithic
|
||||
"""Optional method for subclasses compatible with monolithic
|
||||
FusedMoEExpertsModular kernels.
|
||||
|
||||
Perform any combine plus apply weights and perform a reduction on the
|
||||
@@ -485,8 +463,7 @@ class FusedMoEExperts(ABC):
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
"""
|
||||
moe_config: MoE layer configuration.
|
||||
"""moe_config: MoE layer configuration.
|
||||
quant_config: Quantization parameters for this experts instance.
|
||||
"""
|
||||
if self.activation_format() == FusedMoEActivationFormat.Standard and (
|
||||
@@ -521,8 +498,7 @@ class FusedMoEExperts(ABC):
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
"""
|
||||
Whether or not the PrepareFinalize should defer input quantization
|
||||
"""Whether or not the PrepareFinalize should defer input quantization
|
||||
in the prepare step. If True, then the Experts kernel will
|
||||
execute the input quantization itself.
|
||||
|
||||
@@ -533,8 +509,7 @@ class FusedMoEExperts(ABC):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def activation_format() -> FusedMoEActivationFormat:
|
||||
"""
|
||||
A property which is a tuple of the input and output activation formats
|
||||
"""A property which is a tuple of the input and output activation formats
|
||||
for the 'apply' method.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -595,8 +570,7 @@ class FusedMoEExperts(ABC):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def _supports_current_device() -> bool:
|
||||
"""
|
||||
Whether the kernel supports the current device type
|
||||
"""Whether the kernel supports the current device type
|
||||
(compute cability and current platform).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -604,8 +578,7 @@ class FusedMoEExperts(ABC):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
"""
|
||||
Whether the kernel supports act_and_mul=False, i.e.
|
||||
"""Whether the kernel supports act_and_mul=False, i.e.
|
||||
non-gated MoE models like Nemotron-Nano.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -621,16 +594,13 @@ class FusedMoEExperts(ABC):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
"""
|
||||
Whether the kernel supports a particular act function.
|
||||
"""
|
||||
"""Whether the kernel supports a particular act function."""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
"""
|
||||
Whether the kernel supports deployment in particular parallel config.
|
||||
"""Whether the kernel supports deployment in particular parallel config.
|
||||
|
||||
Can be overridden if a kernel does not support EP, SP or some other
|
||||
configuration.
|
||||
@@ -643,8 +613,7 @@ class FusedMoEExperts(ABC):
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
"""
|
||||
Whether the kernel supports a routing method (e.g. GroupedTopK).
|
||||
"""Whether the kernel supports a routing method (e.g. GroupedTopK).
|
||||
|
||||
Can be overridden by monolithic kernels that execute the router
|
||||
in addition to the experts if certain routers are not supported.
|
||||
@@ -656,8 +625,7 @@ class FusedMoEExperts(ABC):
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
"""
|
||||
Whether a kernel supports a particular dtype for router logits input.
|
||||
"""Whether a kernel supports a particular dtype for router logits input.
|
||||
|
||||
Can be overridden by monolithic kernels that execute the router
|
||||
in addition to the experts if certain dtypes are not supported.
|
||||
@@ -666,16 +634,14 @@ class FusedMoEExperts(ABC):
|
||||
|
||||
@staticmethod
|
||||
def _supports_shape(hidden_dim: int) -> bool:
|
||||
"""
|
||||
Whether a kernel supports a particular shape. Can be overridden if a kernel
|
||||
"""Whether a kernel supports a particular shape. Can be overridden if a kernel
|
||||
has specific shape requirements.
|
||||
"""
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_batch_invariance() -> bool:
|
||||
"""
|
||||
Whether the kernel supports batch invariance, i.e. the output does not
|
||||
"""Whether the kernel supports batch invariance, i.e. the output does not
|
||||
depend on the order of the tokens in the input batch. This is useful
|
||||
for determining if the kernel can used with VLLM_BATCH_INVARIANT=1.
|
||||
"""
|
||||
@@ -764,17 +730,15 @@ class FusedMoEExperts(ABC):
|
||||
return False
|
||||
|
||||
def supports_packed_ue8m0_act_scales(self) -> bool:
|
||||
"""
|
||||
A flag indicating whether or not this class can process packed ue8m0
|
||||
"""A flag indicating whether or not this class can process packed ue8m0
|
||||
activation scales.
|
||||
"""
|
||||
return False
|
||||
|
||||
|
||||
class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
"""
|
||||
An abstract base class for the [Permute-Experts-Unpermute] step described
|
||||
above.
|
||||
"""An abstract base class for the [Permute-Experts-Unpermute] step described
|
||||
above.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@@ -788,8 +752,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
"""
|
||||
Extract the MoE problem size from the given tensor arguments:
|
||||
"""Extract the MoE problem size from the given tensor arguments:
|
||||
- a: The hidden states, input to the MoE layer.
|
||||
- w1: The first set of expert weights.
|
||||
- w2: The second set of expert weights.
|
||||
@@ -825,9 +788,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
return E, M, N, K, topk
|
||||
|
||||
def workspace_dtype(self, act_dtype: torch.dtype) -> torch.dtype:
|
||||
"""
|
||||
Workspace type: The dtype to use for the workspace tensors.
|
||||
"""
|
||||
"""Workspace type: The dtype to use for the workspace tensors."""
|
||||
return act_dtype
|
||||
|
||||
@abstractmethod
|
||||
@@ -842,8 +803,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
expert_tokens_meta: ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
"""
|
||||
Compute the shapes for the temporary and final outputs of the two gemms
|
||||
"""Compute the shapes for the temporary and final outputs of the two gemms
|
||||
and activation in the fused expert function. Since the gemms are
|
||||
independent, the workspace for the first gemm can be shared with the
|
||||
workspace for the last gemm.
|
||||
@@ -873,8 +833,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
|
||||
@staticmethod
|
||||
def adjust_N_for_activation(N: int, activation: MoEActivation) -> int:
|
||||
"""
|
||||
Calculate the output dimension for the activation function.
|
||||
"""Calculate the output dimension for the activation function.
|
||||
|
||||
For *_no_mul activations (e.g. relu2_no_mul),
|
||||
there's no gate/up split, so output size equals input size (N).
|
||||
@@ -888,6 +847,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
|
||||
Returns:
|
||||
The output dimension after activation.
|
||||
|
||||
"""
|
||||
return N if not activation.is_gated else N // 2
|
||||
|
||||
@@ -934,49 +894,50 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
expert_tokens_meta: ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
"""
|
||||
This function computes the intermediate result of a Mixture of Experts
|
||||
"""This function computes the intermediate result of a Mixture of Experts
|
||||
(MoE) layer using two sets of weights, w1 and w2.
|
||||
|
||||
Parameters:
|
||||
- output: (torch.Tensor): The unweighted, unreduced output tensor.
|
||||
- hidden_states: (torch.Tensor): The (quantized) input tensor to the MoE
|
||||
layer.
|
||||
- w1 (torch.Tensor): The first set of expert weights.
|
||||
- w2 (torch.Tensor): The second set of expert weights.
|
||||
- topk_weights: A map of row to expert weights. Some implementations
|
||||
choose to do weight application.
|
||||
- topk_ids (torch.Tensor): A map of row to expert id.
|
||||
- activation (str): The activation function to apply after the first
|
||||
MoE layer.
|
||||
- global_num_experts (int): The total number of experts in the global
|
||||
expert space.
|
||||
- expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
|
||||
from the global expert space to the local expert space of the expert
|
||||
parallel shard.
|
||||
- a1q_scale (Optional[torch.Tensor]): Optional quantized scale to be
|
||||
used for a1. Result of quantization from prepare/finalize and not
|
||||
from the FusedMoEQuantConfig.
|
||||
- workspace13 (torch.Tensor): A scratch tensor used for gemm outputs
|
||||
must be large enough to hold output of either MoE gemm.
|
||||
- workspace2 (torch.Tensor): A scratch tensor used for the activation
|
||||
function.
|
||||
- expert_tokens_meta (Optional[ExpertTokensMetadata]) - An optional
|
||||
ExpertTokensMetadata object containing gpu/cpu tensors
|
||||
as big as the number of local experts with the information about the
|
||||
number of tokens assigned to each local expert.
|
||||
- apply_router_weight_on_input: True if router weights are already
|
||||
applied on the input. This is relevant if the implementation
|
||||
chooses to do weight application.
|
||||
Args:
|
||||
output: (torch.Tensor): The unweighted, unreduced output tensor.
|
||||
hidden_states: (torch.Tensor): The (quantized) input tensor to the MoE
|
||||
layer.
|
||||
w1 (torch.Tensor): The first set of expert weights.
|
||||
w2 (torch.Tensor): The second set of expert weights.
|
||||
topk_weights: A map of row to expert weights. Some implementations
|
||||
choose to do weight application.
|
||||
topk_ids (torch.Tensor): A map of row to expert id.
|
||||
activation (str): The activation function to apply after the first
|
||||
MoE layer.
|
||||
global_num_experts (int): The total number of experts in the global
|
||||
expert space.
|
||||
expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
|
||||
from the global expert space to the local expert space of the
|
||||
expert parallel shard.
|
||||
a1q_scale (Optional[torch.Tensor]): Optional quantized scale to be
|
||||
used for a1. Result of quantization from prepare/finalize and not
|
||||
from the FusedMoEQuantConfig.
|
||||
a2_scale (Optional[torch.Tensor]): Optional quantized scale to be
|
||||
used for the second gemm's activations.
|
||||
workspace13 (torch.Tensor): A scratch tensor used for gemm outputs
|
||||
must be large enough to hold output of either MoE gemm.
|
||||
workspace2 (torch.Tensor): A scratch tensor used for the activation
|
||||
function.
|
||||
expert_tokens_meta (Optional[ExpertTokensMetadata]): An optional
|
||||
ExpertTokensMetadata object containing gpu/cpu tensors
|
||||
as big as the number of local experts with the information about
|
||||
the number of tokens assigned to each local expert.
|
||||
apply_router_weight_on_input: True if router weights are already
|
||||
applied on the input. This is relevant if the implementation
|
||||
chooses to do weight application.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
"""
|
||||
An abstract base class for the [Permute-Experts-Unpermute] step described
|
||||
above, but with the monolithic interface (accepts router logits
|
||||
rather than topk ids and weights).
|
||||
"""An abstract base class for the [Permute-Experts-Unpermute] step described
|
||||
above, but with the monolithic interface (accepts router logits
|
||||
rather than topk ids and weights).
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@@ -985,8 +946,7 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
"""
|
||||
Whether the kernel supports a routing method (e.g. GroupedTopK).
|
||||
"""Whether the kernel supports a routing method (e.g. GroupedTopK).
|
||||
|
||||
Monolithic kernels should explicitly opt-in to support.
|
||||
"""
|
||||
@@ -997,8 +957,7 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
"""
|
||||
Whether the kernel supports a dtype for router logits.
|
||||
"""Whether the kernel supports a dtype for router logits.
|
||||
|
||||
Modular kernels should opt-in to support.
|
||||
"""
|
||||
@@ -1083,8 +1042,7 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""
|
||||
Same as apply(), except uses router_logits as opposed
|
||||
"""Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is useful for kernels
|
||||
with fused router and fused_experts (e.g. FLASHINFER_TRTLLM).
|
||||
"""
|
||||
@@ -1128,8 +1086,7 @@ class FusedMoEKernelModularImpl:
|
||||
expert_tokens_meta: ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Allocate temporary and output buffers for the fused experts op.
|
||||
"""Allocate temporary and output buffers for the fused experts op.
|
||||
Inputs:
|
||||
- out_dtype: output type of workspace and output tensors.
|
||||
- device: the device of the workspace and output tensors.
|
||||
@@ -1223,11 +1180,9 @@ class FusedMoEKernelModularImpl:
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]:
|
||||
"""
|
||||
The _prepare method is a wrapper around self.prepare_finalize.prepare
|
||||
"""The _prepare method is a wrapper around self.prepare_finalize.prepare
|
||||
that handles DBO and async.
|
||||
"""
|
||||
|
||||
if not self.prepare_finalize.supports_async():
|
||||
# We shouldn't be running an a2a kernel that doesn't
|
||||
# support async prepare/finalize
|
||||
@@ -1392,17 +1347,24 @@ class FusedMoEKernelModularImpl:
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
The _finalize method is a wrapper around self.prepare_finalize.finalize
|
||||
"""The _finalize method is a wrapper around self.prepare_finalize.finalize
|
||||
that handles DBO, async and shared expert overlap.
|
||||
|
||||
Args:
|
||||
output: Tensor the finalized result is written into.
|
||||
fused_out: The unweighted, unreduced output of the fused experts.
|
||||
hidden_states: The input tensor to the MoE layer.
|
||||
topk_weights: A map of row to expert weights.
|
||||
topk_ids: A map of row to expert id.
|
||||
apply_router_weight_on_input: True if the router weights were
|
||||
already applied on the input.
|
||||
shared_experts: SharedExperts | None. The shared experts if any.
|
||||
shared_experts_input: Optional separate input for shared experts.
|
||||
When latent MoE is used, hidden_states is the latent-projected
|
||||
tensor (smaller dimension) used by routed experts, while
|
||||
shared_experts_input is the original hidden_states (full
|
||||
dimension) needed by the shared expert MLP.
|
||||
|
||||
"""
|
||||
if not self.prepare_finalize.supports_async():
|
||||
assert not dbo_enabled()
|
||||
@@ -1463,33 +1425,34 @@ class FusedMoEKernelModularImpl:
|
||||
shared_experts: SharedExperts | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a Mixture of Experts (MoE) layer using two sets
|
||||
"""This function computes a Mixture of Experts (MoE) layer using two sets
|
||||
of weights, w1 and w2, and top-k gating mechanism.
|
||||
|
||||
Parameters:
|
||||
- hidden_states: (torch.Tensor): The input tensor to the MoE layer.
|
||||
- w1 (torch.Tensor): The first set of expert weights.
|
||||
- w2 (torch.Tensor): The second set of expert weights.
|
||||
- topk_weights (torch.Tensor): The topk weights applied at the end of the layer.
|
||||
- topk_ids (torch.Tensor): A map of row to expert id.
|
||||
- activation (MoEActivation): The activation function to apply after the first
|
||||
MoE layer.
|
||||
- global_num_experts (int): The total number of experts in the global
|
||||
expert space.
|
||||
- expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
|
||||
from the global expert space to the local expert space of the expert
|
||||
parallel shard.
|
||||
- apply_router_weight_on_input (bool): When true, the topk weights are
|
||||
applied directly on the inputs. This is only applicable when topk is
|
||||
1.
|
||||
- shared_experts: SharedExperts | None. The shared experts if any.
|
||||
- shared_experts_input (Optional[torch.Tensor]): Optional separate
|
||||
input for shared experts. For latent MoE, this is the original
|
||||
hidden_states before latent projection.
|
||||
Args:
|
||||
hidden_states: (torch.Tensor): The input tensor to the MoE layer.
|
||||
w1 (torch.Tensor): The first set of expert weights.
|
||||
w2 (torch.Tensor): The second set of expert weights.
|
||||
topk_ids (torch.Tensor): A map of row to expert id.
|
||||
topk_weights (torch.Tensor): The topk weights applied at the end of
|
||||
the layer.
|
||||
activation (MoEActivation): The activation function to apply after
|
||||
the first MoE layer.
|
||||
global_num_experts (int): The total number of experts in the global
|
||||
expert space.
|
||||
expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
|
||||
from the global expert space to the local expert space of the
|
||||
expert parallel shard.
|
||||
apply_router_weight_on_input (bool): When true, the topk weights are
|
||||
applied directly on the inputs. This is only applicable when
|
||||
topk is 1.
|
||||
shared_experts: SharedExperts | None. The shared experts if any.
|
||||
shared_experts_input (Optional[torch.Tensor]): Optional separate
|
||||
input for shared experts. For latent MoE, this is the original
|
||||
hidden_states before latent projection.
|
||||
|
||||
Returns:
|
||||
- torch.Tensor: The output tensor after applying the MoE layer.
|
||||
torch.Tensor: The output tensor after applying the MoE layer.
|
||||
|
||||
"""
|
||||
output = torch.empty_like(hidden_states)
|
||||
|
||||
@@ -1571,12 +1534,10 @@ class FusedMoEKernelMonolithicImpl:
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""
|
||||
Same as forward(), except uses router_logits as opposed
|
||||
"""Same as forward(), except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is used for kernels
|
||||
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
|
||||
"""
|
||||
|
||||
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
|
||||
hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -1674,8 +1635,7 @@ class FusedMoEKernel:
|
||||
return self.fused_experts.supports_lora()
|
||||
|
||||
def _post_init_setup(self):
|
||||
"""
|
||||
Resolve any leftover setup dependencies between self.prepare_finalize
|
||||
"""Resolve any leftover setup dependencies between self.prepare_finalize
|
||||
and self.fused_experts here.
|
||||
"""
|
||||
self.prepare_finalize.post_init_setup(self.impl.fused_experts)
|
||||
@@ -1685,8 +1645,7 @@ class FusedMoEKernel:
|
||||
)
|
||||
|
||||
def output_is_reduced(self) -> bool:
|
||||
"""
|
||||
Indicates whether or not the output of fused MoE kernel
|
||||
"""Indicates whether or not the output of fused MoE kernel
|
||||
is reduced across all ranks.
|
||||
"""
|
||||
return self.prepare_finalize.output_is_reduced()
|
||||
|
||||
@@ -16,8 +16,7 @@ def moe_align_block_size(
|
||||
pad_sorted_ids: bool = False,
|
||||
ignore_invalid_experts: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Aligns the token distribution across experts to be compatible with block
|
||||
"""Aligns the token distribution across experts to be compatible with block
|
||||
size for matrix multiplication.
|
||||
|
||||
Note: In the case of expert_parallel, moe_align_block_size initially
|
||||
@@ -26,23 +25,23 @@ def moe_align_block_size(
|
||||
the current GPU rank as -1 so the MoE matmuls could skip those blocks.
|
||||
This requires the num_experts input arg to be the num global experts.
|
||||
|
||||
Parameters:
|
||||
- topk_ids: A tensor of shape [total_tokens, top_k] representing the
|
||||
top-k expert indices for each token.
|
||||
- block_size: The block size used in block matrix multiplication.
|
||||
- num_experts: The total number of experts.
|
||||
- expert_map: A tensor of shape [num_experts] that maps the expert index
|
||||
from the global space to the local index space of the current
|
||||
expert parallel shard. If the expert is not in the current expert
|
||||
parallel shard, the mapping is set to -1.
|
||||
- pad_sorted_ids: A flag indicating whether the sorted_token_ids length
|
||||
should be padded to a multiple of block_size,
|
||||
- ignore_invalid_experts: A flag indicating whether to ignore invalid
|
||||
experts. When False, all expert_ids in topk_ids will participate in
|
||||
counting and ranking, but invalid experts in expert_ids will be marked
|
||||
as -1. When True, all invalid expert_ids in topk_ids will be ignored
|
||||
and will not participate in counting or ranking, and there will be no
|
||||
-1 in expert_ids.
|
||||
Args:
|
||||
topk_ids: A tensor of shape [total_tokens, top_k] representing the
|
||||
top-k expert indices for each token.
|
||||
block_size: The block size used in block matrix multiplication.
|
||||
num_experts: The total number of experts.
|
||||
expert_map: A tensor of shape [num_experts] that maps the expert index
|
||||
from the global space to the local index space of the current
|
||||
expert parallel shard. If the expert is not in the current expert
|
||||
parallel shard, the mapping is set to -1.
|
||||
pad_sorted_ids: A flag indicating whether the sorted_token_ids length
|
||||
should be padded to a multiple of block_size,
|
||||
ignore_invalid_experts: A flag indicating whether to ignore invalid
|
||||
experts. When False, all expert_ids in topk_ids will participate in
|
||||
counting and ranking, but invalid experts in expert_ids will be marked
|
||||
as -1. When True, all invalid expert_ids in topk_ids will be ignored
|
||||
and will not participate in counting or ranking, and there will be no
|
||||
-1 in expert_ids.
|
||||
|
||||
Returns:
|
||||
- sorted_token_ids: A tensor containing the sorted token indices according
|
||||
@@ -70,6 +69,7 @@ def moe_align_block_size(
|
||||
the subsequent matrix multiplication.
|
||||
- The padding ensures that the total number of tokens is now divisible
|
||||
by block_size for proper block matrix operations.
|
||||
|
||||
"""
|
||||
max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
|
||||
if pad_sorted_ids:
|
||||
@@ -106,8 +106,7 @@ def moe_align_block_size(
|
||||
def batched_moe_align_block_size(
|
||||
max_tokens_per_batch: int, block_size: int, expert_num_tokens: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Given num_batches, max_tokens_per_batch, block_size and the number of
|
||||
"""Given num_batches, max_tokens_per_batch, block_size and the number of
|
||||
valid-tokens in each batch, prepare sorted_token_ids, expert_ids and
|
||||
num_tokens_post_pad. sorted_token_ids, expert_ids and num_tokens_post_pad
|
||||
have the same semantics as in moe_align_block_size.
|
||||
@@ -115,12 +114,12 @@ def batched_moe_align_block_size(
|
||||
This function is intended to be a drop in replacement for
|
||||
moe_align_batch_size for the batched case.
|
||||
|
||||
Parameters:
|
||||
- max_tokens_per_batch (int): Number of tokens in each batch (both
|
||||
valid and invalid).
|
||||
- block_size (int): block_size to align the data to.
|
||||
- expert_num_tokens (torch.Tensor): expert_num_tokens[i], indicates
|
||||
the number of valid tokens in batch i.
|
||||
Args:
|
||||
max_tokens_per_batch (int): Number of tokens in each batch (both
|
||||
valid and invalid).
|
||||
block_size (int): block_size to align the data to.
|
||||
expert_num_tokens (torch.Tensor): expert_num_tokens[i], indicates
|
||||
the number of valid tokens in batch i.
|
||||
|
||||
Returns:
|
||||
- sorted_token_ids (torch.Tensor): Torch tensor of size
|
||||
@@ -132,6 +131,7 @@ def batched_moe_align_block_size(
|
||||
- num_tokens_post_pad (torch.Tensor): Torch tensor of size 1
|
||||
indicating the number of valid blocks with actual data to
|
||||
process. This is represented in terms of num tokens.
|
||||
|
||||
Example:
|
||||
Let num_batches=5, max_tokens_per_batch=8, block_size=4, and
|
||||
expert_num_tokens=[2, 3, 0, 6, 8]. This expert_num_tokens tensor
|
||||
@@ -166,8 +166,8 @@ def batched_moe_align_block_size(
|
||||
|
||||
num_tokens_post_pad will be 24 as sorted_token_ids has valid entries
|
||||
until 24.
|
||||
"""
|
||||
|
||||
"""
|
||||
B = expert_num_tokens.size(0)
|
||||
device = expert_num_tokens.device
|
||||
|
||||
|
||||
@@ -92,8 +92,7 @@ def moe_fused_mul_sum(
|
||||
expert_map: torch.Tensor | None = None,
|
||||
num_valid_tokens: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Fused kernel for MoE (Mixture of Experts) to perform weighted summation
|
||||
"""Fused kernel for MoE (Mixture of Experts) to perform weighted summation
|
||||
of expert outputs.
|
||||
|
||||
Args:
|
||||
@@ -122,6 +121,7 @@ def moe_fused_mul_sum(
|
||||
Returns:
|
||||
The fused weighted sum of expert outputs.
|
||||
Shape: (num_tokens, hidden_size).
|
||||
|
||||
"""
|
||||
assert inputs.ndim == 3
|
||||
assert topk_weights.ndim == 2
|
||||
|
||||
@@ -57,8 +57,8 @@ def convert_flashinfer_moe_output(
|
||||
|
||||
Raises:
|
||||
ValueError: If FlashInfer returns an unexpected layout.
|
||||
"""
|
||||
|
||||
"""
|
||||
if num_tokens < 0:
|
||||
raise ValueError("num_tokens must be non-negative.")
|
||||
if top_k <= 0:
|
||||
|
||||
@@ -112,20 +112,24 @@ def moe_permute(
|
||||
permuted_hidden_states: torch.Tensor | None = None,
|
||||
scratch: MoEPermuteScratch | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
This function expands and permutes activation to gather uncontinuous tokens
|
||||
"""This function expands and permutes activation to gather uncontinuous tokens
|
||||
for each expert.
|
||||
Parameters:
|
||||
- hidden_states (torch.Tensor): The input tensor to the MoE layer.
|
||||
- a1q_scale (Optional[torch.Tensor]): quant scale for hidden_states
|
||||
- topk_ids (torch.Tensor): topk expert route id for each token.
|
||||
- n_expert (int): The number of expert.
|
||||
- n_local_expert (int): The number of expert in current EP rank.
|
||||
- expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
|
||||
from the global expert space to the local expert space of the expert
|
||||
parallel shard.
|
||||
- permuted_hidden_states (Optional[torch.Tensor]): Optional output tensor.
|
||||
If None, the output tensor will be created in this function.
|
||||
|
||||
Args:
|
||||
hidden_states (torch.Tensor): The input tensor to the MoE layer.
|
||||
a1q_scale (Optional[torch.Tensor]): quant scale for hidden_states
|
||||
topk_ids (torch.Tensor): topk expert route id for each token.
|
||||
n_expert (int): The number of expert.
|
||||
n_local_expert (int): The number of expert in current EP rank.
|
||||
expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
|
||||
from the global expert space to the local expert space of the expert
|
||||
parallel shard.
|
||||
permuted_hidden_states (Optional[torch.Tensor]): Optional output tensor.
|
||||
If None, the output tensor will be created in this function.
|
||||
scratch (Optional[MoEPermuteScratch]): Optional preallocated scratch
|
||||
buffers. Validated against hidden_states and topk_ids when given,
|
||||
otherwise the buffers are allocated in this function.
|
||||
|
||||
Returns:
|
||||
- permuted_hidden_states (torch.Tensor): permuted activation.
|
||||
- a1q_scale (Optional[torch.Tensor]): permuted quant scale for hidden_states
|
||||
@@ -134,6 +138,7 @@ def moe_permute(
|
||||
of each expert for standard grouped gemm.
|
||||
- inv_permuted_idx (torch.Tensor): idx map for moe_unpermute.
|
||||
- permuted_idx (torch.Tensor): idx map from hidden to permuted_hidden.
|
||||
|
||||
"""
|
||||
n_token, n_hidden = hidden_states.size()
|
||||
topk = topk_ids.size(1)
|
||||
@@ -249,19 +254,21 @@ def moe_unpermute(
|
||||
inv_permuted_idx: torch.Tensor,
|
||||
expert_first_token_offset: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
This function expands and permutes activation to gathering uncontinuous
|
||||
"""This function expands and permutes activation to gathering uncontinuous
|
||||
tokens for each expert.
|
||||
Parameters:
|
||||
- out (torch.Tensor): output tensor
|
||||
- permuted_hidden_states (torch.Tensor): permuted activation.
|
||||
- topk_weights (torch.Tensor): topk expert route weight for each token.
|
||||
- inv_permuted_idx (torch.Tensor): row idx map for moe_unpermute.
|
||||
- expert_first_token_offset (Optional[torch.Tensor]): offset of the first
|
||||
token of each expert for grouped gemm.
|
||||
|
||||
Args:
|
||||
out (torch.Tensor): output tensor
|
||||
permuted_hidden_states (torch.Tensor): permuted activation.
|
||||
topk_weights (torch.Tensor): topk expert route weight for each token.
|
||||
inv_permuted_idx (torch.Tensor): row idx map for moe_unpermute.
|
||||
expert_first_token_offset (Optional[torch.Tensor]): offset of the first
|
||||
token of each expert for grouped gemm.
|
||||
|
||||
Returns:
|
||||
- hidden_states (torch.Tensor): The reduced and unpermuted activation
|
||||
tensor.
|
||||
|
||||
"""
|
||||
topk = topk_weights.size(1)
|
||||
n_hidden = permuted_hidden_states.size(-1)
|
||||
|
||||
@@ -55,24 +55,28 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
|
||||
@abstractmethod
|
||||
def backend_enum_cls(self) -> type[BackendT]:
|
||||
"""Return the concrete `Enum` class enumerating this oracle's
|
||||
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`)."""
|
||||
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]:
|
||||
"""Return platform-appropriate backends in priority order for
|
||||
this `moe_config`."""
|
||||
this `moe_config`.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def backend_to_kernel_cls(
|
||||
self, backend: BackendT
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
"""Map a backend enum value to its concrete `FusedMoEExperts`
|
||||
subclasses, in selection priority order."""
|
||||
subclasses, in selection priority order.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
|
||||
"""Map a user-facing `MoEBackend` (from the runner config) to
|
||||
this oracle's enum."""
|
||||
this oracle's enum.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def select_backend(
|
||||
@@ -104,7 +108,8 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Construct the `FusedMoEKernel` (Prepare/Finalize + Experts
|
||||
combinator) for the chosen backend."""
|
||||
combinator) for the chosen backend.
|
||||
"""
|
||||
|
||||
def convert_to_kernel_format(
|
||||
self,
|
||||
|
||||
@@ -72,12 +72,10 @@ def _get_priority_backends(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> list[Fp8MoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
"""Get available backends in priority order based on platform and config.
|
||||
|
||||
This function can be extended to become more complex as needed.
|
||||
"""
|
||||
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Fp8MoeBackend.AITER,
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
@@ -273,8 +271,7 @@ def refine_fp8_moe_block_shape(
|
||||
config: FusedMoEConfig,
|
||||
weight_block_size: list[int],
|
||||
) -> list[int] | None:
|
||||
"""
|
||||
Compute a refined block shape for block-quantized FP8 MoE weights whose
|
||||
"""Compute a refined block shape for block-quantized FP8 MoE weights whose
|
||||
checkpoint blocks cannot be sharded exactly across TP ranks.
|
||||
|
||||
TP shards the intermediate dim of the expert weights, so a per-shard size
|
||||
@@ -308,11 +305,9 @@ def select_fp8_moe_backend(
|
||||
activation_key: QuantKey | None,
|
||||
allow_vllm_cutlass: bool = False,
|
||||
) -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts] | None]:
|
||||
"""
|
||||
Select the primary FP8 MoE backend
|
||||
"""Select the primary FP8 MoE backend
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
|
||||
# NOTE: the kernels are selected in the following order.
|
||||
AVAILABLE_BACKENDS = _get_priority_backends(config, weight_key, activation_key)
|
||||
|
||||
@@ -456,7 +451,8 @@ def _humming_fp8_weight_schema(
|
||||
layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
|
||||
) -> dict[str, Any]:
|
||||
"""Build the humming weight schema from the canonical on-device fp8/mxfp8
|
||||
tensors (scale dtype/shape, block size), not the producing quant method."""
|
||||
tensors (scale dtype/shape, block size), not the producing quant method.
|
||||
"""
|
||||
# mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
|
||||
# no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
|
||||
if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
|
||||
@@ -614,8 +610,7 @@ def make_fp8_moe_quant_config(
|
||||
gemm1_beta: float | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Create FusedMoEQuantConfig for the specified FP8 Backend.
|
||||
"""Create FusedMoEQuantConfig for the specified FP8 Backend.
|
||||
The FusedMoEQuantConfig holds the scales that are used
|
||||
at runtime by the Modular Kernel abstraction.
|
||||
|
||||
@@ -626,7 +621,6 @@ def make_fp8_moe_quant_config(
|
||||
In a future PR, we will have this function should be
|
||||
a method of the modular kernel itself.
|
||||
"""
|
||||
|
||||
# MARLIN and CPU are mixed precision W8A16 config.
|
||||
if fp8_backend == Fp8MoeBackend.MARLIN or fp8_backend == Fp8MoeBackend.CPU:
|
||||
return fp8_w8a16_moe_quant_config(
|
||||
|
||||
@@ -37,9 +37,7 @@ class Int8MoeBackend(Enum):
|
||||
def _get_priority_backends(
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> list[Int8MoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
"""
|
||||
"""Get available backends in priority order based on platform and config."""
|
||||
return [
|
||||
Int8MoeBackend.TRITON,
|
||||
Int8MoeBackend.HUMMING,
|
||||
@@ -100,11 +98,9 @@ def select_int8_moe_backend(
|
||||
weight_key: QuantKey | None = kInt8StaticChannelSym,
|
||||
activation_key: QuantKey | None = kInt8DynamicTokenSym,
|
||||
) -> tuple[Int8MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
"""
|
||||
Select the primary Int8 MoE backend.
|
||||
"""Select the primary Int8 MoE backend.
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
|
||||
AVAILABLE_BACKENDS = _get_priority_backends(config)
|
||||
|
||||
activation_format = (
|
||||
@@ -228,7 +224,8 @@ def _humming_int8_weight_schema(
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor
|
||||
) -> dict[str, Any]:
|
||||
"""Build the humming compressed-tensors int8 schema from the canonical
|
||||
on-device tensors; humming does the signed-int8 -> native conversion."""
|
||||
on-device tensors; humming does the signed-int8 -> native conversion.
|
||||
"""
|
||||
config: dict[str, Any] = {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "int-quantized",
|
||||
|
||||
@@ -115,9 +115,7 @@ def backend_to_kernel_cls(
|
||||
|
||||
|
||||
def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
"""
|
||||
"""Get available backends in priority order based on platform and config."""
|
||||
if current_platform.is_cpu():
|
||||
return [WNA16MoEBackend.CPU]
|
||||
if current_platform.is_xpu():
|
||||
@@ -241,11 +239,13 @@ def select_wna16_moe_backend(
|
||||
quant_config: Quantization structure and checkpoint format description.
|
||||
may_have_zp: Whether the integration can provide weight zero points.
|
||||
may_have_bias: Whether the integration can provide expert bias.
|
||||
allow_tile_padding: Whether backends that require padding the weights
|
||||
up to a tile boundary may be selected.
|
||||
|
||||
Returns:
|
||||
A tuple of (``WNA16MoEBackend``, experts class or ``None``).
|
||||
"""
|
||||
|
||||
"""
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
if config.moe_parallel_config.use_batched_activation_format
|
||||
@@ -512,7 +512,8 @@ def _process_weights_flashinfer(
|
||||
|
||||
def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor:
|
||||
"""Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)``
|
||||
tensor along its last dim, from ``unit`` to ``padded_unit`` columns."""
|
||||
tensor along its last dim, from ``unit`` to ``padded_unit`` columns.
|
||||
"""
|
||||
if padded_unit == unit:
|
||||
return x
|
||||
e, rows, _ = x.shape
|
||||
@@ -1080,7 +1081,8 @@ def _humming_wna16_weight_schema(
|
||||
quant_config: QuantizationConfig | QuantizationArgs | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Humming weight schema for a WNA16 checkpoint, derived from the quant
|
||||
config rather than the running kernel."""
|
||||
config rather than the running kernel.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
|
||||
@@ -1194,6 +1196,7 @@ def _unpack_and_dequant_int4_gptq(
|
||||
|
||||
Returns:
|
||||
Dequantized weight tensor in the requested layout.
|
||||
|
||||
"""
|
||||
E, K_packed, N = w_int32.shape
|
||||
K = K_packed * 8
|
||||
@@ -1256,6 +1259,7 @@ def _unpack_and_dequant_int4_awq(
|
||||
|
||||
Returns:
|
||||
Dequantized weight tensor in the requested layout.
|
||||
|
||||
"""
|
||||
E, K, N_packed = w_int32.shape
|
||||
N = N_packed * 8
|
||||
@@ -1442,6 +1446,15 @@ def convert_to_wna16_moe_kernel_format(
|
||||
layer: the ``MoERunner`` layer whose parameters are being prepared.
|
||||
quant_config: the ``QuantizationConfig`` for this layer.
|
||||
input_dtype: optional activation dtype, usually should be 16 bit.
|
||||
w13: fused gate/up expert weights.
|
||||
w2: down-projection expert weights.
|
||||
w13_scale: quantization scales for ``w13``.
|
||||
w2_scale: quantization scales for ``w2``.
|
||||
w13_qzeros: optional zero points for ``w13``.
|
||||
w2_qzeros: optional zero points for ``w2``.
|
||||
w13_bias: optional bias for ``w13``.
|
||||
w2_bias: optional bias for ``w2``.
|
||||
|
||||
"""
|
||||
if backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
@@ -331,7 +331,8 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
|
||||
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
"""Available backends in priority order, BF16-act variant before
|
||||
activation-quantized variant within each vendor family."""
|
||||
activation-quantized variant within each vendor family.
|
||||
"""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
@@ -355,8 +356,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
|
||||
|
||||
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
|
||||
"""Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
|
||||
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
|
||||
backend-level ``is_supported_config`` check filters by device capability).
|
||||
"""
|
||||
@@ -411,7 +411,8 @@ def _resolve_activation_key(
|
||||
model_activation_key: QuantKey | None,
|
||||
) -> QuantKey | None:
|
||||
"""Combine the model-supplied activation key with the user override.
|
||||
Raises on conflict (both set and disagreeing)."""
|
||||
Raises on conflict (both set and disagreeing).
|
||||
"""
|
||||
user_override = _user_moe_activation_override()
|
||||
if user_override is None:
|
||||
return model_activation_key
|
||||
@@ -461,7 +462,8 @@ def _filter_by_activation(
|
||||
) -> list[Mxfp4MoeBackend]:
|
||||
"""Pick variants matching ``requested_activation_key``; without one,
|
||||
prefer BF16 if the list has any, else keep the list as-is so explicit
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land.
|
||||
"""
|
||||
if requested_activation_key is not None:
|
||||
return [
|
||||
b
|
||||
@@ -526,8 +528,7 @@ def select_mxfp4_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
activation_key: QuantKey | None = None,
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts] | None]:
|
||||
"""
|
||||
Select the primary MXFP4 MoE backend.
|
||||
"""Select the primary MXFP4 MoE backend.
|
||||
|
||||
Args:
|
||||
config: MoE configuration
|
||||
@@ -536,6 +537,7 @@ def select_mxfp4_moe_backend(
|
||||
Use kFp8StaticTensorSym for W4A8 scheme.
|
||||
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
|
||||
"""
|
||||
runner_backend = config.moe_backend
|
||||
requested_activation_key = _resolve_activation_key(activation_key)
|
||||
@@ -648,8 +650,7 @@ def select_mxfp4_moe_backend(
|
||||
def select_deepseek_v4_mxfp4_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts] | None]:
|
||||
"""
|
||||
Select the MXFP4 MoE backend with MXFP8 activation as top priority.
|
||||
"""Select the MXFP4 MoE backend with MXFP8 activation as top priority.
|
||||
Falls back through BF16 and other backends.
|
||||
"""
|
||||
activation_format = (
|
||||
@@ -799,7 +800,6 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
"""Convert loaded weights into backend-specific kernel format."""
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
|
||||
w13_weight,
|
||||
|
||||
@@ -111,8 +111,8 @@ def select_mxfp8_moe_backend(
|
||||
|
||||
Returns:
|
||||
A tuple of (fp8_backend, experts_cls).
|
||||
"""
|
||||
|
||||
"""
|
||||
runner_backend = config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
backend = _BACKEND_NAME_MAP.get(runner_backend)
|
||||
|
||||
@@ -184,11 +184,9 @@ def select_nvfp4_moe_backend(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> tuple[NvFp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
"""
|
||||
Select the primary NvFP4 MoE backend
|
||||
"""Select the primary NvFP4 MoE backend
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
|
||||
# NOTE: the kernels are selected in the following order.
|
||||
# FLASHINFER_B12X is intentionally excluded from auto-selection until
|
||||
# the upstream CUTLASS SM121 MMA op guard is resolved; use
|
||||
|
||||
@@ -46,8 +46,7 @@ class UnquantizedMoeBackend(Enum):
|
||||
|
||||
|
||||
def _get_priority_backends(moe_config: FusedMoEConfig) -> list[UnquantizedMoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
"""Get available backends in priority order based on platform and config.
|
||||
|
||||
This function can be extended to become more complex as needed.
|
||||
"""
|
||||
@@ -208,11 +207,9 @@ def _trtllm_bf16_lora_supported(moe_config: FusedMoEConfig) -> bool:
|
||||
def select_unquantized_moe_backend(
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]:
|
||||
"""
|
||||
Select the primary Unquantized MoE backend.
|
||||
"""Select the primary Unquantized MoE backend.
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
|
||||
if current_platform.is_tpu():
|
||||
return UnquantizedMoeBackend.TPU, None
|
||||
|
||||
|
||||
@@ -32,8 +32,7 @@ class W4A8Int8MoeBackend(Enum):
|
||||
def _get_priority_backends(
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> list[W4A8Int8MoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
"""Get available backends in priority order based on platform and config.
|
||||
|
||||
Currently only CPU INT4 backend is available for W4A8 INT8 MoE.
|
||||
"""
|
||||
@@ -75,8 +74,7 @@ def select_w4a8_int8_moe_backend(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> tuple[W4A8Int8MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
"""
|
||||
Select the primary W4A8 Int8 MoE backend.
|
||||
"""Select the primary W4A8 Int8 MoE backend.
|
||||
|
||||
Args:
|
||||
config: MoE configuration
|
||||
@@ -85,8 +83,8 @@ def select_w4a8_int8_moe_backend(
|
||||
|
||||
Returns:
|
||||
Tuple of (backend, kernel_class)
|
||||
"""
|
||||
|
||||
"""
|
||||
AVAILABLE_BACKENDS = _get_priority_backends(config)
|
||||
|
||||
if not AVAILABLE_BACKENDS:
|
||||
@@ -160,8 +158,7 @@ def select_w4a8_int8_moe_backend(
|
||||
def make_w4a8_int8_moe_quant_config(
|
||||
block_shape: tuple[int, int] | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Create FusedMoEQuantConfig for W4A8 Int8 MoE.
|
||||
"""Create FusedMoEQuantConfig for W4A8 Int8 MoE.
|
||||
|
||||
Args:
|
||||
block_shape: Quantization block shape (row, col).
|
||||
@@ -170,6 +167,7 @@ def make_w4a8_int8_moe_quant_config(
|
||||
|
||||
Returns:
|
||||
FusedMoEQuantConfig with appropriate settings for W4A8 Int8
|
||||
|
||||
"""
|
||||
# W4A8 Int8 uses static weight quantization, dynamic activation quantization
|
||||
# Weights are 4-bit (stored as int8, packed to uint8),
|
||||
@@ -196,8 +194,7 @@ def pack_int4_weights_for_kleidi(
|
||||
out_features: int,
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Pack INT4 weights (stored as int8 in [-8,7]) to KleidiAI format.
|
||||
"""Pack INT4 weights (stored as int8 in [-8,7]) to KleidiAI format.
|
||||
|
||||
Args:
|
||||
int4_as_int8: [out, in] int8 tensor with values in [-8, 7]
|
||||
@@ -209,6 +206,7 @@ def pack_int4_weights_for_kleidi(
|
||||
|
||||
Returns:
|
||||
Packed weight tensor in KleidiAI format
|
||||
|
||||
"""
|
||||
# Shift to unsigned nibble [0, 15]
|
||||
tmp = int4_as_int8.add(8)
|
||||
@@ -250,8 +248,7 @@ def convert_to_w4a8_int8_moe_format(
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
"""
|
||||
Pack INT4 MoE weights to KleidiAI format.
|
||||
"""Pack INT4 MoE weights to KleidiAI format.
|
||||
|
||||
This function packs the INT4 weights (stored as int8 values) into
|
||||
the format expected by the KleidiAI dynamic_4bit_int_moe kernel.
|
||||
@@ -267,6 +264,7 @@ def convert_to_w4a8_int8_moe_format(
|
||||
|
||||
Returns:
|
||||
Tuple of (w13_packed, w2_packed) tensors
|
||||
|
||||
"""
|
||||
# Derive dimensions from tensor shapes
|
||||
E = w13_weight.shape[0] # num_experts
|
||||
@@ -315,8 +313,7 @@ def make_w4a8_int8_moe_kernel(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""
|
||||
Create FusedMoEKernel for W4A8 Int8 MoE.
|
||||
"""Create FusedMoEKernel for W4A8 Int8 MoE.
|
||||
|
||||
Args:
|
||||
moe_quant_config: Quantization configuration
|
||||
@@ -326,6 +323,7 @@ def make_w4a8_int8_moe_kernel(
|
||||
|
||||
Returns:
|
||||
Configured FusedMoEKernel instance
|
||||
|
||||
"""
|
||||
# Create Prepare/Finalize.
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
|
||||
@@ -16,8 +16,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
|
||||
|
||||
class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""
|
||||
A reference prepare/finalize class that reorganizes the tokens into
|
||||
"""A reference prepare/finalize class that reorganizes the tokens into
|
||||
expert batched format, i.e. E x max_num_tokens x K. This is the format
|
||||
that the batched dispatch/combine kernels use.
|
||||
"""
|
||||
|
||||
@@ -27,9 +27,7 @@ from vllm.v1.worker.ubatching import (
|
||||
|
||||
|
||||
class DeepEPHTPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""
|
||||
Prepare/Finalize using DeepEP High-Throughput kernels.
|
||||
"""
|
||||
"""Prepare/Finalize using DeepEP High-Throughput kernels."""
|
||||
|
||||
@staticmethod
|
||||
def maybe_roundup_layer_hidden_size(hidden_size: int, dtype: torch.dtype) -> int:
|
||||
|
||||
@@ -34,9 +34,7 @@ logger = init_logger(__name__)
|
||||
def dequant_fp8(
|
||||
expert_x_fp8: torch.Tensor, expert_x_scales: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Return dequantized tensor in fp32
|
||||
"""
|
||||
"""Return dequantized tensor in fp32."""
|
||||
# TODO (varun) : Optimize leverage num_tokens_per_expert counts
|
||||
assert expert_x_fp8.is_contiguous()
|
||||
expert_x_scales = expert_x_scales.contiguous()
|
||||
@@ -50,9 +48,7 @@ def dequant_fp8(
|
||||
|
||||
|
||||
class DeepEPLLPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""
|
||||
Prepare/Finalize using DeepEP low-latency kernels.
|
||||
"""
|
||||
"""Prepare/Finalize using DeepEP low-latency kernels."""
|
||||
|
||||
# DeepEP low-latency kernels are compiled only for certain
|
||||
# specific hidden sizes.
|
||||
|
||||
@@ -28,8 +28,7 @@ from vllm.v1.worker.ubatching import (
|
||||
def _quantize_before_dispatch(
|
||||
quant_config: FusedMoEQuantConfig, defer_input_quant: bool
|
||||
) -> bool:
|
||||
"""
|
||||
Do quantized dispatch for blockfp8 and mxfp8, unless the
|
||||
"""Do quantized dispatch for blockfp8 and mxfp8, unless the
|
||||
subsequent moe kernel requires bf16 inputs.
|
||||
"""
|
||||
if defer_input_quant:
|
||||
@@ -69,8 +68,7 @@ def _unpack_mxfp8_scale(
|
||||
|
||||
|
||||
class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""
|
||||
Prepare/Finalize using DeepEP v2 ElasticBuffer (unified API).
|
||||
"""Prepare/Finalize using DeepEP v2 ElasticBuffer (unified API).
|
||||
|
||||
Supports two modes controlled by the `use_cudagraph` constructor arg:
|
||||
|
||||
|
||||
@@ -13,9 +13,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""
|
||||
Prepare/Finalize using MoRI kernels.
|
||||
"""
|
||||
"""Prepare/Finalize using MoRI kernels."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -60,8 +58,7 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareResultType:
|
||||
"""
|
||||
Returns a tuple of:
|
||||
"""Returns a tuple of:
|
||||
- quantized + dispatched a.
|
||||
- Optional quantized + dispatched a1_scales.
|
||||
- Optional ExpertTokensMetadata containing gpu/cpu tensors
|
||||
|
||||
@@ -69,8 +69,7 @@ def _unwrap_scale_and_prepare_for_moe(
|
||||
|
||||
|
||||
class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""
|
||||
Naive Prepare/Finalize for Dp/Ep case for Modular Kernels.
|
||||
"""Naive Prepare/Finalize for Dp/Ep case for Modular Kernels.
|
||||
|
||||
Uses Torch AR/RS or AR for dispatch/combine operations, applied
|
||||
to the topk weights and ids.
|
||||
@@ -121,7 +120,6 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareResultType:
|
||||
"""Quantize and Dispatch Topk Weights and Topk Ids."""
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
topk = topk_ids.size(1)
|
||||
assert topk == 1, (
|
||||
@@ -210,8 +208,7 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
|
||||
|
||||
class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):
|
||||
"""
|
||||
Naive Prepare/Finalize for Dp/Ep case for Modular Kernels.
|
||||
"""Naive Prepare/Finalize for Dp/Ep case for Modular Kernels.
|
||||
|
||||
Uses Torch AR/RS or AR for dispatch/combine operations, applied
|
||||
to the router logits (the MoE kernel runs the router internally).
|
||||
@@ -250,7 +247,6 @@ class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMono
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareMonolithicResultType:
|
||||
"""Quantize and Dispatch Router Logits."""
|
||||
|
||||
a1q, scales, a1q_scale_orig = _quantize_and_setup_dispatch(
|
||||
a1, quant_config, defer_input_quant
|
||||
)
|
||||
|
||||
@@ -33,9 +33,7 @@ assert isinstance(NIXL_EP_TOPK_INDICES_DTYPE, torch.dtype)
|
||||
def dequant_fp8(
|
||||
expert_x_fp8: torch.Tensor, expert_x_scales: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Return dequantized tensor in fp32
|
||||
"""
|
||||
"""Return dequantized tensor in fp32."""
|
||||
assert expert_x_fp8.is_contiguous()
|
||||
expert_x_scales = expert_x_scales.contiguous()
|
||||
num_experts = expert_x_fp8.size(0)
|
||||
@@ -48,9 +46,7 @@ def dequant_fp8(
|
||||
|
||||
|
||||
class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""
|
||||
Prepare/Finalize using NIXL EP kernels.
|
||||
"""
|
||||
"""Prepare/Finalize using NIXL EP kernels."""
|
||||
|
||||
# NIXL EP kernels are compiled only for certain specific hidden sizes.
|
||||
# NOTE: Keep this list sorted, maybe_roundup_layer_hidden_size depends
|
||||
|
||||
@@ -44,8 +44,7 @@ class FusedMoeWeightScaleSupported(Enum):
|
||||
|
||||
@PluggableLayer.register("routed_experts")
|
||||
class RoutedExperts(PluggableLayer):
|
||||
"""
|
||||
Container for routed expert weights and execution logic.
|
||||
"""Container for routed expert weights and execution logic.
|
||||
|
||||
This module owns the expert weight parameters (w13_weight, w2_weight, scales, etc.)
|
||||
and handles:
|
||||
@@ -188,8 +187,7 @@ class RoutedExperts(PluggableLayer):
|
||||
quant_config: QuantizationConfig | None,
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> FusedMoEMethodBase:
|
||||
"""
|
||||
Helper method to ensure quant_method is never None and
|
||||
"""Helper method to ensure quant_method is never None and
|
||||
of the proper type.
|
||||
"""
|
||||
quant_method = None
|
||||
@@ -318,8 +316,7 @@ class RoutedExperts(PluggableLayer):
|
||||
param: torch.Tensor,
|
||||
tp_rank: int,
|
||||
):
|
||||
"""
|
||||
Load w13 weight scales assuming that w1 weight scales and w3 weight
|
||||
"""Load w13 weight scales assuming that w1 weight scales and w3 weight
|
||||
scales are stored in the same loaded_weight tensor.
|
||||
"""
|
||||
shard_size = param.shape[shard_dim]
|
||||
@@ -336,8 +333,7 @@ class RoutedExperts(PluggableLayer):
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
):
|
||||
"""
|
||||
Load grouped weight scales for group quantization or model weights
|
||||
"""Load grouped weight scales for group quantization or model weights
|
||||
|
||||
Args:
|
||||
shard_dim: dimension to shard
|
||||
@@ -345,6 +341,7 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_id: either w1, w2, or w3
|
||||
loaded_weight: checkpoint weight to load into the param
|
||||
tp_rank: tensor parallel rank
|
||||
|
||||
"""
|
||||
if shard_id == "w2":
|
||||
self._load_w2(
|
||||
@@ -444,6 +441,7 @@ class RoutedExperts(PluggableLayer):
|
||||
Must be non-negative.
|
||||
shard_dim: The dimension index corresponding to the shard
|
||||
(intermediate) dimension. Defaults to `None`.
|
||||
|
||||
"""
|
||||
dims = (hidden_dim,) if shard_dim is None else (hidden_dim, shard_dim)
|
||||
if loaded_weight.ndim > 0:
|
||||
@@ -990,8 +988,7 @@ class RoutedExperts(PluggableLayer):
|
||||
lora_base_layer_prefix_on_param_name: str = "",
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""
|
||||
Create expert parameter mapping for weight loading with redundant experts.
|
||||
"""Create expert parameter mapping for weight loading with redundant experts.
|
||||
|
||||
This mapping handles the physical-to-logical expert ID conversion needed
|
||||
when loading weights with EPLB redundant experts.
|
||||
@@ -1011,6 +1008,7 @@ class RoutedExperts(PluggableLayer):
|
||||
``make_expert_params_mapping`` indexes the model-wide
|
||||
``params_dict`` (prefix included).
|
||||
include_fused: Prepend the fused pre-fused-checkpoint entries
|
||||
routed_experts_prefix: Prefix of the routed experts submodule
|
||||
|
||||
Returns:
|
||||
List of tuples (param_name, weight_name, expert_id, shard_id)
|
||||
@@ -1019,6 +1017,7 @@ class RoutedExperts(PluggableLayer):
|
||||
- weight_name: Weight name in checkpoint
|
||||
- expert_id: Physical expert ID
|
||||
- shard_id: Shard identifier (w1, w2, w3)
|
||||
|
||||
"""
|
||||
num_physical_experts = num_experts + num_redundant_experts
|
||||
|
||||
@@ -1097,8 +1096,7 @@ class RoutedExperts(PluggableLayer):
|
||||
def _maybe_make_contiguous(
|
||||
name: str, p: torch.nn.Parameter
|
||||
) -> torch.nn.Parameter:
|
||||
"""
|
||||
In some cases, the last 2 dimensions (the non-expert dimensions)
|
||||
"""In some cases, the last 2 dimensions (the non-expert dimensions)
|
||||
of the weight scale tensor are transposed. This function
|
||||
transforms the tensor (view update) so the tensor is contiguous().
|
||||
Example: A non-contiguous scale tensor,
|
||||
@@ -1178,8 +1176,7 @@ class RoutedExperts(PluggableLayer):
|
||||
shared_experts: "SharedExperts | None" = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Execute routed experts using the quantization method's apply function.
|
||||
"""Execute routed experts using the quantization method's apply function.
|
||||
|
||||
This is called by the runner after router selection (for modular kernels)
|
||||
quant_method.apply() which accesses the weights on this RoutedExperts
|
||||
@@ -1194,6 +1191,7 @@ class RoutedExperts(PluggableLayer):
|
||||
|
||||
Returns:
|
||||
Output tensor from routed experts.
|
||||
|
||||
"""
|
||||
assert not self.quant_method.is_monolithic
|
||||
|
||||
@@ -1213,8 +1211,7 @@ class RoutedExperts(PluggableLayer):
|
||||
router_logits: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""
|
||||
Execute routed experts using the quantization method's apply function.
|
||||
"""Execute routed experts using the quantization method's apply function.
|
||||
|
||||
This is called by the runner after router selection (for modular kernels)
|
||||
or with router logits (for monolithic kernels). It delegates to
|
||||
@@ -1228,6 +1225,7 @@ class RoutedExperts(PluggableLayer):
|
||||
|
||||
Returns:
|
||||
Finalized routed states or a deferred-finalize output.
|
||||
|
||||
"""
|
||||
assert self.quant_method.is_monolithic
|
||||
|
||||
|
||||
@@ -136,8 +136,8 @@ class RoutedExpertsCapturer:
|
||||
Args:
|
||||
layer_id: The layer index.
|
||||
topk_ids: Tensor of shape (batch_size, num_routed_experts).
|
||||
"""
|
||||
|
||||
"""
|
||||
ctx = get_forward_context()
|
||||
if ctx.dp_metadata is None: # single dp
|
||||
start_loc = 0
|
||||
@@ -241,6 +241,7 @@ class RoutedExpertsCapturer:
|
||||
slot_mappings: Per-KV-cache-group slot mappings for this step,
|
||||
shape ``(num_kv_cache_groups, max_num_batched_tokens)``.
|
||||
num_tokens: Total number of tokens scheduled in this step.
|
||||
|
||||
"""
|
||||
return RoutedExpertsTensors(
|
||||
routing_data=self.device_buffer[:num_tokens].clone(),
|
||||
@@ -417,6 +418,7 @@ class RoutedExpertsManager:
|
||||
Returns:
|
||||
Array of shape (num_tokens - token_start, num_layers,
|
||||
num_experts_per_tok).
|
||||
|
||||
"""
|
||||
bs = self.block_size
|
||||
block_ids_array = np.array(block_ids, dtype=np.int32)
|
||||
|
||||
@@ -16,8 +16,7 @@ from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
|
||||
|
||||
|
||||
class AiterSharedRoutedFusedMoERouter(BaseRouter):
|
||||
"""
|
||||
ROCm AITER router for models with fused shared experts (e.g. Qwen3-MoE).
|
||||
"""ROCm AITER router for models with fused shared experts (e.g. Qwen3-MoE).
|
||||
|
||||
When the AITER topk_softmax kernel supports sigmoid fusion, the routing
|
||||
softmax and shared-expert sigmoid are computed in a single kernel launch.
|
||||
|
||||
@@ -157,8 +157,7 @@ else:
|
||||
|
||||
|
||||
class BaseRouter(FusedMoERouter):
|
||||
"""
|
||||
Base router class that provides common functionality for all router implementations.
|
||||
"""Base class providing common functionality for all router implementations.
|
||||
|
||||
This class implements the template method pattern where select_experts() handles
|
||||
common pre-processing and post-processing, delegating the actual routing logic
|
||||
@@ -171,11 +170,11 @@ class BaseRouter(FusedMoERouter):
|
||||
global_num_experts: int,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
top_k: Number of experts to select per token
|
||||
global_num_experts: Total number of experts
|
||||
eplb_state: Optional EPLBLayerState for load balancing
|
||||
"""Args:
|
||||
top_k: Number of experts to select per token
|
||||
global_num_experts: Total number of experts
|
||||
eplb_state: Optional EPLBLayerState for load balancing
|
||||
|
||||
"""
|
||||
super().__init__(eplb_state=eplb_state)
|
||||
self.top_k = top_k
|
||||
@@ -241,8 +240,7 @@ class BaseRouter(FusedMoERouter):
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Compute the actual routing logic.
|
||||
"""Compute the actual routing logic.
|
||||
|
||||
This method must be implemented by subclasses to provide the specific
|
||||
routing algorithm (e.g., grouped_topk, fused_topk, custom routing, etc.).
|
||||
@@ -251,9 +249,11 @@ class BaseRouter(FusedMoERouter):
|
||||
hidden_states: Input hidden states
|
||||
router_logits: Router logits for expert selection
|
||||
indices_type: Desired dtype for expert indices (may be None)
|
||||
input_ids: Token ids, for routers that condition on them
|
||||
|
||||
Returns:
|
||||
tuple of (topk_weights, topk_ids)
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -265,8 +265,7 @@ class BaseRouter(FusedMoERouter):
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Route the input hidden states to the top-k experts based on the
|
||||
"""Route the input hidden states to the top-k experts based on the
|
||||
router logits.
|
||||
|
||||
This method implements the template method pattern:
|
||||
@@ -283,6 +282,7 @@ class BaseRouter(FusedMoERouter):
|
||||
**Compatibility**: When EPLB is not enabled, the returned ids are
|
||||
equivalent to global logical ids, so should be compatible with
|
||||
plain MoE implementations without redundant experts.
|
||||
|
||||
"""
|
||||
# Step 1: Validate EPLB state
|
||||
self._validate_eplb_state()
|
||||
|
||||
@@ -10,8 +10,7 @@ from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
|
||||
|
||||
class FusedMoERouter(ABC):
|
||||
"""
|
||||
FusedMoERouter is an abstract class that provides a 'select_experts'
|
||||
"""FusedMoERouter is an abstract class that provides a 'select_experts'
|
||||
method that is used for routing hidden states based on router logits.
|
||||
"""
|
||||
|
||||
@@ -54,8 +53,7 @@ class FusedMoERouter(ABC):
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Route the input hidden states to the top-k experts based on the
|
||||
"""Route the input hidden states to the top-k experts based on the
|
||||
router logits.
|
||||
|
||||
Returns:
|
||||
@@ -66,8 +64,8 @@ class FusedMoERouter(ABC):
|
||||
**Compatibility**: When EPLB is not enabled, the returned ids are
|
||||
equivalent to global logical ids, so should be compatible with
|
||||
plain MoE implementations without redundant experts.
|
||||
"""
|
||||
|
||||
"""
|
||||
topk_weights, topk_ids = self._select_experts(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
|
||||
@@ -217,8 +217,7 @@ def fp32_router_gemm_dispatch_impl(
|
||||
weight: torch.Tensor,
|
||||
allow_bf16x3_router_gemm: bool,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
|
||||
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
|
||||
otherwise optionally run the experimental BF16x3 kernel for medium/large
|
||||
SM100 router batches, then fall back to F.linear.
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
|
||||
@@ -66,8 +66,7 @@ def create_fused_moe_router(
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
) -> FusedMoERouter:
|
||||
"""
|
||||
Factory function to create the appropriate FusedMoERouter subclass based on
|
||||
"""Factory function to create the appropriate FusedMoERouter subclass based on
|
||||
the provided parameters.
|
||||
|
||||
The selection logic follows this priority order:
|
||||
@@ -114,8 +113,8 @@ def create_fused_moe_router(
|
||||
|
||||
Returns:
|
||||
An instance of the appropriate FusedMoERouter subclass
|
||||
"""
|
||||
|
||||
"""
|
||||
routing_strategy = envs.VLLM_MOE_ROUTING_SIMULATION_STRATEGY
|
||||
if routing_strategy != "":
|
||||
return RoutingSimulatorRouter(
|
||||
|
||||
@@ -25,8 +25,7 @@ class RoutingStrategy(ABC):
|
||||
top_k: int,
|
||||
indices_type: torch.dtype | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Route tokens to experts.
|
||||
"""Route tokens to experts.
|
||||
|
||||
Args:
|
||||
hidden_states: Input hidden states [num_tokens, hidden_size]
|
||||
@@ -36,13 +35,13 @@ class RoutingStrategy(ABC):
|
||||
|
||||
Returns:
|
||||
tuple of (topk_weights, topk_ids)
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class DistributionBasedRouting(RoutingStrategy):
|
||||
"""
|
||||
Distribution-based random routing strategy with configurable distributions.
|
||||
"""Distribution-based random routing strategy with configurable distributions.
|
||||
|
||||
This routing strategy randomly selects experts for each token based on
|
||||
different probability distributions. Currently supports uniform and normal
|
||||
@@ -50,8 +49,7 @@ class DistributionBasedRouting(RoutingStrategy):
|
||||
"""
|
||||
|
||||
def __init__(self, distribution: str = "uniform", **distribution_params: Any):
|
||||
"""
|
||||
Initialize distribution-based routing.
|
||||
"""Initialize distribution-based routing.
|
||||
|
||||
Args:
|
||||
distribution: Type of distribution to use for sampling
|
||||
@@ -61,6 +59,7 @@ class DistributionBasedRouting(RoutingStrategy):
|
||||
chosen distribution
|
||||
For "uniform": No additional parameters needed
|
||||
For "normal": mean (default: 0.0), std (default: 1.0)
|
||||
|
||||
"""
|
||||
self.distribution = distribution.lower()
|
||||
self.distribution_params = distribution_params
|
||||
@@ -90,8 +89,7 @@ class DistributionBasedRouting(RoutingStrategy):
|
||||
top_k: int,
|
||||
indices_type: torch.dtype | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Randomly select experts for each token using the specified distribution.
|
||||
"""Randomly select experts for each token using the specified distribution.
|
||||
|
||||
Args:
|
||||
hidden_states: Input hidden states [num_tokens, hidden_size]
|
||||
@@ -103,6 +101,7 @@ class DistributionBasedRouting(RoutingStrategy):
|
||||
tuple of (topk_weights, topk_ids) where:
|
||||
- topk_weights: Weights based on distribution sampling
|
||||
- topk_ids: Expert indices sampled from the distribution
|
||||
|
||||
"""
|
||||
num_tokens = hidden_states.shape[0]
|
||||
num_experts = router_logits.shape[-1]
|
||||
@@ -129,7 +128,6 @@ class DistributionBasedRouting(RoutingStrategy):
|
||||
indices_type: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Sample expert IDs based on the specified distribution."""
|
||||
|
||||
if self.distribution == "uniform":
|
||||
# Generate random scores, and take the top-k to avoid duplicate topk_ids
|
||||
scores = torch.rand(num_tokens, num_experts, device=device)
|
||||
@@ -218,8 +216,7 @@ class DistributionBasedRouting(RoutingStrategy):
|
||||
|
||||
|
||||
class RoutingSimulator:
|
||||
"""
|
||||
Token-to-Expert Routing Simulator.
|
||||
"""Token-to-Expert Routing Simulator.
|
||||
|
||||
This class provides a framework for testing and comparing different
|
||||
routing strategies for MoE models. It can simulate routing behavior
|
||||
@@ -239,22 +236,22 @@ class RoutingSimulator:
|
||||
|
||||
@classmethod
|
||||
def register_strategy(cls, name: str, strategy: RoutingStrategy):
|
||||
"""
|
||||
Register a custom routing strategy.
|
||||
"""Register a custom routing strategy.
|
||||
|
||||
Args:
|
||||
name: Name of the strategy
|
||||
strategy: RoutingStrategy instance
|
||||
|
||||
"""
|
||||
cls._routing_strategies[name] = strategy
|
||||
|
||||
@classmethod
|
||||
def get_available_strategies(cls) -> list[str]:
|
||||
"""
|
||||
Get list of available routing strategy names.
|
||||
"""Get list of available routing strategy names.
|
||||
|
||||
Returns:
|
||||
List of available strategy names
|
||||
|
||||
"""
|
||||
return list(cls._routing_strategies.keys())
|
||||
|
||||
@@ -266,8 +263,7 @@ class RoutingSimulator:
|
||||
top_k: int,
|
||||
indices_type: torch.dtype | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Simulate token-to-expert routing using the specified strategy.
|
||||
"""Simulate token-to-expert routing using the specified strategy.
|
||||
|
||||
Args:
|
||||
hidden_states: Input hidden states [num_tokens, hidden_size]
|
||||
@@ -278,6 +274,7 @@ class RoutingSimulator:
|
||||
|
||||
Returns:
|
||||
tuple of (topk_weights, topk_ids)
|
||||
|
||||
"""
|
||||
if strategy_name not in RoutingSimulator._routing_strategies:
|
||||
raise ValueError(
|
||||
|
||||
@@ -71,7 +71,8 @@ class ZeroExpertRouter(BaseRouter):
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute routing with full bias, compute zero expert output,
|
||||
mask zero expert IDs."""
|
||||
mask zero expert IDs.
|
||||
"""
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
|
||||
@@ -225,8 +225,7 @@ def _unpack(
|
||||
|
||||
|
||||
class MoERunner(MoERunnerInterface):
|
||||
"""
|
||||
Standard MoE runner implementation for executing Mixture of Experts layers.
|
||||
"""Standard MoE runner implementation for executing Mixture of Experts layers.
|
||||
|
||||
This is the primary concrete implementation of MoE execution logic, providing
|
||||
comprehensive support for standard MoE operations. It handles:
|
||||
@@ -704,7 +703,6 @@ class MoERunner(MoERunnerInterface):
|
||||
1. pytorch cannot handle union types in custom op signatures so
|
||||
_moe_forward and _moe_forward_shared must be split.
|
||||
"""
|
||||
|
||||
# Apply transform for routed experts (e.g., latent projection for
|
||||
# latent MoE). When the caller pre-applies the routed input transform
|
||||
# outside the runner (e.g. to overlap it on a separate stream), it
|
||||
@@ -1019,8 +1017,7 @@ class MoERunner(MoERunnerInterface):
|
||||
logical_to_physical_map: torch.Tensor,
|
||||
logical_replica_count: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Register the EPLB state in this layer.
|
||||
"""Register the EPLB state in this layer.
|
||||
|
||||
This is used later in forward pass, where we get the expert mapping
|
||||
and record the load metrics in `expert_load_view`.
|
||||
|
||||
@@ -17,8 +17,7 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
|
||||
|
||||
class MoERunnerInterface(PluggableLayer, ABC):
|
||||
"""
|
||||
Abstract base class for Mixture of Experts (MoE) runners.
|
||||
"""Abstract base class for Mixture of Experts (MoE) runners.
|
||||
|
||||
This class defines the interface that all MoE runner implementations must follow.
|
||||
MoE runners are responsible for executing the forward pass of MoE layers, handling
|
||||
|
||||
@@ -9,8 +9,7 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
|
||||
|
||||
class TopKWeightAndReduceDelegate(mk.TopKWeightAndReduce):
|
||||
"""
|
||||
Useful in the case when some FusedMoEExpertsModular
|
||||
"""Useful in the case when some FusedMoEExpertsModular
|
||||
implementation does not perform weight application and reduction
|
||||
but cannot address the needs of all the compatible PrepareAndFinalize
|
||||
implementations.
|
||||
@@ -42,8 +41,7 @@ class TopKWeightAndReduceDelegate(mk.TopKWeightAndReduce):
|
||||
|
||||
|
||||
class TopKWeightAndReduceNoOP(mk.TopKWeightAndReduce):
|
||||
"""
|
||||
The fused_experts outputs have already been weight applied and reduced.
|
||||
"""The fused_experts outputs have already been weight applied and reduced.
|
||||
This implementation is a no-op.
|
||||
"""
|
||||
|
||||
@@ -78,8 +76,7 @@ class TopKWeightAndReduceNoOP(mk.TopKWeightAndReduce):
|
||||
|
||||
|
||||
class TopKWeightAndReduceContiguous(mk.TopKWeightAndReduce):
|
||||
"""
|
||||
TopKWeightAndReduce implementation for a fused_experts output
|
||||
"""TopKWeightAndReduce implementation for a fused_experts output
|
||||
of shape (m, topk, K)
|
||||
"""
|
||||
|
||||
@@ -122,8 +119,7 @@ class TopKWeightAndReduceContiguous(mk.TopKWeightAndReduce):
|
||||
|
||||
|
||||
class TopKWeightAndReduceNaiveBatched(mk.TopKWeightAndReduce):
|
||||
"""
|
||||
TopKWeightAndReduce implementation for a fused_experts output
|
||||
"""TopKWeightAndReduce implementation for a fused_experts output
|
||||
of shape (num_experts, batch_size, K)
|
||||
"""
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ def resolve_layer_fused_shared_expert(
|
||||
|
||||
Raises:
|
||||
ValueError: If requested shared-expert fusion is quantization-incompatible.
|
||||
|
||||
"""
|
||||
# NOTE: is_fusion_moe_shared_experts_enabled is decorated with @if_aiter_supported
|
||||
# that returns None if AITER is not available.
|
||||
@@ -164,20 +165,19 @@ def _count_expert_num_tokens(
|
||||
def count_expert_num_tokens(
|
||||
topk_ids: torch.Tensor, num_local_experts: int, expert_map: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Count the number to tokens assigned to each expert.
|
||||
"""Count the number to tokens assigned to each expert.
|
||||
|
||||
Parameters:
|
||||
- topk_ids (torch.Tensor): Tensor mapping each token to its
|
||||
list of experts.
|
||||
- num_local_experts (int): Number of experts in this rank.
|
||||
- expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
|
||||
from the global expert space to the local expert space of the expert
|
||||
parallel shard.
|
||||
Args:
|
||||
topk_ids (torch.Tensor): Tensor mapping each token to its list of experts.
|
||||
num_local_experts (int): Number of experts in this rank.
|
||||
expert_map (Optional[torch.Tensor]): A tensor mapping expert indices
|
||||
from the global expert space to the local expert space of the expert
|
||||
parallel shard.
|
||||
|
||||
Returns:
|
||||
A tensor of size num_local_experts, where tensor[i] holds the number
|
||||
of tokens assigned to the ith expert.
|
||||
|
||||
"""
|
||||
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
|
||||
expert_num_tokens = torch.empty(
|
||||
@@ -202,8 +202,7 @@ def count_expert_num_tokens(
|
||||
|
||||
|
||||
def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
|
||||
"""
|
||||
Shrink the given tensor and apply the given view to it. This is
|
||||
"""Shrink the given tensor and apply the given view to it. This is
|
||||
used to resize the intermediate fused_moe caches.
|
||||
"""
|
||||
assert prod(v) <= x.numel(), (
|
||||
@@ -226,8 +225,7 @@ def _fp8_quantize(
|
||||
per_act_token: bool,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Perform fp8 quantization on the inputs. If a block_shape
|
||||
"""Perform fp8 quantization on the inputs. If a block_shape
|
||||
is provided, the output will be blocked.
|
||||
"""
|
||||
if block_shape is None:
|
||||
@@ -252,11 +250,9 @@ def _int8_quantize(
|
||||
per_act_token: bool,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Perform int8 quantization on the inputs. If a block_shape
|
||||
"""Perform int8 quantization on the inputs. If a block_shape
|
||||
is provided, the output will be blocked.
|
||||
"""
|
||||
|
||||
# If weights are per-channel (per_channel_quant=True), then
|
||||
# activations apply per-token quantization. Otherwise, assume
|
||||
# activation tensor-wise fp8/int8 quantization, dynamic or static
|
||||
|
||||
@@ -170,6 +170,7 @@ def fused_qk_rmsnorm_rope_gate(
|
||||
Returns:
|
||||
(q_out, k_out, gate_out) -- all contiguous (n_tokens, heads * head_dim).
|
||||
``gate_out`` is the raw (pre-sigmoid) gate.
|
||||
|
||||
"""
|
||||
if rotary_dim <= 0 or rotary_dim > head_dim or rotary_dim % 2 != 0:
|
||||
raise ValueError(
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Producer side of the QuantizedActivation contract for activation layers.
|
||||
"""Producer side of the QuantizedActivation contract for activation layers.
|
||||
|
||||
Given an activation module and the downstream linear it feeds, fuse the
|
||||
activation with that linear's input quantization into a single kernel when the
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
A QuantizedActivation is a pre-quantized activation produced by a fused kernel
|
||||
"""A QuantizedActivation is a pre-quantized activation produced by a fused kernel
|
||||
and consumed directly by a linear layer, letting the layer skip its own input
|
||||
quantization. A linear advertises the key its kernel can consume via
|
||||
expose_input_quant_key; the kernel validates and reads the activation via
|
||||
|
||||
@@ -107,6 +107,7 @@ def hpc_gated_mla_gemm(
|
||||
|
||||
Returns:
|
||||
``[m, n]`` bfloat16.
|
||||
|
||||
"""
|
||||
from hpc.gemm import gated_mla_gemm
|
||||
|
||||
|
||||
@@ -122,6 +122,7 @@ class HpcIHCPre(HpcModule):
|
||||
When given, its weight/eps are folded into the kernel so a single
|
||||
launch covers pre + RMSNorm (the caller then skips the separate
|
||||
layernorm). None keeps the original pre-only behaviour.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -87,6 +87,7 @@ def deep_select_topk(
|
||||
|
||||
Returns:
|
||||
The (num_rows, topk) indices tensor.
|
||||
|
||||
"""
|
||||
assert input.dim() == 2 and input.stride(1) == 1
|
||||
assert (
|
||||
@@ -162,7 +163,8 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
) -> str:
|
||||
"""Resolve the decode top-k implementation from the configured
|
||||
backend ("auto" = the pre-existing chain, or a validated explicit
|
||||
value)."""
|
||||
value).
|
||||
"""
|
||||
if self._backend == "auto":
|
||||
return self._resolve_auto(logits, topk_tokens, num_rows)
|
||||
|
||||
@@ -209,7 +211,8 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""The pre-existing priority chain: cooperative -> persistent ->
|
||||
per_row. deep_select/flashinfer/torch are opt-in only."""
|
||||
per_row. deep_select/flashinfer/torch are opt-in only.
|
||||
"""
|
||||
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
|
||||
return "cooperative"
|
||||
if self._is_cuda and topk_tokens in (512, 1024, 2048):
|
||||
@@ -271,7 +274,8 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
"""Run the resolved decode top-k implementation, writing into
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
|
||||
"""
|
||||
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
|
||||
if backend == "deep_select":
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
|
||||
@@ -231,6 +231,7 @@ class RMSNormGated(CustomOp):
|
||||
device: Device to create parameters on
|
||||
dtype: Data type for parameters
|
||||
activation: Activation function name for gating
|
||||
|
||||
"""
|
||||
factory_kwargs = {"device": device, "dtype": dtype}
|
||||
super().__init__()
|
||||
@@ -335,9 +336,7 @@ class RMSNormGated(CustomOp):
|
||||
|
||||
|
||||
class LayerNorm(nn.Module):
|
||||
"""
|
||||
Layer Normalization.
|
||||
"""
|
||||
"""Layer Normalization."""
|
||||
|
||||
def __init__(self, dim: int, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
|
||||
@@ -541,8 +541,7 @@ def lightning_attention(
|
||||
block_size: int = 256,
|
||||
kv_history: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Apply lightning attention algorithm
|
||||
"""Apply lightning attention algorithm
|
||||
to compute attention efficiently.
|
||||
|
||||
Args:
|
||||
@@ -556,6 +555,7 @@ def lightning_attention(
|
||||
Returns:
|
||||
output: Attention output
|
||||
kv: Updated key-value history
|
||||
|
||||
"""
|
||||
d = q.shape[-1]
|
||||
e = v.shape[-1]
|
||||
@@ -610,8 +610,7 @@ def _linear_attn_decode_kernel(
|
||||
pad_slot_id: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Kernel for linear attention decoding with KV cache.
|
||||
"""Kernel for linear attention decoding with KV cache.
|
||||
|
||||
This kernel computes attention for a single token using the KV cache.
|
||||
"""
|
||||
@@ -683,8 +682,7 @@ def linear_decode_forward_triton(
|
||||
slot_idx: torch.Tensor,
|
||||
BLOCK_SIZE: int = 32,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Perform linear attention decoding using Triton kernels.
|
||||
"""Perform linear attention decoding using Triton kernels.
|
||||
|
||||
Args:
|
||||
q: Query tensor of shape [B, H, 1, D]
|
||||
@@ -697,6 +695,7 @@ def linear_decode_forward_triton(
|
||||
|
||||
Returns:
|
||||
output: Attention output tensor
|
||||
|
||||
"""
|
||||
B, H, _, D = q.shape
|
||||
assert k.shape == (B, H, 1, D)
|
||||
|
||||
@@ -146,6 +146,7 @@ class LinearMethodBase(QuantizeMethodBase):
|
||||
input_size: Size of the input dim of the weight across all ranks.
|
||||
output_size: Size of the output dim of the weight across all ranks.
|
||||
params_dtype: Datatype of the parameters.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -157,7 +158,8 @@ class LinearMethodBase(QuantizeMethodBase):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the weights in layer to the input tensor.
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -250,6 +252,7 @@ class LinearBase(PluggableLayer):
|
||||
prefix: Prefix for parameter names.
|
||||
return_bias: If true, return bias together with outputs in forward pass.
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
|
||||
"""
|
||||
|
||||
requires_unquantized_input = False
|
||||
@@ -335,6 +338,7 @@ class ReplicatedLinear(LinearBase):
|
||||
(e.g. model.layers.0.qkv_proj)
|
||||
return_bias: If true, return bias together with outputs in forward pass.
|
||||
disable_tp: Take no effect for replicated linear layers.
|
||||
|
||||
"""
|
||||
|
||||
# --8<-- [end:replicated_linear]
|
||||
@@ -454,6 +458,7 @@ class ColumnParallelLinear(LinearBase):
|
||||
shard per rank (see ``DCPGroupColumnParallelLinear``).
|
||||
tp_size: Override the tensor-parallel world size used for sharding.
|
||||
Defaults to the global TP world size.
|
||||
|
||||
"""
|
||||
|
||||
# --8<-- [end:column_parallel_linear]
|
||||
@@ -690,6 +695,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
return_bias: If true, return bias together with outputs in forward pass.
|
||||
disable_tp: If true, all weights matrix won't be sharded, this layer
|
||||
will be treated as a "Replicated" MergedLinear.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -867,8 +873,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
loaded_weight: torch.Tensor,
|
||||
output_sizes: list[int] | None = None,
|
||||
):
|
||||
"""
|
||||
Handle special case for models where MLP layers are already
|
||||
"""Handle special case for models where MLP layers are already
|
||||
fused on disk. In this case, we have no shard id. This function
|
||||
determines the shard id by splitting these layers and then calls
|
||||
the weight loader using the shard id.
|
||||
@@ -876,7 +881,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
An example of a model with these fused layers:
|
||||
https://huggingface.co/microsoft/Phi-3-mini-4k-instruct
|
||||
"""
|
||||
|
||||
current_shard_offset = 0
|
||||
shard_offsets: list[tuple[int, int, int]] = []
|
||||
output_sizes = output_sizes or self.output_sizes
|
||||
@@ -1018,6 +1022,7 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
(e.g. model.layers.0.qkv_proj)
|
||||
return_bias: If true, return bias together with outputs in forward pass.
|
||||
disable_tp: If true, weights matrix won't be sharded through tp rank.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1102,8 +1107,7 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
def _load_fused_module_from_checkpoint(
|
||||
self, param: BasevLLMParameter, loaded_weight: torch.Tensor
|
||||
):
|
||||
"""
|
||||
Handle special case for models where QKV layers are already
|
||||
"""Handle special case for models where QKV layers are already
|
||||
fused on disk. In this case, we have no shard id. This function
|
||||
determines the shard id by splitting these layers and then calls
|
||||
the weight loader using the shard id.
|
||||
@@ -1632,6 +1636,7 @@ class RowParallelLinear(LinearBase):
|
||||
| . |
|
||||
| A_p |
|
||||
- -
|
||||
|
||||
Arguments:
|
||||
input_size: first dimension of matrix A.
|
||||
output_size: second dimension of matrix A.
|
||||
@@ -1651,6 +1656,7 @@ class RowParallelLinear(LinearBase):
|
||||
(e.g. model.layers.0.down_proj)
|
||||
return_bias: If true, return bias together with outputs in forward pass.
|
||||
disable_tp: If true, weights matrix won't be sharded through tp rank.
|
||||
|
||||
"""
|
||||
|
||||
# --8<-- [end:row_parallel_linear]
|
||||
|
||||
@@ -74,9 +74,9 @@ class LogitsProcessor(PluggableLayer):
|
||||
logits_as_input: bool = False,
|
||||
soft_cap: float | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
scale: A scaling factor to apply to the logits.
|
||||
"""Args:
|
||||
scale: A scaling factor to apply to the logits.
|
||||
|
||||
"""
|
||||
super().__init__()
|
||||
self.scale = scale
|
||||
|
||||
@@ -16,8 +16,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
|
||||
|
||||
class MambaBase(AttentionLayerBase):
|
||||
"""
|
||||
Base class for Mamba-like layers which support the v1 engine.
|
||||
"""Base class for Mamba-like layers which support the v1 engine.
|
||||
Inherit from this class if you implement a custom layer.
|
||||
"""
|
||||
|
||||
@@ -44,8 +43,7 @@ class MambaBase(AttentionLayerBase):
|
||||
|
||||
@abstractmethod
|
||||
def get_state_shape(self) -> Iterable[tuple[int, ...]]:
|
||||
"""
|
||||
Defines the shape of the state.
|
||||
"""Defines the shape of the state.
|
||||
For mamba layers this is usually a (conv_state, ssm_state) tuple.
|
||||
In this case, returns (conv_state_shape, ssm_state_shape).
|
||||
"""
|
||||
|
||||
@@ -45,8 +45,7 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
@PluggableLayer.register("olmo_hybrid_gated_delta_net_attention")
|
||||
class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
"""
|
||||
Gated DeltaNet linear attention layer for OLMo Hybrid.
|
||||
"""Gated DeltaNet linear attention layer for OLMo Hybrid.
|
||||
|
||||
This implements the linear attention mechanism that replaces sliding window
|
||||
attention in the hybrid architecture.
|
||||
@@ -294,9 +293,7 @@ class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
a: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
Core attention computation (called by custom op).
|
||||
"""
|
||||
"""Core attention computation (called by custom op)."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
|
||||
|
||||
@@ -639,9 +639,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
mixed_qkvz: torch.Tensor,
|
||||
mixed_ba: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
Derives `query`, `key` and `value` tensors from `mixed_qkvzba`.
|
||||
"""
|
||||
"""Derives `query`, `key` and `value` tensors from `mixed_qkvzba`."""
|
||||
new_tensor_shape_qkvz = mixed_qkvz.size()[:-1] + (
|
||||
self.num_k_heads // self.tp_size,
|
||||
(
|
||||
@@ -692,8 +690,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
mixed_ba: torch.Tensor,
|
||||
num_tokens: int,
|
||||
):
|
||||
"""
|
||||
Derives mixed_qkv, z, b, a from projected qkvz/ba for the GDN custom op.
|
||||
"""Derives mixed_qkv, z, b, a from projected qkvz/ba for the GDN custom op.
|
||||
|
||||
For gqa_interleaved_layout (Qwen3-Next): unpack the interleaved
|
||||
[ng, (hk + hk + np/ng*hv + np/ng*hv)] layout into contiguous qkv.
|
||||
@@ -864,7 +861,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""ROCm forward using AITER Triton fused projection+attention when
|
||||
available, otherwise falling back to the generic CUDA path."""
|
||||
available, otherwise falling back to the generic CUDA path.
|
||||
"""
|
||||
if GDN_AITER_TRITON_AVAILABLE:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
@@ -899,8 +897,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass with three parts:
|
||||
"""Forward pass with three parts:
|
||||
1. Input projection
|
||||
2. Core attention (custom op)
|
||||
3. Output projection
|
||||
@@ -977,8 +974,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass with three parts:
|
||||
"""Forward pass with three parts:
|
||||
1. Input projection
|
||||
2. Core attention (custom op)
|
||||
3. Output projection
|
||||
@@ -1221,6 +1217,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
z_out: **output** buffer for z (num_tokens, num_heads,
|
||||
head_dim); mutated in-place.
|
||||
core_attn_out: Pre-allocated output buffer for attention results.
|
||||
|
||||
"""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
@@ -1279,6 +1276,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
b: beta gating vector (num_tokens, num_heads)
|
||||
a: alpha gating vector (num_tokens, num_heads)
|
||||
core_attn_out: Pre-allocated output buffer for attention results.
|
||||
|
||||
"""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
@@ -1658,9 +1656,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
core_attn_out: torch.Tensor,
|
||||
attn_metadata: GDNAttentionMetadata,
|
||||
):
|
||||
"""
|
||||
Core attention computation with a packed non-spec decode fast path.
|
||||
"""
|
||||
"""Core attention computation with a packed non-spec decode fast path."""
|
||||
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor # noqa: E501
|
||||
self_kv_cache = self.kv_cache
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
@@ -2030,8 +2026,7 @@ def fused_gdn_gating(
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Fused computation of g and beta for Gated Delta Net.
|
||||
"""Fused computation of g and beta for Gated Delta Net.
|
||||
g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
|
||||
beta_output = b.sigmoid()
|
||||
TODO maybe use torch.compile to replace this triton kernel
|
||||
|
||||
@@ -523,7 +523,7 @@ class BailingMoELinearAttention(LinearAttention):
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Forward method called by torch.ops.vllm.linear_attention"""
|
||||
"""Forward method called by torch.ops.vllm.linear_attention."""
|
||||
torch.ops.vllm.linear_attention(
|
||||
hidden_states,
|
||||
output,
|
||||
|
||||
@@ -49,8 +49,7 @@ from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
# --8<-- [start:mamba_mixer]
|
||||
@PluggableLayer.register("mamba_mixer")
|
||||
class MambaMixer(MambaBase, PluggableLayer):
|
||||
"""
|
||||
Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
"""Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
the `contextualized_states`. A, D are input independent
|
||||
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
|
||||
for why A isn't selective) ∆, B, C are input-dependent
|
||||
@@ -237,8 +236,7 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
)
|
||||
|
||||
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
|
||||
"""
|
||||
Run the Mamba-1 SSM pipeline.
|
||||
"""Run the Mamba-1 SSM pipeline.
|
||||
|
||||
Steps
|
||||
-----
|
||||
@@ -258,7 +256,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
decode tokens), both sets of kernels are executed independently
|
||||
and their outputs are concatenated before the final output projection.
|
||||
"""
|
||||
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
|
||||
@@ -252,8 +252,7 @@ def mamba_v2_sharded_weight_loader(
|
||||
# --8<-- [start:mamba_mixer2]
|
||||
@PluggableLayer.register("mamba_mixer2")
|
||||
class MambaMixer2(MambaBase, PluggableLayer):
|
||||
"""
|
||||
Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
"""Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
the `contextualized_states`. A, D are input independent
|
||||
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
|
||||
for why A isn't selective) ∆, B, C are input-dependent
|
||||
@@ -1221,7 +1220,6 @@ def share_replayssm_ring_trackers(
|
||||
The final local layer in each group advances its cursors after every layer
|
||||
in that group has consumed the previous values.
|
||||
"""
|
||||
|
||||
replayssm_mixers: dict[str, MambaMixer2] = {}
|
||||
for layer_name in ordered_layer_names:
|
||||
layer = forward_context[layer_name]
|
||||
|
||||
@@ -261,8 +261,8 @@ class MambaStateShapeCalculator:
|
||||
@classmethod
|
||||
def extra_groups_for_head_shards(cls, ngroups: int, tp_size: int):
|
||||
"""Compute the increase in group numbers to account for
|
||||
replication in order to accompany the head shards."""
|
||||
|
||||
replication in order to accompany the head shards.
|
||||
"""
|
||||
# in the case ngoups % tp_size == 0, this will be zero
|
||||
if ngroups % tp_size == 0:
|
||||
return 0
|
||||
@@ -344,13 +344,13 @@ class MambaStateShapeCalculator:
|
||||
|
||||
@dataclass
|
||||
class MambaCopySpec:
|
||||
"""
|
||||
Data class specifying the memory-copy parameters for Mamba states used for
|
||||
"""Data class specifying the memory-copy parameters for Mamba states used for
|
||||
prefix caching in align mode.
|
||||
|
||||
Attributes:
|
||||
start_addr (int): Starting address for the memory copy operation.
|
||||
num_elements (int): Number of elements to copy from the starting address.
|
||||
|
||||
"""
|
||||
|
||||
start_addr: int
|
||||
|
||||
@@ -497,7 +497,7 @@ def causal_conv1d_fn(
|
||||
metadata=None,
|
||||
validate_data=False,
|
||||
):
|
||||
"""support varlen + continuous batching when x is 2D tensor
|
||||
"""Support varlen + continuous batching when x is 2D tensor.
|
||||
|
||||
x: (dim,cu_seq_len)
|
||||
cu_seq_len = total tokens of all seqs in that batch
|
||||
@@ -1109,8 +1109,7 @@ def causal_conv1d_update(
|
||||
validate_data=False,
|
||||
out: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
x: Input tensor which can take the following shapes:
|
||||
"""x: Input tensor which can take the following shapes:
|
||||
|
||||
- `[batch, dim]` - single token prediction
|
||||
- `[batch, dim, seqlen]` - single or multiple tokens prediction
|
||||
|
||||
@@ -123,8 +123,7 @@ def causal_conv1d_update_torch(
|
||||
bias: torch.Tensor | None = None,
|
||||
activation: str | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Pure PyTorch fallback for causal_conv1d_update.
|
||||
"""Pure PyTorch fallback for causal_conv1d_update.
|
||||
Currently used as a fallback for Arm (aarch64) to leverage
|
||||
oneDNN/ACL F.conv1d kernels for batched decoding.
|
||||
"""
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user