Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
@@ -802,18 +802,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
kv_cache_dtype,
k_scale,
)
if cache is not None:
mirror_target = cache.mirror_write_target(kv_c_normed.shape[0])
if mirror_target is not None:
mirror_cache, mirror_slots = mirror_target
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
kv_c_normed,
k_pe,
mirror_cache,
mirror_slots,
kv_cache_dtype,
k_scale,
)
def prepare_kv_cache_update(
self, attn_metadata: "MLACommonMetadata | None"
@@ -1272,23 +1260,34 @@ class MLAAttention(nn.Module, AttentionLayerBase):
quark_quantize_weight_to_mxfp4,
)
self.W_K, self.W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
# Convert from (L, N, P) to (N, L, P)
self.W_K = self.W_K.transpose(0, 1)
self.W_K_scale = self.W_K_scale.transpose(0, 1)
self.W_V, self.W_V_scale = quark_quantize_weight_to_mxfp4(
W_UV.permute(1, 2, 0)
)
W_K, W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
W_V, W_V_scale = quark_quantize_weight_to_mxfp4(W_UV.permute(1, 2, 0))
# Copy in place on reload so captured CUDA graphs stay valid.
# Convert W_K from (L, N, P) to (N, L, P).
for name, tensor in (
("W_K", W_K.transpose(0, 1)),
("W_K_scale", W_K_scale.transpose(0, 1)),
("W_V", W_V),
("W_V_scale", W_V_scale),
):
replace_parameter(self, name, tensor, prefer_copy=True)
elif self.is_aiter_triton_fp8_bmm_enabled:
W_K = W_UK.transpose(0, 1) # 16 512 128
W_V = W_UV.permute(1, 2, 0) # 16 128 512
self.W_K, self.W_K_scale = dynamic_per_batched_tensor_quant(
W_K, W_K_scale = dynamic_per_batched_tensor_quant(
W_K, dtype=current_platform.fp8_dtype()
)
self.W_V, self.W_V_scale = dynamic_per_batched_tensor_quant(
W_V, W_V_scale = dynamic_per_batched_tensor_quant(
W_V, dtype=current_platform.fp8_dtype()
)
# Copy in place on reload so captured CUDA graphs stay valid.
for name, tensor in (
("W_K", W_K),
("W_K_scale", W_K_scale),
("W_V", W_V),
("W_V_scale", W_V_scale),
):
replace_parameter(self, name, tensor, prefer_copy=True)
# The kernel operates on non-padded inputs. Hence, pre-compiling
# triton kernel to avoid runtime compilation for unseen batch sizes
@@ -1327,8 +1326,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# Convert from (L, N, P) to (N, P, L)
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
if self.dcp_q_replicate:
self.W_UK_T_dcp_qrep = get_dcp_group().all_gather(
self.W_UK_T.contiguous(), dim=0
replace_parameter(
self,
"W_UK_T_dcp_qrep",
get_dcp_group().all_gather(self.W_UK_T.contiguous(), dim=0),
prefer_copy=True,
)
# If we should not load quant weights, we initialize the scales to 1.0
@@ -2384,8 +2386,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.vllm_config = vllm_config
self.device = device
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.non_causal_multi_token_decode = getattr(
kv_cache_spec, "non_causal_multi_token_decode", False
# merge() unions this flag across the KV group, so a shared target/draft
# group would mark the target builder as non-causal.
ctx = self.compilation_config.static_forward_context
self.non_causal_multi_token_decode = any(
getattr(ctx[name], "non_causal_multi_token_decode", False)
for name in layer_names
)
self._validate_dspark_dcp_support(supports_dcp_with_varlen)
@@ -3033,7 +3039,13 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
attn_metadata: MLACommonMetadata,
k_scale: torch.Tensor,
dcp_world_size: int,
fused_mla_kv_concat_fn: Callable[
[torch.Tensor, torch.Tensor, bool], tuple[torch.Tensor, torch.Tensor]
]
| None = None,
):
# fused_mla_kv_concat_fn(kv_nope, k_pe, use_fp8_prefill) -> (k, v) replaces
# the per-chunk cast + split + concat below with one fused kernel.
assert attn_metadata.prefill is not None
prefill_metadata = attn_metadata.prefill
assert prefill_metadata.prefill_backend is not None
@@ -3127,11 +3139,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
)
if use_fp8_prefill:
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
k_pe = k_pe.to(prefill_metadata.q_data_type)
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
k = self._concat_k_nope_k_pe(k_nope, k_pe)
if fused_mla_kv_concat_fn is not None:
k, v = fused_mla_kv_concat_fn(kv_nope, k_pe, use_fp8_prefill)
else:
if use_fp8_prefill:
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
k_pe = k_pe.to(prefill_metadata.q_data_type)
k_nope, v = kv_nope.split(
[self.qk_nope_head_dim, self.v_head_dim], dim=-1
)
k = self._concat_k_nope_k_pe(k_nope, k_pe)
attn_output, attn_softmax_lse = (
prefill_metadata.prefill_backend.run_prefill_context_chunk(
@@ -1293,6 +1293,7 @@ class FusedMoEConfig:
has_bias: bool = False
is_lora_enabled: bool = False
has_hash_routing: bool = False
shared_expert_prefix: str | None = None
# When True, the MoE skips its final cross-rank all-reduce (and the separate
# shared-expert reduce), returning the partial per-rank sum. The caller is
@@ -0,0 +1,164 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 4,
"grf_mode": "256"
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"24": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"48": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"96": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"128": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"256": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 4,
"grf_mode": "256"
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"1536": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 4,
"num_stages": 3,
"grf_mode": "256"
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"3072": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
}
}
@@ -0,0 +1,164 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 16,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"4": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 16,
"num_stages": 4,
"grf_mode": "256"
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"24": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"48": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8,
"num_warps": 16,
"num_stages": 4,
"grf_mode": "256"
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"96": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"128": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"256": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 4,
"grf_mode": "256"
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"1536": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"3072": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
}
}
@@ -0,0 +1,56 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
},
"16": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 16,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3,
"grf_mode": "256"
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8,
"num_warps": 16,
"num_stages": 3,
"grf_mode": "256"
}
}
@@ -0,0 +1,466 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepGEMM MegaMoE backends: fused EP dispatch, expert GEMMs and combine."""
from inspect import signature
from typing import Any
import torch
import torch.nn as nn
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp8Dynamic,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
def ue8m0_uint8_to_float(sf: torch.Tensor) -> torch.Tensor:
"""Reinterpret a uint8 UE8M0 exponent tensor as float32 scale values."""
return (sf.to(torch.int32) << 23).view(torch.float32)
class DeepGemmMegaMoEBackend:
"""Abstract MegaMoE backend that hides deep_gemm-specific details.
A backend advertises its expected hidden-state quantization layout via
``hidden_quant`` and its expected MMA type via ``mma_type``. It is
responsible for the routed-expert weight transform run at load time, the
optional shared-expert weight transform, and the actual MegaMoE kernel
invocation at inference time. Runtime capability checks live in
:func:`get_deep_gemm_mega_moe_backend`: obtaining a backend already implies
that the current device and shapes are supported.
"""
# Human-readable MMA type tag used by deep_gemm helpers. SM100 default.
mma_type: str = "fp8xfp4"
hidden_quant: QuantKey = kMxfp8Dynamic
def transform_weights(
self,
*,
w13_weight: torch.Tensor,
w13_weight_scale: torch.Tensor,
w2_weight: torch.Tensor,
w2_weight_scale: torch.Tensor,
num_local_experts: int,
hidden_size: int,
intermediate_size: int,
activation: str | None = None,
) -> tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]:
raise NotImplementedError
def supports_shared_experts(self, deep_gemm) -> bool:
return False
def transform_shared_expert_weights(
self,
*,
shared_experts: nn.Module,
num_shared_experts: int,
hidden_size: int,
intermediate_size: int,
prefix: str,
) -> (
tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]
| None
):
"""Return ``(l1, l2)`` transformed shared-expert weights, or ``None``.
``shared_experts`` must expose ``gate_up_proj`` and ``down_proj``
linear layers. A ``None`` return means the backend cannot fuse shared
experts for this module; the caller should fall back to a separate
shared MLP.
"""
return None
def run_mega_moe(
self,
*,
y: torch.Tensor,
l1_weights: tuple[torch.Tensor, torch.Tensor],
l2_weights: tuple[torch.Tensor, torch.Tensor],
symm_buffer,
activation_clamp: float | None,
fast_math: bool,
activation: str | None = None,
activation_alpha: float | None = None,
activation_beta: float | None = None,
shared_l1_weights: tuple[torch.Tensor, torch.Tensor] | None = None,
shared_l2_weights: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> None:
raise NotImplementedError
def supports_bf16_mega_gate(self) -> bool:
"""Whether this backend can run the DeepGEMM bf16 MegaMoE gate kernel.
Defaults to ``False`` (the symbol is not guaranteed to exist in every
deep_gemm build). Backends that target a deep_gemm build exposing the
``bf16_mega_gate`` symbol override this and :meth:`bf16_mega_gate`.
"""
return False
def bf16_mega_gate(
self,
*,
x: torch.Tensor,
weight: torch.Tensor,
num_topk: int,
scoring_func: str,
routed_scaling_factor: float,
ep_rank: int,
bias: torch.Tensor | None = None,
image_bias: torch.Tensor | None = None,
image_token_mask: torch.Tensor | None = None,
fix_routing_mask: torch.Tensor | None = None,
unmapped_topk_idx: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
raise NotImplementedError
class DeepGemmSm100MegaMoEBackend(DeepGemmMegaMoEBackend):
"""Default MegaMoE backend targeting SM100 via deep_gemm fp8_fp4_mega_moe."""
mma_type = "fp8xfp4"
hidden_quant = kMxfp8Dynamic
def transform_weights(
self,
*,
w13_weight: torch.Tensor,
w13_weight_scale: torch.Tensor,
w2_weight: torch.Tensor,
w2_weight_scale: torch.Tensor,
num_local_experts: int,
hidden_size: int,
intermediate_size: int,
activation: str | None = None,
) -> tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]:
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
w13_scale = deep_gemm.transform_sf_into_required_layout(
ue8m0_uint8_to_float(w13_weight_scale).contiguous(),
2 * intermediate_size,
hidden_size,
(1, 32),
num_local_experts,
)
w2_scale = deep_gemm.transform_sf_into_required_layout(
ue8m0_uint8_to_float(w2_weight_scale).contiguous(),
hidden_size,
intermediate_size,
(1, 32),
num_local_experts,
)
kwargs = {} if activation is None else {"activation": activation}
return deep_gemm.transform_weights_for_mega_moe(
(w13_weight.view(torch.int8).contiguous(), w13_scale),
(w2_weight.view(torch.int8).contiguous(), w2_scale),
**kwargs,
)
def supports_shared_experts(self, deep_gemm) -> bool:
"""Check the Python API before touching a symmetric-memory group.
This also gives users of an older precompiled vLLM wheel a safe serial
fallback instead of failing halfway through multi-rank buffer setup.
"""
try:
buffer_params = signature(deep_gemm.get_symm_buffer_for_mega_moe).parameters
kernel_params = signature(deep_gemm.fp8_fp4_mega_moe).parameters
except (TypeError, ValueError):
return False
return (
hasattr(deep_gemm, "get_block_m_for_mega_moe")
and hasattr(deep_gemm, "transform_weights_for_mega_moe")
and "num_shared_experts" in buffer_params
and "shared_l1_weights" in kernel_params
and "shared_l2_weights" in kernel_params
)
def transform_shared_expert_weights(
self,
*,
shared_experts: nn.Module,
num_shared_experts: int,
hidden_size: int,
intermediate_size: int,
prefix: str,
) -> (
tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]
| None
):
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
gate_up = shared_experts.gate_up_proj
down = shared_experts.down_proj
gate_up_weight = gate_up.weight.data
gate_up_scale = (
gate_up.weight_scale
if hasattr(gate_up, "weight_scale")
else gate_up.weight_scale_inv
).data
down_weight = down.weight.data
down_scale = (
down.weight_scale
if hasattr(down, "weight_scale")
else down.weight_scale_inv
).data
# MegaMoE's shared FP8 MMA consumes a 1x32 scale for every weight row,
# while the checkpoint uses coarser block-FP8 scales (usually
# 128x128). Build a dedicated, numerically equivalent scale view.
ue8m0_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8, torch.int32)
if (
gate_up_scale.dtype in ue8m0_scale_dtypes
and down_scale.dtype in ue8m0_scale_dtypes
):
gate_up_scale = self._prepare_shared_expert_scale(
deep_gemm,
gate_up,
gate_up_scale,
gate_up_weight.shape[0],
gate_up_weight.shape[1],
prefix,
)
down_scale = self._prepare_shared_expert_scale(
deep_gemm,
down,
down_scale,
down_weight.shape[0],
down_weight.shape[1],
prefix,
)
if gate_up_scale is None or down_scale is None:
return None
shared_intermediate_size = intermediate_size * num_shared_experts
expected_gate_up_shape = (
2 * shared_intermediate_size,
hidden_size,
)
expected_down_shape = (hidden_size, shared_intermediate_size)
if (
gate_up_weight.dtype != torch.float8_e4m3fn
or down_weight.dtype != torch.float8_e4m3fn
or gate_up_scale.dtype != torch.int32
or down_scale.dtype != torch.int32
or tuple(gate_up_weight.shape) != expected_gate_up_shape
or tuple(down_weight.shape) != expected_down_shape
):
logger.warning(
"Disabling native MegaMoE shared-expert fusion for %s: expected "
"replicated block-FP8 weights with gate_up=%s, down=%s, and "
"DeepGEMM int32 scales; got gate_up=%s/%s/%s and down=%s/%s/%s.",
prefix,
expected_gate_up_shape,
expected_down_shape,
tuple(gate_up_weight.shape),
gate_up_weight.dtype,
gate_up_scale.dtype,
tuple(down_weight.shape),
down_weight.dtype,
down_scale.dtype,
)
return None
transformed_l1, transformed_l2 = deep_gemm.transform_weights_for_mega_moe(
(gate_up_weight, gate_up_scale),
(down_weight, down_scale),
)
# L1 interleaving allocates a full copy. Re-home the loader Parameter on
# that storage so the original 2*intermediate*hidden FP8 tensor can be
# released instead of adding roughly 0.7 GiB per rank on DSV4-Flash.
# The generic linear post-load hook may still repack the serial scales,
# but this shared MLP is never called after native fusion is enabled.
gate_up.weight.data = transformed_l1[0]
transformed_l1 = (gate_up.weight.data, transformed_l1[1])
return transformed_l1, transformed_l2
@staticmethod
def _prepare_shared_expert_scale(
deep_gemm,
linear: nn.Module,
scale: torch.Tensor,
mn: int,
k: int,
prefix: str,
) -> torch.Tensor | None:
block_size = getattr(linear, "weight_block_size", None)
if block_size is None or len(block_size) != 2:
logger.warning(
"Disabling native MegaMoE shared-expert fusion for %s: "
"shared FP8 weight block size is unavailable.",
prefix,
)
return None
block_m, block_k = block_size
num_k_blocks = (k + block_k - 1) // block_k
# The linear hook's DeepGEMM layout packs four k-block bytes per int32.
packed = scale.dtype == torch.int32
expected_shape = (
(mn, (num_k_blocks + 3) // 4)
if packed
else ((mn + block_m - 1) // block_m, num_k_blocks)
)
if block_k % 32 != 0 or tuple(scale.shape) != expected_shape:
logger.warning(
"Disabling native MegaMoE shared-expert fusion for %s: "
"cannot convert shared scale shape %s with block size %s "
"to MegaMoE's 1x32 layout for weight (%d, %d).",
prefix,
tuple(scale.shape),
tuple(block_size),
mn,
k,
)
return None
if packed:
row_scale = scale.flatten().view(torch.uint8).view(mn, -1)[:, :num_k_blocks]
else:
row_scale = scale.view(torch.uint8).repeat_interleave(block_m, dim=0)[:mn]
scale_1x32 = (
ue8m0_uint8_to_float(row_scale)
.repeat_interleave(block_k // 32, dim=1)[:, : k // 32]
.contiguous()
)
# The grouped API is used with a singleton dimension to request the
# MN-major, TMA-aligned packed-UE8M0 strides, then squeezed back to the
# 2D layout required for a shared expert.
return deep_gemm.transform_sf_into_required_layout(
scale_1x32.unsqueeze(0),
mn,
k,
(1, 32),
1,
).squeeze(0)
def run_mega_moe(
self,
*,
y: torch.Tensor,
l1_weights: tuple[torch.Tensor, torch.Tensor],
l2_weights: tuple[torch.Tensor, torch.Tensor],
symm_buffer,
activation_clamp: float | None,
fast_math: bool,
activation: str | None = None,
activation_alpha: float | None = None,
activation_beta: float | None = None,
shared_l1_weights: tuple[torch.Tensor, torch.Tensor] | None = None,
shared_l2_weights: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> None:
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
kwargs: dict[str, Any] = {
name: value
for name, value in (
("activation", activation),
("activation_alpha", activation_alpha),
("activation_beta", activation_beta),
)
if value is not None
}
if shared_l1_weights is not None and shared_l2_weights is not None:
kwargs["shared_l1_weights"] = shared_l1_weights
kwargs["shared_l2_weights"] = shared_l2_weights
deep_gemm.fp8_fp4_mega_moe(
y,
l1_weights,
l2_weights,
symm_buffer,
activation_clamp=activation_clamp,
fast_math=fast_math,
**kwargs,
)
def supports_bf16_mega_gate(self) -> bool:
"""SM100 deep_gemm build is expected to provide the bf16 gate kernel."""
return True
def bf16_mega_gate(
self,
*,
x: torch.Tensor,
weight: torch.Tensor,
num_topk: int,
scoring_func: str,
routed_scaling_factor: float,
ep_rank: int,
bias: torch.Tensor | None = None,
image_bias: torch.Tensor | None = None,
image_token_mask: torch.Tensor | None = None,
fix_routing_mask: torch.Tensor | None = None,
unmapped_topk_idx: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
from vllm.utils.deep_gemm import bf16_mega_gate
return bf16_mega_gate(
x,
weight,
num_topk,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
ep_rank=ep_rank,
bias=bias,
image_bias=image_bias,
image_token_mask=image_token_mask,
fix_routing_mask=fix_routing_mask,
unmapped_topk_idx=unmapped_topk_idx,
)
def get_deep_gemm_mega_moe_backend(
device: torch.device,
hidden_size: int,
intermediate_size: int,
) -> DeepGemmMegaMoEBackend:
"""Return the DeepGEMM MegaMoE backend for ``device``.
A successful return implies the backend can run on this device with the
given ``hidden_size`` / ``intermediate_size``. All runtime capability
checks (arch + shape divisibility) live here; there is no separate check
on the returned backend.
Currently only the SM100 deep_gemm backend is registered here. SM90 and
other backends are expected to be provided by new backends or plugins
that override this factory.
Raises:
NotImplementedError: If the device architecture is unsupported.
ValueError: If the shapes are unsupported.
"""
if not current_platform.is_device_capability_family(
100, device_id=device.index or 0
):
raise NotImplementedError("DeepGEMM MegaMoE requires SM100 GPUs.")
if hidden_size % 128 != 0 or intermediate_size % 128 != 0:
raise ValueError(
"DeepGEMM MegaMoE requires hidden and intermediate sizes "
"to be multiples of 128."
)
return DeepGemmSm100MegaMoEBackend()
@@ -119,6 +119,7 @@ def _fwd_kernel_ep_scatter_1(
BLOCK_E: tl.constexpr,
BLOCK_EXPERT_NUM: tl.constexpr,
ALIGN_M: tl.constexpr,
USE_PSUM_LAYOUT: tl.constexpr,
):
cur_expert = tl.program_id(0)
@@ -140,20 +141,22 @@ def _fwd_kernel_ep_scatter_1(
tl.store(expert_start_loc + cur_expert, cur_expert_start)
cur_expert_token_num = tl.load(num_recv_tokens_per_expert + cur_expert)
m_indices_start_ptr = m_indices + cur_expert_start
off_expert = tl.arange(0, BLOCK_E)
if USE_PSUM_LAYOUT:
# Logical ends include preceding experts' alignment gaps.
tl.store(m_indices + cur_expert, cur_expert_start + cur_expert_token_num)
else:
m_indices_start_ptr = m_indices + cur_expert_start
off_expert = tl.arange(0, BLOCK_E)
# any rows in the per-expert aligned region that do not correspond to
# real tokens are left untouched here and should remain initialized to
# -1 so DeepGEMM can skip them
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
offs = start_m + off_expert
mask = offs < cur_expert_token_num
tl.store(
m_indices_start_ptr + offs,
cur_expert,
mask=mask,
)
# Unwritten padding retains the -1 initialization.
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
offs = start_m + off_expert
mask = offs < cur_expert_token_num
tl.store(
m_indices_start_ptr + offs,
cur_expert,
mask=mask,
)
@triton.jit
@@ -285,6 +288,7 @@ def ep_scatter(
align_m: int = 128,
block_size: int = 128,
pack_ue8m0: bool = False,
use_psum_layout: bool = False,
):
# BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m.
BLOCK_E = 128
@@ -295,7 +299,10 @@ def ep_scatter(
# grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts)
grid = num_experts
assert m_indices.shape[0] % align_m == 0
if use_psum_layout:
assert m_indices.shape == (num_experts,)
else:
assert m_indices.shape[0] % align_m == 0
assert expert_start_loc.shape[0] == num_experts
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
@@ -311,6 +318,7 @@ def ep_scatter(
BLOCK_E=BLOCK_E,
BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts),
ALIGN_M=align_m,
USE_PSUM_LAYOUT=use_psum_layout,
)
grid = min(recv_topk.shape[0], 1024 * 8)
@@ -407,7 +415,7 @@ def _fwd_kernel_ep_gather(
tl.store(
output_tensor
+ cur_token * output_tensor_stride0
+ tl.cast(cur_token, tl.int64) * output_tensor_stride0
+ cur_block * BLOCK_D
+ off_d,
accumulator.to(output_tensor.dtype.element_ty),
@@ -465,6 +473,7 @@ def deepgemm_moe_permute(
expert_tokens_meta: mk.ExpertTokensMetadata | None,
aq_out: torch.Tensor | None = None,
block_size: int | None = None,
use_psum_layout: bool = False,
):
assert aq.ndim == 2
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
@@ -509,17 +518,15 @@ def deepgemm_moe_permute(
else:
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
# completely invalid / padded blocks that should be skipped. We always
# initialize expert_ids to -1 so any row that is not explicitly written
# by the scatter kernel will be treated as invalid and skipped by
# DeepGEMM's scheduler.
expert_ids = torch.full(
(M_sum,),
fill_value=-1,
device=device,
dtype=torch.int32,
)
if use_psum_layout:
grouped_layout = torch.empty(
(local_num_experts,), device=device, dtype=torch.int32
)
else:
# Negative expert IDs mark padding for the contiguous scheduler.
grouped_layout = torch.full(
(M_sum,), fill_value=-1, device=device, dtype=torch.int32
)
inv_perm = torch.empty(topk_ids.shape, device=device, dtype=torch.int32)
expert_num_tokens = None
@@ -541,14 +548,15 @@ def deepgemm_moe_permute(
expert_map=expert_map,
output_tensor=aq_out,
output_tensor_scale=aq_scale_out,
m_indices=expert_ids,
m_indices=grouped_layout,
output_index=inv_perm,
align_m=align_used,
block_size=block_k,
pack_ue8m0=pack_ue8m0,
use_psum_layout=use_psum_layout,
)
return aq_out, aq_scale_out, expert_ids, inv_perm, align_used
return aq_out, aq_scale_out, grouped_layout, inv_perm, align_used
def deepgemm_unpermute_and_reduce(
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
MoEPermuteScratch,
get_moe_permute_scratch,
moe_permute,
moe_permute_unpermute_supported,
moe_unpermute,
@@ -304,7 +305,6 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
self.ab_strides2 = ab_strides2
self.c_strides1 = c_strides1
self.c_strides2 = ab_strides1_c_strides2
self._permute_scratch: MoEPermuteScratch | None = None
@staticmethod
def _supports_current_device() -> bool:
@@ -335,15 +335,13 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
return TopKWeightAndReduceDelegate()
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if self._permute_scratch is None and moe_permute_unpermute_supported():
self._permute_scratch = MoEPermuteScratch(
max_num_tokens=self.moe_config.max_num_tokens,
topk=self.moe_config.experts_per_token,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
)
return self._permute_scratch
if not moe_permute_unpermute_supported():
return None
return get_moe_permute_scratch(
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
)
def apply(
self,
@@ -1296,7 +1294,6 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
self.s_strides2[:, 0] = k
self.group_size = group_size
self._permute_scratch: MoEPermuteScratch | None = None
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
@@ -1357,15 +1354,13 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
return self.out_dtype if self.out_dtype is not None else act_dtype
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if self._permute_scratch is None and moe_permute_unpermute_supported():
self._permute_scratch = MoEPermuteScratch(
max_num_tokens=self.moe_config.max_num_tokens,
topk=self.moe_config.experts_per_token,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
)
return self._permute_scratch
if not moe_permute_unpermute_supported():
return None
return get_moe_permute_scratch(
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
)
def workspace_shapes(
self,
@@ -244,7 +244,12 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
return (workspace1, workspace2, output)
def _act_mul_quant(
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
self,
input: torch.Tensor,
output: torch.Tensor,
activation: MoEActivation,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
assert self.block_shape is not None
block_k = self.block_shape[1]
@@ -271,6 +276,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
clamp_limit=self.gemm1_clamp_limit,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
act_out = torch.empty(
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
@@ -294,6 +301,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
group_size=block_k,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
@@ -349,7 +358,20 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
# Both GEMMs and quantization must agree on the live expert ranges.
use_psum_layout = (
not self.mxfp8
and (
current_platform.is_device_capability_family(90)
or (
current_platform.is_device_capability_family(100)
and DeepGemmQuantScaleFMT.from_oracle()
== DeepGemmQuantScaleFMT.UE8M0
)
)
and activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI_UNINTERLEAVE)
)
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -360,17 +382,21 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
# FP8-block keeps the default (128) alignment.
block_size=self.block_shape[1] if self.mxfp8 else None,
use_psum_layout=use_psum_layout,
)
assert a1q.size(0) == M_sum
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
# (1, 32); the FP8 block path keeps the default (128) recipe.
gemm_kwargs = (
gemm_kwargs: dict = (
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
if self.mxfp8
else {}
)
if use_psum_layout:
gemm_kwargs["use_psum_layout"] = True
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
# otherwise the scheduler can pick the wrong expert id from m_indices
# under cudagraph replay.
@@ -380,7 +406,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
(a1q, a1q_scale),
(w1, self.w1_scale),
mm1_out,
expert_ids,
grouped_layout,
**gemm_kwargs,
)
@@ -389,7 +415,11 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
input=mm1_out.view(-1, N),
output=quant_out,
activation=activation,
expert_ends=grouped_layout if use_psum_layout else None,
expert_alignment=align_used if use_psum_layout else 0,
)
mm2_out = _resize_cache(workspace2, (M_sum, K))
@@ -397,7 +427,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
(a2q, a2q_scale),
(w2, self.w2_scale),
mm2_out,
expert_ids,
grouped_layout,
**gemm_kwargs,
)
@@ -258,25 +258,14 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
**kwargs,
)
def _get_permute_scratch(
self, topk: int, indices_only: bool = False
) -> MoEPermuteScratch | None:
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if not moe_permute_unpermute_supported():
return None
max_expanded_rows = (
self.moe_config.max_num_tokens
* self.moe_config.dp_size
* self.moe_config.experts_per_token
)
return get_moe_permute_scratch(
max_num_tokens=math.ceil(max_expanded_rows / topk),
topk=topk,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
hidden_size=None if indices_only else self.moe_config.hidden_dim,
hidden_dtype=None if indices_only else self.moe_config.in_dtype,
)
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
@@ -911,14 +900,18 @@ class HummingGroupedExperts(HummingExpertsBase):
)
buffers["output"] = output
scratch = self._get_permute_scratch()
if a1q_scale is None:
scratch = self._get_permute_scratch(topk_ids.size(1), indices_only=True)
assert scratch is not None
scatter_metadata = moe_prepare_scatter(topk_ids, expert_map, scratch)
expert_offsets, scatter_idx = scatter_metadata
inv_perm = scatter_idx.flatten()
num_valid_tokens = expert_offsets[-1:] if expert_map is not None else None
else:
# Prequantized inputs skip w13 quantization, so its buffer is free
# to hold the permuted hidden states.
permuted_hidden_states = buffers["quanted_gate_up_input"]
assert permuted_hidden_states.dtype == hidden_states.dtype
hidden_states, a1q_scale, expert_offsets, inv_perm, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=a1q_scale,
@@ -926,7 +919,8 @@ class HummingGroupedExperts(HummingExpertsBase):
n_expert=global_num_experts,
n_local_expert=self.num_experts,
expert_map=expert_map,
scratch=self._get_permute_scratch(topk_ids.size(1)),
permuted_hidden_states=permuted_hidden_states,
scratch=scratch,
)
scatter_idx = None
num_valid_tokens = None
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -41,11 +43,11 @@ def _triton_kernel_moe_supports_current_device() -> bool:
p = current_platform
if p.is_cuda():
cap = p.get_device_capability()
# Keep the original `(9, 0) <= cap < (11, 0)` window on
# CUDA (covers Hopper SM90 and Blackwell SM100, excludes
# SM120) — this PR is ROCm-scoped and the broader CUDA
# range was not validated.
return cap is not None and (9, 0) <= (cap.major, cap.minor) < (11, 0)
# Hopper SM90, datacenter Blackwell SM100/103 and consumer Blackwell
# SM120/121. SM12x needs the opt-flag constraints applied by
# `_swizzle_mxfp4` and `_constrain_sm12x_block_m` to fit its 99KB of
# shared memory; it is still ordered after MARLIN by the selector.
return cap is not None and (9, 0) <= (cap.major, cap.minor) < (13, 0)
if p.is_rocm():
from vllm.platforms.rocm import on_gfx1x, on_gfx9
@@ -57,6 +59,36 @@ def _triton_kernel_moe_supports_current_device() -> bool:
return False
@functools.cache
def _is_sm12x() -> bool:
return current_platform.is_cuda() and current_platform.is_device_capability_family(
120
)
# Consumer Blackwell has 99KB of opt-in shared memory: with block_m >= 64 the
# persistent mxfp4 kernel fits a single pipeline stage and runs 2-3x slower
# than with 32, but triton_kernels' block_m heuristic still tops out at 128.
_SM12X_MAX_BLOCK_M = 32
def _sm12x_block_m(num_rows: int, num_experts: int) -> int:
"""triton_kernels' tokens-per-expert block_m heuristic, capped at 32."""
tokens_per_expt = max(1, num_rows // num_experts)
return max(16, min(triton.next_power_of_2(tokens_per_expt), _SM12X_MAX_BLOCK_M))
def _constrain_sm12x_block_m(num_rows: int, num_experts: int) -> None:
"""Pin block_m for the following matmul_ogs calls on SM12x."""
if not _is_sm12x():
return
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
opt_flags.update_opt_flags_constraints(
{"block_m": _sm12x_block_m(num_rows, num_experts)}
)
def _patch_make_bitmatrix_metadata() -> None:
"""Monkey-patch make_bitmatrix_metadata to support non-power-of-2 top_k.
@@ -784,6 +816,7 @@ def triton_kernel_fused_experts(
)
)
gammas = routing_data.gate_scal if routing_data else None
_constrain_sm12x_block_m(M * topk, E)
matmul_ogs(
hidden_states,
@@ -1403,6 +1436,7 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
intermediate_cache2 = _resize_cache(workspace13, (M * topk, activation_out_dim))
gammas = routing_data.gate_scal if routing_data else None
_constrain_sm12x_block_m(M * topk, E)
matmul_ogs(
hidden_states,
@@ -254,6 +254,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.SigmoidRenorm,
RoutingMethodType.MiniMax2,
RoutingMethodType.Sigmoid,
]
@@ -175,6 +175,11 @@ class XPUExperts(mk.FusedMoEExpertsModular):
gemm1_clamp_limit=self.gemm1_clamp_limit,
)
assert self.fused_moe_impl is not None
if self.moe_config.ep_size > 1:
# Avoid topk_ids = -1, remap them to an
# expert not owned by this rank so they are skipped.
sentinel = global_num_experts - 1 if self.moe_config.ep_rank == 0 else 0
topk_ids = torch.where(topk_ids < 0, sentinel, topk_ids)
self.fused_moe_impl.apply(
output=output,
hidden_states=hidden_states,
@@ -118,10 +118,11 @@ def FusedMoEFactory(
has_bias: bool = False,
is_sequence_parallel: bool = False,
reduce_results: bool = True,
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
ckpt_names: tuple[str, str, str | None] = ("gate_proj", "down_proj", "up_proj"),
is_fused_checkpoint_transposed: bool = False,
n_shared_experts: int | None = None,
fuse_shared_experts: bool = False,
shared_expert_prefix: str | None = None,
router_logits_dtype: torch.dtype | None = None,
gate: torch.nn.Module | None = None,
shared_experts: torch.nn.Module | None = None,
@@ -197,6 +198,7 @@ def FusedMoEFactory(
n_shared_experts: Number of shared experts to fuse into the routed
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
fuse_shared_experts: Whether to enable shared-expert fusion.
shared_expert_prefix: Checkpoint prefix for the fused shared expert.
router_logits_dtype: Data type for router logits buffers
gate: Pre-configured gate module
shared_experts: Pre-configured shared experts module
@@ -360,6 +362,7 @@ def FusedMoEFactory(
elastic_ep_max_dp_size=vllm_config.parallel_config.elastic_ep_max_dp_size,
has_bias=has_bias,
is_lora_enabled=vllm_config.lora_config is not None,
shared_expert_prefix=shared_expert_prefix,
activation=moe_activation,
device=vllm_config.device_config.device,
routing_method=router.routing_method_type, # Not ideal
@@ -442,7 +445,7 @@ def fused_moe_make_expert_params_mapping(
model: torch.nn.Module,
ckpt_gate_proj_name: str,
ckpt_down_proj_name: str,
ckpt_up_proj_name: str,
ckpt_up_proj_name: str | None,
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
@@ -197,6 +197,20 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return
def allocate_fused_expert_output(
self,
shape: tuple[int, ...],
dtype: torch.dtype,
device: torch.device,
weight_and_reduce_impl: TopKWeightAndReduce,
) -> torch.Tensor | None:
"""Optionally provide the exact output buffer for the expert kernel.
Implementations returning a tensor must preserve its identity through
finalize; replacing it may silently reintroduce a write-back copy.
"""
return None
@property
@abstractmethod
def activation_format(self) -> FusedMoEActivationFormat:
@@ -1054,13 +1068,14 @@ class FusedMoEKernelModularImpl:
local_num_experts: int,
expert_tokens_meta: ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool]:
"""Allocate temporary and output buffers for the fused experts op.
Inputs:
- out_dtype: output type of workspace and output tensors.
- device: the device of the workspace and output tensors.
See `workspace_shapes` for a description of the remainder of arguments.
Returns a tuple of (workspace13, workspace2, output) tensors.
Returns the workspace tensors, output tensor, and whether the output
uses a prepare/finalize-provided allocation.
"""
assert M_full > 0 and M_chunk > 0
@@ -1090,9 +1105,20 @@ class FusedMoEKernelModularImpl:
activation,
)
fused_out_alias = self.prepare_finalize.allocate_fused_expert_output(
fused_out_shape,
workspace_dtype,
device,
self.fused_experts.finalize_weight_and_reduce_impl(),
)
# We can reuse the memory between cache1 and cache3 because by the
# time we need cache3, we're done with cache1.
# Reuse workspace13 for the output since there is only one chunk.
# Keep the original fused-output reservation even when the output is
# externally aliased. The workspace manager profiles and locks a
# process-wide high-water mark; later graph shapes may need this
# storage for workspace13.
max_shape_size = max(prod(workspace13_shape), prod(fused_out_shape))
if current_platform.is_cpu():
@@ -1110,16 +1136,23 @@ class FusedMoEKernelModularImpl:
)
workspace13 = _resize_cache(common_workspace, workspace13_shape)
fused_out = _resize_cache(common_workspace, fused_out_shape)
return workspace13, workspace2, fused_out
return workspace13, workspace2, fused_out, False
common_workspace, workspace2 = current_workspace_manager().get_simultaneous(
((max_shape_size,), workspace_dtype),
(workspace2_shape, workspace_dtype),
)
workspace13 = _resize_cache(common_workspace, workspace13_shape)
fused_out = _resize_cache(common_workspace, fused_out_shape)
if fused_out_alias is None:
fused_out = _resize_cache(common_workspace, fused_out_shape)
else:
assert fused_out_alias.shape == fused_out_shape
assert fused_out_alias.dtype == workspace_dtype
assert fused_out_alias.device == device
assert fused_out_alias.is_contiguous()
fused_out = fused_out_alias
return workspace13, workspace2, fused_out
return workspace13, workspace2, fused_out, fused_out_alias is not None
def _maybe_apply_shared_experts(
self,
@@ -1251,22 +1284,25 @@ class FusedMoEKernelModularImpl:
if M_full == 0:
return torch.empty_like(a1q, dtype=in_dtype)
workspace13, workspace2, fused_out = self._allocate_buffers(
in_dtype,
a1q.device,
M_full,
M_full,
N,
K,
top_k,
global_num_experts,
local_num_experts,
expert_tokens_meta,
activation,
workspace13, workspace2, fused_out, fused_out_is_specialized = (
self._allocate_buffers(
in_dtype,
a1q.device,
M_full,
M_full,
N,
K,
top_k,
global_num_experts,
local_num_experts,
expert_tokens_meta,
activation,
)
)
use_output_alias = (
output_alias is not None
not fused_out_is_specialized
and output_alias is not None
and output_alias.shape == fused_out.shape
and output_alias.dtype == fused_out.dtype
and output_alias.device == fused_out.device
@@ -13,90 +13,70 @@ from vllm.v1.worker.workspace import (
@dataclass
class MoEPermuteScratch:
# Reused metadata buffers for repeated grouped-MoE permutes.
max_num_tokens: int
topk: int
"""Reused metadata buffers for repeated grouped-MoE permutes.
Capacity follows the largest input seen, like the shared workspace: it grows
during warmup and is fixed once the workspace manager is locked.
"""
num_experts: int
num_local_experts: int
device: torch.device
hidden_size: int | None = None
hidden_dtype: torch.dtype | None = None
max_expanded_rows: int = field(init=False, default=0)
token_expert_indices: torch.Tensor = field(init=False)
expert_first_token_offset: torch.Tensor = field(init=False)
permuted_idx: torch.Tensor = field(init=False)
inv_permuted_idx: torch.Tensor = field(init=False)
permuted_hidden_states: torch.Tensor | None = field(init=False, default=None)
sort_workspace: torch.Tensor = field(init=False)
permuted_experts_id: torch.Tensor = field(init=False)
sorted_row_idx: torch.Tensor = field(init=False)
topk_ids_int32: torch.Tensor = field(init=False)
topk_ids_for_sort: torch.Tensor = field(init=False)
max_expanded_rows: int = field(init=False)
def __post_init__(self) -> None:
assert self.max_num_tokens > 0
assert self.topk > 0
assert self.num_experts > 0
assert self.num_local_experts > 0
if self.hidden_size is None:
assert self.hidden_dtype is None
else:
assert self.hidden_dtype is not None
self.max_expanded_rows = self.max_num_tokens * self.topk
self.token_expert_indices = torch.arange(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.expert_first_token_offset = torch.empty(
self.num_local_experts + 1, dtype=torch.int64, device=self.device
)
self.permuted_idx = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.inv_permuted_idx = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
if self.hidden_size is not None:
hidden_numel = self.max_expanded_rows * self.hidden_size
self.permuted_hidden_states = torch.empty(
hidden_numel, dtype=self.hidden_dtype, device=self.device
# torch.device("cuda") in config, after initialized,
# will be changed to cuda:{index}, so we need to refresh here.
self.device = self.expert_first_token_offset.device
def reserve(self, expanded_rows: int) -> None:
"""Ensure capacity for ``expanded_rows`` (num_tokens * topk) rows."""
if expanded_rows <= self.max_expanded_rows:
return
if (
is_workspace_manager_initialized()
and current_workspace_manager().is_locked()
):
raise AssertionError(
f"Workspace is locked but MoE permute scratch requires "
f"{expanded_rows} rows, current capacity is "
f"{self.max_expanded_rows} rows. Scratch growth is not allowed "
"after locking."
)
self.permuted_experts_id = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.sorted_row_idx = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.topk_ids_int32 = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.topk_ids_for_sort = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
def empty_rows() -> torch.Tensor:
return torch.empty(expanded_rows, dtype=torch.int32, device=self.device)
self.token_expert_indices = torch.arange(
expanded_rows, dtype=torch.int32, device=self.device
)
self.permuted_idx = empty_rows()
self.inv_permuted_idx = empty_rows()
self.permuted_experts_id = empty_rows()
self.sorted_row_idx = empty_rows()
self.topk_ids_int32 = empty_rows()
self.topk_ids_for_sort = empty_rows()
sorter_size = torch.ops._moe_C.moe_permute_sort_workspace_size(
self.max_expanded_rows, self.num_experts
expanded_rows, self.num_experts
)
self.sort_workspace = torch.empty(
sorter_size, dtype=torch.int8, device=self.device
)
# torch.device("cuda") in config, after initialized,
# will be changed to cuda:{index}, so we need to refresh here.
self.device = self.token_expert_indices.device
def validate(self, hidden_states: torch.Tensor, topk_ids: torch.Tensor) -> None:
n_token, n_hidden = hidden_states.shape
assert hidden_states.device == self.device
assert topk_ids.device == self.device
assert n_token <= self.max_num_tokens
assert topk_ids.size(1) == self.topk
assert topk_ids.size(0) == n_token
if self.hidden_size is not None:
assert n_hidden == self.hidden_size
assert hidden_states.dtype == self.hidden_dtype
assert self.permuted_hidden_states is not None
def token_expert_indices_view(self, n_token: int) -> torch.Tensor:
return self.token_expert_indices[: n_token * self.topk].view(n_token, self.topk)
self.max_expanded_rows = expanded_rows
def prepare_topk_ids(self, topk_ids: torch.Tensor) -> torch.Tensor:
if topk_ids.dtype == torch.int32:
@@ -114,17 +94,16 @@ def moe_prepare_scatter(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Generate expert offsets and shared scatter/unpermute destination indices."""
assert topk_ids.device == scratch.device
assert topk_ids.size(0) <= scratch.max_num_tokens
assert topk_ids.size(1) == scratch.topk
n_token, topk = topk_ids.shape
expanded_rows = topk_ids.numel()
scratch.reserve(expanded_rows)
inverse = scratch.inv_permuted_idx[:expanded_rows].view(n_token, topk)
if expanded_rows == 0:
scratch.expert_first_token_offset.zero_()
return scratch.expert_first_token_offset, inverse
torch.ops._moe_C.moe_prepare_scatter(
scratch.prepare_topk_ids(topk_ids),
scratch.token_expert_indices_view(n_token),
scratch.token_expert_indices[:expanded_rows].view(n_token, topk),
expert_map,
scratch.num_experts,
scratch.num_local_experts,
@@ -140,13 +119,9 @@ def moe_prepare_scatter(
def get_moe_permute_scratch(
*,
max_num_tokens: int,
topk: int,
num_experts: int,
num_local_experts: int,
device: torch.device,
hidden_size: int | None = None,
hidden_dtype: torch.dtype | None = None,
) -> MoEPermuteScratch:
"""Share scratch across sequential layers in the current ubatch and lane.
@@ -155,28 +130,15 @@ def get_moe_permute_scratch(
def create_scratch() -> MoEPermuteScratch:
return MoEPermuteScratch(
max_num_tokens=max_num_tokens,
topk=topk,
num_experts=num_experts,
num_local_experts=num_local_experts,
device=device,
hidden_size=hidden_size,
hidden_dtype=hidden_dtype,
)
if not is_workspace_manager_initialized():
return create_scratch()
key = (
MoEPermuteScratch,
max_num_tokens,
topk,
num_experts,
num_local_experts,
device,
hidden_size,
hidden_dtype,
)
key = (MoEPermuteScratch, num_experts, num_local_experts, device)
return current_workspace_manager().get_persistent_resource(key, create_scratch)
@@ -204,9 +166,8 @@ def moe_permute(
parallel shard.
permuted_hidden_states (Optional[torch.Tensor]): Optional output tensor.
If None, the output tensor will be created in this function.
scratch (Optional[MoEPermuteScratch]): Optional preallocated scratch
buffers. Validated against hidden_states and topk_ids when given,
otherwise the buffers are allocated in this function.
scratch (Optional[MoEPermuteScratch]): Optional reusable metadata
buffers. Otherwise they are allocated in this function.
Returns:
- permuted_hidden_states (torch.Tensor): permuted activation.
@@ -227,20 +188,11 @@ def moe_permute(
if n_local_expert == -1:
n_local_expert = n_expert
if permuted_hidden_states is None:
if scratch is None:
permuted_hidden_states = torch.empty(
(permuted_row_size, n_hidden),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
else:
scratch.validate(hidden_states, topk_ids)
hidden_numel = permuted_row_size * n_hidden
scratch_hidden_states = scratch.permuted_hidden_states
assert scratch_hidden_states is not None
permuted_hidden_states = scratch_hidden_states[:hidden_numel].view(
permuted_row_size, n_hidden
)
permuted_hidden_states = torch.empty(
(permuted_row_size, n_hidden),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
assert permuted_hidden_states.size() == (permuted_row_size, n_hidden), (
f"Expected permuted hidden states to be {(permuted_row_size, n_hidden)}"
f" but got {permuted_hidden_states.size()}"
@@ -278,10 +230,15 @@ def moe_permute(
permuted_idx,
)
else:
scratch.validate(hidden_states, topk_ids)
assert hidden_states.device == scratch.device
assert topk_ids.device == scratch.device
assert topk_ids.size(0) == n_token
assert n_expert == scratch.num_experts
assert n_local_expert == scratch.num_local_experts
token_expert_indices = scratch.token_expert_indices_view(n_token)
scratch.reserve(permuted_row_size)
token_expert_indices = scratch.token_expert_indices[:permuted_row_size].view(
n_token, topk
)
expert_first_token_offset = scratch.expert_first_token_offset
permuted_idx = scratch.permuted_idx[:permuted_row_size]
permuted_idx.fill_(permuted_row_size)
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
prepare_fp8_moe_layer_for_deepgemm,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_fp8_moe_layer_for_marlin,
)
@@ -37,6 +38,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
@@ -98,6 +100,7 @@ def _get_priority_backends(
Fp8MoeBackend.CPU,
Fp8MoeBackend.HPC,
]
_AVAILABLE_BACKENDS = prioritize_humming(_AVAILABLE_BACKENDS)
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
backends.insert(0, backends.pop(backends.index(backend)))
@@ -607,6 +610,21 @@ def convert_to_fp8_moe_kernel_format(
convert_to_humming_moe_kernel_format,
)
# Online quantization has not installed the quantized tensors yet.
scale_name = (
"weight_scale_inv"
if hasattr(layer, "w13_weight_scale_inv")
else "weight_scale"
)
for prefix, weight, scale, input_scale in (
("w13", w13, w13_scale, w13_input_scale),
("w2", w2, w2_scale, w2_input_scale),
):
replace_parameter(layer, f"{prefix}_weight", weight)
replace_parameter(layer, f"{prefix}_{scale_name}", scale)
if input_scale is not None:
replace_parameter(layer, f"{prefix}_input_scale", input_scale)
convert_to_humming_moe_kernel_format(
layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
)
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
TrtLlmMxint4ExpertsMonolithic,
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_config,
marlin_act_int8_process_scales,
@@ -129,7 +130,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
return [
backends = [
# Native HIP kernel, gated on gfx1100 by _supports_current_device().
WNA16MoEBackend.RDNA3,
WNA16MoEBackend.FLASHINFER_TRTLLM,
@@ -139,6 +140,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
WNA16MoEBackend.HUMMING,
WNA16MoEBackend.EMULATION,
]
return prioritize_humming(backends)
def _backend_incompatibility_reason(
@@ -924,15 +926,18 @@ def _process_weights_rdna3(
]:
"""RDNA3 (gfx1100) W4A16 weight post-processing.
Interleaves the packed nibbles per expert (the exllama shuffle the dense
RDNA3 kernel also uses) and synthesizes the symmetric zero points that
``moe_gptq_gemm_rdna3`` dequantizes with. The packed layout
``[E, K // 8, N]`` and the ``[E, groups, N]`` scales are already what the
kernel wants, so neither is repacked.
Transposes the canonical N-first inputs to the K-first layout of
``moe_gptq_gemm_rdna3`` (weights ``[E, K // 8, N]``, scales
``[E, groups, N]``), interleaves the packed nibbles per expert (the exllama
shuffle the dense RDNA3 kernel also uses) and synthesizes the symmetric
zero points the kernel dequantizes with.
"""
device = w13.device
num_experts = w13.size(0)
w13 = w13.transpose(1, 2).contiguous()
w2 = w2.transpose(1, 2).contiguous()
for e in range(num_experts):
w13_e = w13[e].contiguous()
ops.gptq_shuffle(w13_e, 4)
@@ -948,8 +953,8 @@ def _process_weights_rdna3(
return (
w13,
w2,
w13_scale.contiguous(),
w2_scale.contiguous(),
w13_scale.transpose(1, 2).contiguous(),
w2_scale.transpose(1, 2).contiguous(),
_qzeros(w13),
_qzeros(w2),
None, # w13_input_global_scale
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
swap_w13_to_w31,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
_swizzle_mxfp4,
)
@@ -113,6 +114,18 @@ def _pack_deepgemm_mxfp4_scales(
)
def _swizzle_cutlass_mxfp4_scales(scale: torch.Tensor) -> torch.Tensor:
"""Swizzle [E, N, K // 32] block scales into the CUTLASS tiled layout."""
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
swizzle_mxfp4_scales,
)
n, scale_k = scale.shape[1:]
return torch.stack(
[swizzle_mxfp4_scales(s, n, scale_k * 32).reshape(n, scale_k) for s in scale]
)
class Mxfp4MoeBackend(Enum):
NONE = "None"
# b12x backends
@@ -126,6 +139,8 @@ class Mxfp4MoeBackend(Enum):
# FlashInfer CUTLASS backends
FLASHINFER_CUTLASS_MXFP4_MXFP8 = "FLASHINFER_CUTLASS_MXFP4_MXFP8"
FLASHINFER_CUTLASS_MXFP4_BF16 = "FLASHINFER_CUTLASS_MXFP4_BF16"
# vLLM CUTLASS backend
CUTLASS_MXFP4_MXFP4 = "CUTLASS_MXFP4_MXFP4"
# Marlin
BATCHED_MARLIN = "BATCHED_MARLIN"
MARLIN = "MARLIN"
@@ -217,6 +232,13 @@ def backend_to_kernel_cls(
return [FlashInferExperts]
elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
return [CutlassExpertsMxfp4]
elif backend == Mxfp4MoeBackend.TRITON:
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
OAITritonExperts,
@@ -377,21 +399,32 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.TRITON,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
# TRITON_UNFUSED has bug with MTP support
# TODO re-enable after kernel is fixed
# TRITON_UNFUSED
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.HUMMING,
Mxfp4MoeBackend.XPU,
Mxfp4MoeBackend.CPU,
Mxfp4MoeBackend.EMULATION,
]
return _AVAILABLE_BACKENDS
if current_platform.is_cuda() and current_platform.is_device_capability_family(120):
# Consumer Blackwell: the OAI Triton kernels run but their 99KB shared
# memory budget caps them at small tiles, so MARLIN prefills ~10%
# faster at equal decode throughput. Keep TRITON as an opt-in.
_AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.TRITON)
_AVAILABLE_BACKENDS.insert(
_AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1,
Mxfp4MoeBackend.TRITON,
)
return prioritize_humming(_AVAILABLE_BACKENDS)
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
"""Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
TRTLLM MXFP8; SM90 falls through to Humming or Marlin (the
backend-level ``is_supported_config`` check filters by device capability).
"""
if current_platform.is_rocm():
@@ -411,8 +444,9 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
# TRITON_UNFUSED
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.HUMMING,
]
return _AVAILABLE_BACKENDS
return prioritize_humming(_AVAILABLE_BACKENDS)
def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
@@ -428,7 +462,10 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
return kMxfp8Dynamic
if backend == Mxfp4MoeBackend.AITER_MXFP4_FP8:
return kFp8StaticTensorSym
if backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
if backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
):
return kMxfp4Dynamic
return None # BF16 activation
@@ -462,6 +499,22 @@ def _make_log_backend(backend: Mxfp4MoeBackend) -> str:
return f"Using '{backend.value}' Mxfp4 MoE backend."
def _warn_on_bf16_fallback(
backend: Mxfp4MoeBackend, requested_activation_key: QuantKey | None
) -> None:
if (
requested_activation_key is not None
and backend != Mxfp4MoeBackend.EMULATION
and _backend_activation_key(backend) is None
):
logger.warning_once(
"No Mxfp4 MoE backend supports activation=%s for the deployment "
"configuration. Falling back to '%s' with BF16 activations.",
requested_activation_key,
backend.value,
)
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
base = (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
@@ -493,17 +546,19 @@ def _filter_by_activation(
backends: list[Mxfp4MoeBackend],
requested_activation_key: QuantKey | None,
) -> list[Mxfp4MoeBackend]:
"""Pick variants matching ``requested_activation_key``; without one,
prefer BF16 if the list has any, else keep the list as-is so explicit
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
"""Pick variants matching ``requested_activation_key``, followed by BF16
variants as a higher-precision fallback; without one, prefer BF16 if the
list has any, else keep the list as-is so explicit non-BF16 picks (e.g.
the ``_afp8`` aliases) still land."""
bf16 = [b for b in backends if _backend_activation_key(b) is None]
if requested_activation_key is not None:
return [
matching = [
b
for b in backends
if _backend_activation_key(b) == requested_activation_key
or b == Mxfp4MoeBackend.EMULATION
]
bf16 = [b for b in backends if _backend_activation_key(b) is None]
return matching + [b for b in bf16 if b not in matching]
return bf16 if bf16 else backends
@@ -564,8 +619,9 @@ def select_mxfp4_moe_backend(
Args:
config: MoE configuration
activation_key: Optional activation quantization key. If provided,
overrides the default activation key for backend selection.
activation_key: Optional activation quantization key declared by
the checkpoint. Backends that match it are preferred, with BF16
activation backends as a fallback.
Use kFp8StaticTensorSym for W4A8 scheme.
Note: Shape-specific fallbacks may still occur at runtime.
@@ -602,7 +658,7 @@ def select_mxfp4_moe_backend(
else _backend_activation_key(requested_backend)
)
try:
return _return_or_raise(
selected = _return_or_raise(
requested_backend,
config,
kMxfp4Static,
@@ -611,6 +667,9 @@ def select_mxfp4_moe_backend(
)
except ValueError as e:
last_error = e
else:
_warn_on_bf16_fallback(requested_backend, requested_activation_key)
return selected
assert last_error is not None
raise last_error
@@ -636,10 +695,10 @@ def select_mxfp4_moe_backend(
unsupported_reasons = []
for backend in AVAILABLE_BACKENDS:
# Use requested_activation_key if provided, otherwise use backend default
# EMULATION takes the requested key, other backends use their own
act_key = (
requested_activation_key
if requested_activation_key is not None
if backend == Mxfp4MoeBackend.EMULATION
else _backend_activation_key(backend)
)
for k_cls in backend_to_kernel_cls(backend):
@@ -648,6 +707,7 @@ def select_mxfp4_moe_backend(
)
if supported:
logger.info_once(_make_log_backend(backend))
_warn_on_bf16_fallback(backend, requested_activation_key)
return backend, k_cls
else:
logger.debug_once(_make_log_unsupported(backend, reason))
@@ -777,6 +837,10 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
# DeepGEMM requires M/N/K alignment
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 128)
elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
# CUTLASS scale tiles cover 128 rows by 128 elements
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 128)
elif backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
intermediate_size = round_up(intermediate_size, 128)
if current_platform.is_xpu():
@@ -1421,7 +1485,8 @@ def convert_weight_to_mxfp4_moe_kernel_format(
]:
"""Convert loaded weights into backend-specific kernel format.
Supports DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin backends.
Supports CUTLASS, DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin
backends.
"""
is_gfx1250 = False
if current_platform.is_rocm():
@@ -1439,6 +1504,16 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
if mxfp4_backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
return (
w13_weight.data,
w2_weight.data,
_swizzle_cutlass_mxfp4_scales(w13_weight_scale.data),
_swizzle_cutlass_mxfp4_scales(w2_weight_scale.data),
w13_bias,
w2_bias,
)
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
w13_weight,
@@ -1993,7 +2068,10 @@ def make_mxfp4_moe_quant_config(
block_shape=None,
gemm1_clamp_limit=swiglu_limit,
)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
elif mxfp4_backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
):
return ocp_mx_moe_quant_config(
quant_dtype="mxfp4",
w1_bias=w1_bias,
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
backend_to_kernel_cls,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Dynamic,
kMxfp8Static,
@@ -129,7 +130,8 @@ def select_mxfp8_moe_backend(
return backend, _select_kernel_cls(backend, config)
# Auto-select: pick the first supported backend.
for backend in _SUPPORTED_BACKENDS:
backends = prioritize_humming(list(_SUPPORTED_BACKENDS))
for backend in backends:
try:
experts_cls = _select_kernel_cls(backend, config)
except ValueError:
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_nvfp4_moe_layer_for_marlin,
)
@@ -209,6 +210,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.HUMMING,
NvFp4MoeBackend.EMULATION,
]
AVAILABLE_BACKENDS = prioritize_humming(AVAILABLE_BACKENDS)
NVFP4_BACKENDS_WITH_CLAMP = {
NvFp4MoeBackend.B12X,
@@ -332,6 +334,7 @@ def convert_to_nvfp4_moe_kernel_format(
a2_scale: torch.Tensor | None,
is_act_and_mul: bool,
use_a16: bool = False,
trtllm_hidden_alignment: int = 256,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -417,6 +420,7 @@ def convert_to_nvfp4_moe_kernel_format(
w2_scale_2=w2_scale_2,
a2_scale=a2_scale,
is_act_and_mul=is_act_and_mul,
trtllm_hidden_alignment=trtllm_hidden_alignment,
)
elif nvfp4_backend == NvFp4MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming import (
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceContiguous,
TopKWeightAndReduceDelegate,
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
@@ -108,6 +109,30 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
def output_is_reduced(self) -> bool:
return False
def _allocate_combine_input(
self,
shape: tuple[int, ...],
dtype: torch.dtype,
device: torch.device,
) -> torch.Tensor | None:
return get_ep_group().allocate_combine_input(
shape,
dtype,
device,
is_sequence_parallel=self.is_sequence_parallel,
)
def allocate_fused_expert_output(
self,
shape: tuple[int, ...],
dtype: torch.dtype,
device: torch.device,
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> torch.Tensor | None:
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP):
return self._allocate_combine_input(shape, dtype, device)
return None
def prepare(
self,
a1: torch.Tensor,
@@ -194,16 +219,29 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceDelegate):
weight_and_reduce_impl = TopKWeightAndReduceContiguous()
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP):
# allocate_fused_expert_output already selected the combine input.
# Reuse it directly; the symmetric allocator may return a new view
# object for the same high-water backing allocation.
combine_input = fused_expert_output
else:
combine_input = self._allocate_combine_input(
(topk_ids.shape[0], fused_expert_output.shape[-1]),
fused_expert_output.dtype,
fused_expert_output.device,
)
out = weight_and_reduce_impl.apply(
output=None,
output=combine_input,
fused_expert_output=fused_expert_output,
topk_weights=topk_weights,
topk_ids=topk_ids,
apply_router_weight_on_input=apply_router_weight_on_input,
)
output.copy_(
get_ep_group().combine(out, is_sequence_parallel=self.is_sequence_parallel)
get_ep_group().combine_into_output(
out,
output,
is_sequence_parallel=self.is_sequence_parallel,
)
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
resolve_quant_method,
@@ -87,7 +88,7 @@ class RoutedExperts(PluggableLayer):
expert_map_manager: ExpertMapManager,
ckpt_gate_proj_name: str = "gate_proj",
ckpt_down_proj_name: str = "down_proj",
ckpt_up_proj_name: str = "up_proj",
ckpt_up_proj_name: str | None = "up_proj",
is_fused_checkpoint_transposed: bool = False,
#
# Extra params that are needed by quant_methods, pass along for now
@@ -114,6 +115,13 @@ class RoutedExperts(PluggableLayer):
self.ckpt_gate_proj_name = ckpt_gate_proj_name
self.ckpt_down_proj_name = ckpt_down_proj_name
self.ckpt_up_proj_name = ckpt_up_proj_name
if moe_config.is_act_and_mul == (ckpt_up_proj_name is None):
raise ValueError(
f"{layer_name}: ckpt_up_proj_name={ckpt_up_proj_name!r} is "
f"inconsistent with activation {moe_config.activation.value!r}. "
"Gated MoE needs an up projection name; non-gated MoE must "
"pass None."
)
self.is_fused_checkpoint_transposed = is_fused_checkpoint_transposed
self.expert_map_manager = expert_map_manager
self.hidden_size = moe_config.hidden_dim
@@ -146,6 +154,7 @@ class RoutedExperts(PluggableLayer):
self.apply_router_weight_on_input = apply_router_weight_on_input
# End random parameters
self._loaded_expert_biases: set[str] = set()
self._fused_shared_expert_quantizer = self._get_fused_shared_expert_quantizer()
self.quant_method = self._get_quant_method(
self.layer_name,
@@ -623,6 +632,31 @@ class RoutedExperts(PluggableLayer):
# _to_scalar's reshape(()) would reject the size-2 weight_shape.
param_data[expert_id] = loaded_weight
def _get_fused_shared_expert_quantizer(
self,
) -> Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]] | None:
"""Return the quantizer for an online-quantized fused shared expert."""
online_config = getattr(self.quant_config, "online_quantization_config", None)
prefix = self.moe_config.shared_expert_prefix
if (
online_config is None
or prefix is None
or self.expert_map_manager.num_fused_shared_experts == 0
):
return None
# FSE compatibility checks require every shared-expert projection to
# use the routed experts' weight key.
resolved = online_config.resolve_quant_method_cls(
LinearBase, f"{prefix}.down_proj"
)
if resolved is None:
return None
from vllm.model_executor.layers.quantization.online.base import (
ONLINE_SHARED_EXPERT_QUANTIZERS,
)
return ONLINE_SHARED_EXPERT_QUANTIZERS[resolved[3].weight]
@overload
def weight_loader(
self,
@@ -680,6 +714,35 @@ class RoutedExperts(PluggableLayer):
return False if return_success else None
# Hereafter, `expert_id` is local physical id
# A full-precision fused shared expert is quantized into the checkpoint
# layout, then loaded like a pre-quantized expert weight and scale.
if (
self._fused_shared_expert_quantizer is not None
and self.moe_config.num_logical_experts
<= global_expert_id
< self.moe_config.num_logical_experts
+ self.expert_map_manager.num_fused_shared_experts
and weight_name.endswith("_weight")
and loaded_weight.is_floating_point()
and loaded_weight.dtype != param.dtype
):
weight, weight_scale = self._fused_shared_expert_quantizer(
loaded_weight.to(self.moe_config.device)
)
stem = "w2" if shard_id == "w2" else "w13"
self.weight_loader(
getattr(self, f"{stem}_weight_scale"),
weight_scale,
f"{weight_name}_scale",
shard_id,
global_expert_id,
return_success=True,
)
loaded = self.weight_loader(
param, weight, weight_name, shard_id, global_expert_id, True
)
return loaded if return_success else None
# is_transposed: if the dim to shard the weight
# should be flipped. Required by GPTQ/AWQ (K-first format).
# should be whatever dimension intermediate_size_per_partition is
@@ -971,7 +1034,7 @@ class RoutedExperts(PluggableLayer):
loaded_weight,
self.is_fused_checkpoint_transposed and uses_weight_layout,
)
if shard_id in {"w1", "w3"}:
if shard_id in {"w1", "w3"} and self.moe_config.is_act_and_mul:
# Repurpose expert_id for deconcatenating w1 and w3
experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
else:
@@ -1029,6 +1092,7 @@ class RoutedExperts(PluggableLayer):
routed_experts_prefix="",
lora_base_layer_prefix=self.lora_base_layer_prefix,
include_fused=include_fused,
is_gated=moe_config.is_act_and_mul,
)
@staticmethod
@@ -1036,7 +1100,7 @@ class RoutedExperts(PluggableLayer):
model: torch.nn.Module,
ckpt_gate_proj_name: str,
ckpt_down_proj_name: str,
ckpt_up_proj_name: str,
ckpt_up_proj_name: str | None,
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
@@ -1067,13 +1131,14 @@ class RoutedExperts(PluggableLayer):
def build_expert_params_mapping(
ckpt_gate_proj_name: str,
ckpt_down_proj_name: str,
ckpt_up_proj_name: str,
ckpt_up_proj_name: str | None,
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
lora_base_layer_prefix: str = "",
lora_base_layer_prefix_on_param_name: str = "",
include_fused: bool = False,
is_gated: bool = True,
) -> list[tuple[str, str, int, str]]:
"""Create expert parameter mapping for weight loading with redundant experts.
@@ -1095,6 +1160,8 @@ class RoutedExperts(PluggableLayer):
``make_expert_params_mapping`` indexes the model-wide
``params_dict`` (prefix included).
include_fused: Prepend the fused pre-fused-checkpoint entries
is_gated: Whether w13 holds gate and up (gated activation) or a
single up projection. Selects the fused checkpoint layout.
routed_experts_prefix: Prefix of the routed experts submodule
Returns:
@@ -1129,7 +1196,13 @@ class RoutedExperts(PluggableLayer):
fused_mapping = []
if include_fused:
gate_up = None
if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
w13_shards: tuple[str, ...] = ("w1", "w3")
if not is_gated:
# Non-gated: the stacked checkpoint tensor is the up projection
# itself, nothing to split into gate and up.
gate_up = ckpt_gate_proj_name
w13_shards = ("w1",)
elif ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
gate_up = "gate_up_proj"
elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3":
gate_up = "w13"
@@ -1143,10 +1216,16 @@ class RoutedExperts(PluggableLayer):
if gate_up is not None:
fused_mapping = [
# (param_name, weight_name, expert_id, shard_id)
(f"{w13}weight", f"experts.{gate_up}", 0, "w1"),
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
# expert_id doubles as the chunk index when splitting w13.
*(
(f"{w13}weight", f"experts.{gate_up}", chunk, shard_id)
for chunk, shard_id in enumerate(w13_shards)
),
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
]
if gate_up is not None and is_gated:
# Per-expert gate_up_proj tensors; the non-gated equivalent is
# already covered by the per-expert mapping below.
fused_mapping.extend(
(
w13,
@@ -1175,6 +1254,7 @@ class RoutedExperts(PluggableLayer):
("w2", ckpt_down_proj_name),
("w3", ckpt_up_proj_name),
]
if weight_name is not None
]
return fused_mapping + per_expert_mapping
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm BF16x3 MoE router GEMM for fp32 gate weights.
Gates whose weights ship in fp32 (MiniMax-M2/M3, HunYuan-V3) otherwise land in
the ``F.linear`` fallback, which casts the bf16 activations up to fp32 and runs
the GEMM on the fp32 MFMA pipe. Splitting the weight into three bf16 terms
whose sum reproduces it exactly moves the GEMM onto the bf16 pipe and is more
accurate than the path it replaces, which rounds the activations on the way to
fp32 while this one leaves them untouched.
The terms are stacked along the expert dimension into one ``(TERMS * E, K)``
weight, so the router costs a single hipBLASLt GEMM plus a reduction over the
``TERMS`` column blocks of its output.
"""
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
__all__ = [
"BF16X3_TERMS",
"MIN_TOKENS",
"bf16x3_router_gemm",
"is_supported",
"platform_supported",
"split_bf16x3",
]
BF16X3_TERMS = 3
# Below this the fp32 fallback is faster: the GEMM plus reduction has a ~32us
# floor from launch and allocation that small batches cannot amortize.
# Measured on gfx950; the crossover is 1536 at (K=6144, E=128) and (K=3072,
# E=256), and this leaves margin for untested shapes.
MIN_TOKENS = 2048
_MAX_INT32 = (1 << 31) - 1
def split_bf16x3(weight: torch.Tensor) -> torch.Tensor:
"""Split an fp32 weight into bf16 terms whose sum reproduces it exactly.
Returns a contiguous ``(BF16X3_TERMS, E, K)`` tensor. Three
round-to-nearest bf16 terms carry 24 mantissa bits, matching fp32's 24, so
the reconstruction is bit-exact for finite normals down to ``2**-108``.
Raises ValueError on non-finite input: ``inf`` rounds to bf16 ``inf`` and
``inf - inf`` is NaN, which would poison every logit rather than just the
affected experts.
"""
if weight.dtype != torch.float32:
raise ValueError(f"expected fp32 weight, got {weight.dtype}")
if not torch.isfinite(weight).all():
raise ValueError("router weight contains inf/NaN")
terms = []
resid = weight.detach().clone()
for _ in range(BF16X3_TERMS):
hi = resid.to(torch.bfloat16)
terms.append(hi)
resid -= hi.to(torch.float32)
return torch.stack(terms).contiguous()
def platform_supported() -> bool:
"""Whether this platform has a benchmarked bf16x3 router path.
gfx942 has a different fp32:bf16 MFMA ratio and is not yet evaluated.
"""
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
def is_supported(x: torch.Tensor, weight: torch.Tensor) -> bool:
"""Whether this path can serve ``x @ weight.T`` -> fp32 profitably."""
if x.dim() != 2 or weight.dim() != 2:
return False
n, k = weight.shape
m = x.shape[0]
return (
m >= MIN_TOKENS
and x.dtype == torch.bfloat16
and weight.dtype == torch.float32
and x.shape[1] == k
# The weight may be offloaded while the activations are not; the
# fallback this guards would then fail on a device mismatch.
and x.device == weight.device
# torch.mm accepts any stride, but a non-K-contiguous activation costs
# a hidden copy that gives back the win over the fallback.
and x.stride(1) == 1
and 0 < m * BF16X3_TERMS * n <= _MAX_INT32
)
@triton.jit
def _reduce_terms_kernel(
y_ptr,
out_ptr,
M,
stride_ym,
stride_om,
N: tl.constexpr,
TERMS: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
):
pid = tl.program_id(0)
offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = tl.arange(0, BLOCK_N)
mask = (offs_m < M)[:, None] & (offs_n < N)[None, :]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for t in tl.static_range(TERMS):
acc += tl.load(
y_ptr + offs_m[:, None] * stride_ym + t * N + offs_n[None, :],
mask=mask,
other=0.0,
)
tl.store(out_ptr + offs_m[:, None] * stride_om + offs_n[None, :], acc, mask=mask)
def bf16x3_router_gemm(
x: torch.Tensor,
weight_split: torch.Tensor,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Router logits in fp32 from bf16 activations and a bf16x3 split weight."""
m, k = x.shape
terms, n, k_w = weight_split.shape
assert terms == BF16X3_TERMS, f"expected {BF16X3_TERMS} terms, got {terms}"
assert k == k_w, f"K mismatch: x has {k}, weight_split has {k_w}"
assert weight_split.dtype == torch.bfloat16, weight_split.dtype
assert weight_split.is_contiguous(), "weight_split must be contiguous"
assert x.dtype == torch.bfloat16, x.dtype
if out is None:
out = torch.empty((m, n), dtype=torch.float32, device=x.device)
else:
assert out.shape == (m, n) and out.is_contiguous(), "bad out layout"
# Rows [t*E, (t+1)*E) of the (TERMS * E, K) view hold term t, so the GEMM
# emits one column block per term and the reduction sums them.
y = torch.empty((m, terms * n), dtype=torch.float32, device=x.device)
torch.mm(x, weight_split.view(terms * n, k).T, out_dtype=torch.float32, out=y)
# 64 rows/program keeps the accumulator small enough to stay in registers
# at the largest supported expert count.
block_m = 64
_reduce_terms_kernel[(triton.cdiv(m, block_m),)](
y,
out,
m,
y.stride(0),
out.stride(0),
N=n,
TERMS=terms,
BLOCK_M=block_m,
BLOCK_N=triton.next_power_of_2(n),
num_warps=4,
)
return out
@@ -4,6 +4,7 @@ import torch
from torch.nn.parameter import Parameter
import vllm._custom_ops as ops
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.linear import (
ReplicatedLinear,
@@ -13,6 +14,8 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
@PluggableLayer.register("gate_linear")
class GateLinear(ReplicatedLinear):
@@ -23,8 +26,10 @@ class GateLinear(ReplicatedLinear):
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
M<=32, model-specific shapes)
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
5. F.linear via ReplicatedLinear (ultimate fallback)
4. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
M>=2048)
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
6. F.linear via ReplicatedLinear (ultimate fallback)
The ``out_dtype`` attribute is mutable and can be set after init
(e.g. when the required dtype depends on the expert quantization
@@ -140,6 +145,33 @@ class GateLinear(ReplicatedLinear):
and self.out_dtype == torch.float32
)
# ROCm bf16x3 router GEMM eligibility. out_dtype may still be None here
# and is folded in by set_out_dtype; the split itself is only built
# once the tier is known to be on (see _GateLinearMethod).
from vllm.model_executor.layers.fused_moe.router import (
bf16x3_router_gemm_rocm,
)
self._rocm_bf16x3_weight_eligible = (
self._router_gemm_no_bias
and self.weight.dtype == torch.float32
and bf16x3_router_gemm_rocm.platform_supported()
)
self.allow_rocm_bf16x3_router_gemm = (
self._rocm_bf16x3_weight_eligible and self.out_dtype == torch.float32
)
# Filled in by process_weights_after_loading. Not persistent: it is
# derived from `weight` rather than loaded from a checkpoint.
self.register_buffer("_bf16x3_weight", None, persistent=False)
if self._rocm_bf16x3_weight_eligible:
# Safe to swap after create_weights() because
# UnquantizedLinearMethod holds no per-layer state. Asserted
# because the swap would silently drop quantization if a
# quant_config were ever plumbed into this layer.
assert isinstance(self.quant_method, UnquantizedLinearMethod)
self.quant_method = _GateLinearMethod()
logger.info_once("Enabled ROCm BF16x3 router GEMM.")
# cuteDSL ll_bf16_gemm eligibility. Any dims supported, but SM90+ required bc:
# 1. PDL support. Both dot-product and split-K kernels.
# 2. Thread Block Clusters. Split-K kernel for cross-CTA reduction.
@@ -173,6 +205,10 @@ class GateLinear(ReplicatedLinear):
self.allow_cublas_router_gemm = self.weight.dtype == torch.bfloat16
# out_dtype may start as None -> recompute eligibility here
self.allow_rocm_bf16x3_router_gemm = (
self._rocm_bf16x3_weight_eligible and out_dtype == torch.float32
)
if self.allow_specialized_router_gemm:
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
is_available,
@@ -205,6 +241,23 @@ class GateLinear(ReplicatedLinear):
output = ll_bf16_gemm(x, self.weight)
return self._return(output)
# Tier 4: ROCm bf16x3 router GEMM. Checked before tier 2 because both
# tiers claim the same gfx950 shapes and tier 2 returns
# unconditionally; the custom op below picks between them on the
# runtime num_tokens.
if self.allow_rocm_bf16x3_router_gemm and x.dtype == torch.bfloat16:
if self._bf16x3_weight is None:
logger.warning_once(
"ROCm BF16x3 router GEMM is enabled for %s but its weight "
"split is missing; falling back to fp32.",
self.prefix,
)
else:
output = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(
x, self.weight, self._bf16x3_weight
)
return self._return(output)
# Tier 2: fp32 specialized kernel (model-specific shapes, M<=32)
# Dispatch is wrapped in a custom op so that torch.compile/CUDA-graph
# capture does not freeze the runtime num_tokens branch.
@@ -226,12 +279,12 @@ class GateLinear(ReplicatedLinear):
output = bf16x3_router_gemm(x, self.weight)
return self._return(output)
# Tier 4: cuBLAS bf16→fp32
# Tier 5: cuBLAS bf16→fp32
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
return self._return(output)
# Tier 5: F.linear (ReplicatedLinear)
# Tier 6: F.linear (ReplicatedLinear)
if (
self.out_dtype is not None
and self.is_unquantized
@@ -249,9 +302,90 @@ class GateLinear(ReplicatedLinear):
return output if not self.return_bias else (output, output_bias)
class _GateLinearMethod(UnquantizedLinearMethod):
"""UnquantizedLinearMethod plus the ROCm bf16x3 weight split.
Building the split here rather than lazily in forward() charges it to the
weights memory pool ahead of KV cache profiling, and rebuilds it on weight
reload. The latter matters because an in-place ``param.data.copy_()``
leaves both ``data_ptr()`` and ``_version`` untouched, so a stale split
cannot be detected from inside forward().
``allow_rocm_bf16x3_router_gemm`` is final by the time this runs: the only
caller of ``set_out_dtype`` does so during model construction.
"""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# The base CPU branch may delete layer.weight, so it must run first;
# nothing breaks today only because platform_supported() is False on
# CPU.
super().process_weights_after_loading(layer)
if not layer.allow_rocm_bf16x3_router_gemm:
return
try:
from vllm.model_executor.layers.fused_moe.router import (
bf16x3_router_gemm_rocm,
)
layer._bf16x3_weight = bf16x3_router_gemm_rocm.split_bf16x3(layer.weight)
except ValueError as exc:
layer.allow_rocm_bf16x3_router_gemm = False
logger.warning_once(
"Disabling ROCm BF16x3 router GEMM for %s: %s", layer.prefix, exc
)
_FP32_ROUTER_GEMM_MAX_TOKENS = GateLinear.FP32_MAX_TOKENS
def rocm_bf16x3_router_gemm_dispatch_impl(
x: torch.Tensor,
weight: torch.Tensor,
weight_split: torch.Tensor,
) -> torch.Tensor:
"""Run the ROCm bf16x3 router GEMM, or fall back for small batches.
This must be a custom op because our torch.compile integration does not
support runtime dispatching on num_tokens: models carrying fp32 router
weights (MiniMax-M2, HunYuan-V3) call the gate from
``@support_torch_compile`` model code and vLLM drops all Dynamo guards, so
a plain Python branch on ``x.shape[0]`` would be frozen at first trace.
"""
from vllm.model_executor.layers.fused_moe.router import (
bf16x3_router_gemm_rocm,
)
if bf16x3_router_gemm_rocm.is_supported(x, weight):
return bf16x3_router_gemm_rocm.bf16x3_router_gemm(x, weight_split)
# Below MIN_TOKENS, hand back to the low-M gfx950 kernel this tier is
# checked ahead of.
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
can_use_rocm_fp32_router_gemm,
rocm_fp32_router_gemm,
)
x = x.contiguous()
if can_use_rocm_fp32_router_gemm(x, weight):
return rocm_fp32_router_gemm(x, weight)
return torch.nn.functional.linear(x.float(), weight)
def rocm_bf16x3_router_gemm_dispatch_fake(
x: torch.Tensor,
weight: torch.Tensor,
weight_split: torch.Tensor,
) -> torch.Tensor:
return x.new_empty((x.shape[0], weight.shape[0]), dtype=torch.float32)
direct_register_custom_op(
op_name="rocm_bf16x3_router_gemm_dispatch",
op_func=rocm_bf16x3_router_gemm_dispatch_impl,
fake_impl=rocm_bf16x3_router_gemm_dispatch_fake,
)
def fp32_router_gemm_dispatch_impl(
x: torch.Tensor,
weight: torch.Tensor,
@@ -84,6 +84,40 @@ def resolve_layer_fused_shared_expert(
else (True, None)
)
is_fused_shared_expert_enabled = fse_requested and fse_compatible
# In online shared_expert quantization is used, register it to
# `online_quantization_config.quantized_layers` ahead of weight loading.
if is_fused_shared_expert_enabled and quant_config is not None:
online_quantization_config = quant_config.online_quantization_config
if online_quantization_config is not None:
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
for projection_name in ("gate_up_proj", "down_proj"):
projection_prefix = f"{prefix}.{shared_expert_name}.{projection_name}"
quant_method_metadata = (
online_quantization_config.resolve_quant_method_cls(
LinearBase, projection_prefix
)
)
if quant_method_metadata is None:
continue
source, quant_key_str, target_pattern, _, quant_method_cls = (
quant_method_metadata
)
if quant_method_cls in (None, UnquantizedLinearMethod):
continue
online_quantization_config.quantized_layers[projection_prefix] = (
source.value,
quant_key_str,
target_pattern,
)
if fse_requested and not is_fused_shared_expert_enabled:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
@@ -63,10 +63,10 @@ def _silu_and_mul_fp8_dynamic_block(
x: torch.Tensor, linear: LinearBase, group_size: int, quant_key: QuantKey
) -> QuantizedActivation:
"""SiluAndMul + FP8 dynamic per-block quantization."""
assert x.ndim == 2, f"Input must be 2D [batch, hidden*2], got {x.shape}"
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
x = x.reshape(-1, x.shape[-1])
num_tokens = x.shape[0]
num_groups = d // group_size
@@ -102,10 +102,10 @@ def _silu_and_mul_nvfp4_dynamic(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + NVFP4 dynamic quantization."""
assert x.ndim == 2, f"Input must be 2D [batch, hidden*2], got {x.shape}"
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
x = x.reshape(-1, x.shape[-1])
num_tokens = x.shape[0]
# NVFP4 packs 2 values into 1 byte
@@ -152,6 +152,7 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams:
for key in ("do_normalize", "do_rescale", "image_mean", "image_std")
):
config = model_config.hf_image_processor_config
config = config.get("media_proc_cfg", config)
has_norm_params = "image_mean" in config and "image_std" in config
do_normalize = bool(config.get("do_normalize", has_norm_params))
@@ -312,19 +313,15 @@ class FusedMMInputNorm(CustomOp):
def forward_xpu(
self, pixel_values: torch.Tensor, visual_dtype: torch.dtype
) -> torch.Tensor:
"""XPU fused custom kernel path.
On XPU, fuse the whole rescale + normalise into a single custom
kernel. The eager path materializes an fp32 intermediate and then
casts back, which adds device-side compute that cancels the
bandwidth saving of transferring uint8 pixel_values. The fused
kernel reads uint8 directly and writes ``visual_dtype`` in one pass.
"""XPU fused custom kernel path for uint8 and Triton kernel path
for others.
"""
# The out-of-tree XPU kernel only supports the uint8 input that
# device-side normalisation guarantees; fail loudly otherwise.
assert pixel_values.dtype == torch.uint8, (
f"xpu_fused_input_norm requires uint8 input, got {pixel_values.dtype}"
)
if pixel_values.dtype != torch.uint8:
return self.forward_cuda(pixel_values, visual_dtype)
# For uint8 input, falls to xpu_fused_input_norm from xpu kernels
import vllm._xpu_ops # noqa: F401
return torch.ops.vllm.xpu_fused_input_norm(
pixel_values, self.weight, self.bias, visual_dtype
)
+5 -5
View File
@@ -29,12 +29,10 @@ IDX_OOB_FILL_VALUE = -1
try:
import vllm._deepselect_C # noqa: F401 (registers torch.ops.deep_select)
except ImportError as e:
from vllm.logger import init_logger
init_logger(__name__).warning(
"Failed to import the DeepSelect extension (vllm._deepselect_C): %s", e
)
_HAS_DEEP_SELECT_C = True
except ImportError:
_HAS_DEEP_SELECT_C = False
@functools.lru_cache(maxsize=1)
@@ -196,6 +194,8 @@ class SparseIndexerTopk(torch.nn.Module):
failures.append("requires a CUDA platform")
elif not self._has_deep_select:
failures.append("requires SM100a/SM103a (10.x device family)")
elif not _HAS_DEEP_SELECT_C:
failures.append("vllm._deepselect_C could not be imported")
elif not is_deep_select_supported(logits, topk_tokens):
failures.append(
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
+13 -2
View File
@@ -25,6 +25,7 @@ def compute_mamba_prefill_checkpoints(
mamba_block_size: int,
checkpoint_alignment: int | None,
drop_eagle_block: bool,
cache_hit_alignment_tokens: int,
) -> tuple[list[int], list[int]]:
"""Per-row internal prefill checkpoint offsets and cache block columns.
@@ -43,7 +44,7 @@ def compute_mamba_prefill_checkpoints(
for seq_len, query_len in zip(seq_lens, query_lens):
query_start = seq_len - query_len
position = get_mamba_prefill_checkpoint_position(
seq_len, hash_block_size, drop_eagle_block=drop_eagle_block
seq_len, cache_hit_alignment_tokens, drop_eagle_block=drop_eagle_block
)
valid = is_mamba_prefill_checkpoint_valid(
query_start=query_start,
@@ -70,6 +71,9 @@ def _gather_checkpoint_state_indices(
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Host copy of ``checkpoint_offsets``, for backends that lay out the
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
offsets: list[int] | None = None
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
@@ -103,12 +107,17 @@ class MambaPrefillCheckpointBuilder:
return None
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
return None
cache_config = self.vllm_config.cache_config
hash_block_size = cache_config.hash_block_size
cache_hit_alignment_tokens = cache_config.cache_hit_alignment_tokens
if hash_block_size is None or cache_hit_alignment_tokens is None:
# Unresolved only for the minimal KV cache built for profiling.
return None
assert m.seq_lens_cpu_upper_bound is not None
all_query_lens = m.query_start_loc_cpu.diff().tolist()
query_lens = [all_query_lens[row] for row in request_rows]
seq_lens = m.seq_lens_cpu_upper_bound.tolist()
block_size = self.kv_cache_spec.block_size
hash_block_size = self.vllm_config.cache_config.prefix_match_unit or block_size
speculative_config = self.vllm_config.speculative_config
drop_eagle_block = (
speculative_config is not None and speculative_config.use_eagle_block_drop()
@@ -120,6 +129,7 @@ class MambaPrefillCheckpointBuilder:
mamba_block_size=block_size,
checkpoint_alignment=self.kv_cache_spec.prefill_checkpoint_alignment,
drop_eagle_block=drop_eagle_block,
cache_hit_alignment_tokens=cache_hit_alignment_tokens,
)
if not any(checkpoint_offsets):
return None
@@ -139,6 +149,7 @@ class MambaPrefillCheckpointBuilder:
_gather_checkpoint_state_indices(
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
offsets=checkpoint_offsets,
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
@@ -8,6 +8,7 @@ from typing import Literal
import torch
from einops import rearrange
from torch import nn
from transformers import Qwen3NextConfig
from vllm import _custom_ops as ops
from vllm import envs
@@ -58,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import (
)
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
@@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]:
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
"""Resolve GDN prefill backend.
FlashInfer's GDN prefill kernel is chosen when:
@@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend(
In-tree CuteDSL GDN prefill kernel is chosen when:
* "cutedsl" is requested; (opt-in only)
* Blackwell (SM10.x) with ``head_k_dim == 128``;
AITER FlyDSL GDN prefill kernels are chosen when:
* "aiter_flydsl" is requested; (opt-in only)
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
"""
additional_config = vllm_config.additional_config
backend_cfg = (
@@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend(
)
backend = str(backend_cfg).strip().lower()
if not current_platform.is_cuda():
return backend, "triton"
head_k_dim = getattr(
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
)
head_v_dim = getattr(
vllm_config.model_config.hf_text_config, "linear_value_head_dim", None
)
if current_platform.is_rocm():
if backend == "aiter_flydsl":
if not rocm_aiter_ops.is_gdn_flydsl_prefill_available():
raise RuntimeError(
"GDN prefill backend 'aiter_flydsl' was requested but the "
"AITER FlyDSL prefill kernels are not importable. Check "
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
"newer GPU, and that the installed AITER exports them."
)
if head_k_dim != 128 or head_v_dim != 128:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
"Falling back to Triton/FLA.",
head_k_dim,
head_v_dim,
)
return backend, "triton"
if vllm_config.model_config.dtype != torch.bfloat16:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"model dtype is %s; FlyDSL requires bfloat16. "
"Falling back to Triton/FLA.",
vllm_config.model_config.dtype,
)
return backend, "triton"
return backend, "aiter_flydsl"
return backend, "triton"
if backend == "aiter_flydsl":
raise RuntimeError(
"GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm "
f"AITER backend and the current platform is {current_platform.device_name}."
)
if not current_platform.is_cuda():
return backend, "triton"
supports_flashinfer = False
supports_cutedsl = False
@@ -172,6 +215,7 @@ def _log_gdn_backend_decision(
chosen = {
"flashinfer": "FlashInfer",
"cutedsl": "CuteDSL",
"aiter_flydsl": "AITER FlyDSL",
"triton": "Triton/FLA",
}[active_backend]
logger.info_once(
@@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp):
self._forward_method = self.forward_cuda
elif active_backend == "cutedsl":
self._forward_method = self.forward_cutedsl
elif active_backend == "aiter_flydsl":
self._forward_method = self.forward_aiter_flydsl
else:
self._forward_method = self.forward_native
@@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
o, final_state = fi_chunk_gated_delta_rule(
q=q,
@@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
return fla_chunk_gated_delta_rule(
q=q,
@@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
chunk_gated_delta_rule_cutedsl,
@@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp):
final_state = None
return o, final_state
def forward_aiter_flydsl(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
output_final_state: bool,
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
o, final_state = rocm_aiter_ops.gdn_flydsl_prefill(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=output_final_state,
cu_seqlens=cu_seqlens,
aiter_prefill_metadata=aiter_prefill_metadata,
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
)
if core_attn_out is not None:
o_flat = o.reshape(-1)
core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat)
return o, final_state
@PluggableLayer.register("qwen_gated_delta_net_attention")
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
@@ -1193,12 +1275,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
# CuteDSL kernels require metadata
chunk_indices = None
chunk_offsets = None
aiter_prefill_metadata = None
if self.gdn_prefill_backend == "cutedsl":
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
prepare_metadata_cutedsl,
)
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T)
elif self.gdn_prefill_backend == "aiter_flydsl":
aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
[T],
cu_seqlens=cu_seqlens,
)
try:
self.chunk_gated_delta_rule(
@@ -1213,6 +1301,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=aiter_prefill_metadata,
)
except Exception:
logger.warning(
@@ -1243,6 +1332,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
cu_seqlens,
chunk_indices,
chunk_offsets,
aiter_prefill_metadata,
)
torch.accelerator.empty_cache()
@@ -1634,6 +1724,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
# Init cache
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
@@ -3,11 +3,12 @@
from collections.abc import Sequence
from dataclasses import replace
import torch
from torch import nn
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.config.mamba import MambaBackendEnum
from vllm.distributed import (
divide,
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
selective_state_update_replayssm_output_only,
)
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import store_prefill_checkpoint
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
@@ -66,7 +68,7 @@ from vllm.utils.torch_utils import (
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec
logger = init_logger(__name__)
@@ -766,6 +768,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
query_start_loc_d = attn_metadata.query_start_loc_d
num_decodes = attn_metadata.num_decodes
num_decode_tokens = attn_metadata.num_decode_tokens
checkpoint_chunk_idx = attn_metadata.checkpoint_chunk_idx
checkpoint_meta = attn_metadata.checkpoint_meta
if attn_metadata is None:
# V1 profile run -- warm up SSD kernels so that autotuning
@@ -842,6 +846,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
# NOTE: final output is an in-place update of out tensor
assert preallocated_ssm_out_p is not None
has_checkpoints = checkpoint_chunk_idx is not None
varlen_states = mamba_chunk_scan_combined_varlen(
hidden_states_p.view(
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
@@ -859,6 +864,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
cu_chunk_seqlens=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
initial_states=initial_states,
return_intermediate_states=has_checkpoints,
dt_softplus=True,
dt_limit=(0.0, float("inf")),
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
@@ -869,7 +875,24 @@ class MambaMixer2(MambaBase, PluggableLayer):
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
final_states = (
varlen_states[last_chunk_indices_p]
if has_checkpoints
else varlen_states
)
if has_checkpoints:
assert checkpoint_meta is not None
assert query_start_loc_p is not None
store_prefill_checkpoint(
checkpoint_meta,
checkpoint_chunk_idx,
x,
conv_state,
varlen_states,
ssm_state,
query_start_loc_p,
)
ssm_state[state_indices_tensor_p] = final_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
@@ -1070,6 +1093,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
is_blackwell=self.is_blackwell,
)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
spec = super().get_kv_cache_spec(vllm_config)
if not isinstance(spec, MambaSpec):
return spec
# The block-boundary state can be exported from inside one forward
# pass, so the scheduler need not split the prefill; every reader
# acts on this only in align mode. The metadata builder puts a chunk
# end exactly on the checkpoint, so any token position works.
return replace(
spec, num_prefill_checkpoint_blocks=1, prefill_checkpoint_alignment=1
)
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
assert self.model_config is not None
assert self.cache_config is not None
@@ -0,0 +1,51 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,51 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"32": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"16": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"32": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"64": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"128": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"256": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"8192": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"16384": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"32768": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 8,
"num_warps": 8
},
"16": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"32": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"64": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"256": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"48": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"96": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"1536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"3072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"6144": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"12288": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"24576": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"49152": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,39 @@
{
"triton_version": "3.8.0",
"48": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"384": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"768": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"1536": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"3072": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"6144": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"12288": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"24576": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"49152": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"32": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"256": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"80": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"640": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"1280": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2560": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"5120": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"10240": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"20480": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"40960": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"81920": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"122880": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"163840": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"80": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"640": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"1280": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2560": {
"BLOCK_SIZE_M": 8,
"num_warps": 4
},
"5120": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"10240": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"20480": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"40960": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"81920": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"122880": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"163840": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
}
}
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.triton_utils import tl, triton
@triton.jit
def _store_prefill_checkpoint_kernel(
x_ptr,
conv_state_ptr,
chunk_states_ptr,
ssm_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
state_indices_ptr,
chunk_idx_ptr,
stride_x_dim,
stride_x_token,
stride_conv_block,
stride_conv_dim,
stride_conv_token,
stride_chunk_states,
stride_ssm_block,
DIM: tl.constexpr,
STATE_LEN: tl.constexpr,
SSM_ROW_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
checkpoint_offset = tl.load(checkpoint_offsets_ptr + row)
valid = checkpoint_offset > 0
dst = tl.load(state_indices_ptr + row).to(tl.int64)
conv_mask = valid & (cols < DIM * STATE_LEN)
dim_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
token_idx = (
tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx
).to(tl.int64)
conv = tl.load(
x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask
)
tl.store(
conv_state_ptr
+ dst * stride_conv_block
+ dim_idx * stride_conv_dim
+ history_idx * stride_conv_token,
conv,
mask=conv_mask,
)
ssm_mask = valid & (cols < SSM_ROW_SIZE)
src = tl.load(chunk_idx_ptr + row).to(tl.int64)
ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask)
tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask)
def store_prefill_checkpoint(
checkpoint: MambaPrefillCheckpointMetadata,
chunk_idx: torch.Tensor,
x: torch.Tensor,
conv_state: torch.Tensor,
chunk_states: torch.Tensor,
ssm_state: torch.Tensor,
query_start_loc: torch.Tensor,
) -> None:
"""Write each prefill row's mid-prefill checkpoint into its paged block.
Rows without a checkpoint (offset 0) write nothing.
Args:
checkpoint: Per-row offsets and destination blocks.
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
chunk_states: SSD states at every chunk end.
ssm_state: Paged SSM state.
query_start_loc: Prefill query start locations.
"""
dim = x.size(0)
state_len = conv_state.size(-1)
ssm_row_size = chunk_states[0].numel()
block_size = 1024
grid = (
chunk_idx.numel(),
triton.cdiv(max(dim * state_len, ssm_row_size), block_size),
)
_store_prefill_checkpoint_kernel[grid](
x,
conv_state,
chunk_states,
ssm_state,
query_start_loc,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
chunk_idx,
x.stride(0),
x.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
chunk_states.stride(0),
ssm_state.stride(0),
DIM=dim,
STATE_LEN=state_len,
SSM_ROW_SIZE=ssm_row_size,
BLOCK_SIZE=block_size,
)
@@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC):
"""
return
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
"""Materialize a serialized quantized weight for requantization."""
raise NotImplementedError(
f"The quantization method {type(self)} does not implement "
"dequantize_weight. Please open an issue."
)
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
"""Not all quant methods have embedding implemented, so we need to check that
@@ -294,13 +301,16 @@ def resolve_quant_method(
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
# No online configuration: retain the checkpoint method.
return base_quant_method
# Online quantization currently supports only LinearBase and RoutedExperts.
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
if not isinstance(layer, (LinearBase, RoutedExperts)):
# Online quantization only supports linear and routed MoE layers.
return base_quant_method
quant_config.online_quantization_config.packed_modules_mapping = (
@@ -310,16 +320,36 @@ def resolve_quant_method(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
layer, prefix
type(layer), prefix
)
if checkpoint_is_quantized:
if online_target is not None:
raise ValueError(
f"Cannot apply requested online quantization {online_target[3]} to "
f"pre-quantized layer {prefix}: {base_quant_method} was already "
"selected by the checkpoint quantization config."
if online_target is None:
# The checkpoint quant method is applied as there is no online override.
return base_quant_method
if isinstance(layer, RoutedExperts):
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
return base_quant_method
online_quant_method = quant_config.online_quantization_config.get_quant_method(
layer, prefix
)
assert base_quant_method is not None and not isinstance(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
assert isinstance(online_quant_method, OnlineLinearBase)
online_quant_method.set_requantization_source(base_quant_method)
# The online method dequantizes the checkpoint method before requantizing.
return online_quant_method
if online_target is None:
# The layer is unquantized and no online target applies.
return base_quant_method
# Quantize an unquantized layer with the online method.
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
@@ -450,10 +450,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
if w2_bias is not None:
replace_parameter(layer, "w2_bias", w2_bias)
# CPU fused_experts_cpu requires zero points even for symmetric quant.
# CPU fused_experts_cpu and the RDNA3 HIP kernel require zero points
# even for symmetric quant (the oracle synthesizes them).
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
if (
not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU
not self.symmetric
or self.wna16_backend in (WNA16MoEBackend.CPU, WNA16MoEBackend.RDNA3)
) and self.wna16_backend != WNA16MoEBackend.EMULATION:
assert w13_qzeros is not None and w2_qzeros is not None
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
@@ -12,7 +12,7 @@ from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.config import get_current_vllm_config_or_none
from vllm.config.quantization import QuantSpec
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_fp8_linear_kernel,
@@ -63,6 +63,9 @@ from vllm.model_executor.layers.quantization.base_config import (
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
trtllm_nvfp4_hidden_alignment,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
FP8_SCALE_SENTINEL,
process_fp8_input_tensor_strategy_moe,
@@ -76,6 +79,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
dequant_mxfp8_to_bf16,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP4_DTYPE,
@@ -89,6 +93,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Dynamic,
kMxfp8Static,
kNvfp4Dynamic,
kNvfp4DynamicToken,
kNvfp4Static,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -723,12 +728,18 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
kv_cache_quant_algo: str | None = None,
exclude_modules: list[str] | None = None,
group_size: int = 16,
*,
quantization_args: QuantizationConfigArgs | None = None,
) -> None:
if exclude_modules is None:
exclude_modules = []
super().__init__(exclude_modules)
self.quant_method = quant_method
self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized
moe_spec = quantization_args.moe if quantization_args is not None else None
self.moe_activation_override: QuantKey | None = (
moe_spec.activation if moe_spec is not None else None
)
if is_checkpoint_nvfp4_serialized:
self.group_size = group_size
self.kv_cache_quant_algo = kv_cache_quant_algo
@@ -814,6 +825,7 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
kv_cache_quant_method,
exclude_modules,
group_size,
quantization_args=original_config.get("_online_quantization_args"),
)
@@ -834,18 +846,36 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
) -> None:
super().__init__(moe_config)
self.quant_config = quant_config
# W4A16 mode fires for W4A16_NVFP4 on-disk checkpoints. With
# activation_key=None every W4A4 backend's _supports_quant_scheme
# rejects itself (they all require (kNvfp4Static, kNvfp4Dynamic)
# exactly); only Marlin survives. Marlin's MoE path drops
# activation scales in convert_to_nvfp4_moe_kernel_format, so no
# other change is needed.
self.use_a16 = quant_config.quant_method == "W4A16_NVFP4"
activation_key = None if self.use_a16 else kNvfp4Dynamic
if quant_config.moe_activation_override is not None:
if self.use_a16:
raise ValueError(
"NVFP4 activation overrides require a W4A4 checkpoint."
)
activation_key = quant_config.moe_activation_override
if activation_key not in (kNvfp4Dynamic, kNvfp4DynamicToken):
raise ValueError(
"Unsupported ModelOpt NVFP4 MoE activation override: "
f"{activation_key}. Use quantization_config.moe.activation="
"'nvfp4_per_token' or omit the override."
)
self.per_token_activation = activation_key == kNvfp4DynamicToken
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,
activation_key=None if self.use_a16 else kNvfp4Dynamic,
activation_key=activation_key,
)
if (
self.per_token_activation
and self.nvfp4_backend != NvFp4MoeBackend.FLASHINFER_TRTLLM
):
raise ValueError(
"Per-token NVFP4 activation for pre-quantized weights "
"requires the FlashInfer TRTLLM MoE backend."
)
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
self.nvfp4_backend
@@ -983,6 +1013,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
self._build_moe_kernel(layer)
return
if self.per_token_activation:
layer.w13_input_scale.data.fill_(1.0)
layer.w2_input_scale.data.fill_(1.0)
# Use a single gscale for w13.
if self.moe.is_act_and_mul and not torch.allclose(
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
@@ -1015,6 +1049,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
a2_scale=layer.w2_input_scale,
is_act_and_mul=self.moe.is_act_and_mul,
use_a16=self.use_a16,
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
per_token_activation=self.per_token_activation,
is_act_and_mul=self.moe.is_act_and_mul,
),
)
replace_parameter(layer, "w13_weight", w13)
@@ -1038,6 +1076,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
per_token_activation=self.per_token_activation,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -1587,6 +1626,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
group_size: int | None,
**kwargs: Any,
) -> "ModelOptMixedPrecisionConfig":
quantization_args = original_config.get("_online_quantization_args")
if "quantization" in original_config:
quantized_layers = original_config["quantization"].get(
"quantized_layers", {}
@@ -1625,6 +1665,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
kv_cache_quant_algo=kv_cache_quant_method,
exclude_modules=[],
group_size=group_size,
quantization_args=quantization_args,
)
# Sibling config for layers that declare quant_algo: "W4A16_NVFP4".
# get_quant_method resolves this sub-config to the (kNvfp4Static, None)
@@ -1637,6 +1678,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
kv_cache_quant_algo=kv_cache_quant_method,
exclude_modules=[],
group_size=group_size,
quantization_args=quantization_args,
)
mxfp8_config = ModelOptMxFp8Config(
@@ -2572,6 +2614,17 @@ class ModelOptLinearMethod(LinearMethodBase):
layer._nvfp4_group_size_for_gather = self.ctx.group_size
self.kernel.process_weights_after_loading(layer)
def dequantize_weight(self, layer: torch.nn.Module) -> torch.Tensor:
"""Reconstruct serialized weights for online requantization."""
if self.wkey.key is kMxfp8Static:
return dequant_mxfp8_to_bf16(
layer.weight.contiguous(), layer.weight_scale.contiguous()
)
else:
raise NotImplementedError(
"ModelOpt weight dequantization is only supported for MXFP8."
)
def apply(self, layer, x, bias=None):
return self.fmt.apply(
layer,
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from enum import Enum
from types import MappingProxyType
from typing import Any
@@ -60,6 +60,7 @@ from vllm.model_executor.layers.quantization.utils.config_utils import (
find_matching_patterns,
get_layer_name_after_index,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Static128BlockSym,
@@ -104,6 +105,15 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = {
kNvfp4Static: Nvfp4OnlineMoEMethod,
}
# Quantizers for full-precision fused shared-expert weights, keyed by the routed
# experts' weight QuantKey. Each shard is quantized to the checkpoint layout on
# arrival, so only block-local schemes are supported.
ONLINE_SHARED_EXPERT_QUANTIZERS: dict[
QuantKey, Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]]
] = {
kMxfp4Static: mxfp4_quantize,
}
def _find_matching_targets(
prefix: str,
@@ -156,6 +166,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
self.args = args
self.ignored_layers: list[str] = args.ignore
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
self.online_quantization_time = 0.0
@property
def quantized_layer_summaries(self) -> list[str]:
@@ -216,14 +227,14 @@ class OnlineQuantizationConfig(QuantizationConfig):
self,
spec: QuantSpec | None,
table: dict[QuantKey, type],
layer: torch.nn.Module,
layer_cls: type[torch.nn.Module],
) -> type | None:
"""Resolve the online method class for a layer's quantization spec.
Args:
spec: Quantization specification to resolve.
table: Mapping from weight quantization keys to method classes.
layer: Layer that will use the resolved method.
layer_cls: Type of the layer that will use the resolved method.
Returns:
The matching method class, or None when ``spec`` has no weight
@@ -238,7 +249,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
cls = table.get(spec.weight)
if cls is None:
raise ValueError(
f"online quantization for {type(layer).__name__} with "
f"online quantization for {layer_cls.__name__} with "
f"weight={spec.weight} is not supported; supported weight "
f"keys: {sorted(str(k) for k in table)}"
)
@@ -253,12 +264,12 @@ class OnlineQuantizationConfig(QuantizationConfig):
return cls
def resolve_quant_method_cls(
self, layer: torch.nn.Module, prefix: str
self, layer_cls: type[torch.nn.Module], prefix: str
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
"""Resolve quantization metadata and method class without instantiating it.
Args:
layer: Layer for which to resolve online quantization.
layer_cls: Type of layer for which to resolve online quantization.
prefix: Fully qualified layer name.
Returns:
@@ -270,17 +281,17 @@ class OnlineQuantizationConfig(QuantizationConfig):
quant_spec: QuantSpec | None
if self.args.targets is not None:
resolved_pattern = self._resolve_targets_quant_method_metadata(
prefix, layer
prefix, layer_cls
)
if resolved_pattern is None:
return None
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
else:
if isinstance(layer, LinearBase):
if issubclass(layer_cls, LinearBase):
source = OnlineQuantizationSource.linear
quant_spec = self.args.linear
table = _ONLINE_LINEAR_METHODS
elif isinstance(layer, RoutedExperts):
elif issubclass(layer_cls, RoutedExperts):
source = OnlineQuantizationSource.moe
quant_spec = self.args.moe
table = _ONLINE_MOE_METHODS
@@ -297,14 +308,14 @@ class OnlineQuantizationConfig(QuantizationConfig):
quant_key_str = str(quant_spec)
target_pattern = None
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
quant_method_cls = self._get_method_cls(quant_spec, table, layer_cls)
if quant_method_cls is None:
return None
assert quant_spec is not None
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
def _resolve_targets_quant_method_metadata(
self, prefix: str, layer: torch.nn.Module
self, prefix: str, layer_cls: type[torch.nn.Module]
) -> (
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
| None
@@ -313,7 +324,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
Args:
prefix: Fully qualified layer name.
layer: Layer matched against configured target patterns.
layer_cls: Type of layer matched against configured target patterns.
Returns:
A tuple of source, quantization key string, target pattern, spec,
@@ -348,22 +359,22 @@ class OnlineQuantizationConfig(QuantizationConfig):
target_pattern = matches[0]
quant_key_str = self.args.targets[target_pattern]
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
if isinstance(layer, LinearBase):
if issubclass(layer_cls, LinearBase):
quant_spec = shorthand.linear
table = _ONLINE_LINEAR_METHODS
elif isinstance(layer, RoutedExperts):
elif issubclass(layer_cls, RoutedExperts):
quant_spec = shorthand.moe
table = _ONLINE_MOE_METHODS
else:
raise ValueError(
f"Layer {prefix} was matched by quantization_config.targets "
f"({target_pattern}), but online quantization is not supported for "
f"{type(layer).__name__}."
f"{layer_cls.__name__}."
)
if quant_spec is None:
raise ValueError(
f"targets pattern {target_pattern} = {quant_key_str} does "
f"not define a QuantSpec for {type(layer).__name__} layers "
f"not define a QuantSpec for {layer_cls.__name__} layers "
f"(matched at {prefix})."
)
return (
@@ -378,7 +389,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# `targets` takes precedence over `moe` and `linear` and is exclusive.
resolved = self.resolve_quant_method_cls(layer, prefix)
resolved = self.resolve_quant_method_cls(type(layer), prefix)
if resolved is not None:
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
self.quantized_layers[prefix] = (
@@ -12,8 +12,6 @@ if TYPE_CHECKING:
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.config import get_current_vllm_config
@@ -34,11 +32,13 @@ from vllm.model_executor.layers.fusion.quant_activation import (
from vllm.model_executor.layers.linear import (
LinearMethodBase,
)
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
amax_for_moe_weight_quant,
amax_for_tp_weight_quant,
create_fp8_quant_key,
@@ -121,10 +121,18 @@ class OnlineLinearBase(LinearMethodBase):
weights onto meta device and materializes them just-in-time."""
uses_meta_device: bool = True
activation_quant_key: QuantKey | None
def __init__(self):
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
self.requantization_source: QuantizeMethodBase | None = None
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Configure serialized-weight conversion before online quantization."""
self.requantization_source = source_method
self.uses_meta_device = False
def create_weights(
self,
@@ -136,6 +144,24 @@ class OnlineLinearBase(LinearMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
if self.requantization_source is not None:
existing_parameter_names = set(layer._parameters)
self.requantization_source.create_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
)
self.requantization_source_parameters = {
name: parameter
for name, parameter in layer._parameters.items()
if name not in existing_parameter_names and parameter is not None
}
return
output_size_per_partition = sum(output_partition_sizes)
weight_loader = extra_weight_attrs.get("weight_loader")
layer.logical_widths = output_partition_sizes
@@ -159,11 +185,28 @@ class OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
"""Release checkpoint parameters after successful requantization."""
for name, source_parameter in self.requantization_source_parameters.items():
if layer._parameters.get(name) is source_parameter:
delattr(layer, name)
self.requantization_source_parameters.clear()
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
"""Return checkpoint weights materialized for online quantization."""
if self.requantization_source is None:
return layer.weight
return self.requantization_source.dequantize_weight(layer)
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
activation_quant_key = (
kFp8DynamicTokenSym if cutlass_fp8_supported() else kFp8DynamicTensorSym
)
def __init__(self):
super().__init__()
@@ -172,11 +215,6 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
self.use_marlin = False
self.marlin_input_dtype = None
self.weight_quant_key = kFp8StaticTensorSym
# Use per-token quantization for better perf if dynamic and cutlass
if cutlass_fp8_supported():
self.activation_quant_key = kFp8DynamicTokenSym
else:
self.activation_quant_key = kFp8DynamicTensorSym
def create_weights(
self,
@@ -198,6 +236,8 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
**extra_weight_attrs,
)
# TODO: init_fp8_linear_kernel should support activation_quant_key=None
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
@@ -213,10 +253,11 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
return
layer.input_scale = None
amax = weight_amax(layer.weight).reshape(1)
weight = self.get_weight_for_quantization(layer)
amax = weight_amax(weight).reshape(1)
amax = amax_for_tp_weight_quant(amax, _is_tp_sharded(layer))
weight_scale = _fp8_scale(amax)
qweight, _ = ops.scaled_fp8_quant(layer.weight, scale=weight_scale)
qweight, _ = ops.scaled_fp8_quant(weight, scale=weight_scale)
# Update layer with new values.
replace_parameter(layer, "weight", qweight.t().data)
@@ -265,13 +306,11 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading."""
activation_quant_key = kFp8Dynamic128Sym
def __init__(self):
super().__init__()
self.weight_block_size = [128, 128]
self.activation_quant_key = create_fp8_quant_key(
static=False,
group_shape=GroupShape(1, self.weight_block_size[0]),
)
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.weight_block_size)
)
@@ -297,6 +336,8 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
)
layer.weight_block_size = self.weight_block_size
# TODO: init_fp8_linear_kernel should support activation_quant_key=None
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
@@ -312,14 +353,18 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
layer.input_scale = None
block_size = self.weight_block_size
weight = self.get_weight_for_quantization(layer)
qweight, weight_scale_inv = per_block_cast_to_fp8(
layer.weight, block_size=block_size, use_ue8m0=False
weight, block_size=block_size, use_ue8m0=False
)
replace_parameter(layer, "weight", qweight.data)
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
if self.requantization_source is not None and hasattr(layer, "weight_scale"):
del layer.weight_scale
self.fp8_linear.process_weights_after_loading(layer)
# Prevent duplicate processing (e.g., during weight reload)
@@ -395,18 +440,20 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
return
layer.input_scale = None
amax = weight_amax(layer.weight, dim=-1, keepdim=True)
weight = self.get_weight_for_quantization(layer)
amax = weight_amax(weight, dim=-1, keepdim=True)
amax = amax_for_tp_weight_quant(
amax, _is_tp_sharded(layer, reduces_output_dim=False)
)
weight_scale = _fp8_channel_scale(amax)
qweight = _fp8_quant_per_channel(layer.weight, weight_scale)
qweight = _fp8_quant_per_channel(weight, weight_scale)
replace_parameter(layer, "weight", qweight.t())
replace_parameter(layer, "weight_scale", weight_scale)
self.fp8_linear.process_weights_after_loading(layer)
expose_input_quant_key(layer, self.fp8_linear)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe import (
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.reload.layerwise import (
initialize_online_processing,
)
@@ -24,6 +25,12 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
uses_meta_device: bool = True
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Reject requantization from a checkpoint-quantized MoE method."""
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
def create_weights(
self,
layer: torch.nn.Module,
@@ -119,7 +119,8 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
weight_fp4, weight_scale = mxfp4_quantize(layer.weight.contiguous())
weight = self.get_weight_for_quantization(layer)
weight_fp4, weight_scale = mxfp4_quantize(weight.contiguous())
layer.input_scale = None
replace_parameter(layer, "weight", weight_fp4.data)
@@ -9,6 +9,7 @@ import torch
from torch.nn import Module
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
if TYPE_CHECKING:
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -42,6 +43,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
FP8 with block-32 scales) during weight loading.
"""
activation_quant_key: QuantKey | None = None
def create_weights(
self,
layer: torch.nn.Module,
@@ -75,7 +78,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
weight_fp8, weight_scale = mxfp8_e4m3_quantize(layer.weight.contiguous())
weight = self.get_weight_for_quantization(layer)
weight_fp8, weight_scale = mxfp8_e4m3_quantize(weight.contiguous())
layer.input_scale = None
replace_parameter(layer, "weight", weight_fp8.data)
@@ -19,6 +19,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
trtllm_nvfp4_hidden_alignment,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
FLOAT4_E2M1_MAX,
)
@@ -87,6 +90,8 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
(SM100) only.
"""
activation_quant_key = kNvfp4DynamicToken
def __init__(
self,
*,
@@ -100,9 +105,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,
activation_key=kNvfp4DynamicToken,
activation_key=self.activation_quant_key,
)
@property
def per_token_activation(self) -> bool:
return self.activation_quant_key == kNvfp4DynamicToken
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
@@ -156,6 +165,10 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
w2_scale_2=layer.w2_weight_scale_2,
a2_scale=layer.w2_input_scale,
is_act_and_mul=self.moe.is_act_and_mul,
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
per_token_activation=self.per_token_activation,
is_act_and_mul=self.moe.is_act_and_mul,
),
)
replace_parameter(layer, "w13_weight", w13)
@@ -176,7 +189,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
per_token_activation=True,
per_token_activation=self.per_token_activation,
)
else:
# Reload creates new scale tensors; derived kernel scales must use
@@ -8,9 +8,13 @@ from typing import TYPE_CHECKING
import regex as re
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
logger = init_logger(__name__)
def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
@@ -70,18 +74,100 @@ def is_shared_expert_quant_fse_compatible(
A compatibility flag and, when incompatible, the reason.
"""
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
if quant_config is None:
return True, None
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
online_quant_config = quant_config.online_quantization_config
shared_expert_targets = None
if online_quant_config is not None:
from vllm.model_executor.layers.linear import LinearBase
online_quant_config.packed_modules_mapping = quant_config.packed_modules_mapping
shared_expert_targets = [
online_quant_config.resolve_quant_method_cls(
LinearBase, f"{shared_expert_prefix}.{projection_name}"
)
for projection_name in projection_names
]
if all(target is None for target in shared_expert_targets):
shared_expert_targets = None
if shared_expert_targets is not None:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.base import (
ONLINE_SHARED_EXPERT_QUANTIZERS,
)
# NOTE: online shared experts quantization check is only implemented for quark
# quant method at the moment. This can be extended here for other quant methods.
# Requirement is:
# - Retrieve routed weight quant key
# - Retrieve routed activation quant key
# and ensure the online quantization set on shared expert match them.
if isinstance(quant_config, QuarkConfig):
routed_weight_key, routed_activation_key, _ = (
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
)
else:
reason = (
"online quantization FSE compatibility check is not implemented for "
f"{type(quant_config).__name__}"
)
logger.warning_once(reason)
return False, reason
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
return (
False,
"online shared-expert quantization is not supported for routed "
f"expert weight key {routed_weight_key}",
)
for shared_expert_target in shared_expert_targets:
if shared_expert_target is None:
return (
False,
"online quantization targets only part of the shared expert at "
f"{shared_expert_prefix}",
)
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
if shared_method_cls in (None, UnquantizedLinearMethod):
return (
False,
"online quantization excludes FSE weights at "
f"{shared_expert_prefix}",
)
assert issubclass(shared_method_cls, OnlineLinearBase)
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
# does not define the default activation quant key.
# `online_quant_config.resolve_quant_method_cls` should return the
# activation quant key as well. Only some online methods define it
# on the class, so compare the weight key first.
if (
shared_quant_spec.weight != routed_weight_key
or shared_method_cls.activation_quant_key != routed_activation_key
):
return (
False,
"online shared-expert quantization keys do not match the "
"routed expert keys",
)
return True, None
if isinstance(quant_config, DeepseekV4FP8Config):
from vllm.config import get_current_vllm_config
@@ -344,6 +344,7 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
w2_scale_2: torch.Tensor,
a2_scale: torch.Tensor,
is_act_and_mul: bool,
trtllm_hidden_alignment: int = 256,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -398,7 +399,11 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
w13,
w13_scale,
w2,
w2_scale,
min_alignment=trtllm_hidden_alignment,
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
@@ -280,6 +280,20 @@ def align_fp4_moe_weights_for_fi(
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
def trtllm_nvfp4_hidden_alignment(
per_token_activation: bool, is_act_and_mul: bool
) -> int:
"""Hidden-dim alignment for the TRTLLM-Gen NVFP4 MoE cubins.
The per-token (dynamic activation scale) non-gated variant has far fewer
tile configs for 256-aligned K and its heuristic default tactic fails with
"No valid config found" (e.g. Nemotron's 2688 -> 2816), so pad to 512.
"""
if per_token_activation and not is_act_and_mul:
return 512
return 256
def align_fp4_moe_hidden_dim_for_fi(
w13: torch.Tensor,
w13_scale: torch.Tensor,
@@ -142,6 +142,7 @@ def _silu_mul_quant_fp8_packed_kernel(
input_ptr,
output_q_ptr,
output_scale_ptr,
expert_ends_ptr,
M,
input_stride_m,
output_q_stride_m,
@@ -158,6 +159,7 @@ def _silu_mul_quant_fp8_packed_kernel(
PACKS_PER_CTA: tl.constexpr,
BLOCK_M: tl.constexpr,
HAS_CLAMP: tl.constexpr,
EXPERT_ALIGNMENT: tl.constexpr,
):
GROUPS_PER_PACK: tl.constexpr = 4
hidden_size: tl.constexpr = N // 2
@@ -166,6 +168,13 @@ def _silu_mul_quant_fp8_packed_kernel(
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
if EXPERT_ALIGNMENT:
expert = tl.program_id(2)
previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0)
expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT
row_start += expert_start.to(tl.int64)
M = tl.load(expert_ends_ptr + expert)
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
col_start = pack_tile * elems_per_cta
@@ -254,7 +263,19 @@ def silu_mul_quant_fp8_packed_triton(
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
*,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fuse activation and FP8 quantization, optionally skipping expert padding.
``expert_ends`` uses DeepGEMM's prefix-sum layout: each entry is the
exclusive end of an expert's live rows, including earlier alignment gaps.
An expert starts at the preceding end rounded up to ``expert_alignment``.
Padding outputs are left unwritten and must not be consumed as live rows.
Without ``expert_ends``, the original dense launch configuration is used
and the metadata loads are compiled out.
"""
assert input.dim() == 2
assert input.is_contiguous()
@@ -293,13 +314,29 @@ def silu_mul_quant_fp8_packed_triton(
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
grid_m = min(triton.cdiv(M, BM), 4096)
grid = (grid_n, grid_m)
grid: tuple[int, ...] = (grid_n, grid_m)
if expert_ends is not None:
assert expert_ends.ndim == 1 and expert_ends.numel() > 0
assert expert_ends.dtype == torch.int32
assert expert_ends.device == input.device and expert_ends.is_contiguous()
assert expert_alignment > 0
num_experts = expert_ends.numel()
# Bound empty CTAs while retaining enough parallelism for full experts.
max_ctas = 4096 if group_size < 128 else 8192
grid_m = min(
triton.cdiv(M, BM * num_experts),
max(1, triton.cdiv(max_ctas, grid_n * num_experts)),
)
grid = (grid_n, grid_m, num_experts)
else:
assert expert_alignment == 0
has_clamp = clamp_limit is not None
_silu_mul_quant_fp8_packed_kernel[grid](
input,
output_q,
output_scale_packed,
expert_ends,
M,
input.stride(0),
output_q.stride(0),
@@ -316,6 +353,7 @@ def silu_mul_quant_fp8_packed_triton(
PACKS_PER_CTA=packs_per_cta,
BLOCK_M=BM,
HAS_CLAMP=has_clamp,
EXPERT_ALIGNMENT=expert_alignment,
num_warps=num_warps,
num_stages=num_stages,
)
@@ -328,6 +366,7 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
y_ptr, # [M, N]
y_q_ptr, # [M, N // 2]
y_s_ptr, # [M, (N // 2) // GROUP_SIZE]
expert_ends_ptr,
M, # num tokens
N, # intermediate size
# Stride
@@ -345,8 +384,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
GROUP_SIZE: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
EXPERT_ALIGNMENT: tl.constexpr,
):
# TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks.
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
the outputs tensors at the right positions.
@@ -357,56 +396,77 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
m_offset = pid_m.to(tl.int64) * BLOCK_M
n_offset = pid_n.to(tl.int64) * BLOCK_N
if m_offset >= M:
return
row_step = tl.num_programs(0).to(tl.int64) * BLOCK_M
num_iterations = 1
if EXPERT_ALIGNMENT:
expert = tl.program_id(2)
previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0)
expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT
m_offset += expert_start.to(tl.int64)
M = tl.load(expert_ends_ptr + expert)
num_iterations = tl.cdiv(tl.maximum(M - m_offset, 0), row_step)
offs_n = tl.arange(0, BLOCK_N).to(tl.int64)
offs_m = tl.arange(0, BLOCK_M).to(tl.int64)
base_y_ptr = y_ptr + m_offset * N + n_offset
for _ in range(num_iterations):
base_y_ptr = y_ptr + m_offset * N + n_offset
act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :]
act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :]
act_in = tl.load(act_in_ptrs)
mul_in = tl.load(act_in_ptrs + N_2)
if EXPERT_ALIGNMENT:
live = m_offset + offs_m < M
act_in = tl.load(act_in_ptrs, live[:, None], other=0)
mul_in = tl.load(act_in_ptrs + N_2, live[:, None], other=0)
else:
act_in = tl.load(act_in_ptrs)
mul_in = tl.load(act_in_ptrs + N_2)
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
# input dtype, run silu in fp32 then narrow, and do the mul at input
# precision so HAS_CLAMP True/False share the same multiplication path.
if HAS_CLAMP:
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
y_ptr.dtype.element_ty
)
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
y_ptr.dtype.element_ty
)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
act_in = act_in.to(tl.float32)
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
y = (glu * up).to(tl.float32)
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
# input dtype, run silu in fp32 then narrow, and do the mul at input
# precision so HAS_CLAMP True/False share the same multiplication path.
if HAS_CLAMP:
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
y_ptr.dtype.element_ty
)
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
y_ptr.dtype.element_ty
)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
act_in = act_in.to(tl.float32)
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
y = (glu * up).to(tl.float32)
# quant
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
scale_raw = _absmax * (1.0 / fp8_max)
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
y_s = tl.reshape(y_s, (BLOCK_M, 1))
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
# quant
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
scale_raw = _absmax * (1.0 / fp8_max)
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
y_s = tl.reshape(y_s, (BLOCK_M, 1))
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
# store y_q
base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset
y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :]
tl.store(y_q_ptrs, y_q)
# store y_q
base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset
y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :]
if EXPERT_ALIGNMENT:
tl.store(y_q_ptrs, y_q, live[:, None])
else:
tl.store(y_q_ptrs, y_q)
# store y_s
group_id = n_offset // GROUP_SIZE
base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset
y_s_ptrs = base_y_s_ptr + offs_m
y_s = tl.reshape(y_s, (BLOCK_M,))
tl.store(y_s_ptrs, y_s)
# store y_s
group_id = n_offset // GROUP_SIZE
base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset
y_s_ptrs = base_y_s_ptr + offs_m
y_s = tl.reshape(y_s, (BLOCK_M,))
if EXPERT_ALIGNMENT:
tl.store(y_s_ptrs, y_s, live)
else:
tl.store(y_s_ptrs, y_s)
m_offset += row_step
def silu_mul_per_token_group_quant_fp8_colmajor(
@@ -418,9 +478,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
group_size: int = 128,
alpha: float = 1.0,
beta: float = 0.0,
*,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
):
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
Optional expert endpoints follow DeepGEMM's prefix-sum layout. Padding
values and scales are left unwritten and must not be consumed as live rows.
"""
GROUP_SIZE = group_size
assert input.ndim == 2
@@ -458,13 +524,27 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
# Force even division so we can avoid edgecases within the kernel.
assert M % BLOCK_M == 0
assert N_2 % BLOCK_N == 0
grid = (M // BLOCK_M, N_2 // BLOCK_N)
grid: tuple[int, ...] = (M // BLOCK_M, N_2 // BLOCK_N)
if expert_ends is not None:
assert expert_ends.ndim == 1 and expert_ends.numel() > 0
assert expert_ends.dtype == torch.int32
assert expert_ends.device == input.device and expert_ends.is_contiguous()
assert expert_alignment > 0 and expert_alignment % BLOCK_M == 0
num_experts = expert_ends.numel()
grid_m = min(
triton.cdiv(M, BLOCK_M * num_experts),
max(1, triton.cdiv(8192, grid[1] * num_experts)),
)
grid = (grid_m, grid[1], num_experts)
else:
assert expert_alignment == 0
has_clamp = clamp_limit is not None
_silu_mul_per_token_group_quant_fp8_colmajor[grid](
input,
output,
output_scales,
expert_ends,
M,
N,
output_scales.stride(-1),
@@ -479,6 +559,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
GROUP_SIZE,
BLOCK_M,
BLOCK_N,
expert_alignment,
)
return output, output_scales
@@ -133,6 +133,11 @@ def is_layer_gptq_quantized(
quantized_layers = flatten_list(quantized_layers)
# Empty means no metadata could be read at all, not that the checkpoint
# holds no quantized layer, and loading GPTQ weights unquantized cannot work.
if not quantized_layers:
return True
# Fused layers like gate_up_proj or qkv_proj will not be fused
# in the safetensors checkpoint. So, we convert the name
# from the fused version to unfused + check to make sure that
@@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import (
make_humming_moe_quant_config,
select_humming_moe_experts,
)
from vllm.model_executor.layers.quantization.utils.humming.priority import (
prefers_humming,
prioritize_humming,
)
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
humming_is_layer_skipped,
@@ -41,6 +45,8 @@ __all__ = [
"convert_linear_layer_to_humming_standard",
"get_humming_linear_compute_config",
"prepare_humming_linear_layer_config",
"prefers_humming",
"prioritize_humming",
"make_humming_moe_kernel",
"select_humming_moe_experts",
"input_schema_to_quant_key",
@@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config(
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight and input schemas to humming standard format
source_tensors = dict(layer.named_parameters())
source_tensors = dict(layer.named_parameters(recurse=False))
weight_schema, tensors = weight_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
@@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config(
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
layer.register_buffer("zero_bias", zero_bias)
for name, _ in list(layer.named_parameters()):
for name, _ in list(layer.named_parameters(recurse=False)):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Humming kernel selection priority."""
from enum import Enum
from typing import TypeVar
_KernelT = TypeVar("_KernelT", bound=type | Enum)
# Compute capabilities where Humming outranks Marlin.
_HUMMING_PREFERRED_CAPABILITIES = frozenset({90})
def prefers_humming(compute_capability: int | None = None) -> bool:
"""Whether Humming outranks Marlin. A missing compute capability uses the
current device."""
if compute_capability is None:
from vllm.platforms import current_platform
if current_platform.is_cuda():
cc = current_platform.get_device_capability()
compute_capability = cc.to_int() if cc is not None else None
return compute_capability in _HUMMING_PREFERRED_CAPABILITIES
def prioritize_humming(
kernels: list[_KernelT],
compute_capability: int | None = None,
) -> list[_KernelT]:
"""Move Humming directly ahead of Marlin where Humming is preferred.
Every other entry keeps its relative order and the input list is not
modified. Match kernel class names or MoE backend enum names.
"""
if not prefers_humming(compute_capability):
return kernels
names = [
(kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
for kernel in kernels
]
humming = next((i for i, name in enumerate(names) if "humming" in name), None)
marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
if humming is None or marlin is None or humming < marlin:
return kernels
kernels = kernels.copy()
kernels.insert(marlin, kernels.pop(humming))
return kernels
@@ -322,6 +322,22 @@ def check_and_fallback_input_schema(
input_group_size = input_schema.input_scale_group_size
input_scale_dtype = input_schema.input_scale_dtype
def is_compatible(candidate: "HummingInputSchema") -> bool:
check_weight_schema = weight_schema
if (
candidate.a_dtype in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
and weight_schema.b_dtype == humming_dtypes.float4e2m1
and weight_group_size == 16
and weight_schema.bs_dtype
in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
):
# Humming rejects folded NVFP4 scales despite A16 software dequant
# supporting them. Relax only this check, preserving kernel metadata.
check_weight_schema = dataclasses.replace(
weight_schema, bs_dtype=humming_dtypes.float8e4m3
)
return candidate.is_compatible_with(check_weight_schema, param_dtype)
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
if dtype is None or dtype.num_bits == 16:
return False
@@ -333,7 +349,7 @@ def check_and_fallback_input_schema(
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
if input_schema.is_compatible_with(weight_schema, param_dtype):
if is_compatible(input_schema):
if not allow_fallback:
if is_deprecated(a_dtype):
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
@@ -390,7 +406,7 @@ def check_and_fallback_input_schema(
input_scale_dtype=scale_dtype,
input_quant_mode=quant_mode,
)
if candidate.is_compatible_with(weight_schema, param_dtype):
if is_compatible(candidate):
return candidate
raise ValueError(
@@ -120,7 +120,14 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
K = scale.shape[-1]
pad_k = -K % 4
scale = torch.nn.functional.pad(scale, (0, pad_k))
elif current_platform.is_device_capability_family(100):
elif current_platform.is_device_capability_family(
100
) or current_platform.is_device_capability_family(120):
# Native MXFP (cap >= 10.0) requires the persistent kernel. On
# SM12x, epilogue_subtile=1 also keeps the heuristic away from a
# triton_kernels 3.5.1 codegen bug (SWAP_XW with a subtiled
# epilogue); block_m is pinned per call in
# gpt_oss_triton_kernels_moe to fit the 99KB shared memory.
constraints = {
"is_persistent": True,
"epilogue_subtile": 1,
+1 -4
View File
@@ -128,16 +128,13 @@ def _can_use_flashinfer_cutedsl_bf16(
if (
k <= 0
or n <= 0
or x.numel() == 0
or weight.shape[1] != k
or k % 128 != 0
or x.data_ptr() % 32 != 0
or weight.data_ptr() % 32 != 0
):
return False
m = x.numel() // k
if not 1 <= m <= 32:
return False
return bias is None or (
bias.is_cuda
and bias.device == x.device