Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
@@ -609,6 +609,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
vllm_config = get_current_vllm_config()
parallel_config = vllm_config.parallel_config
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
@@ -784,6 +785,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
slot_mapping,
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
self.use_pcp,
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
)
assert slot_mapping is not None
if cache is not None:
@@ -1350,6 +1352,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
common_kwargs = dict(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_size,
dtype=kv_cache_dtype,
cache_dtype_str=self.kv_cache_dtype,
@@ -165,7 +165,6 @@ T = TypeVar("T", bound=SparseMLACommonMetadata)
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
metadata_cls: type[T]
require_uniform_decodes: ClassVar[bool] = False
hisparse_supports_multi_token_decode: ClassVar[bool] = False
def __init__(
self,
@@ -259,8 +258,6 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
) -> None:
if self.vllm_config.attention_config.hisparse_config is not None:
reorder_batch_threshold = 1
if not self.hisparse_supports_multi_token_decode:
supports_spec_as_decode = False
super()._init_reorder_batch_threshold(
reorder_batch_threshold,
supports_spec_as_decode,
@@ -21,6 +21,8 @@ class AttentionLayerBase(ABC):
impl: "AttentionImpl"
supports_dcp: bool = True
# Set by the speculator on its draft model's layers.
is_draft_layer: bool = False
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
"""Bind the allocated KV cache tensor to this layer.
@@ -6,6 +6,7 @@ from typing import Union
import torch
import vllm.envs as envs
from vllm.config import ParallelConfig, SchedulerConfig
from vllm.config.kernel import MoEBackend
from vllm.distributed import get_dp_group, get_pcp_group, get_tensor_model_parallel_rank
@@ -1247,6 +1248,12 @@ class FusedMoEParallelConfig:
)
# Model types validated for VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1. This flag
# also changes the MXFP4 weight shuffle layout, so using it on other models
# can produce garbled output.
_AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES = ("deepseek_v41", "deepseek_v41_text")
# Adapted from pplx-kernels tests/all_to_all_utils.py
@dataclass
class FusedMoEConfig:
@@ -1314,6 +1321,11 @@ class FusedMoEConfig:
tp_shard_with_padding: bool = False
rocm_aiter_fmoe_enabled: bool = False
aiter_fmoe_shared_expert_enabled: bool = False
# Whether to force MXFP4 (a4w4) MoE activations for DeepSeek V4.1 on
# ROCm/AITER. Opt-in via VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1; rejected
# for any other model type. Resolved here, not in the forward path,
# because get_current_vllm_config() isn't set there.
use_mxfp4_w4a4_dsv4: bool = False
def __post_init__(self):
from vllm._aiter_ops import rocm_aiter_ops
@@ -1345,6 +1357,29 @@ class FusedMoEConfig:
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if self.rocm_aiter_fmoe_enabled and envs.VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4:
from vllm.config import get_current_vllm_config_or_none
vllm_config = get_current_vllm_config_or_none()
model_type = (
getattr(vllm_config.model_config.hf_config, "model_type", None)
if vllm_config is not None
else None
)
if model_type not in _AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES:
raise ValueError(
f"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 only supports "
f"model_type in {_AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES}, "
f"got {model_type!r}. Unset this env var for this model."
)
if not rocm_aiter_ops.fused_moe_supports_quant_dtype_a():
raise ValueError(
"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 needs an AITER build "
"with fused_moe(quant_dtype_a=...) support "
"(ROCm/aiter#5439+). Upgrade AITER or unset this env var."
)
self.use_mxfp4_w4a4_dsv4 = True
if self.use_mori_kernels:
assert self.rocm_aiter_fmoe_enabled, (
"Mori needs to be used with aiter fused_moe for now."
@@ -295,6 +295,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
)
@staticmethod
@@ -344,6 +345,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
]
@staticmethod
@@ -373,11 +375,20 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
score_mode = (
"sqrtsoftplus"
if self.moe_config.routing_method == RoutingMethodType.DeepseekV4
else None
)
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
score_mode = "sqrtsoftplus"
elif routing_method == RoutingMethodType.DeepSeekV3:
# MiMo-V2.6: an ungrouped sigmoid router with a per-expert
# correction bias. aiter's flat top-k implements this natively
# (select on sigmoid(logits) + bias, return the unbiased sigmoid
# scores).
assert e_score_correction_bias is not None, (
"DeepSeekV3 routing requires e_score_correction_bias"
)
score_mode = "sigmoid"
else:
score_mode = None
return aiter_triton_kernel_w4a16_moe_forward(
hidden_states=hidden_states,
w1=w1,
@@ -51,6 +51,32 @@ from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.math_utils import round_up
logger = init_logger(__name__)
# ===========================================================================
# Weight layout
# ===========================================================================
def swigluoai_w13_interleave_perm(
experts_cls: type[mk.FusedMoEExperts],
activation: MoEActivation,
two_i: int,
device: torch.device,
) -> torch.Tensor | None:
"""Index taking a half-split w13 ``[gate | up]`` to the interleaved order
ZenDNN's ``swiglu_oai_mul`` expects, or None when no reorder is needed.
Callers apply it along whichever axis holds their output channels."""
if not getattr(experts_cls, "requires_interleaved_w13", False):
return None
if activation != MoEActivation.SWIGLUOAI:
return None
i = two_i // 2
return torch.stack(
[torch.arange(0, i, device=device), torch.arange(i, two_i, device=device)],
dim=1,
).flatten()
# ===========================================================================
# Unquantized (BF16/FP16/FP32) MoE
# ===========================================================================
@@ -1476,15 +1502,17 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
"ZenCPUExpertsInt8 requires per-channel weight scales on the layer."
)
num_experts = self.w1_scale.shape[0]
self._w1_scale_bf16 = (
# ZenDNN's fast routed MoE kernel only accepts f32 expert scales; bf16
# silently drops the layer onto the slower per-expert path.
self._w1_scale_f32 = (
self.w1_scale.detach()
.to(torch.bfloat16)
.to(torch.float32)
.reshape(num_experts, -1)
.contiguous()
)
self._w2_scale_bf16 = (
self._w2_scale_f32 = (
self.w2_scale.detach()
.to(torch.bfloat16)
.to(torch.float32)
.reshape(num_experts, -1)
.contiguous()
)
@@ -1611,8 +1639,8 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
topk_ids.to(torch.int32).contiguous(),
apply_router_weight_on_input, # skip_weighted
str(activation.value).lower(),
self._w1_scale_bf16,
self._w2_scale_bf16,
self._w1_scale_f32,
self._w2_scale_f32,
)
@@ -14,11 +14,14 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
activation_to_flashinfer_int,
quantize_nvfp4_per_token_input,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4DynamicToken,
kNvfp4Static,
)
from vllm.platforms import current_platform
@@ -40,6 +43,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
per_token_activation: bool = False,
):
super().__init__(
moe_config=moe_config,
@@ -63,6 +67,14 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
self.situ_beta = moe_config.activation_situ_beta
self.situ_linear_beta = moe_config.activation_situ_linear_beta
self.per_token_activation = per_token_activation
self.per_token_global_scale = None
if per_token_activation:
assert quant_config.a2_gscale is not None
self.per_token_global_scale = quant_config.a2_gscale.new_full(
(1,),
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
@@ -92,6 +104,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
) -> bool:
SUPPORTED_W_A = [
(kNvfp4Static, kNvfp4Dynamic),
(kNvfp4Static, kNvfp4DynamicToken),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@@ -114,6 +127,10 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
@property
def expects_unquantized_inputs(self) -> bool:
return self.per_token_activation
def workspace_shapes(
self,
M: int,
@@ -127,8 +144,8 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
workspace1 = (0,)
workspace2 = (0,)
# K is packed (K//2 for uint8), so output uses hidden_dim.
assert self.hidden_dim == K * 2
expected_hidden_dim = K if self.expects_unquantized_inputs else K * 2
assert self.hidden_dim == expected_hidden_dim
output = (M, self.hidden_dim)
return (workspace1, workspace2, output)
@@ -151,13 +168,26 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
apply_router_weight_on_input: bool | None,
):
assert self.quant_dtype == "nvfp4"
assert a1q_scale is not None
assert self.w1_scale is not None
assert self.w2_scale is not None
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
quantize_nvfp4_per_token_input(hidden_states)
)
fc2_input_scale = self.per_token_global_scale
else:
assert a1q_scale is not None
block_scale = a1q_scale
per_token_scale = None
fc2_input_scale = self.a2_gscale
assert block_scale is not None
assert fc2_input_scale is not None
# a1q_scale is (M, K//16) float8_e4m3fn from fp4_quantize.
# The functional API expects x_sf with trailing dim: (M, K//16, 1).
x_sf = a1q_scale.unsqueeze(-1)
x_sf = block_scale.unsqueeze(-1)
# The kernel defaults swiglu_{alpha,beta,limit} to the plain-SwiGLU
# values, so only forward the ones the model actually sets.
@@ -188,6 +218,9 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
}
swiglu_kwargs = {k: v for k, v in swiglu_params.items() if v is not None}
per_token_kwargs = (
{"per_token_scale": per_token_scale} if self.per_token_activation else {}
)
flashinfer_cute_dsl_fused_moe_nvfp4(
x=hidden_states,
x_sf=x_sf,
@@ -196,7 +229,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
w1_weight=w1,
w1_weight_sf=self.w1_scale,
w1_alpha=self.g1_alphas,
fc2_input_scale=self.a2_gscale,
fc2_input_scale=fc2_input_scale,
w2_weight=w2,
w2_weight_sf=self.w2_scale,
w2_alpha=self.g2_alphas,
@@ -209,4 +242,5 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
MoEActivation.SILU if activation == MoEActivation.SITU else activation
),
**swiglu_kwargs,
**per_token_kwargs,
)
@@ -375,15 +375,13 @@ def rocm_aiter_fused_experts(
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
)
# https://github.com/ROCm/aiter/pull/3123 specialized the AITER stage1 GEMMs
# for interleaved vs separated gate and up weights.
# For gpt-oss i.e. use_mxfp4_w4a16=True, the weights are shuffled by
# `rocm_aiter_ops.shuffle_weight_a16w4` in `oracle/mxfp4.py`,
# which always sets `is_guinterleave=True`.
# Hence, we pass in GateMode.INTERLEAVE to match the weight shuffling.
# AITER's stage1 GEMM needs gate_mode to match how weights were
# shuffled at load time (oracle/mxfp4.py). gpt-oss uses INTERLEAVE;
# DeepSeek V4.1 a4w4 uses SEPARATED (see below).
from aiter.ops.flydsl.moe_common import GateMode
gate_mode = ""
q_dtype_a = None
if activation == MoEActivation.SITU:
gate_mode = (
GateMode.INTERLEAVE.value
@@ -391,7 +389,15 @@ def rocm_aiter_fused_experts(
else GateMode.SEPARATED.value
)
elif quant_config.use_mxfp4_w4a16:
gate_mode = GateMode.INTERLEAVE.value
if moe_config.use_mxfp4_w4a4_dsv4:
# Opt-in a4w4 for DeepSeek V4.1 (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1).
# Must match the SEPARATED weight shuffle from load time.
from aiter import dtypes
gate_mode = GateMode.SEPARATED.value
q_dtype_a = dtypes.fp4x2
else:
gate_mode = GateMode.INTERLEAVE.value
elif activation_interleave is not None:
gate_mode = (
GateMode.INTERLEAVE.value
@@ -433,6 +439,7 @@ def rocm_aiter_fused_experts(
shared_w1_scale=shared_w1_scale,
shared_w2_scale=shared_w2_scale,
shared_expert_id=shared_expert_id,
q_dtype_a=q_dtype_a,
)
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_int,
has_flashinfer_situ_activation,
quantize_nvfp4_per_token_input,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -37,10 +38,6 @@ from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
logger = init_logger(__name__)
# Base scale for per-token NVFP4 activation quant; the kernel folds the
# per-token global scale (from the activation amax) on top of it.
_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
class TrtLlmNvFp4ExpertsBase:
"""NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface."""
@@ -259,23 +256,6 @@ class TrtLlmNvFp4ExpertsBase:
def expects_unquantized_inputs(self) -> bool:
return self.per_token_activation
def _quantize_per_token_input(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""NVFP4-quantize activations with a per-token global scale.
Returns ``(packed_fp4, block_scale, per_token_scale)``.
"""
from flashinfer import SfLayout, nvfp4_quantize
hs_fp4, hs_block_scale, per_token_scale = nvfp4_quantize(
hidden_states,
_PER_TOKEN_BASE_GLOBAL_SCALE,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
)
return hs_fp4, hs_block_scale, per_token_scale
def _get_chunk_size(self) -> int:
MAX_GRID_Y = 65535
MAX_TILE_TOKENS_DIM = 128
@@ -361,9 +341,9 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
# Per-token: input is unquantized, quantize it here. Otherwise it was
# already quantized in prepare() with the static global scale.
if self.per_token_activation:
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
self._quantize_per_token_input(hidden_states)
quantize_nvfp4_per_token_input(hidden_states)
)
else:
block_scale, per_token_scale = a1q_scale, None
@@ -437,7 +417,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
) -> UnfinalizedMoEOutput | None:
assert self._supports_activation(activation)
# Per-token defers input quant to _invoke_kernel, so a1q_scale is None.
assert a1q_scale is not None or self.per_token_activation
assert a1q_scale is not None or self.expects_unquantized_inputs
# DeepEP produces int64 indexes.
topk_ids = topk_ids.to(dtype=torch.int32)
@@ -539,7 +519,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
import flashinfer
assert self._supports_activation(activation)
assert a1q_scale is not None or self.per_token_activation
assert a1q_scale is not None or self.expects_unquantized_inputs
assert self.quant_config.w1_scale is not None
assert self.quant_config.w2_scale is not None
assert (
@@ -551,9 +531,9 @@ class TrtLlmNvFp4ExpertsMonolithic(
)
# Per-token: input is unquantized, quantize it here (see modular apply).
if self.per_token_activation:
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
self._quantize_per_token_input(hidden_states)
quantize_nvfp4_per_token_input(hidden_states)
)
else:
block_scale, per_token_scale = a1q_scale, None
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zentorch fused MoE experts for AMD Zen CPUs."""
from collections.abc import Callable
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
_moe_activation_to_str,
has_zentorch_op,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import select_experts
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
)
class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
"""DA8W4 (W4A8) group-quantized monolithic MoE experts."""
# swiglu_oai_mul reads gate/up interleaved, not in the half-split order the
# weight loader leaves behind.
requires_interleaved_w13 = True
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
# Defaults hold until process_weights_after_loading sees the layer.
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
)
self.custom_routing_function: Callable | None = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Capture the router config that the monolithic apply() signature
cannot carry; Custom routing has no other source for it."""
self.renormalize = layer.renormalize
self.custom_routing_function = layer.custom_routing_function
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return has_zentorch_op(["zentorch_fused_moe", "zentorch_woq_repack_weight"])
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return supported, reason
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
# The grouped GEMM needs tokens spread over at least two experts, which
# top_k=1 cannot satisfy once decode drops to a single token.
if moe_config.experts_per_token < 2:
return False, "kernel requires experts_per_token >= 2"
return True, None
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return _moe_activation_to_str(activation) in _ZENTORCH_MOE_ACTIVATIONS
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
# apply() passes select_experts' global ids straight to the kernel, so
# every expert has to be resident on the rank.
return not moe_parallel_config.use_ep
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) in [
(kInt4Static, None),
(kInt4Static32, None),
]
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.Custom,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
"ZentorchExpertsInt4 does not support "
"apply_router_weight_on_input=True."
)
if (
self.moe_config.routing_method == RoutingMethodType.Custom
and self.custom_routing_function is None
):
raise RuntimeError(
"ZentorchExpertsInt4 needs the model's custom_routing_function "
"for RoutingMethodType.Custom."
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
custom_routing_function=self.custom_routing_function,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
output = torch.empty_like(hidden_states)
torch.ops.zentorch.zentorch_fused_moe(
output,
hidden_states,
w1,
w2,
self.w1_bias,
self.w2_bias,
topk_weights,
topk_ids,
False, # skip_weighted
_moe_activation_to_str(activation),
self.w1_scale,
self.w2_scale,
)
return output
__all__ = ["ZentorchExpertsInt4"]
@@ -12,6 +12,7 @@ from compressed_tensors.quantization import (
import vllm._custom_ops as ops
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config.kernel import MoEBackend
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
@@ -55,6 +56,7 @@ class WNA16MoEBackend(Enum):
BATCHED_MARLIN = "BATCHED_MARLIN"
HUMMING = "HUMMING"
CPU = "CPU"
ZEN_CPU = "ZEN_CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
TRITON = "TRITON"
XPU = "XPU"
@@ -98,6 +100,12 @@ def backend_to_kernel_cls(
)
return [CPUExpertsInt4]
elif backend == WNA16MoEBackend.ZEN_CPU:
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
return [ZentorchExpertsInt4]
elif backend == WNA16MoEBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
Int4EmulationTritonExperts,
@@ -117,7 +125,7 @@ def backend_to_kernel_cls(
def _get_priority_backends() -> list[WNA16MoEBackend]:
"""Get available backends in priority order based on platform and config."""
if current_platform.is_cpu():
return [WNA16MoEBackend.CPU]
return [WNA16MoEBackend.ZEN_CPU, WNA16MoEBackend.CPU]
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
@@ -160,6 +168,27 @@ def _backend_incompatibility_reason(
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
if backend == WNA16MoEBackend.ZEN_CPU:
if not envs.VLLM_CPU_INT4_W4A8:
return "VLLM_CPU_INT4_W4A8=0 disables the DA8W4 path"
if may_have_zp:
return "zero points are not supported"
if isinstance(quant_config, MoeWNA16Config):
return "the MoeWNA16 weight layout is not supported"
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
return "GPTQ activation ordering is not supported"
if (
isinstance(quant_config, QuantizationArgs)
and quant_config.actorder == "group"
):
return "group activation ordering is not supported"
group_size = getattr(quant_config, "group_size", None)
if group_size is None or group_size <= 0:
return "DA8W4 requires group-quantized weights"
# AOCL sym_quant requires the group size to be a multiple of 4.
if group_size % 4 != 0:
return f"group size {group_size} is not a multiple of 4"
if backend == WNA16MoEBackend.TRITON:
if may_have_bias:
return "expert bias is not supported"
@@ -403,10 +432,14 @@ def make_wna16_moe_kernel(
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsWNA16,
)
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming
# grouped/indexed experts, and Int4EmulationTritonExperts
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4,
# ZentorchExpertsInt4, the Humming grouped/indexed experts, and
# Int4EmulationTritonExperts
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
MarlinExperts,
BatchedMarlinExperts,
@@ -414,6 +447,7 @@ def make_wna16_moe_kernel(
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
ZentorchExpertsInt4,
Int4EmulationTritonExperts,
Rdna3WNA16Experts,
)
@@ -1034,6 +1068,64 @@ def _process_weights_cpu(
)
def _zen_repack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
"""Repack CT int4 ``[E, N, K//8]`` into zentorch s4 ``[E, N, K//8]``."""
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
_import_unpack_from_int32,
)
num_experts, out_features, _ = packed.shape
# packed_dim indexes the 2D expert slice, which is N-first, so K is dim 1.
unpacked = _import_unpack_from_int32()(
packed,
4,
torch.Size([num_experts, out_features, in_features]),
packed_dim=1,
)
# CT is already N-first, which is what the repack consumes.
repack_op = torch.ops.zentorch.zentorch_woq_repack_weight.default
return torch.stack([repack_op(w) for w in unpacked])
def _process_weights_zen_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, # w13_qweight
torch.Tensor, # w2_qweight
torch.Tensor, # w13_scales
torch.Tensor, # w2_scales
torch.Tensor | None, # w13_qzeros
torch.Tensor | None, # w2_qzeros
torch.Tensor | None, # w13_input_global_scale
torch.Tensor | None, # w2_input_global_scale
torch.Tensor | None, # w13_bias
torch.Tensor | None, # w2_bias
]:
"""Zen CPU INT4 DA8W4 (W4A8) weight post-processing."""
# CT WNA16 buffers are N-first, so the scales are [E, N, G].
hidden_size = w2_scale.shape[1]
intermediate_size = w13_scale.shape[1] // 2
# The kernel reads per-group scales as [E, G, N], so N-first needs a swap.
return (
_zen_repack_s4(w13.data, hidden_size),
_zen_repack_s4(w2.data, intermediate_size),
w13_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
w2_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
None, # w13_qzeros (symmetric)
None, # w2_qzeros (symmetric)
None, # w13_input_global_scale
None, # w2_input_global_scale
w13_bias,
w2_bias,
)
def _process_weights_xpu(
layer: torch.nn.Module,
quant_config: QuantizationConfig,
@@ -1584,6 +1676,15 @@ def convert_to_wna16_moe_kernel_format(
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.ZEN_CPU:
return _process_weights_zen_cpu(
w13,
w2,
w13_scale,
w2_scale,
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
return _process_weights_flashinfer(
w13,
@@ -1382,6 +1382,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias: torch.Tensor | None = None,
_cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None,
activation: MoEActivation | None = None,
use_separated_a4w4: bool = False,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -1626,10 +1627,16 @@ def convert_weight_to_mxfp4_moe_kernel_format(
from aiter.ops.shuffle import shuffle_scale as _shuf_s
from aiter.ops.shuffle import shuffle_weight as _shuf_w
# DeepSeek V4.1 a4w4 uses ATOM's SEPARATED gate/up layout instead of
# the default INTERLEAVE shuffle (INTERLEAVE + fp4x2 has no tuned
# kernel and produces garbage output). Must match GateMode.SEPARATED
# in rocm_aiter_moe.py.
is_guinterleave = not use_separated_a4w4
w13_weight = torch.nn.Parameter(
_shuf_w(
w13_weight.data.view(torch.float4_e2m1fn_x2),
is_guinterleave=True,
is_guinterleave=is_guinterleave,
gate_up=True,
),
requires_grad=False,
@@ -1637,14 +1644,14 @@ def convert_weight_to_mxfp4_moe_kernel_format(
shuffled_w13_scale = _shuf_s(
w13_weight_scale.reshape(-1, w13_weight_scale.shape[-1]),
num_experts,
True,
is_guinterleave,
True,
)
w2_weight = torch.nn.Parameter(
_shuf_w(
w2_weight.data.view(torch.float4_e2m1fn_x2),
is_guinterleave=True,
is_guinterleave=is_guinterleave,
gate_up=False,
),
requires_grad=False,
@@ -1653,7 +1660,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
shuffled_w2_scale = _shuf_s(
w2_weight_scale.reshape(-1, w2_weight_scale.shape[-1]),
num_experts,
True,
is_guinterleave,
False,
)
@@ -612,7 +612,14 @@ def make_nvfp4_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM and per_token_activation:
if (
backend
in (
NvFp4MoeBackend.FLASHINFER_TRTLLM,
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
)
and per_token_activation
):
extra_kwargs["per_token_activation"] = True
# Create Experts.
@@ -371,6 +371,16 @@ def select_unquantized_moe_backend(
)
def aiter_moe_intermediate_alignment(intermediate: int) -> int:
"""Intermediate-size alignment required by AITER's CK 2stages MoE kernel.
AITER dispatches on ``inter_dim <= 192``: below the threshold both stages
use 64-wide tiles, above it at least one stage uses a 128-wide tile, and
CK's ``IsSupportedArgument`` rejects a size not divisible by that width.
"""
return 64 if intermediate <= 192 else 128
def unquantized_round_up_hidden_size_and_intermediate_size(
backend: UnquantizedMoeBackend,
hidden_size: int,
@@ -379,9 +389,30 @@ def unquantized_round_up_hidden_size_and_intermediate_size(
"""Round up dimensions before allocation to satisfy the selected kernel."""
if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
intermediate_size = round_up(intermediate_size, 128)
elif backend == UnquantizedMoeBackend.AITER:
intermediate_size = round_up(
intermediate_size, aiter_moe_intermediate_alignment(intermediate_size)
)
return hidden_size, intermediate_size
def _zero_intermediate_padding(
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
) -> None:
"""Zero intermediate padding; reloads rewrite only the checkpoint slices."""
unpadded = moe_config.intermediate_size_per_partition_unpadded
assert unpadded is not None
padded = moe_config.intermediate_size_per_partition
if padded <= unpadded:
return
w13_weight[:, unpadded:padded].zero_()
if moe_config.is_act_and_mul:
w13_weight[:, padded + unpadded :].zero_()
w2_weight[:, :, unpadded:].zero_()
def convert_to_unquantized_kernel_format(
unquantized_backend: UnquantizedMoeBackend,
moe_config: FusedMoEConfig,
@@ -424,6 +455,7 @@ def convert_to_unquantized_kernel_format(
return layout.full_gate_weight, layout.full_down_weight
if unquantized_backend == UnquantizedMoeBackend.AITER:
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
w13_weight.is_shuffled = True
w2_weight.is_shuffled = True
@@ -445,15 +477,7 @@ def convert_to_unquantized_kernel_format(
)
moe_config.intermediate_size_per_partition = padded_intermediate
# Reloads only overwrite checkpoint slices. An earlier in-place
# permutation can leave nonzero values in the raw padding slots.
unpadded = moe_config.intermediate_size_per_partition_unpadded
assert unpadded is not None
if padded_intermediate > unpadded:
w13_weight[:, unpadded:padded_intermediate].zero_()
if is_act_and_mul:
w13_weight[:, padded_intermediate + unpadded :].zero_()
w2_weight[:, :, unpadded:].zero_()
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
convert_moe_weights_to_flashinfer_trtllm_block_layout(
@@ -27,6 +27,7 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
self.num_dispatchers_ = num_dispatchers
self.max_tokens_per_rank = max_tokens_per_rank
self.use_fp8_dispatch = use_fp8_dispatch
self._dispatch_topk_ids: torch.Tensor | None = None
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
@@ -92,6 +93,12 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
# mori expects one scale slot per token; broadcast.
scale = scale.expand(a1.shape[0], 1).contiguous()
# mori's combine() reduces over this rank's own [num_tokens, topk]
# routing. The modular kernel rebinds topk_ids to the dispatched ids
# returned below before it calls finalize(), so hold a reference to
# the pre-dispatch tensor (ROCm/mori#475).
self._dispatch_topk_ids = topk_ids
(
dispatch_a1,
dispatch_weights,
@@ -122,9 +129,13 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> None:
num_token = output.shape[0]
assert self._dispatch_topk_ids is not None, (
"finalize() called before prepare(); mori combine() needs the "
"routing tensor that dispatch() was given"
)
result = self.mori_op.combine(
fused_expert_output,
None,
topk_ids,
self._dispatch_topk_ids,
)[0]
output.copy_(result[:num_token])
@@ -691,7 +691,10 @@ class RoutedExperts(PluggableLayer):
)
and is_transposed
):
loaded_weight = loaded_weight.t().contiguous()
# Transpose on the parameter's device: a CPU transpose of the
# mmap view reads the file column-wise, which is very slow on
# NFS. The contiguous .to() reads it sequentially.
loaded_weight = loaded_weight.to(param.device).t().contiguous()
if shard_id not in ("w1", "w2", "w3"):
raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.")
@@ -19,6 +19,11 @@ from vllm.model_executor.layers.fused_moe.router.dsv4_topk import (
dsv4_topk,
)
# AITER instantiates biased_grouped_topk only for these NUM_GRP values, with at
# most this many experts per group.
AITER_SUPPORTED_NUM_GRP = (1, 2, 4, 8)
AITER_MAX_EXPERTS_PER_GROUP = 32
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
@@ -104,17 +109,40 @@ def vllm_topk_softplus_sqrt(
@functools.lru_cache(maxsize=8)
def _aiter_get_num_expert_group(num_experts: int) -> int:
_AITER_MAX_EXPERTS_PER_GROUP = 32
g = max(1, -(-num_experts // _AITER_MAX_EXPERTS_PER_GROUP))
g = max(1, -(-num_experts // AITER_MAX_EXPERTS_PER_GROUP))
while num_experts % g != 0:
g += 1
assert num_experts % g == 0, f"{num_experts=} not divisible by {g=}"
assert num_experts // g <= _AITER_MAX_EXPERTS_PER_GROUP, (
f"group size {num_experts // g} exceeds limit {_AITER_MAX_EXPERTS_PER_GROUP}"
# Grouping is a no-op here (topk_group == num_expert_group), so any divisor
# within the limit routes identically and we are free to pick one AITER has
# a kernel for. If none fits, the unsupported value is returned as-is --
# callers must gate on _aiter_can_use_biased_grouped_topk.
if g not in AITER_SUPPORTED_NUM_GRP:
g = next(
(
c
for c in sorted(AITER_SUPPORTED_NUM_GRP, reverse=True)
if num_experts % c == 0
and num_experts // c <= AITER_MAX_EXPERTS_PER_GROUP
),
g,
)
assert num_experts // g <= AITER_MAX_EXPERTS_PER_GROUP, (
f"group size {num_experts // g} exceeds limit {AITER_MAX_EXPERTS_PER_GROUP}"
)
return g
def _aiter_can_use_biased_grouped_topk(num_experts: int, topk: int) -> bool:
"""Whether AITER has an instantiated kernel for this routing shape.
Both checks are needed: an unsupported group count can still be small
enough to satisfy ``topk >= g`` (num_experts=33 gives g=3).
"""
g = _aiter_get_num_expert_group(num_experts)
return topk >= g and g in AITER_SUPPORTED_NUM_GRP
def fused_topk_bias(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
@@ -229,7 +257,7 @@ def fused_topk_bias(
M = hidden_states.size(0)
num_experts = gating_output.shape[-1]
num_expert_group = _aiter_get_num_expert_group(num_experts)
if topk >= num_expert_group:
if _aiter_can_use_biased_grouped_topk(num_experts, topk):
topk_weights = torch.empty(
M, topk, dtype=torch.float32, device=hidden_states.device
)
@@ -720,7 +720,7 @@ class MoERunner(MoERunnerInterface):
)
# Record before `_maybe_pad_hidden_states` pads activations to match
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
# `moe_config.hidden_dim`, e.g. after `align_fp4_moe_hidden_dim_for_fi`
# so routed output can be trimmed before
# shared+routed add / latent up proj if needed.
@@ -147,6 +147,11 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams:
rescale_factor)`` from the processor config."""
config = get_processor_config(model_config.model, revision=model_config.revision)
config = config.get("image_processor", config)
if not any(
key in config
for key in ("do_normalize", "do_rescale", "image_mean", "image_std")
):
config = model_config.hf_image_processor_config
has_norm_params = "image_mean" in config and "image_std" in config
do_normalize = bool(config.get("do_normalize", has_norm_params))
@@ -241,6 +246,10 @@ class FusedMMInputNorm(CustomOp):
self.register_buffer("weight", (rescale_factor / std).to(device))
self.register_buffer("bias", (-mean / std).to(device))
@classmethod
def enabled(cls) -> bool:
return True
@property
def input_dtype(self) -> torch.dtype | None:
return torch.uint8
+73 -16
View File
@@ -211,6 +211,79 @@ class GemmaRMSNorm(CustomOp):
return out
@CustomOp.register("layer_norm")
class LayerNorm(CustomOp):
"""Standard (mean-centered) LayerNorm.
Drop-in for a bare `nn.LayerNorm` that dispatches to a fused XPU kernel
when one is available, and to the native implementation otherwise.
Normalization runs in the wider of the input and parameter dtypes and the
result is cast back to the input dtype, so `dtype=torch.float32` gives an
fp32 reduction for a lower-precision input.
"""
def __init__(
self,
hidden_size: int,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
weight_dtype = dtype or torch.get_default_dtype()
self.weight: nn.Parameter | None = None
self.bias: nn.Parameter | None = None
if elementwise_affine:
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype))
if bias:
self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype))
def _compute_dtype(self, x: torch.Tensor) -> torch.dtype:
if self.weight is None:
return x.dtype
return torch.promote_types(x.dtype, self.weight.dtype)
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
compute_dtype = self._compute_dtype(x)
if compute_dtype != x.dtype:
return F.layer_norm(
x.to(compute_dtype),
self.normalized_shape,
self.weight,
self.bias,
self.eps,
).type_as(x)
return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_layer_norm
if (
self.weight is None
or self.bias is None
or not hasattr(torch.ops._C, "layer_norm")
# The kernel reduces in the input dtype, so a widened reduction
# takes the native path.
or self._compute_dtype(x) != x.dtype
):
return self.forward_native(x)
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_layer_norm(out, x, self.weight, self.bias, self.eps)
return out
def extra_repr(self) -> str:
return f"hidden_size={self.normalized_shape[0]}, eps={self.eps}"
# --8<-- [start:rms_norm_gated]
@CustomOp.register("rms_norm_gated")
class RMSNormGated(CustomOp):
@@ -350,19 +423,3 @@ class RMSNormGated(CustomOp):
self, x: torch.Tensor, z: torch.Tensor | None = None
) -> torch.Tensor:
return self.forward_cuda(x, z)
class LayerNorm(nn.Module):
"""Layer Normalization."""
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.dim = dim
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32))
self.bias = nn.Parameter(torch.zeros(dim, dtype=torch.float32))
def forward(self, x: torch.Tensor):
return F.layer_norm(
x.float(), (self.dim,), self.weight, self.bias, self.eps
).type_as(x)
+31 -10
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
from dataclasses import dataclass, replace
import torch
@@ -58,10 +58,33 @@ def compute_mamba_prefill_checkpoints(
return offsets, cols
def _gather_checkpoint_state_indices(
block_table: torch.Tensor, request_rows: torch.Tensor, block_cols: torch.Tensor
) -> torch.Tensor:
return torch.where(
block_cols >= 0, block_table[request_rows, block_cols], NULL_BLOCK_ID
)
@dataclass
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
block_cols: torch.Tensor | None = None
def regather_state_indices(
self, block_table: torch.Tensor
) -> "MambaPrefillCheckpointMetadata":
assert self.request_rows is not None and self.block_cols is not None
return replace(
self,
state_indices=_gather_checkpoint_state_indices(
block_table, self.request_rows, self.block_cols
),
)
class MambaPrefillCheckpointBuilder:
@@ -76,6 +99,8 @@ class MambaPrefillCheckpointBuilder:
m: CommonAttentionMetadata,
request_rows: list[int],
) -> MambaPrefillCheckpointMetadata | None:
if self.vllm_config.cache_config.mamba_cache_mode != "align":
return None
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
return None
assert m.seq_lens_cpu_upper_bound is not None
@@ -109,17 +134,13 @@ class MambaPrefillCheckpointBuilder:
checkpoint_cols_tensor = async_tensor_h2d(
checkpoint_cols, dtype=torch.int64, device=m.query_start_loc.device
)
checkpoint_state_indices = m.block_table_tensor[
request_rows_tensor, checkpoint_cols_tensor
]
checkpoint_state_indices = torch.where(
checkpoint_cols_tensor >= 0,
checkpoint_state_indices,
NULL_BLOCK_ID,
)
return MambaPrefillCheckpointMetadata(
checkpoint_offsets_tensor,
checkpoint_state_indices,
_gather_checkpoint_state_indices(
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
@@ -260,8 +260,6 @@ class MambaMixer(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -297,8 +295,6 @@ class MambaMixer(MambaBase, PluggableLayer):
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
num_decode_tokens = attn_metadata.num_decode_tokens
num_prefills = attn_metadata.num_prefills # request count
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
has_prefill = num_prefill_tokens > 0
has_decode = num_decode_tokens > 0
num_actual_tokens = num_prefill_tokens + num_decode_tokens
@@ -314,34 +310,6 @@ class MambaMixer(MambaBase, PluggableLayer):
gate_p = prefill_decode_split.gate_p
gate_d = prefill_decode_split.gate_d
if is_mamba_cache_all:
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_d = None
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
num_computed_tokens_p = None
ssm_outputs = []
if has_prefill:
@@ -355,11 +323,6 @@ class MambaMixer(MambaBase, PluggableLayer):
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
query_start_loc=query_start_loc_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
@@ -383,10 +346,6 @@ class MambaMixer(MambaBase, PluggableLayer):
cache_indices=state_indices_tensor_p,
has_initial_state=has_initial_states_p,
query_start_loc=query_start_loc_p,
block_size=mamba_block_size,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
cu_chunk_seqlen=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
)
@@ -396,16 +355,6 @@ class MambaMixer(MambaBase, PluggableLayer):
# state_indices_tensor_d is assigned when attn_metadata is not None,
# and has_decode is only True when attn_metadata is not None
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
# 2. Convolution sequence transformation
conv_out_d = causal_conv1d_update(
hidden_states_BC_d.transpose(0, 1),
@@ -414,8 +363,6 @@ class MambaMixer(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
).transpose(0, 1)
# 3. State Space Model sequence transformation.
@@ -437,8 +384,8 @@ class MambaMixer(MambaBase, PluggableLayer):
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=scan_outputs_d,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
+126 -198
View File
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
commit_replayssm_ring_trackers,
reset_replayssm_ring_trackers,
selective_state_update,
selective_state_update_replayssm_flashinfer,
@@ -69,6 +70,13 @@ from vllm.v1.kv_cache_interface import KVCacheGroupSpec
logger = init_logger(__name__)
def _view_mtp_decode_tensor(
tensor: torch.Tensor, decode_batch: int, spec_query_len: int
) -> torch.Tensor:
return tensor.view(decode_batch, spec_query_len, *tensor.shape[1:])
# Added by the IBM Team, 2024
@@ -232,13 +240,14 @@ def mamba_v2_sharded_weight_loader(
# - the ignore is for a mundane mypy error as it does not
# seem to handle slices well.
# https://github.com/python/mypy/issues/2410
target_slice = param.data[boundary : (boundary + take), ...]
param.data[
boundary : (boundary + take), ... # type: ignore[misc]
] = loaded_weight[
loaded_start_idx : (
loaded_start_idx + take
) # type: ignore[misc]
] # type: ignore[misc]
].view_as(target_slice)
# move indexing boundaries
boundary += shard_size
@@ -440,17 +449,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
tp_rank,
)
# Apply the custom weight loader to in_proj.weight
# Apply the custom weight loader to in_proj.weight and its scales.
# Works for both non-quantized (Parameter) and quantized
# (ModelWeightParameter which extends BasevLLMParameter)
if isinstance(self.in_proj.weight, BasevLLMParameter):
# For BasevLLMParameter subclasses (quantized layers like FP8)
# These have a weight_loader property that can be directly set
self.in_proj.weight.weight_loader = mamba_loader
else:
# For standard Parameter (non-quantized layers)
delattr(self.in_proj.weight, "weight_loader")
set_weight_attrs(self.in_proj.weight, {"weight_loader": mamba_loader})
for attr_name in ("weight", "weight_scale", "input_scale"):
param = getattr(self.in_proj, attr_name, None)
if param is not None:
if isinstance(param, BasevLLMParameter):
param.weight_loader = mamba_loader
else:
if hasattr(param, "weight_loader"):
delattr(param, "weight_loader")
set_weight_attrs(param, {"weight_loader": mamba_loader})
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
@@ -534,15 +544,11 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.kv_cache = tuple(torch.tensor([]) for _ in range(_n_state))
self._replayssm_ring_start = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_query_len = torch.empty(0, dtype=torch.int32)
self._commits_replayssm_trackers = True
self._updates_replayssm_trackers = True
self.num_spec = vllm_config.num_speculative_tokens
if self.num_spec > 0:
self.register_buffer(
"_decode_state_offsets",
torch.arange(1 + self.num_spec, dtype=torch.int32).unsqueeze(0),
persistent=False,
)
# Pre-compute sizes for forward pass
self.tped_intermediate_size = self.intermediate_size // self.tp_size
@@ -720,9 +726,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
ring_start = prev_num_accepted = None
use_spec_decode = self.num_spec > 0
ring_start = prev_num_accepted = prev_query_len = None
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -744,6 +749,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
ring_start = self._replayssm_ring_start
prev_num_accepted = self._replayssm_prev_num_accepted
prev_query_len = self._replayssm_prev_query_len
else:
x_cache = dt_cache = B_cache = None
has_initial_states_p = attn_metadata.has_initial_states_p
@@ -755,6 +761,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
replayssm_state_indices_d = attn_metadata.replayssm_state_indices_d
num_accepted_tokens = attn_metadata.num_accepted_tokens
query_start_loc_d = attn_metadata.query_start_loc_d
num_decodes = attn_metadata.num_decodes
@@ -788,45 +795,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
dim=0,
)
if is_mamba_cache_all:
# If prefix caching is enabled, retrieve the relevant variables
# for prefill and decode
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
if attn_metadata.block_idx_last_scheduled_token_prev_step is not None:
block_idx_last_scheduled_token_prev_step_d, _ = torch.split(
attn_metadata.block_idx_last_scheduled_token_prev_step,
[num_decodes, num_prefills],
dim=0,
)
else:
block_idx_last_scheduled_token_prev_step_d = None
# Prefill-only variables:
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_computed_token_d = None
block_idx_last_scheduled_token_prev_step_d = None
num_computed_tokens_p = None
preallocated_ssm_out_d, preallocated_ssm_out_p = torch.split(
output[:num_actual_tokens],
[num_decode_tokens, num_prefill_tokens],
@@ -843,11 +811,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
# to by "state_indices_tensor_p".
# In particular, it will always write the state at the
# sequence end.
# In addition, "block_idx_first_scheduled_token_p" and
# "block_idx_last_scheduled_token_p"
# are provided (which are pointers into
# "state_indices_tensor_p"), it will write additional cache
# states aligned at "block_size_to_align".
x = hidden_states_B_C_p.transpose(
0, 1
) # this is the form that causal-conv see
@@ -859,11 +822,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
query_start_loc=query_start_loc_p,
).transpose(0, 1)[:num_prefill_tokens]
@@ -876,14 +834,9 @@ class MambaMixer2(MambaBase, PluggableLayer):
initial_states = None
if has_initial_states_p is not None and prep_initial_states:
assert state_indices_tensor_p is not None
kernel_ssm_indices = state_indices_tensor_p
if is_mamba_cache_all:
kernel_ssm_indices = state_indices_tensor_p.gather(
1, block_idx_last_computed_token_p.unsqueeze(1)
).squeeze(1)
initial_states = torch.where(
has_initial_states_p[:, None, None, None],
ssm_state[kernel_ssm_indices],
ssm_state[state_indices_tensor_p],
0,
)
@@ -906,128 +859,56 @@ class MambaMixer2(MambaBase, PluggableLayer):
cu_chunk_seqlens=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
initial_states=initial_states,
return_intermediate_states=is_mamba_cache_all,
dt_softplus=True,
dt_limit=(0.0, float("inf")),
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
state_dtype=ssm_state.dtype,
)
if is_mamba_cache_all:
assert mamba_block_size is not None
assert state_indices_tensor_p is not None
assert block_idx_first_scheduled_token_p is not None
assert block_idx_last_scheduled_token_p is not None
assert last_chunk_indices_p is not None
assert num_computed_tokens_p is not None
# The chunk_stride is the number of chunks per mamba block
# e.g., if mamba_block_size = 512 and chunk_size = 256,
# then chunk_stride = 2
chunk_stride = mamba_block_size // chunk_size
# Save state for sequences with more than just final state,
# batched over sequences so that the number of kernel launches
# does not scale with the number of prefills.
n_blocks_to_fill = (
block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p
).clamp(min=0)
total_fill = attn_metadata.num_state_writes_p
if total_fill > 0:
first_chunk = torch.zeros_like(last_chunk_indices_p)
first_chunk[1:] = last_chunk_indices_p[:-1] + 1
# The chunk that completes this sequence's first mamba block.
# A block spans chunk_stride chunks. When the sequence resumed
# mid-block, the chunks before this step are not in varlen_states,
# so subtract them: the boundary sits earlier by their count.
num_unaligned_computed_tokens = (
num_computed_tokens_p % mamba_block_size
)
unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size
first_aligned_chunk = (
first_chunk + (chunk_stride - 1) - unaligned_chunk_shift
)
# Flatten all writes into one list: fill_counts=[2,1,2]
# -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1]
fill_counts = n_blocks_to_fill.long()
# output_size, or repeat_interleave reads the size off the device
seq_ids = torch.repeat_interleave(
torch.arange(num_prefills, device=ssm_state.device),
fill_counts,
output_size=total_fill,
)
write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts
write_offsets = torch.arange(
total_fill, device=ssm_state.device
) - torch.repeat_interleave(
write_starts, fill_counts, output_size=total_fill
)
cache_blocks_to_fill = state_indices_tensor_p[
seq_ids,
block_idx_first_scheduled_token_p[seq_ids] + write_offsets,
]
# Write the states. Each row goes to its own cache block,
# so the indexed copy has no duplicate destinations.
ssm_state[cache_blocks_to_fill] = varlen_states[
first_aligned_chunk[seq_ids] + write_offsets * chunk_stride
]
# For all seqs, store the last state (note: might be partial):
assert state_indices_tensor_p is not None
ssm_state[
state_indices_tensor_p.gather(
1, block_idx_last_scheduled_token_p.unsqueeze(1)
).squeeze(1)
] = varlen_states[last_chunk_indices_p]
else:
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_indices_tensor_p,
)
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_indices_tensor_p,
)
# Process decode requests
if has_decode:
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
if self.num_spec > 0:
assert block_idx_last_scheduled_token_prev_step_d is not None
input_indices = (
block_idx_last_scheduled_token_prev_step_d.unsqueeze(1)
+ self._decode_state_offsets
)
output_indices = (
block_idx_last_scheduled_token_d.unsqueeze(1)
+ self._decode_state_offsets
)
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, input_indices
)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, output_indices
)
else:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
# Without caching, read and write in-place to the same blocks:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
if (
self.use_replayssm
and self.mamba_config.backend == MambaBackendEnum.FLASHINFER
and use_spec_decode
and self._commits_replayssm_trackers
):
assert ring_start is not None
assert prev_num_accepted is not None
assert prev_query_len is not None
assert replayssm_state_indices_d is not None
assert num_accepted_tokens is not None
assert query_start_loc_d is not None
assert x_cache is not None
assert self.replayssm_buffer_len is not None
# The previous MTP query's accepted prefix is known only now.
# Commit it once, before this cache group's layers evaluate the
# current speculative query with the shared ring positions.
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
replayssm_state_indices_d,
num_accepted_tokens,
query_start_loc_d,
logical_window=self.replayssm_buffer_len,
ring_buffer_len=x_cache.size(2),
)
# 2. Convolution sequence transformation
hidden_states_B_C_d = causal_conv1d_update(
@@ -1037,11 +918,15 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
max_query_len=state_indices_tensor_d.size(-1),
# ReplaySSM keeps one physical state block while a speculative
# decode call still processes the full target + draft window.
max_query_len=(
1 + self.num_spec
if self.use_replayssm and use_spec_decode
else state_indices_tensor_d.size(-1)
),
)
hidden_states_d, B_d, C_d = self.split_hidden_states_B_C_fn(
@@ -1077,7 +962,35 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
assert ring_start is not None
assert prev_num_accepted is not None
assert prev_query_len is not None
assert attn_metadata.replayssm_scratch is not None
fi_cu_seqlens = query_start_loc_d
fi_max_seqlen = None
if use_spec_decode:
spec_query_len = 1 + self.num_spec
fi_max_seqlen = spec_query_len
assert replayssm_state_indices_d is not None
decode_batch = replayssm_state_indices_d.size(0)
if num_decode_tokens == decode_batch * spec_query_len:
hidden_states_d = _view_mtp_decode_tensor(
hidden_states_d, decode_batch, spec_query_len
)
dt_d = _view_mtp_decode_tensor(
dt_d, decode_batch, spec_query_len
)
B_d = _view_mtp_decode_tensor(
B_d, decode_batch, spec_query_len
)
C_d = _view_mtp_decode_tensor(
C_d, decode_batch, spec_query_len
)
preallocated_ssm_out_d = _view_mtp_decode_tensor(
preallocated_ssm_out_d,
decode_batch,
spec_query_len,
)
fi_cu_seqlens = None
fi_max_seqlen = None
selective_state_update_replayssm_flashinfer(
ssm_state,
hidden_states_d,
@@ -1091,19 +1004,25 @@ class MambaMixer2(MambaBase, PluggableLayer):
dt_cache,
ring_start,
prev_num_accepted,
prev_query_len,
logical_window=self.replayssm_buffer_len,
D=D_d,
dt_bias=dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=replayssm_state_indices_d,
scratch=attn_metadata.replayssm_scratch,
update_trackers=self._updates_replayssm_trackers,
update_trackers=(
self._updates_replayssm_trackers and not use_spec_decode
),
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
),
stochastic_rounding_philox_rounds=(
self.mamba_config.stochastic_rounding_philox_rounds
),
cu_seqlens=fi_cu_seqlens,
max_seqlen=fi_max_seqlen,
enable_pdl=False,
)
else:
selective_state_update_replayssm_output_only(
@@ -1123,7 +1042,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
write_pos=attn_metadata.write_pos_d,
is_flush=attn_metadata.is_flush_d,
max_cache_len=self.replayssm_buffer_len,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
@@ -1143,8 +1062,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
D_d,
dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=query_start_loc_d,
@@ -1185,6 +1104,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
tp_world_size=tp_world_size,
logical_window=self.replayssm_buffer_len,
backend=self.mamba_config.backend,
num_speculative_tokens=self.num_spec,
)
return base_shape
@@ -1202,9 +1122,10 @@ def share_replayssm_ring_trackers(
Layers backed by one KV-cache group use the same physical block indices and
can therefore share cursors. Different KV-cache groups may assign different
block indices to the same request and must keep separate cursor tensors.
The final local layer in each group advances its cursors after every layer
in that group has consumed the previous values.
block indices to the same request and must keep separate cursor tensors. For
speculative decode, the first local layer commits the preceding acceptance;
for standard decode, the final local layer advances after all layers consume
the previous values.
"""
replayssm_mixers: dict[str, MambaMixer2] = {}
for layer_name in ordered_layer_names:
@@ -1230,6 +1151,7 @@ def share_replayssm_ring_trackers(
groups_by_namespace.setdefault(namespace, []).append(layer_name)
for group_layer_names in groups_by_namespace.values():
first_layer_name = group_layer_names[0]
last_layer_name = group_layer_names[-1]
first_mixer = replayssm_mixers[group_layer_names[0]]
@@ -1244,11 +1166,17 @@ def share_replayssm_ring_trackers(
ring_start = torch.zeros(num_blocks, dtype=torch.int32, device=device)
prev_num_accepted = torch.zeros_like(ring_start)
prev_query_len = torch.zeros_like(ring_start)
for layer_name in group_layer_names:
mixer = replayssm_mixers[layer_name]
mixer._replayssm_ring_start = ring_start
mixer._replayssm_prev_num_accepted = prev_num_accepted
mixer._updates_replayssm_trackers = layer_name == last_layer_name
mixer._replayssm_prev_query_len = prev_query_len
mixer._commits_replayssm_trackers = False
mixer._updates_replayssm_trackers = False
replayssm_mixers[first_layer_name]._commits_replayssm_trackers = True
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
def mamba_mixer2(
@@ -227,6 +227,7 @@ class MambaStateShapeCalculator:
tp_world_size: int,
logical_window: int,
backend: MambaBackendEnum,
num_speculative_tokens: int = 0,
) -> tuple[
tuple[int, int],
tuple[int, int, int],
@@ -241,8 +242,8 @@ class MambaStateShapeCalculator:
"""
ring_buffer_len = logical_window
if backend == MambaBackendEnum.FLASHINFER:
# FlashInfer keeps the live window and appended token together.
ring_buffer_len += 1
# FlashInfer keeps the live window and current verify window together.
ring_buffer_len += 1 + num_speculative_tokens
local_nheads, head_dim, state_size = base_shapes[1]
local_ngroups = divide(n_groups, tp_world_size)
return (
@@ -14,7 +14,6 @@ from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
)
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_cpu,
causal_conv1d_update_torch,
)
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import (
@@ -56,7 +55,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices: torch.Tensor,
has_initial_state: torch.Tensor,
silu_activation: bool,
is_vnni: bool,
is_weight_packed: bool,
) -> torch.Tensor:
if not is_conv_state_dim_first():
return ops.causal_conv1d_fwd_cpu(
@@ -68,7 +67,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=silu_activation,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
scratch, identity_indices = _stage_ds_conv_states(conv_states, cache_indices)
@@ -81,7 +80,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices=identity_indices,
has_initial_state=has_initial_state,
silu_activation=silu_activation,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
_scatter_ds_conv_states(conv_states, cache_indices, scratch)
return out
@@ -95,7 +94,7 @@ def _causal_conv1d_update_cpu(
bias: torch.Tensor | None,
silu_activation: bool,
conv_state_indices: torch.Tensor,
is_vnni: bool,
is_weight_packed: bool,
num_accepted_tokens: torch.Tensor | None = None,
) -> torch.Tensor:
if not is_conv_state_dim_first():
@@ -106,7 +105,7 @@ def _causal_conv1d_update_cpu(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
@@ -118,13 +117,20 @@ def _causal_conv1d_update_cpu(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=identity_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
_scatter_ds_conv_states(conv_states, conv_state_indices, scratch)
return out
def is_arm_bf16() -> bool:
return (
current_platform.get_cpu_architecture() == CpuArchEnum.ARM
and torch.cpu.get_capabilities().get("bf16", False)
)
def cpu_gdn_attention_core(
mixed_qkv: torch.Tensor,
b: torch.Tensor,
@@ -201,9 +207,10 @@ def _cpu_gdn_attention_nonspec(
assert state_indices_tensor is not None
assert query_start_loc is not None
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
use_cpp_conv = torch.cpu._is_avx512_bf16_supported()
# C++ conv (conv.cpp) runs on aarch64 with bf16 support and on x86 uses VDPBF16PS,
# not AMX tiles, so it runs on any AVX-512BF16 CPU. The weights are packed on this
# same predicate at load time.
use_cpp_conv = torch.cpu._is_avx512_bf16_supported() or is_arm_bf16()
conv_state = layer.kv_cache[0]
if use_cpp_conv:
@@ -244,28 +251,17 @@ def _cpu_gdn_attention_nonspec(
bias=layer.conv1d.bias,
silu_activation=(layer.activation == "silu"),
conv_state_indices=decode_state_indices,
is_vnni=True,
is_weight_packed=True,
)
else:
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
decode_conv_state = conv_state[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
@@ -315,7 +311,7 @@ def _cpu_gdn_attention_nonspec(
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
prefill_mixed_qkv = causal_conv1d_torch(
@@ -383,9 +379,9 @@ def _ssm_state_view(layer) -> torch.Tensor:
def _unpacked_conv_weight(layer) -> torch.Tensor:
"""Return the plain (dim, width) conv weight.
On AMX the conv1d weight is VNNI-packed in place at load time (only usable
by the AMX C++ kernel), so the torch spec-decode path relies on the
un-packed copy stashed by ``dispatch_cpu_unquantized_gemm``.
On AMX and aarch64+bf16 the conv1d weight is packed in place at load time,
so the torch spec-decode path relies on the un-packed copy stashed by
``dispatch_cpu_unquantized_gemm``.
"""
w = getattr(layer.conv1d, "_cpu_unpacked_conv_weight", None)
if w is not None:
@@ -505,13 +501,15 @@ def _spec_forward(
seq_starts = spec_qsl_cpu[:-1]
seq_lens = spec_qsl_cpu[1:] - spec_qsl_cpu[:-1]
is_amx = torch.cpu._is_amx_tile_supported()
# ---- 1. Convolution (per-sequence rolling buffer) ----
dim = mixed_qkv_spec.size(-1)
bias = layer.conv1d.bias
silu = layer.activation == "silu"
can_use_native_conv = (
torch.cpu._is_amx_tile_supported()
(is_amx or is_arm_bf16())
and width == 4
and num_spec_decodes > 0
and bool(torch.all(seq_lens == seq_lens[0]).item())
@@ -528,7 +526,7 @@ def _spec_forward(
conv_state_indices=spec_state_indices[:num_spec_decodes, 0]
.to("cpu", torch.int32)
.contiguous(),
is_vnni=True,
is_weight_packed=True,
num_accepted_tokens=num_accepted[:num_spec_decodes].to("cpu", torch.int32),
).view_as(mixed_qkv_spec)
else:
@@ -603,9 +601,9 @@ def _spec_aware_nonspec(
assert query_start_loc is not None
state_indices_tensor = state_indices_tensor.contiguous()
is_amx = torch.cpu._is_amx_tile_supported()
can_use_native_conv = torch.cpu._is_amx_tile_supported() or is_arm_bf16()
if not is_amx:
if not can_use_native_conv:
conv_weights = _unpacked_conv_weight(layer)
num_decodes = attn_metadata_i.num_decodes
@@ -618,7 +616,8 @@ def _spec_aware_nonspec(
decode_b = b[:num_decode_tokens]
decode_a = a[:num_decode_tokens]
decode_state_indices = state_indices_tensor[:num_decodes]
if is_amx:
if can_use_native_conv:
decode_mixed_qkv = _causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_states=conv_buf,
@@ -626,30 +625,19 @@ def _spec_aware_nonspec(
bias=layer.conv1d.bias,
silu_activation=layer.activation == "silu",
conv_state_indices=decode_state_indices,
is_vnni=True,
is_weight_packed=True,
)
else:
# Only the first ``width-1`` columns hold the real conv state.
conv_state_view = conv_buf[:, :, : width - 1]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state_view[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state_view,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state_view,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu(
@@ -685,7 +673,8 @@ def _spec_aware_nonspec(
prefill_has_initial_state = has_initial_state[
num_decodes : num_decodes + num_prefills
]
if is_amx:
if can_use_native_conv:
prefill_mixed_qkv = _causal_conv1d_fwd_cpu(
x=prefill_mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
@@ -695,7 +684,7 @@ def _spec_aware_nonspec(
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
prefill_mixed_qkv = causal_conv1d_torch(
@@ -757,8 +746,9 @@ def _spec_aware_nonspec_subset(
prefill_state_indices = prefill_state_indices.contiguous()
is_amx = torch.cpu._is_amx_tile_supported()
can_use_native_conv = is_amx or is_arm_bf16()
if is_amx:
if can_use_native_conv:
conv_out = _causal_conv1d_fwd_cpu(
x=mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
@@ -768,7 +758,7 @@ def _spec_aware_nonspec_subset(
cache_indices=prefill_state_indices,
has_initial_state=has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
conv_weights = _unpacked_conv_weight(layer)
@@ -31,6 +31,7 @@ logger = init_logger(__name__)
def _update_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
state_batch_indices_stride,
n_slots,
@@ -52,6 +53,9 @@ def _update_replayssm_ring_trackers_kernel(
if RESET:
tl.store(ring_start + slots, 0, mask=valid)
tl.store(prev_num_accepted + slots, 0, mask=valid)
# MTP alone reads prev_query_len. Standard decode deliberately leaves
# it unchanged; reset clears the pending MTP window after prefill.
tl.store(prev_query_len + slots, 0, mask=valid)
else:
prev = tl.load(prev_num_accepted + slots, mask=valid, other=0)
start = tl.load(ring_start + slots, mask=valid, other=0)
@@ -66,9 +70,70 @@ def _update_replayssm_ring_trackers_kernel(
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
@triton.jit(
do_not_specialize=["n_slots", "state_batch_indices_stride"],
do_not_specialize_on_alignment=["state_batch_indices"],
)
def _commit_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
num_accepted_tokens,
query_start_loc,
state_batch_indices_stride,
n_slots,
num_states,
logical_window: tl.constexpr,
ring_buffer_len: tl.constexpr,
pad_slot_id: tl.constexpr,
BLOCK: tl.constexpr,
) -> None:
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < n_slots
slots = tl.load(
state_batch_indices + offsets * state_batch_indices_stride,
mask=mask,
other=pad_slot_id,
)
valid = mask & (slots != pad_slot_id) & (slots >= 0) & (slots < num_states)
accepted_since_checkpoint = tl.load(prev_num_accepted + slots, mask=valid, other=0)
current_ring_start = tl.load(ring_start + slots, mask=valid, other=0)
previous_speculative_query_len = tl.load(
prev_query_len + slots, mask=valid, other=0
)
accepted_from_previous_query = tl.load(
num_accepted_tokens + offsets, mask=mask, other=0
)
must_checkpoint = (previous_speculative_query_len > 0) & (
accepted_since_checkpoint + previous_speculative_query_len > logical_window
)
next_start = tl.where(
must_checkpoint,
(current_ring_start + accepted_since_checkpoint) % ring_buffer_len,
current_ring_start,
)
next_prev = tl.where(
previous_speculative_query_len == 0,
0,
tl.where(
must_checkpoint,
accepted_from_previous_query,
accepted_since_checkpoint + accepted_from_previous_query,
),
)
current_speculative_query_len = tl.load(
query_start_loc + offsets + 1, mask=mask, other=0
) - tl.load(query_start_loc + offsets, mask=mask, other=0)
tl.store(ring_start + slots, next_start, mask=valid)
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
tl.store(prev_query_len + slots, current_speculative_query_len, mask=valid)
def update_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
logical_window: int | None = None,
ring_buffer_len: int | None = None,
@@ -90,10 +155,15 @@ def update_replayssm_ring_trackers(
_update_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
state_batch_indices.stride(0),
n_slots,
min(ring_start.numel(), prev_num_accepted.numel()),
min(
ring_start.numel(),
prev_num_accepted.numel(),
prev_query_len.numel(),
),
logical_window,
ring_buffer_len,
pad_slot_id,
@@ -105,6 +175,7 @@ def update_replayssm_ring_trackers(
def reset_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
@@ -112,11 +183,62 @@ def reset_replayssm_ring_trackers(
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
pad_slot_id=pad_slot_id,
)
def commit_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
num_accepted_tokens: torch.Tensor,
query_start_loc: torch.Tensor,
logical_window: int,
ring_buffer_len: int,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
"""Commit the preceding speculative window and record the current one.
MTP evaluates a target token and its draft tokens together, but the number
accepted from that query is available only on the next forward pass. This
function then advances the per-request ring by the accepted prefix and
records the current query length for the following pass. A zero
``prev_query_len`` means that reset/prefill left no prior MTP query to
commit; slot validity is handled independently by the kernel mask.
Standard single-token decode needs no delayed commit: its ReplaySSM kernel
advances the shared ring trackers directly after every token.
"""
if state_batch_indices.dim() > 1:
state_batch_indices = state_batch_indices[:, 0]
n_slots = state_batch_indices.numel()
if n_slots == 0:
return
block = 128
_commit_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
num_accepted_tokens,
query_start_loc,
state_batch_indices.stride(0),
n_slots,
min(
ring_start.numel(),
prev_num_accepted.numel(),
prev_query_len.numel(),
),
logical_window,
ring_buffer_len,
pad_slot_id,
BLOCK=block,
)
class MambaSSUBackend(ABC):
"""Abstract base class for Mamba SSU backends."""
@@ -385,6 +507,7 @@ def selective_state_update_replayssm_flashinfer(
dt_cache: torch.Tensor,
ring_start: torch.Tensor,
prev_num_accepted_tokens: torch.Tensor,
prev_query_len: torch.Tensor,
logical_window: int,
D: torch.Tensor | None = None,
dt_bias: torch.Tensor | None = None,
@@ -395,6 +518,9 @@ def selective_state_update_replayssm_flashinfer(
update_trackers: bool = True,
enable_stochastic_rounding: bool = False,
stochastic_rounding_philox_rounds: int = 0,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: int | None = None,
enable_pdl: bool = False,
) -> torch.Tensor:
"""Run FlashInfer checkpointing SSU and optionally advance shared trackers."""
if _flashinfer_replayssm_kernel is None:
@@ -404,11 +530,12 @@ def selective_state_update_replayssm_flashinfer(
)
if x.dim() == 3:
x = x.unsqueeze(1)
dt = dt.unsqueeze(1)
B = B.unsqueeze(1)
C = C.unsqueeze(1)
out = out.unsqueeze(1)
dim = 0 if cu_seqlens is not None else 1
x = x.unsqueeze(dim)
dt = dt.unsqueeze(dim)
B = B.unsqueeze(dim)
C = C.unsqueeze(dim)
out = out.unsqueeze(dim)
indices = state_batch_indices
if indices is not None and indices.dim() > 1:
@@ -443,6 +570,9 @@ def selective_state_update_replayssm_flashinfer(
pad_slot_id=null_block_id,
rand_seed=rand_seed,
philox_rounds=stochastic_rounding_philox_rounds or 10,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
enable_pdl=enable_pdl,
cb_scaled=cb_scaled,
cumAdt_vec=cumAdt_vec,
cb_old=cb_old,
@@ -451,6 +581,7 @@ def selective_state_update_replayssm_flashinfer(
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted_tokens,
prev_query_len,
indices,
logical_window=logical_window,
ring_buffer_len=x_cache.size(2),
@@ -1208,9 +1208,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
"""Override the default vLLM placeholder scales with the llm-compressor loaded
scales. Zero points are not used as only symmetric quantization is supported.
"""
layer._k_scale = layer.k_scale
layer._v_scale = layer.v_scale
layer._q_scale = layer.q_scale
# Buffers (not Parameters) so sleep snapshots them; ATTN_HEAD changes shape.
layer.register_buffer("_k_scale", layer.k_scale.detach())
layer.register_buffer("_v_scale", layer.v_scale.detach())
layer.register_buffer("_q_scale", layer.q_scale.detach())
# Set the _float variants that the attention backend uses.
def _to_scalar(tensor: torch.Tensor) -> float:
@@ -210,6 +210,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
# Per-expert biases, as gpt-oss carries.
if self.moe.has_bias:
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
w13_num_shards * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
requires_grad=False,
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
self.group_size = -1
@@ -350,7 +371,41 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
routing_tables=layer._expert_routing_tables(),
)
if self.wna16_backend == WNA16MoEBackend.ZEN_CPU:
# Monolithic experts route inside apply(), so like the unquantized
# CPU path they need the router config that only the layer carries.
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
swigluoai_w13_interleave_perm,
)
# Reorder before the conversion below, which repacks w13 for the kernel.
# The CT WNA16 buffers are N-first, so 2I is dim 1 of the packed weight
# [E, 2I, K/pack] and of its group scales [E, 2I, G], as for the bias.
perm = swigluoai_w13_interleave_perm(
self.experts_cls,
self.moe.activation,
layer.w13_weight_packed.size(1),
layer.w13_weight_packed.device,
)
if perm is not None:
replace_parameter(
layer,
"w13_weight_packed",
layer.w13_weight_packed.data[:, perm].contiguous(),
)
replace_parameter(
layer,
"w13_weight_scale",
layer.w13_weight_scale.data[:, perm].contiguous(),
)
if self.moe.has_bias:
replace_parameter(
layer, "w13_bias", layer.w13_bias.data[:, perm].contiguous()
)
# Process weights using the shared oracle infrastructure
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
@@ -363,6 +418,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_scale=layer.w2_weight_scale,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
if converted is None:
@@ -378,8 +435,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
w13_bias,
w2_bias,
) = converted
# Replace common parameters
@@ -388,6 +445,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
if w13_bias is not None:
replace_parameter(layer, "w13_bias", w13_bias)
if w2_bias is not None:
replace_parameter(layer, "w2_bias", w2_bias)
# CPU fused_experts_cpu requires zero points even for symmetric quant.
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
if (
@@ -437,6 +499,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
num_bits=self.num_bits,
w1_zp=getattr(layer, "w13_weight_zero_point", None),
w2_zp=getattr(layer, "w2_weight_zero_point", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
@@ -736,6 +736,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w2_bias=w2_bias,
_cache_permute_indices=self._cache_permute_indices,
activation=self.moe.activation,
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
)
)
@@ -220,6 +220,7 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
w2_weight_scale=w2_scale,
w13_bias=w13_bias,
w2_bias=w2_bias,
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
)
)
@@ -9,8 +9,8 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_hidden_dim_for_fi,
align_fp4_moe_weights_for_fi,
align_trtllm_fp4_moe_hidden_dim_for_fi,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
swizzle_blockscale,
@@ -128,9 +128,9 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
]:
"""Prepare weights for the CuteDSL wrapper-based NvFP4 MoE backend.
Converts weight scale factors to MMA layout expected by CuteDslMoEWrapper,
and interleaves w13 gate/linear rows for gated activations. Non-gated
activations use a single w13 projection and keep its row order unchanged.
Pads the runtime expert tensors to the kernel's GEMM alignment, converts
weight scale factors to the MMA layout expected by CuteDslMoEWrapper, and
interleaves w13 gate/linear rows for gated activations.
"""
# Global scaling factors (same as other FlashInfer backends).
num_experts = w13.shape[0]
@@ -140,12 +140,38 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
)
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(num_experts)
if layer.activation.is_gated:
gated = layer.activation.is_gated
if gated:
w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
layer.activation, w13, w13_scale
)
# Interleave up/gate rows for w13 weights and scales.
# GEMM1's output dimension must be a multiple of 128: 2I for gated
# activations (also required by interleaving), but only I for non-gated.
# Keep the checkpoint tensors unchanged and pad only the kernel's runtime
# representation. Zero rows also make the padded GEMM2 contraction a no-op.
w13, w13_scale, w2, w2_scale, padded_intermediate = align_fp4_moe_weights_for_fi(
w13,
w13_scale,
w2,
w2_scale,
is_act_and_mul=gated,
min_alignment=64 if gated else 128,
)
layer.moe_config.intermediate_size_per_partition = padded_intermediate
# GEMM1 gathers full 256-element K tiles of activations and their scales
# without a K-tail predicate. Pad runtime weights to match; the MoE runner
# pads activations and trims the output using the original hidden size.
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
layer.moe_config.hidden_dim = padded_hidden
if gated:
# Interleave up/gate rows for the fused gated activation.
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
@@ -371,8 +397,8 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
w13, w13_scale, w2, w2_scale, padded_hidden = (
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
@@ -13,6 +13,8 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
def activation_to_flashinfer_int(activation: MoEActivation) -> int:
return activation_to_flashinfer_type(activation).value
@@ -50,6 +52,20 @@ def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType"
return ACTIVATION_TO_FI_ACTIVATION[activation]
def quantize_nvfp4_per_token_input(
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Quantize NVFP4 activations with one FP32 decode scale per token."""
from flashinfer import SfLayout, nvfp4_quantize
return nvfp4_quantize(
hidden_states,
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
)
def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
return (
x.reshape(-1, 2, x.shape[-2] // 2, x.shape[-1]).flip(dims=[1]).reshape(x.shape)
@@ -264,7 +280,7 @@ def align_fp4_moe_weights_for_fi(
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
def align_trtllm_fp4_moe_hidden_dim_for_fi(
def align_fp4_moe_hidden_dim_for_fi(
w13: torch.Tensor,
w13_scale: torch.Tensor,
w2: torch.Tensor,
@@ -279,7 +295,7 @@ def align_trtllm_fp4_moe_hidden_dim_for_fi(
return w13, w13_scale, w2, w2_scale, hidden_size
logger.warning_once(
"Padding hidden size from %d to %d for TRTLLM NVFP4 MoE weights. "
"Padding hidden size from %d to %d for FlashInfer NVFP4 MoE weights. "
"This requires activation slicing at runtime and may cause "
"performance degradation.",
hidden_size,
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse attention indexer layers on aiter's paged MXFP4 kernels, for ROCm.
With ``indexer_kv_dtype="mxfp4"`` on gfx950, the model writes the indexer K
cache in the order aiter's MQA-logits kernel reads (see
`rocm_paged_mxfp4_cache_layout`), and the kernel scores the cache in place.
Requires `RocmMxfp4IndexerMetadataBuilder` metadata.
"""
import torch
from torch import nn
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import (
rocm_mxfp4_sparse_attn_indexer,
rocm_mxfp4_sparse_mqa_indexer,
)
class RocmSparseAttnIndexer(SparseAttnIndexer):
"""`SparseAttnIndexer` whose HIP path walks the paged MXFP4 cache with
aiter's kernel. A two-level indexer's source layer also publishes its
candidate pool here."""
def forward_hip(
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
):
assert isinstance(q_quant, tuple) and self.skip_k_cache_insert, (
"the ROCm MXFP4 indexer takes (values, scales) Q and a K cache "
"the model already wrote"
)
q_values, q_scale = q_quant
return rocm_mxfp4_sparse_attn_indexer(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_values,
q_scale,
weights,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.topk_indices_buffer,
compress_ratio=self.compress_ratio,
candidate_blocks=self.candidate_blocks,
candidate_block_size=self.candidate_block_size,
candidate_write=self.candidate_write,
)
class RocmSparseMQAIndexer(nn.Module):
"""DeepSeek-V4.1's candidate-consuming indexer on aiter's paged MXFP4
MQA-logits kernel: the consumers score only the candidate pool the source
layer published, read straight from the paged cache.
Built and called as `SparseMQAIndexer` is, so the model can take either.
"""
weights_dtype = torch.float32
"""The scorer's weights and logits stay fp32: gfx950 has no packed bf16
arithmetic, so its head reduce costs the same in either dtype."""
def __init__(
self,
k_cache,
topk_tokens: int,
head_dim: int,
max_total_seq_len: int,
topk_indices_buffer: torch.Tensor,
candidate_blocks: torch.Tensor,
candidate_block_size: int,
):
super().__init__()
self.k_cache = k_cache
self.topk_tokens = topk_tokens
self.head_dim = head_dim
# aiter's kernel bounds a row by the model length in compressed
# positions, not by the prefill buffer SparseMQAIndexer sizes.
model_config = get_current_vllm_config().model_config
self.max_model_len = model_config.max_model_len // k_cache.compress_ratio
self.topk_indices_buffer = topk_indices_buffer
self.candidate_blocks = candidate_blocks
self.candidate_block_size = candidate_block_size
self.num_candidate_cols = candidate_blocks.shape[1] * candidate_block_size
def forward(
self,
hidden_states: torch.Tensor,
q_quant: tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
) -> torch.Tensor:
assert k is None, "the model writes the indexer K cache"
q_values, q_scale = q_quant
return rocm_mxfp4_sparse_mqa_indexer(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_values,
q_scale,
weights,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.topk_indices_buffer,
self.k_cache.compress_ratio,
self.candidate_blocks,
self.candidate_block_size,
self.num_candidate_cols,
)
@@ -335,6 +335,7 @@ def sparse_attn_indexer(
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_pcp: bool,
pcp_shard_decode_requests: bool,
dense_mha_metadata_layer_name: LayerNameType,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
@@ -404,6 +405,7 @@ def sparse_attn_indexer(
topk_indices_buffer,
skip_k_cache_insert,
use_pcp,
pcp_shard_decode_requests,
dense_mha_metadata_layer_name,
use_fp4_cache,
candidate_blocks=candidate_blocks,
@@ -442,6 +444,7 @@ def sparse_attn_indexer(
slot_mapping,
num_decode_tokens,
use_pcp,
pcp_shard_decode_requests=pcp_shard_decode_requests,
)
# scale_fmt can be None, but the function expects str
assert scale_fmt is not None
@@ -817,6 +820,7 @@ def sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_pcp: bool,
pcp_shard_decode_requests: bool,
dense_mha_metadata_layer_name: LayerNameType,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
@@ -898,6 +902,7 @@ class SparseAttnIndexer(CustomOp):
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
self._cp_kv_cache_interleave_size: int | None = None
if current_platform.is_cuda() and not has_deep_gemm():
raise RuntimeError(
@@ -994,6 +999,7 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer,
self.skip_k_cache_insert,
self.use_pcp,
self.pcp_shard_decode_requests,
_encode_layer_name(self.dense_mha_metadata_layer_name),
self.use_fp4_cache,
self.dcp_rank,
+22 -8
View File
@@ -475,13 +475,27 @@ def dispatch_cpu_unquantized_gemm(
# Skip CPU GEMM dispatch for non-2D weights (e.g. MoE 3D expert weights).
# These layers are handled by their own specialized methods.
if layer.weight.ndim != 2:
# this is not a linear layer
# For now it should be a causal_conv1d op or MoE 3D expert weights
# The C++ causal_conv1d kernels use VDPBF16PS (no AMX tiles), so the
# VNNI weight prepack applies to any AVX-512BF16 CPU, not just AMX
# (e.g. AMD Zen5/Turin).
if torch.cpu._is_avx512_bf16_supported() and hasattr(
ops, "causal_conv1d_weight_pack"
# This is not a linear layer.
# For now it should be a causal_conv1d op or MoE 3D expert weights.
# Causal-conv weights use [dim, 1, width].
# The C++ causal_conv1d kernels support aarch64 with bf16 and
# x86 CPUs that use VDPBF16PS (no AMX tiles). So the
# weight prepack applies to aarch64 with bf16 and any AVX-512BF16 CPU,
# not just AMX (e.g. AMD Zen5/Turin).
is_causal_conv1d_weight = (
layer.weight.ndim == 3
and layer.weight.size(1) == 1
and layer.weight.size(2) == 4
and layer.weight.dtype == torch.bfloat16
)
is_arm_bf16 = (
current_platform.get_cpu_architecture() == CpuArchEnum.ARM
and torch.cpu.get_capabilities().get("bf16", False)
)
if (
(is_arm_bf16 or torch.cpu._is_avx512_bf16_supported())
and is_causal_conv1d_weight
and hasattr(ops, "causal_conv1d_weight_pack")
):
# prepack conv weight
unpacked = (
@@ -493,7 +507,7 @@ def dispatch_cpu_unquantized_gemm(
.clone()
)
# Stash the un-packed (dim, width) weight so the speculative-decode
# GDN path (which uses torch conv, not the C++ kernel) can use it.
# and unsupported-layout fallbacks can still use ordinary weights.
layer._cpu_unpacked_conv_weight = unpacked
layer.weight.data = ops.causal_conv1d_weight_pack(unpacked)
return