[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -11,7 +11,6 @@ from vllm.model_executor.layers.attention.cross_attention import CrossAttention
from vllm.model_executor.layers.attention.encoder_only_attention import (
EncoderOnlyAttention,
)
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.attention.prefill_prefix_lm_attention import (
PrefillPrefixLMAttention,
@@ -35,7 +34,6 @@ __all__ = [
"ChunkedLocalAttention",
"CrossAttention",
"EncoderOnlyAttention",
"MLAAttention",
"MMEncoderAttention",
"PrefillPrefixLMAttention",
"RSWAAttention",
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,25 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.hpc.gated_mla import (
hpc_gated_mla_gemm,
hpc_gated_mla_supported,
)
from vllm.model_executor.layers.hpc.hpc_ihc import (
HpcIHCHead,
HpcIHCPost,
HpcIHCPre,
)
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy
__all__ = [
"HpcIHCHead",
"HpcIHCPost",
"HpcIHCPre",
"HpcModule",
"HpcRopeNorm",
"QkNormPolicy",
"hpc_gated_mla_gemm",
"hpc_gated_mla_supported",
]
-114
View File
@@ -1,114 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Enablement gate for the HPC fused gated-MLA GEMM.
``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the
sigmoid and the elementwise product -- into a single kernel launch, replacing
the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence.
Kernel constraints:
- Requires VLLM_ENABLE_HPC_OPS=1
- Requires the hpc package (.so) built for the current arch
- Only sm100 / sm103 (compute capability 100, 103)
- All three operands must be bfloat16 and contiguous
- The gate width must be a multiple of 256
- Only elementwise gating (headwise has no fused form)
"""
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.utils.hpc import has_hpc
logger = init_logger(__name__)
# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch.
# It is built for sm100/sm103 and needs all three operands bf16 + contiguous,
# with the gate width a multiple of 256.
_HPC_GATED_GEMM_N_ALIGN = 256
_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103})
def hpc_gated_mla_supported(
gating_type: str | None, attn_output_gate: torch.nn.Module | None
) -> bool:
"""Gate for running the MLA output gating through hpc.gated_mla_gemm."""
if not envs.VLLM_ENABLE_HPC_OPS:
return False
if attn_output_gate is None:
return False
if not has_hpc():
logger.warning_once(
"HPC gated MLA disabled: 'hpc' package is not installed. "
"Please install the HPC library (.so) to enable the fused kernel."
)
return False
from vllm.platforms import current_platform
if not current_platform.is_cuda():
logger.warning_once("HPC gated MLA disabled: only CUDA is supported.")
return False
capability = current_platform.get_device_capability()
if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES:
logger.warning_once(
"HPC gated MLA disabled: compute capability %s not in %s.",
capability,
_HPC_GATED_GEMM_CAPABILITIES,
)
return False
# Headwise gating broadcasts one scalar per head over v_head_dim; the
# kernel only implements the elementwise product.
if gating_type != "elementwise":
logger.warning_once(
"HPC gated MLA disabled: gating_type '%s' has no fused form.",
gating_type,
)
return False
weight = getattr(attn_output_gate, "weight", None)
if weight is None or weight.dtype != torch.bfloat16:
logger.warning_once(
"HPC gated MLA disabled: gate weight dtype is %s, the kernel needs "
"bfloat16 (keep the layer out of quantization).",
None if weight is None else weight.dtype,
)
return False
if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0:
logger.warning_once(
"HPC gated MLA disabled: gate width %d is not a multiple of %d.",
weight.shape[0],
_HPC_GATED_GEMM_N_ALIGN,
)
return False
logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.")
return True
def hpc_gated_mla_gemm(
x: torch.Tensor,
weight: torch.Tensor,
atten_output: torch.Tensor,
) -> torch.Tensor:
"""Fused ``atten_output * sigmoid(x @ weight.T)``.
Args:
x: ``[m, k]`` bfloat16, contiguous.
weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel
transposes it). ``n`` must be a multiple of 256.
atten_output: ``[m, n]`` bfloat16, contiguous.
Returns:
``[m, n]`` bfloat16.
"""
from hpc.gemm import gated_mla_gemm
return gated_mla_gemm(x, weight, atten_output)
-237
View File
@@ -1,237 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4.
Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies
with single-kernel HPC implementations:
pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum
post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :]
head : same as pre but hc_mult projection rows and no H_post output
The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is
a single kernel.
Constraints:
- Requires VLLM_ENABLE_HPC_OPS=1
- Requires the hpc package (.so) built for the current arch
- Only sm100 / sm103 (compute capability 100, 103)
- Only hc_mult == 4 and hidden_size in {4096, 6144}
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
keeps them out of fp8 quantization via modules_to_not_convert)
The fused modules own no parameters: their forwards read the weights of the
eager layer they replace directly via the captured owner reference.
NOTE: The reference implementation also offers a cross-layer post+pre fusion
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
It requires reworking the decoder-layer forward scheduling and is not ported
here. TODO: add it once the decoder dataflow is restructured for it.
"""
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.utils.hpc import has_hpc
logger = init_logger(__name__)
# Shapes the HPC kernels are instantiated for. Other shapes launch nothing,
# so gate on this rather than relying on a silent no-op.
_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144})
_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4})
_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103})
def _ihc_supported(hc_mult: int, hidden_size: int) -> bool:
"""Shared gate for all three iHC ops."""
if not envs.VLLM_ENABLE_HPC_OPS:
return False
if not has_hpc():
logger.warning_once(
"HPC iHC disabled: 'hpc' package is not installed. "
"Please install the HPC library (.so) to enable fused kernels."
)
return False
from vllm.platforms import current_platform
if not current_platform.is_cuda():
logger.warning_once("HPC iHC disabled: only CUDA is supported.")
return False
capability = current_platform.get_device_capability()
if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES:
logger.warning_once(
"HPC iHC disabled: compute capability %s not in %s.",
capability,
_SUPPORTED_CAPABILITIES,
)
return False
if hc_mult not in _SUPPORTED_HC_MULTS:
logger.warning_once(
"HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS
)
return False
if hidden_size not in _SUPPORTED_HIDDEN_SIZES:
logger.warning_once(
"HPC iHC disabled: hidden_size=%d not in %s.",
hidden_size,
_SUPPORTED_HIDDEN_SIZES,
)
return False
if envs.VLLM_BATCH_INVARIANT:
# Batch-invariant mode relies on torch's deterministic reductions; the
# HPC kernel reduces in a different order, so keep the two regimes
# apart.
logger.warning_once(
"HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT."
)
return False
logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.")
return True
class HpcIHCPre(HpcModule):
"""Fused iHC pre block.
Computes, in one kernel:
x_flat = x.flatten(1)
r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps)
mixes = (x_flat @ w.T) * r
H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps
H_post = magnitude * sigmoid(
mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps
y = sum_i H_pre[:, i] * x[:, i, :]
Args:
hc_mult: HC expand ratio.
hidden_size: Model hidden dimension.
magnitude: H_post multiplier (config.hc_magnitude).
hc_eps: Epsilon added to both gates (config.hc_eps).
norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps).
fallback_op: The eager HYV4HCPreLayer to source weights from.
norm_owner: Optional RMSNorm that immediately follows this pre block.
When given, its weight/eps are folded into the kernel so a single
launch covers pre + RMSNorm (the caller then skips the separate
layernorm). None keeps the original pre-only behaviour.
"""
def __init__(
self,
hc_mult: int,
hidden_size: int,
magnitude: float,
hc_eps: float,
norm_eps: float,
fallback_op: torch.nn.Module,
norm_owner: torch.nn.Module | None = None,
) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
self.magnitude = magnitude
self.hc_eps = hc_eps
self.norm_eps = norm_eps
# Stash the owner outside nn.Module's attribute machinery: callers pass
# their own `self` here, and registering that as a submodule would make
# the module tree cyclic (state_dict() then hits RecursionError).
object.__setattr__(self, "_fallback_op", fallback_op)
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
object.__setattr__(self, "_norm_owner", norm_owner)
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
import hpc
owner = self._fallback_op
norm_owner = self._norm_owner
return hpc.fuse_ihc_pre(
x,
owner.hc_fn.weight,
owner.hc_scale,
owner.hc_base,
self.norm_eps,
self.hc_eps,
self.magnitude,
norm_owner.weight if norm_owner is not None else None,
norm_owner.variance_epsilon if norm_owner is not None else 0.0,
)
class HpcIHCPost(HpcModule):
"""Fused iHC post block: H_post gating plus multi-channel residual add.
Unlike mHC there is no comb matrix, so each output channel only needs its
own residual channel and the whole thing is one fused multiply-add per
element.
"""
def __init__(self, hc_mult: int, hidden_size: int) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def forward(
self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor
) -> torch.Tensor:
import hpc
return hpc.fuse_ihc_post(x, residual, H_post)
class HpcIHCHead(HpcModule):
"""Fused iHC head block: merge the hc channels back into one hidden state.
Same structure as HpcIHCPre but the projection emits only hc_mult gate
logits and there is no H_post output. Called once per forward, after the
last decoder layer.
"""
def __init__(
self,
hc_mult: int,
hidden_size: int,
hc_eps: float,
norm_eps: float,
fallback_op: torch.nn.Module,
) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
self.hc_eps = hc_eps
self.norm_eps = norm_eps
# See HpcIHCPre: keep the owner out of the module tree.
object.__setattr__(self, "_fallback_op", fallback_op)
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def forward(self, x: torch.Tensor) -> torch.Tensor:
import hpc
owner = self._fallback_op
return hpc.fuse_ihc_head(
x,
owner.hc_head_fn.weight,
owner.hc_head_scale,
owner.hc_head_base,
self.norm_eps,
self.hc_eps,
)
@@ -1,15 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch.nn as nn
class HpcModule(nn.Module):
def __init__(self):
super().__init__()
@classmethod
def support(cls, *args, **kwargs):
return True
def forward(self, *args, **kwargs):
pass
-575
View File
@@ -1,575 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
Decoupled from HpcAttentionImpl; extra params are passed via layer attrs.
"""
from __future__ import annotations
import importlib.util
from enum import IntEnum
from typing import Any
import torch
from vllm.config import get_current_vllm_config_or_none
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.hpc_attn import HpcAttnMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
logger = init_logger(__name__)
_hpc_rope_norm_instances: dict[str, HpcRopeNorm] = {}
class QkNormPolicy(IntEnum):
"""Order of QK-RMSNorm relative to RoPE in the fused HPC rope_norm kernel.
The values are part of the HPC kernel ABI (passed through as ints), so they
must stay in sync with the kernel's expectations.
"""
# No QK-Norm: apply RoPE only.
NONE = 0
# Apply RoPE first, then QK-RMSNorm.
ROPE_THEN_NORM = 1
# Apply QK-RMSNorm first, then RoPE (e.g. HunYuan V3).
NORM_THEN_ROPE = 2
def hpc_rope_norm_forward(
qkv: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
"""Top-level custom op: RoPE + QK-Norm + KV-Cache-Write + FP8 Q quant.
Fully opaque to torch.compile (dynamo).
"""
forward_context: ForwardContext = get_forward_context()
attn_metadata: Any = forward_context.attn_metadata
if isinstance(attn_metadata, dict):
attn_metadata = attn_metadata[layer_name]
if attn_metadata is None:
output.zero_()
return
attn_layer = forward_context.no_compile_layers[layer_name]
# bind_kv_cache stores the per-layer KV cache as a single tensor
# (num_blocks, num_kv_heads, block_size, 2 * head_size), so use it directly.
kv_cache = attn_layer.kv_cache
if kv_cache.numel() == 0:
output.zero_()
return
assert kv_cache.dim() == 4, (
f"Expected kv_cache to have 4 dims, got {tuple(kv_cache.shape)}"
)
rope_norm = _hpc_rope_norm_instances[layer_name]
rope_norm._forward_impl(qkv, kv_cache, attn_metadata, attn_layer, output)
direct_register_custom_op(
op_name="hpc_rope_norm_forward",
op_func=hpc_rope_norm_forward,
mutates_args=["output"],
)
@CustomOp.register("hpc_rope_norm")
class HpcRopeNorm(CustomOp, HpcModule):
"""HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
Registered as a sub-module in model layers (e.g. HunYuanAttention).
The QK-Norm weights are read directly from the fallback norm modules
(built in float32 when HPC is active), eliminating the derived copy.
forward() is dispatched by CustomOp framework:
- In compiled mode: forward_cuda() calls torch.ops.vllm.hpc_rope_norm_forward
as a splitting point — internal Python control flow is opaque
to torch.compile (its kernel launches can still be CUDA-graph captured).
- In eager/native mode: forward_native() falls back to forward_cuda().
"""
def __init__(
self,
num_heads: int,
num_kv_heads: int,
head_dim: int,
cos_sin_cache: torch.Tensor,
use_qk_norm: bool,
fallback_qnorm: torch.nn.Module | None,
fallback_knorm: torch.nn.Module | None,
kv_cache_dtype: str,
layer_name: str,
qk_norm_policy: QkNormPolicy = QkNormPolicy.ROPE_THEN_NORM,
) -> None:
super().__init__()
if importlib.util.find_spec("hpc") is None:
raise ImportError(
"HPCRopeNorm requires the hpc module to be installed. "
"Please install it from https://github.com/Tencent/hpc-ops"
)
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_dim = head_dim
self.q_size = num_heads * head_dim
self.kv_size = num_kv_heads * head_dim
# Register as a non-persistent buffer so it participates in sleep
# level-2 save/restore (CuMemAllocator) but is excluded from the
# checkpoint state_dict.
self.register_buffer("cos_sin_cache", cos_sin_cache.float(), persistent=False)
self.fallback_qnorm = fallback_qnorm
self.fallback_knorm = fallback_knorm
self.head_per_group = num_heads // num_kv_heads
self.use_fp8 = "fp8" in kv_cache_dtype
# The RMSNorm/RoPE ordering is model dependent (e.g. HunYuan V3 applies
# QK-Norm before RoPE -> NORM_THEN_ROPE), so it is supplied by the
# caller. When QK-Norm is disabled the policy is forced to NONE.
self.qk_norm_policy = qk_norm_policy if use_qk_norm else QkNormPolicy.NONE
if self.qk_norm_policy != QkNormPolicy.NONE and (
fallback_qnorm is None or fallback_knorm is None
):
raise ValueError(
"QK-Norm is enabled but fallback_qnorm/fallback_knorm were not "
"supplied; the fused kernel reads their weights directly."
)
# Register layer_name + add self to the global instance registry so the
# module-level custom op (hpc_rope_norm_forward) can route back here.
self.layer_name: str | None = None
self.register_layer_name(layer_name)
import hpc
if self.use_fp8:
self._quant_type = (
hpc.QuantType.QPERTOKEN_PERHEAD_KPERTENSOR_VPERTENSOR.value
)
else:
self._quant_type = None
@classmethod
def support(
cls,
num_heads: int,
num_kv_heads: int,
head_dim: int,
kv_cache_dtype: str,
) -> bool:
"""Check whether HpcRopeNorm is supported for the given config."""
# HpcRopeNorm is only enabled together with the HPC attention backend.
vllm_config = get_current_vllm_config_or_none()
if (
vllm_config is None
or vllm_config.attention_config.backend != AttentionBackendEnum.HPC_ATTN
):
return False
if kv_cache_dtype not in ("fp8_e4m3", "auto", "bfloat16"):
logger.warning_once(
f"hpc rope_norm not support kv_cache_dtype:{kv_cache_dtype}, "
"only support fp8_e4m3, auto, bfloat16"
)
return False
if head_dim not in (128,):
logger.warning_once("hpc rope_norm only support head_dim == 128.")
return False
head_per_group = num_heads // num_kv_heads
if head_per_group not in (4, 8):
logger.warning_once("hpc rope_norm only support head_per_group in [4, 8].")
return False
logger.info_once("enable hpc rope_norm")
return True
def register_layer_name(self, layer_name: str) -> None:
"""Register layer_name and add self to the global registry.
The global registry is needed because the bottom-level torch op
(hpc_rope_norm_forward) is a module-level function and needs to
route back to the correct instance via layer_name.
"""
self.layer_name = layer_name
_hpc_rope_norm_instances[layer_name] = self
logger.debug(
"[rope_norm] registered HpcRopeNorm for layer: %s",
layer_name,
)
def forward_native(
self,
qkv: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
"""Native fallback path: delegates to forward_cuda().
For now, the default native path will use CUDA backend path.
Other platforms may override via OOT registration.
"""
return self.forward_cuda(qkv, layer_name)
def forward_cuda(
self,
qkv: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
"""CUDA path: invoke the torch custom op as a compile splitting point."""
num_tokens = qkv.shape[0]
output = torch.empty(
(num_tokens, self.num_heads, self.head_dim),
dtype=torch.float8_e4m3fn if self.use_fp8 else qkv.dtype,
device=qkv.device,
)
torch.ops.vllm.hpc_rope_norm_forward(qkv, output, layer_name)
return output
def _kv_write_scratch(self, cache: torch.Tensor) -> torch.Tensor:
"""Throwaway dense K/V pages for the stride-aware fallback path.
``hpc.rope_norm_store_kv[_fp8]`` always writes the paged cache itself
(rotated K/V *and* zero padding for the tail of every request's last
block), addressing it densely. When the real pages are not dense those
writes would land on unrelated blocks, so they are redirected here by
pairing this scratch cache with an all-zero block table; the real data
is taken out through ``out_k``/``out_v`` instead.
Returns a (2, 1, block_size, num_kv_heads, head_dim) tensor: index 0 is
the key scratch page, index 1 the value scratch page.
"""
block_size = cache.shape[1]
key = (cache.dtype, block_size, cache.device)
if getattr(self, "_kv_scratch_key", None) != key:
self._kv_scratch = torch.zeros(
2,
1,
block_size,
self.num_kv_heads,
self.head_dim,
dtype=cache.dtype,
device=cache.device,
)
self._kv_scratch_key = key
return self._kv_scratch
@staticmethod
def _zero_pad_last_blocks(
key_cache: torch.Tensor,
value_cache: torch.Tensor,
seq_lens: torch.Tensor,
block_table: torch.Tensor,
) -> None:
"""Zero the unused tail slots of each request's last block.
The HPC attention kernels read whole blocks, so the slots past
``seq_len`` must be zero. The fused op normally does this itself; on the
fallback path its (densely addressed) padding writes are discarded, so
the padding is reapplied here through the real, strided cache views.
Shapes are static and no value is inspected on the host, so this stays
CUDA-graph capturable. It is also self-contained: each request's last
block is read, masked and written back, so slots that must be preserved
(and requests padded in for graph capture, whose ``seq_len`` is 0) keep
their previous contents instead of being redirected somewhere.
"""
block_size = key_cache.shape[1]
seq_lens = seq_lens.long()
# seq_len == 0 only happens for graph-capture padding; clamping keeps
# the gather in bounds and the all-False mask below makes the write a
# no-op for those rows.
last_block_idx = (seq_lens - 1).clamp_min(0) // block_size
last_block_idx = last_block_idx.clamp_max(block_table.shape[1] - 1)
last_block = block_table.gather(1, last_block_idx.unsqueeze(1)).squeeze(1)
last_block = last_block.long()
remainder = seq_lens % block_size
# A block that ends exactly on the boundary has no tail to clear.
remainder = torch.where(remainder == 0, block_size, remainder)
offsets = torch.arange(block_size, device=key_cache.device)
is_tail = (offsets.unsqueeze(0) >= remainder.unsqueeze(1)) & (
seq_lens > 0
).unsqueeze(1)
is_tail = is_tail[..., None, None]
for cache in (key_cache, value_cache):
view = cache
if view.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
view = view.view(torch.uint8)
blocks = view.index_select(0, last_block)
blocks = torch.where(is_tail, blocks.new_zeros(()), blocks)
view.index_copy_(0, last_block, blocks)
@staticmethod
def _scatter_kv_cache(
cache: torch.Tensor,
src: torch.Tensor,
slot_mapping: torch.Tensor,
) -> None:
"""Write dense per-token K (or V) into a paged cache view.
Used on the stride-aware fallback path (see ``_forward_impl``).
``cache`` is a (num_blocks, block_size, num_kv_heads, head_dim) view
with arbitrary block/token/head strides; ``src`` is the dense
(num_tokens, num_kv_heads, head_dim) buffer produced by the fused HPC
kernel. FP8 payloads are moved as raw bytes because ``index_copy_``
has no FP8 kernel.
Padded tokens carry ``PAD_SLOT_ID`` (-1); clamping sends them to the
reserved null block instead of tripping ``index_copy_``'s bounds check.
This keeps the op CUDA-graph safe (static shape, no host sync), unlike
masking the padded rows out.
"""
# (num_blocks, block_size, ...) -> (num_slots, num_kv_heads, head_dim).
# Valid as a view for any layout whose block stride is a whole number
# of token strides, which holds for every layout the backend accepts.
flat = cache.flatten(0, 1)
if flat.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
flat = flat.view(torch.uint8)
src = src.view(torch.uint8)
flat.index_copy_(0, slot_mapping.clamp_min(NULL_BLOCK_ID), src)
def _forward_impl(
self,
qkv: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata: HpcAttnMetadata,
attn_layer: torch.nn.Module,
output: torch.Tensor,
) -> None:
"""Actual forward logic called by the custom op.
Writes processed q into *output* and attaches extra params
(e.g. FP8 scales) to *attn_layer* as attributes.
"""
import hpc
# KV cache for the FP8 path is stored as uint8; view it as fp8 so the
# rope_norm_store_kv_fp8 kernel can write quantized K/V in-place.
# Must happen *before* deriving the K/V views, so that they (and any
# buffer whose dtype is taken from them) carry the FP8 dtype.
if self.use_fp8:
kv_cache = kv_cache.view(torch.float8_e4m3fn)
# (B, H, N, 2*hs) -> two (B, N, H, hs) K/V views, as in HpcAttentionImpl.
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_dim, dim=-1)
# hpc.rope_norm_store_kv[_fp8] addresses its K/V caches as *dense*
# NHD tensors and ignores their strides, so it can only write the cache
# in place when both views are contiguous. Packed layouts (the backend's
# LBNHC pages interleave K and V inside the state-content dim, so each
# split view has a token stride of 2 * head_dim) must instead take the
# rotated K/V out through out_k/out_v and be scattered separately.
fused_kv_write = key_cache.is_contiguous() and value_cache.is_contiguous()
if not fused_kv_write:
logger.warning_once(
"HpcRopeNorm: KV cache pages interleave K and V, which "
"hpc.rope_norm_store_kv[_fp8] cannot address, so K/V are "
"written by a separate scatter. Update hpc-ops to a build with "
"strided KV cache support to re-enable the fully fused write."
)
slot_mapping = attn_metadata.slot_mapping
num_actual_tokens = attn_metadata.num_actual_tokens
num_prefill_reqs = attn_metadata.num_prefills
num_decode_reqs = attn_metadata.num_decodes
num_decode_tokens = attn_metadata.num_decode_tokens
qkv = qkv[:num_actual_tokens]
num_prefill_tokens = num_actual_tokens - num_decode_tokens
# Per-tensor K/V scales (shape [1]) used by the FP8 kernel.
k_scale = attn_layer._k_scale.reshape(1)
v_scale = attn_layer._v_scale.reshape(1)
if self.qk_norm_policy != QkNormPolicy.NONE:
assert self.fallback_qnorm is not None
assert self.fallback_knorm is not None
q_norm_weight = self.fallback_qnorm.weight
k_norm_weight = self.fallback_knorm.weight
else:
q_norm_weight = None
k_norm_weight = None
# --- Prefill ---
if num_prefill_reqs > 0:
seq_lens_prefill = attn_metadata.seq_lens[num_decode_reqs:]
cu_seqlens_prefill = attn_metadata.qo_indptr
max_seqlens = attn_metadata.max_query_len
block_table_prefill = attn_metadata.block_table_tensor[num_decode_reqs:]
qkv_prefill = qkv[num_decode_tokens:]
out_q_prefill = output[
num_decode_tokens : num_decode_tokens + num_prefill_tokens
]
kv_dtype = key_cache.dtype
if fused_kv_write:
out_k = out_v = None
kcache_arg, vcache_arg = key_cache, value_cache
kv_indices_arg = block_table_prefill
else:
out_k = torch.empty(
num_prefill_tokens,
self.num_kv_heads,
self.head_dim,
dtype=kv_dtype,
device=qkv.device,
)
out_v = torch.empty_like(out_k)
# Send the kernel's own cache writes to scratch (all block ids
# remapped to the single scratch page).
scratch = self._kv_write_scratch(key_cache)
kcache_arg, vcache_arg = scratch[0], scratch[1]
kv_indices_arg = torch.zeros_like(block_table_prefill)
if self.use_fp8:
_, q_scale, split_k_flag = hpc.rope_norm_store_kv_fp8(
key_cache=kcache_arg,
value_cache=vcache_arg,
qkv=qkv_prefill,
cos_sin=self.cos_sin_cache,
num_seqlen_per_req=seq_lens_prefill,
q_index=cu_seqlens_prefill,
kvcache_indices=kv_indices_arg,
is_prefill=True,
k_scale=k_scale,
v_scale=v_scale,
quant_policy=self._quant_type,
max_seqlens=max_seqlens,
q_norm_weight=q_norm_weight,
k_norm_weight=k_norm_weight,
qk_norm_policy=self.qk_norm_policy,
out_q=out_q_prefill,
out_k=out_k,
out_v=out_v,
)
attn_metadata.hpc_prefill_q_scale = q_scale
else:
hpc.rope_norm_store_kv(
kcache_arg,
vcache_arg,
qkv_prefill,
self.cos_sin_cache,
seq_lens_prefill,
cu_seqlens_prefill,
kv_indices_arg,
True, # is_prefill
q_norm_weight=q_norm_weight,
k_norm_weight=k_norm_weight,
out_q=out_q_prefill,
out_k=out_k,
out_v=out_v,
qk_norm_policy=self.qk_norm_policy,
)
if out_k is not None:
self._zero_pad_last_blocks(
key_cache, value_cache, seq_lens_prefill, block_table_prefill
)
prefill_slots = slot_mapping[num_decode_tokens:num_actual_tokens]
self._scatter_kv_cache(key_cache, out_k, prefill_slots)
self._scatter_kv_cache(value_cache, out_v, prefill_slots)
# --- Decode ---
if num_decode_reqs > 0:
num_seq_kvcache = attn_metadata.seq_lens[:num_decode_reqs]
block_table_decode = attn_metadata.block_table_tensor[:num_decode_reqs]
qkv_decode = qkv[:num_decode_tokens]
# Single-token decode: q_index is the per-request prefix sum
# [0, 1, ..., num_decode_reqs].
decode_query_len = attn_metadata.decode_query_len
out_q_decode = output[:num_decode_tokens]
if fused_kv_write:
out_k = out_v = None
kcache_arg, vcache_arg = key_cache, value_cache
kv_indices_arg = block_table_decode
else:
out_k = torch.empty(
num_decode_tokens,
self.num_kv_heads,
self.head_dim,
dtype=key_cache.dtype,
device=qkv.device,
)
out_v = torch.empty_like(out_k)
scratch = self._kv_write_scratch(key_cache)
kcache_arg, vcache_arg = scratch[0], scratch[1]
kv_indices_arg = torch.zeros_like(block_table_decode)
if self.use_fp8:
_, q_scale, split_k_flag = hpc.rope_norm_store_kv_fp8(
key_cache=kcache_arg,
value_cache=vcache_arg,
qkv=qkv_decode,
cos_sin=self.cos_sin_cache,
num_seqlen_per_req=num_seq_kvcache,
q_index=attn_metadata.qo_indptr_decode,
kvcache_indices=kv_indices_arg,
is_prefill=False,
k_scale=k_scale,
v_scale=v_scale,
quant_policy=self._quant_type,
max_seqlens=decode_query_len,
q_norm_weight=q_norm_weight,
k_norm_weight=k_norm_weight,
qk_norm_policy=self.qk_norm_policy,
out_q=out_q_decode,
out_k=out_k,
out_v=out_v,
)
attn_metadata.hpc_decode_q_scale = q_scale
if split_k_flag is not None:
attn_metadata.hpc_split_k_flag = split_k_flag
else:
hpc.rope_norm_store_kv(
kcache_arg,
vcache_arg,
qkv_decode,
self.cos_sin_cache,
num_seq_kvcache,
attn_metadata.qo_indptr_decode,
kv_indices_arg,
False, # is_prefill
q_norm_weight=q_norm_weight,
k_norm_weight=k_norm_weight,
out_q=out_q_decode,
out_k=out_k,
out_v=out_v,
qk_norm_policy=self.qk_norm_policy,
)
if out_k is not None:
self._zero_pad_last_blocks(
key_cache, value_cache, num_seq_kvcache, block_table_decode
)
decode_slots = slot_mapping[:num_decode_tokens]
self._scatter_kv_cache(key_cache, out_k, decode_slots)
self._scatter_kv_cache(value_cache, out_v, decode_slots)
# Signal HpcAttentionImpl that KV cache has already been written by
# rope_norm_store_kv[_fp8] above, so it should skip its own
# reshape_and_cache_flash. Set after the fused kernels ran (either
# prefill or decode branch); otherwise the standard KV-write path in
# the attention impl would kick in (which is what non-HpcRopeNorm
# models rely on).
attn_metadata.hpc_kv_written = True
@@ -1,646 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import TYPE_CHECKING
import torch
from einops import rearrange
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig
from vllm.distributed import divide, get_tensor_model_parallel_rank
from vllm.forward_context import get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
sharded_weight_loader,
)
from vllm.model_executor.parameter import BasevLLMParameter
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from ...linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from ..mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from ..ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
from ..ops.gather_initial_states import gather_initial_states
# Empirical lower bound for the KDA gate to avoid numerical underflow.
_KDA_GATE_LOGBOUND_MIN = -5.0
def a_log_weight_loader(
shard_axis: int,
) -> Callable[[torch.Tensor, torch.Tensor], None]:
"""Load KDA A_log stored as either old 4D or current 1D weights."""
def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
tp_rank = get_tensor_model_parallel_rank()
shard_size = param.data.shape[shard_axis]
start_idx = tp_rank * shard_size
if loaded_weight.dim() == 4:
assert loaded_weight.shape[:2] == (1, 1), (
f"Expected old A_log shape (1, 1, H, 1), got {loaded_weight.shape}"
)
assert loaded_weight.shape[-1] == 1, (
f"Expected old A_log last dim to be 1, got {loaded_weight.shape}"
)
loaded_weight = loaded_weight.view(loaded_weight.shape[2])
loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
return default_weight_loader(param, loaded_weight)
return loader
def _make_fused_conv1d_weight_loader(
dims: list[int],
tp_size: int,
tp_rank: int,
) -> Callable[..., None]:
sharded_dims = [dim // tp_size for dim in dims]
def weight_loader(
param: torch.Tensor,
loaded_weight: torch.Tensor,
loaded_shard_id: int,
) -> None:
if loaded_weight.dim() == 2:
loaded_weight = loaded_weight.unsqueeze(1)
shard_size = sharded_dims[loaded_shard_id]
source_start = tp_rank * shard_size
target_start = sum(sharded_dims[:loaded_shard_id])
loaded_shard = loaded_weight[source_start : source_start + shard_size]
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
return weight_loader
class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
"""Merged projection with one output replicated across TP ranks.
The replicated shard is represented as ``size * tp_size`` so the merged
parameter reserves ``size`` local rows on every rank. Loading that shard
from rank zero then gives every rank the complete checkpoint weight.
"""
def __init__(
self,
input_size: int,
output_sizes: list[int],
replicated_shard_id: int,
tp_size: int,
**kwargs,
) -> None:
self.replicated_shard_id = replicated_shard_id
output_sizes = output_sizes.copy()
output_sizes[replicated_shard_id] *= tp_size
super().__init__(input_size, output_sizes, **kwargs)
def weight_loader(
self,
param: Parameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
) -> None:
tp_rank = self.tp_rank
param_tp_rank = getattr(param, "tp_rank", None)
if loaded_shard_id == self.replicated_shard_id:
self.tp_rank = 0
if param_tp_rank is not None:
param.tp_rank = 0
try:
super().weight_loader(param, loaded_weight, loaded_shard_id)
finally:
self.tp_rank = tp_rank
if param_tp_rank is not None:
param.tp_rank = param_tp_rank
def weight_loader_v2(
self,
param: BasevLLMParameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
) -> None:
tp_rank = self.tp_rank
param_tp_rank = getattr(param, "tp_rank", None)
if loaded_shard_id == self.replicated_shard_id:
self.tp_rank = 0
if param_tp_rank is not None:
param.tp_rank = 0
try:
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
finally:
self.tp_rank = tp_rank
if param_tp_rank is not None:
param.tp_rank = param_tp_rank
@PluggableLayer.register("kimi_gated_delta_net_attention")
class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
def get_state_dtype(
self,
) -> tuple[torch.dtype, torch.dtype]:
if self.model_config is None or self.cache_config is None:
raise ValueError("model_config and cache_config must be set")
return MambaStateDtypeCalculator.kda_state_dtype(
self.model_config.dtype, self.cache_config.mamba_cache_dtype
)
def get_state_shape(
self,
) -> tuple[tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.kda_state_shape(
self.tp_size,
self.num_heads,
self.head_dim,
conv_kernel_size=self.conv_size,
num_spec=self.num_spec,
)
def __init__(
self,
config: KimiLinearConfig,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
super().__init__(config, vllm_config, prefix)
kda_config = config.linear_attn_config # type: ignore[attr-defined]
assert kda_config is not None, "linear_attn_config must be set"
self.head_dim = kda_config["head_dim"]
self.num_heads = kda_config["num_heads"]
assert self.num_heads % self.tp_size == 0
self.local_num_heads = divide(self.num_heads, self.tp_size)
self.projection_size = self.head_dim * self.num_heads
self.local_projection_size = divide(self.projection_size, self.tp_size)
self.conv_size = kda_config["short_conv_kernel_size"]
self.use_full_rank_gate = kda_config.get("use_full_rank_gate", False)
if self.use_full_rank_gate:
# Keep f_a before the narrow beta shard, then pad each TP-local row
# to select the aligned BF16 GEMM path. The padding also avoids an
# Inductor correctness issue seen with the row-strided G view.
qkvg_output_sizes = [self.projection_size] * 4
in_proj_output_sizes = qkvg_output_sizes + [
self.head_dim,
self.num_heads,
]
local_output_size = (
4 * self.local_projection_size + self.head_dim + self.local_num_heads
)
self.in_proj_padding = -local_output_size % 16
if self.in_proj_padding:
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
else:
in_proj_output_sizes = [self.projection_size] * 3 + [
self.num_heads,
self.head_dim,
]
self.in_proj_padding = 0
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
self.hidden_size,
in_proj_output_sizes,
replicated_shard_id=4,
tp_size=self.tp_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.in_proj_qkvgfab",
)
if self.in_proj_padding:
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
self.f_b_proj = ColumnParallelLinear(
self.head_dim,
self.projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.f_b_proj",
)
self.dt_bias = nn.Parameter(
torch.empty(self.local_projection_size, dtype=torch.float32)
)
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
# One packed parameter and cache let decode run a single conv update.
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
self.conv1d = ColumnParallelLinear(
input_size=self.conv_size,
output_size=3 * self.projection_size,
bias=False,
params_dtype=torch.float32,
prefix=f"{prefix}.conv1d",
)
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
delattr(self.conv1d.weight, "weight_loader")
set_weight_attrs(
self.conv1d.weight,
{
"weight_loader": _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
},
)
self.A_log = nn.Parameter(
torch.empty(self.local_num_heads, dtype=torch.float32)
)
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
if self.gate_lower_bound is not None:
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
"KDA gate lower bound must be in "
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
f"Got {self.gate_lower_bound}."
)
self.use_safe_gate = self.gate_lower_bound is not None
additional_config = vllm_config.additional_config
backend = (
additional_config.get("kda_prefill_backend", "auto")
if isinstance(additional_config, dict)
else "auto"
)
backend = "triton" if backend == "auto" else backend
assert backend == "triton", (
"The shared Kimi GDN layer only supports the Triton KDA "
f"prefill backend, got {backend!r}."
)
if not self.use_full_rank_gate:
self.g_a_proj = ReplicatedLinear(
self.hidden_size,
self.head_dim,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_a_proj",
)
self.g_b_proj = ColumnParallelLinear(
self.head_dim,
self.projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_b_proj",
)
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
self.o_proj = RowParallelLinear(
self.projection_size,
self.hidden_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.o_proj",
)
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def rearrange_mixed_qkv(
self, mixed_qkv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
seq_len = mixed_qkv.shape[0]
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
# Materialize all three row-strided inputs with one token-major to
# QKV-major permutation. Each unbound tensor is then contiguous.
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
return qkv.unbind(0)
def forward(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
) -> torch.Tensor:
num_tokens = hidden_states.size(0)
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
if self.use_full_rank_gate:
split_sizes = [
3 * self.local_projection_size,
self.local_projection_size,
self.head_dim,
self.local_num_heads,
]
if self.in_proj_padding:
split_sizes.append(self.in_proj_padding)
projected = projected_qkvgfab.split(split_sizes, dim=-1)
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
else:
mixed_qkv, beta, f_a = projected_qkvgfab.split(
[
3 * self.local_projection_size,
self.local_num_heads,
self.head_dim,
],
dim=-1,
)
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
g1 = self.f_b_proj(f_a)[0]
beta = beta.unsqueeze(0)
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
core_attn_out = torch.empty(
(1, num_tokens, self.local_num_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
self._forward(
mixed_qkv=mixed_qkv,
g1=g1,
g2=g2,
beta=beta,
core_attn_out=core_attn_out,
)
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
return self.o_proj(core_attn_out)[0]
@eager_break_during_capture
def _forward(
self,
mixed_qkv: torch.Tensor,
g1: torch.Tensor,
g2: torch.Tensor,
beta: torch.Tensor,
core_attn_out: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
return
# Vendor-specific KDA kernels: AMD/ROCm and NVIDIA keep their own copies
# under kimi_k3/{amd,nvidia}/ops so each can diverge independently.
# These copies may have different signatures for Kimi-K3, but they agree
# on the arguments used here.
if TYPE_CHECKING:
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
elif current_platform.is_rocm():
from vllm.models.kimi_k3.amd.ops.third_party.kda import ( # type: ignore[assignment]
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
else:
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
assert isinstance(attn_metadata_raw, dict)
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
if attn_metadata_narrowed is None:
# Profile/warmup dummy runs skip mamba-family metadata.
return
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
m = attn_metadata_narrowed
has_initial_state = m.has_initial_state
non_spec_query_start_loc = m.non_spec_query_start_loc
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
spec_sequence_masks = m.spec_sequence_masks
spec_token_indx = m.spec_token_indx
non_spec_token_indx = m.non_spec_token_indx
spec_state_indices_tensor = m.spec_state_indices_tensor
spec_query_start_loc = m.spec_query_start_loc
num_accepted_tokens = m.num_accepted_tokens
num_actual_tokens = m.num_actual_tokens
mixed_qkv = mixed_qkv[:num_actual_tokens]
g1 = g1[:, :num_actual_tokens]
beta = beta[:, :num_actual_tokens]
constant_caches = self.kv_cache
conv_state, recurrent_state = constant_caches
# conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a transpose.
if not is_conv_state_dim_first():
conv_state = conv_state.transpose(-1, -2)
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
self.local_projection_size, dim=0
)
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
self.local_projection_size, dim=-2
)
# Split tokens into the multi-query spec-decode part and the remaining
# (prefill / plain decode) part.
if spec_sequence_masks is not None:
if m.num_prefills == 0 and m.num_decodes == 0:
mixed_qkv_spec = mixed_qkv
g1_spec, beta_spec = g1, beta
mixed_qkv_ns = g1_ns = beta_ns = None
else:
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
g1_spec = g1.index_select(1, spec_token_indx)
beta_spec = beta.index_select(1, spec_token_indx)
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
g1_ns = g1.index_select(1, non_spec_token_indx)
beta_ns = beta.index_select(1, non_spec_token_indx)
else:
mixed_qkv_spec = g1_spec = beta_spec = None
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
# ---------- spec-decode multi-query path ----------
core_attn_out_spec = None
if spec_sequence_masks is not None:
assert spec_state_indices_tensor is not None
assert spec_query_start_loc is not None
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
spec_max_query_len = spec_state_indices_tensor.size(-1)
# Sibling beta and, for full-rank gates, output-gate views remain
# live, so write the convolution output separately.
spec_conv_out = torch.empty(
mixed_qkv_spec.shape,
dtype=mixed_qkv_spec.dtype,
device=mixed_qkv_spec.device,
)
mixed_qkv_spec = causal_conv1d_update(
mixed_qkv_spec,
conv_state,
conv_weights,
self.conv1d.bias,
activation="silu",
conv_state_indices=spec_conv_indices,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=spec_query_start_loc,
max_query_len=spec_max_query_len,
validate_data=False,
out=spec_conv_out,
)
q_spec, k_spec, v_spec = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
)
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
# Spec-only batches write directly into core_attn_out.
spec_out = (
core_attn_out[:, : q_spec.shape[1]]
if m.num_prefills == 0 and m.num_decodes == 0
else None
)
core_attn_out_spec, _ = fused_recurrent_kda(
q=q_spec,
k=k_spec,
v=v_spec,
raw_g=g1_spec,
raw_beta=beta_spec,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
cu_seqlens=spec_cu_seqlens,
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
out=spec_out,
)
# ---------- non-spec path (prefill or plain decode) ----------
core_attn_out_non_spec = None
if mixed_qkv_ns is not None:
assert g1_ns is not None and beta_ns is not None
if m.num_prefills > 0:
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
self.local_projection_size, dim=-1
)
# Packed prefill conv would require copying V solely to make
# it dense for KDA. Separate calls accept the strided inputs
# and produce dense Q/K/V without that extra traffic.
# TODO: Use packed conv once every KDA prefill backend accepts
# row-strided Q/K/V directly.
def _prefill_conv(
x: torch.Tensor,
state: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
return causal_conv1d_fn(
x.transpose(0, 1),
weight,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=m,
).transpose(0, 1)
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
q_ns, k_ns, v_ns = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in (q_ns, k_ns, v_ns)
)
assert non_spec_state_indices_tensor is not None
assert has_initial_state is not None
initial_state = gather_initial_states(
recurrent_state,
non_spec_state_indices_tensor,
has_initial_state,
)
(
core_attn_out_non_spec,
last_recurrent_state,
) = chunk_kda_with_fused_gate(
q=q_ns,
k=k_ns,
v=v_ns,
raw_g=g1_ns,
raw_beta=beta_ns,
A_log=self.A_log,
g_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=initial_state,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=non_spec_query_start_loc,
)
# Init cache
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
else:
# pure-decode non-spec batch
assert non_spec_state_indices_tensor is not None
decode_conv_indices = non_spec_state_indices_tensor[
: mixed_qkv_ns.size(0)
]
# Sibling beta and, for full-rank gates, output-gate views
# remain live, so write the conv output separately.
packed_conv_out = torch.empty(
mixed_qkv_ns.shape,
dtype=mixed_qkv_ns.dtype,
device=mixed_qkv_ns.device,
)
mixed_qkv_ns = causal_conv1d_update(
mixed_qkv_ns,
conv_state,
conv_weights,
self.conv1d.bias,
activation="silu",
conv_state_indices=decode_conv_indices,
validate_data=True,
out=packed_conv_out,
)
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
mixed_qkv=mixed_qkv_ns,
raw_g=g1_ns,
raw_beta=beta_ns,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
state_indices=decode_conv_indices,
)
# ---------- merge spec and non-spec outputs ----------
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
# Mixed batches require indexed placement in the original order.
merged = torch.empty(
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
dtype=core_attn_out_spec.dtype,
device=core_attn_out_spec.device,
)
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
elif core_attn_out_non_spec is not None:
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
]
else:
assert core_attn_out_spec is not None
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
@@ -1,626 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from einops import rearrange
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
VllmConfig,
get_current_vllm_config,
)
from vllm.distributed import (
divide,
)
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.layernorm import RMSNormGated
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.model_loader.weight_utils import (
sharded_weight_loader,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops import (
chunk_gated_delta_rule,
fused_recurrent_gated_delta_rule,
)
from vllm.triton_utils import tl, triton
from vllm.triton_utils.allocation import set_triton_allocator
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
@PluggableLayer.register("olmo_hybrid_gated_delta_net_attention")
class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
"""Gated DeltaNet linear attention layer for OLMo Hybrid.
This implements the linear attention mechanism that replaces sliding window
attention in the hybrid architecture.
"""
def get_state_shape(
self,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.gated_delta_net_state_shape(
self.tp_size,
self.num_k_heads,
self.num_v_heads,
self.head_k_dim,
self.head_v_dim,
self.conv_kernel_size,
self.num_spec,
)
def __init__(
self,
config,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
super().__init__(config, vllm_config, prefix=prefix)
assert getattr(config, "linear_use_gate", True), (
"OlmoHybridGatedDeltaNet requires linear_use_gate=True"
)
self.num_k_heads = config.linear_num_key_heads
self.num_v_heads = config.linear_num_value_heads
self.head_k_dim = config.linear_key_head_dim
self.head_v_dim = config.linear_value_head_dim
self.conv_kernel_size = config.linear_conv_kernel_dim
self.key_dim = self.head_k_dim * self.num_k_heads
self.value_dim = self.head_v_dim * self.num_v_heads
self.allow_neg_eigval = getattr(config, "linear_allow_neg_eigval", False)
# Fused QKVG projection: 1 matmul instead of 4
self.in_proj_qkvg = MergedColumnParallelLinear(
input_size=self.hidden_size,
output_sizes=[self.key_dim, self.key_dim, self.value_dim, self.value_dim],
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.in_proj_qkvg",
)
# Separate B and A projections to preserve numerical precision.
# Fusing these into one matmul changes FP accumulation order for the
# gating scalars, which compounds through the GDN recurrent state.
self.b_proj = ColumnParallelLinear(
input_size=self.hidden_size,
output_size=self.num_v_heads,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.b_proj",
)
self.a_proj = ColumnParallelLinear(
input_size=self.hidden_size,
output_size=self.num_v_heads,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.a_proj",
)
# Fused conv1d: single parameter instead of 3
self.conv_dim = self.key_dim * 2 + self.value_dim
self.conv1d = ColumnParallelLinear(
input_size=self.conv_kernel_size,
output_size=self.conv_dim,
bias=False,
prefix=f"{prefix}.conv1d",
)
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
delattr(self.conv1d.weight, "weight_loader")
set_weight_attrs(
self.conv1d.weight,
{
"weight_loader": _make_fused_conv1d_weight_loader(
[self.key_dim, self.key_dim, self.value_dim],
self.tp_size,
self.tp_rank,
)
},
)
self.dt_bias = nn.Parameter(
torch.ones(self.num_v_heads // self.tp_size),
)
self.A_log = nn.Parameter(
torch.empty(
divide(self.num_v_heads, self.tp_size),
)
)
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(0)})
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
# use eps=1e-5 to match FLA's FusedRMSNormGated
self.o_norm = RMSNormGated(
self.head_v_dim,
eps=1e-5,
group_size=None,
norm_before_gate=True,
device=current_platform.current_device(),
dtype=config.torch_dtype if hasattr(config, "torch_dtype") else None,
)
self.o_proj = RowParallelLinear(
self.value_dim,
self.hidden_size,
bias=False,
input_is_parallel=True,
quant_config=self.quant_config,
prefix=f"{prefix}.o_proj",
)
# FLA triton kernels need a PyTorch-backed allocator for scratch
# memory (required by triton >= 3.x autotuner). Set once at init.
set_triton_allocator(current_platform.current_device())
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def rearrange_mixed_qkv(self, mixed_qkv):
if mixed_qkv is None:
return None, None, None
query, key, value = torch.split(
mixed_qkv,
[
self.key_dim // self.tp_size,
self.key_dim // self.tp_size,
self.value_dim // self.tp_size,
],
dim=-1,
)
num_k_heads = self.num_k_heads // self.tp_size
num_v_heads = self.num_v_heads // self.tp_size
query = rearrange(query, "l (h d) -> 1 l h d", h=num_k_heads, d=self.head_k_dim)
key = rearrange(key, "l (h d) -> 1 l h d", h=num_k_heads, d=self.head_k_dim)
value = rearrange(value, "l (h d) -> 1 l h d", h=num_v_heads, d=self.head_v_dim)
# GQA expansion if needed
if num_v_heads > num_k_heads:
expand_ratio = num_v_heads // num_k_heads
query = query.unsqueeze(3).expand(-1, -1, -1, expand_ratio, -1)
query = query.reshape(1, query.shape[1], num_v_heads, self.head_k_dim)
key = key.unsqueeze(3).expand(-1, -1, -1, expand_ratio, -1)
key = key.reshape(1, key.shape[1], num_v_heads, self.head_k_dim)
return query.contiguous(), key.contiguous(), value.contiguous()
def forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
# NOTE: We wrap the ENTIRE linear attention forward (projections +
# core recurrence + output norm + output projection) in a single
# custom op, rather than just wrapping the recurrent core like
# other GDN models (e.g. Qwen3Next) do.
#
# Why: torch.compile with inductor generates fused kernels for
# matmuls and pointwise ops. These fused kernels can differ in
# floating-point accumulation order from eager-mode cuBLAS,
# introducing small numerical differences (~1e-7 per op). For
# standard transformer attention this is harmless because each
# position is computed independently. But for the GDN recurrent
# state, these tiny input differences compound at every timestep
# across the full sequence length, causing severe logprob
# divergence (e.g. ~15% top-1 agreement with eager baseline).
#
# By making the full forward opaque to inductor, the projections
# and output norm run with eager-mode kernels (cuBLAS, triton),
# preserving numerical consistency. The tradeoff is reduced
# compilation speedup (~1.5x vs ~3x), but logprob agreement
# improves from ~15% to ~83% top-1 vs eager.
#
# The remaining ~17% divergence comes from inductor compiling
# the MLP and transformer attention layers that are NOT wrapped
# in custom ops -- their small precision differences propagate
# as inputs to the GDN layers from outside.
torch.ops.vllm.olmo_hybrid_gdn_full_forward(
hidden_states,
output,
self.prefix,
)
def _full_forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
num_tokens = hidden_states.size(0)
# ============================================================
# Part 1: Input Projection (2 fused matmuls instead of 6)
# ============================================================
projected_qkvg, _ = self.in_proj_qkvg(hidden_states)
conv_dim_sharded = (self.key_dim * 2 + self.value_dim) // self.tp_size
mixed_qkv = projected_qkvg[..., :conv_dim_sharded]
gate = projected_qkvg[..., conv_dim_sharded:]
b, _ = self.b_proj(hidden_states)
a, _ = self.a_proj(hidden_states)
# ============================================================
# Part 2: Core Attention
# ============================================================
core_attn_out = torch.zeros(
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
self._forward_core(
mixed_qkv=mixed_qkv,
b=b,
a=a,
core_attn_out=core_attn_out,
)
# ============================================================
# Part 3: Output Projection
# ============================================================
gate = gate.view(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim)
core_attn_out_flat = core_attn_out.reshape(-1, core_attn_out.shape[-1])
gate_flat = gate.reshape(-1, gate.shape[-1])
core_attn_out_normed = self.o_norm(core_attn_out_flat, gate_flat)
core_attn_out = core_attn_out_normed.view(
num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim
)
core_attn_out = rearrange(core_attn_out, "l h d -> l (h d)")
output[:num_tokens], _ = self.o_proj(core_attn_out)
def _forward_core(
self,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
core_attn_out: torch.Tensor,
):
"""Core attention computation (called by custom op)."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
# V1 profile run
return
assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment]
if attn_metadata is None:
# Profile/warmup dummy runs skip mamba-family metadata.
return
assert isinstance(attn_metadata, GDNAttentionMetadata)
has_initial_state = attn_metadata.has_initial_state
spec_query_start_loc = attn_metadata.spec_query_start_loc
non_spec_query_start_loc = attn_metadata.non_spec_query_start_loc
spec_sequence_masks = attn_metadata.spec_sequence_masks
spec_token_indx = attn_metadata.spec_token_indx
non_spec_token_indx = attn_metadata.non_spec_token_indx
spec_state_indices_tensor = attn_metadata.spec_state_indices_tensor
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor
self_kv_cache = self.kv_cache
# conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a transpose.
conv_state = (
self_kv_cache[0]
if is_conv_state_dim_first()
else self_kv_cache[0].transpose(-1, -2)
)
ssm_state = self_kv_cache[1]
num_actual_tokens = attn_metadata.num_actual_tokens
num_accepted_tokens = attn_metadata.num_accepted_tokens
mixed_qkv = mixed_qkv[:num_actual_tokens]
b = b[:num_actual_tokens]
a = a[:num_actual_tokens]
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
if spec_sequence_masks is not None:
if attn_metadata.num_prefills == 0 and attn_metadata.num_decodes == 0:
mixed_qkv_spec = mixed_qkv
mixed_qkv_non_spec = None
else:
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
mixed_qkv_non_spec = mixed_qkv.index_select(0, non_spec_token_indx)
else:
mixed_qkv_spec = None
mixed_qkv_non_spec = mixed_qkv
if spec_sequence_masks is not None:
assert spec_query_start_loc is not None
assert spec_state_indices_tensor is not None
assert num_accepted_tokens is not None
mixed_qkv_spec = causal_conv1d_update(
mixed_qkv_spec,
conv_state,
conv_weights,
None, # no bias
self.activation,
conv_state_indices=spec_state_indices_tensor[:, 0][
: attn_metadata.num_spec_decodes
],
num_accepted_tokens=num_accepted_tokens,
query_start_loc=spec_query_start_loc,
max_query_len=spec_state_indices_tensor.size(-1),
validate_data=False,
)
if attn_metadata.num_prefills > 0:
assert mixed_qkv_non_spec is not None
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
mixed_qkv_non_spec = causal_conv1d_fn(
mixed_qkv_non_spec_T,
conv_weights,
None,
activation=self.activation,
conv_states=conv_state,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=attn_metadata,
).transpose(0, 1)
elif attn_metadata.num_decodes > 0:
assert non_spec_state_indices_tensor is not None
mixed_qkv_non_spec = causal_conv1d_update(
mixed_qkv_non_spec,
conv_state,
conv_weights,
None,
self.activation,
conv_state_indices=non_spec_state_indices_tensor[
: attn_metadata.num_decodes
],
validate_data=True,
)
else:
mixed_qkv_non_spec = None
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
query_non_spec, key_non_spec, value_non_spec = self.rearrange_mixed_qkv(
mixed_qkv_non_spec
)
g, beta = fused_olmo_hybrid_gdn_gating(
self.A_log, a, b, self.dt_bias, self.allow_neg_eigval
)
if spec_sequence_masks is not None:
assert spec_token_indx is not None
assert non_spec_token_indx is not None
if attn_metadata.num_prefills == 0 and attn_metadata.num_decodes == 0:
g_spec = g
beta_spec = beta
g_non_spec = None
beta_non_spec = None
else:
g_spec = g.index_select(1, spec_token_indx)
beta_spec = beta.index_select(1, spec_token_indx)
g_non_spec = g.index_select(1, non_spec_token_indx)
beta_non_spec = beta.index_select(1, non_spec_token_indx)
else:
g_spec = None
beta_spec = None
g_non_spec = g
beta_non_spec = beta
if spec_sequence_masks is not None:
assert spec_query_start_loc is not None
assert spec_state_indices_tensor is not None
assert num_accepted_tokens is not None
core_attn_out_spec, last_recurrent_state = fused_recurrent_gated_delta_rule(
q=query_spec,
k=key_spec,
v=value_spec,
g=g_spec,
beta=beta_spec,
initial_state=ssm_state,
inplace_final_state=True,
cu_seqlens=spec_query_start_loc[: attn_metadata.num_spec_decodes + 1],
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
use_qk_l2norm_in_kernel=True,
)
else:
core_attn_out_spec, last_recurrent_state = None, None
if attn_metadata.num_prefills > 0:
assert non_spec_state_indices_tensor is not None
assert has_initial_state is not None
assert non_spec_query_start_loc is not None
initial_state = ssm_state[non_spec_state_indices_tensor].contiguous()
initial_state[~has_initial_state, ...] = 0
(
core_attn_out_non_spec,
last_recurrent_state,
) = chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=non_spec_query_start_loc,
use_qk_l2norm_in_kernel=True,
)
ssm_state[non_spec_state_indices_tensor] = last_recurrent_state.to(
ssm_state.dtype
)
elif attn_metadata.num_decodes > 0:
assert non_spec_query_start_loc is not None
assert non_spec_state_indices_tensor is not None
core_attn_out_non_spec, last_recurrent_state = (
fused_recurrent_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=ssm_state,
inplace_final_state=True,
cu_seqlens=non_spec_query_start_loc[
: attn_metadata.num_decodes + 1
],
ssm_state_indices=non_spec_state_indices_tensor,
use_qk_l2norm_in_kernel=True,
)
)
else:
core_attn_out_non_spec, last_recurrent_state = None, None
if spec_sequence_masks is not None and core_attn_out_non_spec is not None:
merged_out = torch.empty(
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
dtype=core_attn_out_non_spec.dtype,
device=core_attn_out_non_spec.device,
)
merged_out.index_copy_(1, spec_token_indx, core_attn_out_spec)
merged_out.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[:num_actual_tokens] = merged_out.squeeze(0)
elif spec_sequence_masks is not None:
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
else:
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
"""Weight loader for loading separate HF conv weights into a fused conv1d.
dims: list of original (un-sharded) dims per section,
e.g. [key_dim, key_dim, value_dim]
"""
sharded_dims = [d // tp_size for d in dims]
def weight_loader(param, loaded_weight, loaded_shard_id=None):
if loaded_weight.dim() == 2:
loaded_weight = loaded_weight.unsqueeze(1)
dim = dims[loaded_shard_id]
shard_size = dim // tp_size
tp_start = tp_rank * shard_size
sharded_weight = loaded_weight[tp_start : tp_start + shard_size]
offset = sum(sharded_dims[:loaded_shard_id])
param.data[offset : offset + shard_size].copy_(sharded_weight)
return weight_loader
@eager_break_during_capture
def olmo_hybrid_gdn_full_forward(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
"""Full linear attention forward wrapped as a custom op.
Prevents inductor from compiling the projections around the GDN core,
which would introduce numerical divergence that compounds through
the recurrent state.
"""
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._full_forward(
hidden_states=hidden_states,
output=output,
)
direct_register_custom_op(
op_name="olmo_hybrid_gdn_full_forward",
op_func=olmo_hybrid_gdn_full_forward,
mutates_args=["output"],
)
@triton.jit
def fused_olmo_hybrid_gdn_gating_kernel(
g,
beta_output,
A_log,
a,
b,
dt_bias,
seq_len,
allow_neg_eigval: tl.constexpr,
NUM_HEADS: tl.constexpr,
beta: tl.constexpr,
threshold: tl.constexpr,
BLK_HEADS: tl.constexpr,
):
i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
mask = head_off < NUM_HEADS
blk_A_log = tl.load(A_log + head_off, mask=mask)
blk_a = tl.load(a + off, mask=mask)
blk_b = tl.load(b + off, mask=mask)
blk_bias = tl.load(dt_bias + head_off, mask=mask)
# g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
softplus_x = tl.where(
beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
)
blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
# beta = self.b_proj(hidden_states).sigmoid()
# if self.allow_neg_eigval: beta = beta * 2.0
blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
if allow_neg_eigval:
blk_beta_output = blk_beta_output * 2.0
tl.store(
beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask
)
def fused_olmo_hybrid_gdn_gating(
A_log: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
dt_bias: torch.Tensor,
allow_neg_eigval: bool = False,
beta: float = 1.0,
threshold: float = 20.0,
) -> tuple[torch.Tensor, torch.Tensor]:
batch, num_heads = a.shape
seq_len = 1
grid = (batch, seq_len, triton.cdiv(num_heads, 8))
g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
beta_output = torch.empty(1, batch, num_heads, dtype=torch.float32, device=b.device)
fused_olmo_hybrid_gdn_gating_kernel[grid](
g,
beta_output,
A_log,
a,
b,
dt_bias,
seq_len,
allow_neg_eigval,
num_heads,
beta,
threshold,
8,
num_warps=1,
)
return g, beta_output
@@ -54,7 +54,12 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.inc import INCConfig
try:
# inc/ loader was removed in the SM86 mini-vLLM cut.
from vllm.model_executor.layers.quantization.inc import INCConfig
except ImportError:
INCConfig = type("AbsentINCConfig", (), {})
from vllm.model_executor.model_loader.weight_utils import (
sharded_weight_loader,
)
@@ -1,247 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import torch
from cuda.bindings.driver import CUstream
from cutlass import Int32, cute
from quack.compile_utils import make_fake_tensor
from vllm.triton_utils import triton
from .kernel_h import h_cutedsl
from .kernel_kkt_inv_uw import kkt_inv_uw_cutedsl
from .kernel_o import o_cutedsl
class PrepMetaKernel:
def __init__(self, BT: int) -> None:
self.BT = BT
self.num_warps = 8
@cute.jit
def __call__(
self,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
chunk_offsets: cute.Tensor,
stream: CUstream,
):
block = (self.num_warps * 32, 1, 1)
self.kernel(
cu_seqlens,
chunk_indices,
chunk_offsets,
).launch(grid=(1, 1, 1), block=block, stream=stream)
@cute.kernel
def kernel(
self,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
chunk_offsets: cute.Tensor,
):
tid, _, _ = cute.arch.thread_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
num_seqs = cu_seqlens.shape[0] - 1
num_warps = self.num_warps
tb_size = num_warps * 32
if tid == 0:
chunk_offsets[0] = 0
coarsen = cute.ceil_div(num_seqs, tb_size)
seq_start = tid * coarsen
num_iters = cutlass.min(seq_start + coarsen, num_seqs) - seq_start
# First pass: compute this thread's total chunk count.
thread_sum = Int32(0)
for i in range(num_iters):
seq_id = seq_start + i
seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id]
thread_sum += cute.ceil_div(seqlen, self.BT)
# warp parallel scan
cu_num_chunks = thread_sum
for i in cutlass.range_constexpr(5):
offset = cutlass.const_expr(1 << i)
lower = cute.arch.shuffle_sync_up(
cu_num_chunks, offset=offset, mask_and_clamp=0
)
if lane_id >= offset:
cu_num_chunks += lower
# cross-warp cumsum (CTA-wide)
smem = cutlass.utils.SmemAllocator()
warp_num_chunks = smem.allocate_array(Int32, num_warps)
if lane_id == 31:
warp_num_chunks[warp_id] = cu_num_chunks
cute.arch.sync_threads()
for i in cutlass.range_constexpr(1, num_warps):
if warp_id >= i:
cu_num_chunks += warp_num_chunks[i - 1]
chunk_start = cu_num_chunks - thread_sum
# Second pass: recompute per-sequence chunk counts and write results.
for i in range(num_iters):
seq_id = seq_start + i
seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id]
num_chunks = cute.ceil_div(seqlen, self.BT)
chunk_end = chunk_start + num_chunks
chunk_offsets[seq_id + 1] = chunk_end
for chunk_id in range(num_chunks):
chunk_indices[chunk_start + chunk_id, 0] = seq_id
chunk_indices[chunk_start + chunk_id, 1] = chunk_id
chunk_start = chunk_end
@cache
@staticmethod
def compile(BT: int):
cu_entries = cute.sym_int()
upper_bound_chunks = cute.sym_int()
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
chunk_indices = make_fake_tensor(Int32, (upper_bound_chunks, 2), divisibility=2)
chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
kernel = PrepMetaKernel(BT)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
cu_seqlens,
chunk_indices,
chunk_offsets,
stream,
options="--enable-tvm-ffi",
)
def _upper_bound_chunks(num_seqs: int, total_tokens: int, chunk_size: int) -> int:
return (num_seqs - 1) + triton.cdiv(total_tokens - (num_seqs - 1), chunk_size)
def prepare_metadata_cutedsl(
cu_seqlens: torch.Tensor,
total_tokens: int,
chunk_size: int = 64,
) -> tuple[torch.Tensor, torch.Tensor]:
num_seqs = cu_seqlens.numel() - 1
upper_bound_chunks = _upper_bound_chunks(num_seqs, total_tokens, chunk_size)
chunk_offsets = cu_seqlens.new_empty(num_seqs + 1, dtype=torch.int32)
chunk_indices = cu_seqlens.new_empty((upper_bound_chunks, 2), dtype=torch.int32)
PrepMetaKernel.compile(chunk_size)(cu_seqlens, chunk_indices, chunk_offsets)
return chunk_indices, chunk_offsets
def chunk_gated_delta_rule_cutedsl(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
cu_seqlens: torch.Tensor,
chunk_indices: torch.Tensor,
chunk_offsets: torch.Tensor,
core_attn_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the GDN chunk CuteDSL prefill kernels.
Args:
q: Query tensor with shape ``[1, T, H, K]``.
k: Key tensor with shape ``[1, T, H, K]``.
v: Value tensor with shape ``[1, T, Hv, V]``.
g: Log-space decay tensor with shape ``[1, T, Hv]``.
beta: Delta-rule beta tensor with shape ``[1, T, Hv]``.
initial_state: Recurrent state with shape ``[N, Hv, V, K]``.
cu_seqlens: Cumulative sequence lengths with shape ``[N + 1]``.
chunk_indices: Chunk index metadata with shape ``[NT, 2]``.
chunk_offsets: Cumulative chunk offsets with shape ``[N + 1]``.
core_attn_out: Optional output buffer with shape ``[T, Hv, V]``.
Returns:
A tuple ``(output, final_state)`` where ``output`` has shape
``[1, T, Hv, V]`` and ``final_state`` has shape ``[N, Hv, V, K]``.
When ``core_attn_out`` is provided, ``output`` is an unsqueezed view of
that buffer.
"""
q = q.squeeze(0)
k = k.squeeze(0)
v = v.squeeze(0)
g = g.squeeze(0)
beta = beta.squeeze(0)
_, _, K_dim = k.shape
_, num_v_heads, V_dim = v.shape
chunk_size = 64
upper_bound_chunks = chunk_indices.shape[0]
pad_t = upper_bound_chunks * chunk_size
total_chunks_ptr = chunk_offsets[-1:]
g_cu = torch.empty_like(g, dtype=torch.float32)
u = q.new_empty(pad_t, num_v_heads, V_dim)
w = q.new_empty(pad_t, num_v_heads, K_dim)
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
kkt_inv_uw_cutedsl(
k,
v,
u,
w,
g,
beta,
g_cu,
cu_seqlens,
chunk_indices,
total_chunks_ptr,
num_sms=num_sms,
)
h = k.new_empty(upper_bound_chunks, num_v_heads, V_dim, K_dim)
v_new = q.new_empty(pad_t, num_v_heads, V_dim)
final_state = torch.empty_like(initial_state)
h_cutedsl(
k,
u,
w,
v_new,
g_cu,
h,
initial_state,
final_state,
cu_seqlens,
chunk_offsets,
)
output = core_attn_out if core_attn_out is not None else torch.empty_like(v)
scale = K_dim**-0.5
o_cutedsl(
q,
k,
v_new,
h,
g_cu,
output,
cu_seqlens,
chunk_indices,
total_chunks_ptr,
scale,
num_sms=num_sms,
)
return output.unsqueeze(0), final_state
__all__ = [
"chunk_gated_delta_rule_cutedsl",
"prepare_metadata_cutedsl",
]
@@ -1,751 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
from cutlass.cute.nvgpu import cpasync, warp
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import (
EVICT_FIRST,
_tcgen05,
cvt,
fence_before_tma_store,
simple_tma_copy,
)
class Sm100ChunkHKernel:
"""For each sequence, compute the chunk recurrent update.
The input V tile is the U output from the KKT/UW kernel. For each chunk:
V_new = U - W @ H.T
(we actually do V_new.T = U.T - H @ W.T instead)
H_scaled = H * exp(g_last)
V_scaled = V_new * exp(g_last - g)
H_new = H_scaled + V_scaled.T @ K
"""
def __init__(
self,
H: int,
Hv: int,
K_dim: int,
V_dim: int,
h_dtype: cutlass.Numeric = Float32,
BT: int = 64,
num_stages: int = 2,
) -> None:
assert Hv % H == 0
assert K_dim == V_dim == 128
assert BT == 64
self.H = H
self.Hv = Hv
self.K_dim = K_dim
self.V_dim = V_dim
self.h_dtype = h_dtype
self.BT = BT
self.num_stages = num_stages
self.num_warps = 10
@cute.jit
def _make_bf16_tma_args(
self,
tensor: cute.Tensor,
dim: cutlass.Constexpr[int],
op: cpasync.TmaCopyOp,
stages: cutlass.Constexpr[int],
):
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(self.BT, 1, (64, dim // 64), stages),
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, 64)),
slayout,
cta_tiler=(self.BT, 1, dim),
)
@cute.jit
def _make_h_tma_args(self, tensor: cute.Tensor, op: cpasync.TmaCopyOp):
# number of elements to fill 128B
num_elems = 128 // (tensor.element_type.width // 8)
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(1, 1, self.V_dim, (num_elems, self.K_dim // num_elems)),
stride=(0, 0, num_elems, (1, self.V_dim * num_elems)),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, None, num_elems)),
slayout,
cta_tiler=(1, 1, self.V_dim, self.K_dim),
)
@cute.jit
def __call__(
self,
K: cute.Tensor,
V: cute.Tensor,
W: cute.Tensor,
V_new: cute.Tensor,
g_cu: cute.Tensor,
h: cute.Tensor,
h0: cute.Tensor,
ht: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_offsets: cute.Tensor,
stream: CUstream,
):
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
K_tma = self._make_bf16_tma_args(K, self.K_dim, tma_g2s, self.num_stages)
V_tma = self._make_bf16_tma_args(V, self.V_dim, tma_g2s, self.num_stages)
W_tma = self._make_bf16_tma_args(W, self.K_dim, tma_g2s, self.num_stages)
V_new_tma = self._make_bf16_tma_args(V_new, self.V_dim, tma_s2g, 1)
H0_tma = self._make_h_tma_args(h0, tma_g2s)
HT_tma = self._make_h_tma_args(ht, tma_s2g)
H_tma = self._make_h_tma_args(h, tma_s2g)
grid = (self.Hv, h0.shape[0], 1)
block = (self.num_warps * 32, 1, 1)
self.kernel(
K_tma,
V_tma,
W_tma,
V_new_tma,
H0_tma,
HT_tma,
H_tma,
g_cu,
cu_seqlens,
chunk_offsets,
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
@cute.kernel
def kernel(
self,
K_tma: cpasync.TmaInfo,
V_tma: cpasync.TmaInfo,
W_tma: cpasync.TmaInfo,
V_new_tma: cpasync.TmaInfo,
H0_tma: cpasync.TmaInfo,
HT_tma: cpasync.TmaInfo,
H_tma: cpasync.TmaInfo,
g_cu: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_offsets: cute.Tensor,
):
tid, _, _ = cute.arch.thread_idx()
head_id, seq_id, _ = cute.arch.block_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
BT = self.BT
V_dim = self.V_dim
K_dim = self.K_dim
num_stages = self.num_stages
is_f32 = self.h_dtype == Float32
def allocate_tensor(smem, dtype, layout):
return smem.allocate_tensor(
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
)
smem = cutlass.utils.SmemAllocator()
# remove size=1 modes
sW = allocate_tensor(smem, BFloat16, W_tma.smem_layout)[None, 0, None, None]
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
sH0 = allocate_tensor(smem, self.h_dtype, H0_tma.smem_layout)[0, 0, None, None]
sH = allocate_tensor(smem, BFloat16, H_tma.smem_layout)[0, 0, None, None]
sV_new = allocate_tensor(smem, BFloat16, V_new_tma.smem_layout)[
None, 0, None, 0
]
s_v_scale = smem.allocate_array(Float32, BT)
tma_mbar = smem.allocate_array(Int64, num_stages)
wh_in_mbar = smem.allocate_array(Int64, num_stages)
wh_done_mbar = smem.allocate_array(Int64, num_stages)
vk_in_mbar = smem.allocate_array(Int64, num_stages)
vk_done_mbar = smem.allocate_array(Int64, num_stages)
h0_mbar = smem.allocate_array(Int64, 1)
taddr = smem.allocate(Int32, 4)
wh_tmem = 0
vk_tmem = wh_tmem + BT
h_tmem_base = vk_tmem + K_dim
v_tmem_base = h_tmem_base + K_dim // 2
if warp_id == 0:
with cute.arch.elect_one():
for i in cutlass.range_constexpr(num_stages):
cute.arch.mbarrier_init(tma_mbar + i, 1)
cute.arch.mbarrier_init(wh_in_mbar + i, 256)
cute.arch.mbarrier_init(wh_done_mbar + i, 1)
cute.arch.mbarrier_init(vk_in_mbar + i, 256)
cute.arch.mbarrier_init(vk_done_mbar + i, 1)
cute.arch.mbarrier_init(h0_mbar, 1)
cute.arch.mbarrier_init_fence()
elif warp_id == 1:
cpasync.prefetch_descriptor(H0_tma.atom)
cpasync.prefetch_descriptor(W_tma.atom)
cpasync.prefetch_descriptor(V_tma.atom)
cpasync.prefetch_descriptor(K_tma.atom)
cpasync.prefetch_descriptor(HT_tma.atom)
cpasync.prefetch_descriptor(H_tma.atom)
cpasync.prefetch_descriptor(V_new_tma.atom)
cute.arch.sync_threads()
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
seqlen = eos - bos
num_chunks = cute.ceil_div(seqlen, BT)
if warp_id == 9:
# TMA warp
stage_id = 0
parity = 1
k_head_id = head_id // (self.Hv // self.H)
chunk_offset = chunk_offsets[seq_id]
# load H0
with cute.arch.elect_one():
H0_size = V_dim * K_dim * self.h_dtype.width // 8
cute.arch.mbarrier_arrive_and_expect_tx(h0_mbar, H0_size)
simple_tma_copy(
H0_tma.atom,
H0_tma.tma_tensor[seq_id, head_id, None, None],
sH0,
h0_mbar,
)
# shape: ((BT, num_BT_tiles), (64, 2))
gW_tiles = cute.logical_divide(
W_tma.tma_tensor[None, head_id, None], (BT, None)
)
gV_tiles = cute.logical_divide(
V_tma.tma_tensor[None, head_id, None], (BT, None)
)
gK_tiles = cute.logical_divide(
cute.domain_offset((bos, 0), K_tma.tma_tensor[None, k_head_id, None]),
(BT, None),
)
for chunk_id in range(num_chunks):
mbar = tma_mbar + stage_id
gW = gW_tiles[(None, chunk_offset + chunk_id), None]
gV = gV_tiles[(None, chunk_offset + chunk_id), None]
gK = gK_tiles[(None, chunk_id), None]
# wait for MMA to release the buffer
cute.arch.mbarrier_wait(vk_done_mbar + stage_id, parity)
# load W, V (i.e. U), and K
with cute.arch.elect_one():
STAGE_SIZE = BT * (K_dim + V_dim + K_dim) * 2
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
simple_tma_copy(
W_tma.atom, gW, sW[None, None, stage_id], mbar, EVICT_FIRST
)
simple_tma_copy(
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
)
simple_tma_copy(K_tma.atom, gK, sK[None, None, stage_id], mbar)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id == 8:
# MMA warp
_tcgen05.alloc(taddr)
stage_id = 0
parity = 0
wh_idesc = _tcgen05.make_bf16_idesc(V_dim, BT, negate_A=True)
vk_idesc = _tcgen05.make_bf16_idesc(V_dim, K_dim, transpose_B=True)
# LBO=BT*128 is ignored for K-major
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
# when using BF16 state, H is read from smem for the 1st iteration
# variable names in this conditional branch can't be the same as those
# in the mainloop below due to CuteDSL restrictions.
if cutlass.const_expr(not is_f32):
##### 1st MMA: V_new.T = V.T - H @ W.T #####
Haddr0 = sH0[None, None].iterator.toint()
Waddr0 = sW[None, None, stage_id].iterator.toint()
hdesc0_base = sdesc_template | (Haddr0 >> 4)
wdesc0_base = sdesc_template | (Waddr0 >> 4)
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
hdesc0 = hdesc0_base | ((i * V_dim * 128 + j * 32) >> 4)
wdesc0 = wdesc0_base | ((i * BT * 128 + j * 32) >> 4)
_tcgen05.mma_f16(wh_tmem, hdesc0, wdesc0, wh_idesc, True)
_tcgen05.commit(wh_done_mbar + stage_id)
##### 2nd MMA: H_new = H + V_new.T @ K #####
Kaddr0 = sK[None, None, stage_id].iterator.toint()
kdesc0_base = sdesc_template | (Kaddr0 >> 4)
cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for k in cutlass.range_constexpr(BT // 16):
vtmem0 = v_tmem_base + k * 8
kdesc0 = kdesc0_base | ((k * 16 * 128) >> 4)
_tcgen05.mma_ts_f16(vk_tmem, vtmem0, kdesc0, vk_idesc, True)
_tcgen05.commit(vk_done_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
num_iters = num_chunks - int(not is_f32)
for _ in range(num_iters):
##### 1st MMA: V_new.T = V.T - H @ W.T #####
Waddr = sW[None, None, stage_id].iterator.toint()
wdesc_base = sdesc_template | (Waddr >> 4)
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
htmem = h_tmem_base + i * 32 + j * 8
wdesc = wdesc_base | ((i * BT * 128 + j * 32) >> 4)
_tcgen05.mma_ts_f16(wh_tmem, htmem, wdesc, wh_idesc, True)
_tcgen05.commit(wh_done_mbar + stage_id)
##### 2nd MMA: H_new = H + V_new.T @ K #####
Kaddr = sK[None, None, stage_id].iterator.toint()
kdesc_base = sdesc_template | (Kaddr >> 4)
cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for k in cutlass.range_constexpr(BT // 16):
vtmem = v_tmem_base + k * 8
kdesc = kdesc_base | ((k * 16 * 128) >> 4)
_tcgen05.mma_ts_f16(vk_tmem, vtmem, kdesc, vk_idesc, True)
_tcgen05.commit(vk_done_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id >= 4:
# H warps
tid_ = tid % 128
warp_id_ = warp_id % 4
chunk_offset = chunk_offsets[seq_id]
stage_id = 0
vk_stage_id = 0
vk_parity = 0
op = cute.nvgpu.CopyUniversalOp()
cp_16B = cute.make_copy_atom(op, Float32, num_bits_per_copy=128)
##### chunk_id = 0 #####
if True:
chunk_id = 0
end_t = min(bos + (chunk_id + 1) * BT, eos)
last_idx = end_t - 1
h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True)
# for 1st chunk, wait for H0 transfer from gmem
if warp_id_ == 0:
cute.arch.mbarrier_wait(h0_mbar, 0)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
# when H0 is FP32, we need to pack it to BF16
# also store to smem for TMA store later.
if cutlass.const_expr(is_f32):
for i in cutlass.range_constexpr(K_dim // 32):
# H0 smem layout: (V_dim, (32, K_dim/32))
h_f32 = cute.make_rmem_tensor(32, Float32)
cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32)
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
h_bf16.store(h_f32.load().to(BFloat16))
_tcgen05.st(
warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16
)
# H smem layout: (V_dim, (64, K_dim/64))
dst = cute.local_tile(sH[tid_, None], (32,), (i,))
cute.copy(cp_16B, h_bf16, dst)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
# scale H for 2nd MMA
for i in cutlass.range_constexpr(K_dim // 32):
h_f32 = cute.make_rmem_tensor(32, Float32)
if cutlass.const_expr(is_f32):
cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32)
else:
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
sH_src = cute.local_tile(sH0[tid_, None], (32,), (i,))
cute.copy(cp_16B, sH_src, h_bf16)
h_f32.store(
cvt.bf16x2_to_fp32x2(
cute.recast_tensor(h_bf16, Uint32)
).load()
)
for j in cutlass.range(32, vectorize=True):
h_f32[j] *= h_scale
_tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
# for BF16 H0, we issue TMA store from H0 smem
# for FP32 H0, we issue TMA store from H smem (after packing)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
fence_before_tma_store()
if warp_id_ == 3:
h_src = sH if cutlass.const_expr(is_f32) else sH0
h_dst = H_tma.tma_tensor[
chunk_offset + chunk_id, head_id, None, None
]
simple_tma_copy(H_tma.atom, h_src, h_dst)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
# When H0 is BF16, and there is only 1 chunk, storing
# the final state to sH0 can race before this store
# has finished. hence, we need to wait here.
if cutlass.const_expr(not is_f32):
cute.arch.cp_async_bulk_wait_group(0, read=True)
stage_id = (stage_id + 1) % num_stages
##### subsequent chunks #####
for chunk_id in range(1, num_chunks):
end_t = min(bos + (chunk_id + 1) * BT, eos)
last_idx = end_t - 1
h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True)
# wait for H from previous vk MMA
if warp_id_ == 0:
cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity)
vk_stage_id = (vk_stage_id + 1) % num_stages
if vk_stage_id == 0:
vk_parity ^= 1
elif warp_id_ == 3:
with cute.arch.elect_one():
cute.arch.cp_async_bulk_wait_group(0, read=True)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
# load FP32 H from tmem, convert to BF16, store to tmem for 1st MMA,
# store to smem for TMA store later.
for i in cutlass.range_constexpr(K_dim // 32):
h_f32 = _tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
h_bf16.store(h_f32.to(BFloat16))
_tcgen05.st(
warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16
)
# H smem layout: (V_dim, (64, K_dim/64))
dst = cute.local_tile(sH[tid_, None], (32,), (i,))
cute.copy(cp_16B, h_bf16, dst)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
# scale H for 2nd MMA
for i in cutlass.range_constexpr(K_dim // 32):
h_f32 = cute.make_rmem_tensor(32, Float32)
h_f32.store(
_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)
)
for j in cutlass.range(32, vectorize=True):
h_f32[j] *= h_scale
_tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
# issue TMA store for O kernel
cute.arch.barrier(barrier_id=1, number_of_threads=128)
fence_before_tma_store()
if warp_id_ == 3:
h_dst = H_tma.tma_tensor[
chunk_offset + chunk_id, head_id, None, None
]
simple_tma_copy(H_tma.atom, sH, h_dst)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
stage_id = (stage_id + 1) % num_stages
# handle final state. reuse H0 smem.
if warp_id_ == 0:
cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 32):
h_f32 = cute.make_rmem_tensor(32, Float32)
h_f32.store(_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32))
if cutlass.const_expr(is_f32):
cute.copy(cp_16B, h_f32, sH0[tid_, (None, i)])
else:
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
h_bf16.store(h_f32.load().to(BFloat16))
sH0_dst = cute.local_tile(sH0[tid_, None], (32,), (i,))
cute.copy(cp_16B, h_bf16, sH0_dst)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
if warp_id_ == 0:
ht_dst = HT_tma.tma_tensor[seq_id, head_id, None, None]
simple_tma_copy(HT_tma.atom, sH0, ht_dst)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
if warp_id_ == 1:
_tcgen05.dealloc()
else:
# V warps
stage_id = 0
parity = 0
chunk_offset = chunk_offsets[seq_id]
ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True)
stsm_trans_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=True)
ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16)
stsm_trans_atom = cute.make_copy_atom(stsm_trans_op, BFloat16)
# ((BT, num_BT_tiles), V_dim)
gV_new_tiles = cute.logical_divide(
V_new_tma.tma_tensor[None, head_id, None], (BT, None)
)
# sV shape: [BT, (64, V_dim/64), num_stages]
# sV_view shape: [BT, (8, (8,2)), num_stages]
sV_view = cute.logical_divide(sV, (None, 8, None))
sV_new_view = cute.logical_divide(sV_new, (None, 8))
# [BT, 8, num_stages]
s_col = warp_id * 4 + (lane_id // 8)
sV_view = sV_view[None, (None, s_col), None]
sV_new_view = sV_new_view[None, (None, s_col)]
for chunk_id in range(num_chunks):
# wait for V to arrive
if warp_id == 0:
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
cute.arch.barrier(barrier_id=2, number_of_threads=128)
# unpack V BF16->FP32, then store to tmem for 1st MMA
# V smem layout: [BT, (64, V_dim/64)] / [BT, V_dim]
# each iteration, CTA loads [8, V_dim] tile
# (warp loads [8, 32] tile)
for i in cutlass.range_constexpr(BT // 8):
s_row = i * 8 + (lane_id % 8)
v_bf16 = cute.make_rmem_tensor(8, BFloat16)
cute.copy(ldsm_trans_atom, sV_view[s_row, None, stage_id], v_bf16)
v_fp32 = cvt.bf16x2_to_fp32x2(cute.recast_tensor(v_bf16, Uint32))
v_fp32 = cute.logical_divide(v_fp32, 4) # (4, 2)
tcol = wh_tmem + i * 8
_tcgen05.st(warp_id * 32 + 0, tcol, "16x256b", 1, v_fp32[None, 0])
_tcgen05.st(warp_id * 32 + 16, tcol, "16x256b", 1, v_fp32[None, 1])
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
# load g_cu for scaling
if tid < BT:
end_t = min(bos + (chunk_id + 1) * BT, eos)
last_idx = end_t - 1
t = bos + chunk_id * BT + tid
val = Float32(0.0)
if t < eos:
val = cute.math.exp(
g_cu[last_idx, head_id] - g_cu[t, head_id],
fastmath=True,
)
s_v_scale[tid] = val
# wait for 1st MMA to finish
if warp_id == 2:
cute.arch.mbarrier_wait(wh_done_mbar + stage_id, parity)
elif warp_id == 3:
with cute.arch.elect_one():
cute.arch.cp_async_bulk_wait_group(0, read=True)
cute.arch.barrier(barrier_id=2, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(BT // 8):
v_new = cute.make_rmem_tensor((4, 2), Float32)
tcol = wh_tmem + i * 8
v_new[None, 0].store(
_tcgen05.ld(warp_id * 32 + 0, tcol, "16x256b", 1)
)
v_new[None, 1].store(
_tcgen05.ld(warp_id * 32 + 16, tcol, "16x256b", 1)
)
v_new_bf16 = cute.make_rmem_tensor(8, BFloat16)
v_new_bf16.store(v_new.load().to(BFloat16))
# scale V_new for 2nd MMA
scale0 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 0]
scale1 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 1]
v_scaled = cute.make_rmem_tensor(8, Float32)
for k in cutlass.range_constexpr(4):
v_scaled[k * 2] = v_new[k * 2] * scale0
v_scaled[k * 2 + 1] = v_new[k * 2 + 1] * scale1
v_scaled_bf16 = v_scaled.load().to(BFloat16).reshape((4, 2))
# store V_new BF16 for O kernel
s_row = i * 8 + (lane_id % 8)
cute.copy(stsm_trans_atom, v_new_bf16, sV_new_view[s_row, None])
# store to tmem
tcol = v_tmem_base + i * 4
_tcgen05.st(
warp_id * 32 + 0, tcol, "16x128b", 1, v_scaled_bf16[None, 0]
)
_tcgen05.st(
warp_id * 32 + 16, tcol, "16x128b", 1, v_scaled_bf16[None, 1]
)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
# issue TMA store for V_new
cute.arch.barrier(barrier_id=2, number_of_threads=128)
fence_before_tma_store()
if warp_id == 3:
gV = gV_new_tiles[(None, chunk_offset + chunk_id), None]
simple_tma_copy(V_new_tma.atom, sV_new, gV)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
@cache
@staticmethod
def compile(
H: int,
Hv: int,
K_dim: int,
V_dim: int,
h_dtype: cutlass.Numeric = Float32,
BT: int = 64,
num_stages: int = 2,
):
total_t = cute.sym_int()
pad_t = cute.sym_int()
total_chunks_n = cute.sym_int()
num_sequences = cute.sym_int()
cu_entries = cute.sym_int()
K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
V = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16)
V_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
h = make_fake_tensor(
BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16
)
h0 = make_fake_tensor(
h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16
)
ht = make_fake_tensor(
h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16
)
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
kernel = Sm100ChunkHKernel(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
K,
V,
W,
V_new,
g_cu,
h,
h0,
ht,
cu_seqlens,
chunk_offsets,
stream,
options="--enable-tvm-ffi",
)
def h_cutedsl(
K: torch.Tensor,
V: torch.Tensor,
W: torch.Tensor,
V_new: torch.Tensor,
g_cu: torch.Tensor,
h: torch.Tensor,
h0: torch.Tensor,
ht: torch.Tensor,
cu_seqlens: torch.Tensor,
chunk_offsets: torch.Tensor,
BT: int = 64,
num_stages: int = 2,
) -> None:
"""Compute H/V_new with the same argument order as the CUDA wrapper."""
_, H, K_dim = K.shape
_, Hv, V_dim = V.shape
h_dtype = {
torch.bfloat16: BFloat16,
torch.float32: Float32,
}[h0.dtype]
Sm100ChunkHKernel.compile(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)(
K,
V,
W,
V_new,
g_cu,
h,
h0,
ht,
cu_seqlens,
chunk_offsets,
)
h_v2b_cutedsl = h_cutedsl
@@ -1,925 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
from cutlass.cute.nvgpu import cpasync, warp
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import (
EVICT_FIRST,
_bf16x2_neg,
_bf16x2_sub,
_tcgen05,
cvt,
fence_before_tma_store,
mma_sync,
simple_tma_copy,
)
class Sm100ChunkUWKernel:
"""Compute per-chunk KKT inverse preprocessing and U/W tiles.
Gamma[i,j] = exp(g_cu[i] - g_cu[j])
A = strictLower(beta * (K @ K.T) * Gamma)
Ai = inverse(I + A)
U = (Ai * beta) @ V
W = (Ai * beta * exp(g_cu)) @ K
"""
def __init__(
self,
H: int,
Hv: int,
K_dim: int,
V_dim: int,
num_stages: int = 2,
) -> None:
assert Hv % H == 0
assert K_dim == V_dim == 128
self.H = H
self.Hv = Hv
self.K_dim = K_dim
self.V_dim = V_dim
self.num_stages = num_stages
# hard-code
self.BT = 64
self.num_warps = 4 + 4 + 4
@cute.jit
def _make_tma_args(
self,
tensor: cute.Tensor,
dim: cutlass.Constexpr[int],
num_stages: int,
op: cpasync.TmaCopyOp,
):
# logical layout: [BT, dim]
# permute for TMA: [dim/64, BT, 64] with swizzling
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(self.BT, 1, (64, dim // 64), num_stages),
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
# we need to convert gmem layout to (T, H, (64, D/64)) for make_tiled_tma_atom()
# to emit a single 4D TMA. otherwise, it will emit (D/64)x 3D TMA.
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, 64)),
slayout,
cta_tiler=(self.BT, 1, dim),
)
@cute.jit
def __call__(
self,
K: cute.Tensor,
V: cute.Tensor,
U: cute.Tensor,
W: cute.Tensor,
g: cute.Tensor,
beta: cute.Tensor,
g_cu: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
total_chunks: cute.Tensor,
num_sms: Int32,
stream: CUstream,
):
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
K_tma = self._make_tma_args(K, self.K_dim, self.num_stages, tma_g2s)
V_tma = self._make_tma_args(V, self.V_dim, self.num_stages, tma_g2s)
U_tma = self._make_tma_args(U, self.V_dim, 1, tma_s2g)
W_tma = self._make_tma_args(W, self.K_dim, 1, tma_s2g)
grid = (num_sms // self.Hv, self.Hv, 1)
block = (self.num_warps * 32, 1, 1)
self.kernel(
K_tma,
V_tma,
U_tma,
W_tma,
g,
beta,
g_cu,
cu_seqlens,
chunk_indices,
total_chunks,
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
@cute.kernel
def kernel(
self,
K_tma: cpasync.TmaInfo,
V_tma: cpasync.TmaInfo,
U_tma: cpasync.TmaInfo,
W_tma: cpasync.TmaInfo,
g: cute.Tensor,
beta: cute.Tensor,
g_cu: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
total_chunks: cute.Tensor,
):
tid, _, _ = cute.arch.thread_idx()
bid, head_id, _ = cute.arch.block_idx()
grid_x, _, _ = cute.arch.grid_dim()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
k_head_id = head_id // (self.Hv // self.H)
BT = self.BT
K_dim = self.K_dim
V_dim = self.V_dim
num_stages = self.num_stages
INV_BAR = 1
EPI_BAR = 2
SCAN_BAR = 3
TMEM_ALLOC_BAR = 4
def allocate_tensor(smem, dtype, layout):
return smem.allocate_tensor(
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
)
smem = cutlass.utils.SmemAllocator()
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
sU = allocate_tensor(smem, BFloat16, U_tma.smem_layout)[None, 0, None, 0]
sW = allocate_tensor(smem, BFloat16, W_tma.smem_layout)[None, 0, None, 0]
sA_ptr = smem.allocate_array(BFloat16, BT * BT, byte_alignment=16)
sAi_ptr = smem.allocate_array(BFloat16, BT * BT, byte_alignment=16)
s_beta = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
s_g_cu = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
s_beta_g = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
tma_mbar = smem.allocate_array(Int64, num_stages)
prep_mbar = smem.allocate_array(Int64, num_stages)
mma_kkt_mbar = smem.allocate_array(Int64, num_stages)
inv_mbar = smem.allocate_array(Int64, num_stages)
mma_u_mbar = smem.allocate_array(Int64, num_stages)
mma_w_mbar = smem.allocate_array(Int64, num_stages)
epi_mbar = smem.allocate_array(Int64, num_stages)
taddr = smem.allocate(Int32, 4)
kkt_tmem = 0
U_tmem_base = kkt_tmem + BT
Ab_tmem_base = U_tmem_base + V_dim * num_stages
assert Ab_tmem_base + (BT // 2) <= 512
# prepare ldmatrix/stmatrix ops
ldsm_op = warp.LdMatrix8x8x16bOp(num_matrices=4)
stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4)
ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True)
ldsm_atom = cute.make_copy_atom(ldsm_op, BFloat16)
stsm_atom = cute.make_copy_atom(stsm_op, BFloat16)
ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16)
if warp_id == 0:
with cute.arch.elect_one():
for i in cutlass.range_constexpr(num_stages):
cute.arch.mbarrier_init(tma_mbar + i, 1)
cute.arch.mbarrier_init(prep_mbar + i, 64)
cute.arch.mbarrier_init(mma_kkt_mbar + i, 1)
cute.arch.mbarrier_init(inv_mbar + i, 128)
cute.arch.mbarrier_init(mma_u_mbar + i, 1)
cute.arch.mbarrier_init(mma_w_mbar + i, 1)
cute.arch.mbarrier_init(epi_mbar + i, 128)
cute.arch.mbarrier_init_fence()
elif warp_id == 1:
cpasync.prefetch_descriptor(K_tma.atom)
cpasync.prefetch_descriptor(V_tma.atom)
cpasync.prefetch_descriptor(U_tma.atom)
cpasync.prefetch_descriptor(W_tma.atom)
cute.arch.sync_threads()
num_global_chunks = total_chunks[0]
if warp_id == 11:
# TMA warp
stage_id = 0
parity = 1
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
# since off_t is not a multiple of BT, we need to use
# domain_offset() to shift the pointer first.
mbar = tma_mbar + stage_id
gK = cute.local_tile(
cute.domain_offset(
(bos, 0), K_tma.tma_tensor[None, k_head_id, None]
),
tiler=(BT, K_dim),
coord=(chunk_id, 0),
)
gV = cute.local_tile(
cute.domain_offset((bos, 0), V_tma.tma_tensor[None, head_id, None]),
tiler=(BT, V_dim),
coord=(chunk_id, 0),
)
# when UW MMA is done, K and V TMA buffers are released
cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity)
with cute.arch.elect_one():
STAGE_SIZE = BT * (K_dim + V_dim) * 2
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
simple_tma_copy(K_tma.atom, gK, sK[None, None, stage_id], mbar)
simple_tma_copy(
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id == 10:
# MMA warp
_tcgen05.alloc(taddr)
cute.arch.barrier(barrier_id=TMEM_ALLOC_BAR, number_of_threads=160)
stage_id = 0
parity = 0
kkt_idesc = _tcgen05.make_bf16_idesc(BT, BT)
u_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True)
w_idesc = _tcgen05.make_bf16_idesc(BT, K_dim, transpose_B=True)
# LBO=BT*128 is ignored for K-major
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
for global_chunk_id in range(bid, num_global_chunks, grid_x):
U_tmem = U_tmem_base + V_dim * stage_id
W_tmem = U_tmem | (16 << 16)
Ab_tmem = Ab_tmem_base
Abg_tmem = Ab_tmem_base | (16 << 16)
##### KKT MMA: KKT = K @ K.T #####
kaddr = sK[None, None, stage_id].iterator.toint()
kdesc_base = sdesc_template | (kaddr >> 4)
# wait for TMA data to arrive
# kkt tmem is guaranteed to be free as this is issued
# after the previous kkt's consumer (inv warps)
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4)
enable_d = (i > 0) or (j > 0)
_tcgen05.mma_f16(kkt_tmem, kdesc, kdesc, kkt_idesc, enable_d)
_tcgen05.commit(mma_kkt_mbar + stage_id)
##### U/W MMA: U = Ab @ V, W = Abg @ K #####
vaddr = sV[None, None, stage_id].iterator.toint()
vdesc = sdesc_template | (vaddr >> 4)
kdesc = sdesc_template | (kaddr >> 4)
# wait for epilogue to release tmem buffer
cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1)
cute.arch.mbarrier_wait(inv_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(BT // 16):
_tcgen05.mma_ts_f16(W_tmem, Abg_tmem + i * 8, kdesc, w_idesc, i > 0)
kdesc += (16 * 128) >> 4
_tcgen05.commit(mma_w_mbar + stage_id)
for i in cutlass.range_constexpr(BT // 16):
_tcgen05.mma_ts_f16(U_tmem, Ab_tmem + i * 8, vdesc, u_idesc, i > 0)
vdesc += (16 * 128) >> 4
_tcgen05.commit(mma_u_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1)
_tcgen05.dealloc()
elif warp_id >= 8:
# dedicated prep warps for beta and gate, consumed by INV warps
stage_id = 0
parity = 0
tid_ = tid % 128
warp_id_ = warp_id % 4
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
off_t = bos + chunk_id * BT
t = off_t + tid_
in_bounds = t < eos
beta_val = beta[t, head_id] if in_bounds else Float32(0.0)
g_val = g[t, head_id] if in_bounds else Float32(0.0)
# warp-local prefix scan
for i in cutlass.range_constexpr(5):
offset = cutlass.const_expr(1 << i)
lower = cute.arch.shuffle_sync_up(g_val, offset, mask_and_clamp=0)
if lane_id >= offset:
g_val += lower
# Delay the stage-reuse wait until just before touching smem:
# global loads and the warp-local scan do not use staged buffers.
if warp_id_ == 0:
cute.arch.mbarrier_wait(inv_mbar + stage_id, parity ^ 1)
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
# Store beta and the per-warp scan totals for the cross-warp fixup.
s_beta[tid_, stage_id] = beta_val
if lane_id == 31:
s_g_cu[warp_id_, stage_id] = g_val
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
# Add the sum from the lower prep warp.
if warp_id_ == 1:
g_val += s_g_cu[0, stage_id]
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
if in_bounds:
g_cu[t, head_id] = g_val
s_g_cu[tid_, stage_id] = g_val
s_beta_g[tid_, stage_id] = beta_val * cute.math.exp(g_val)
cute.arch.mbarrier_arrive(prep_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id >= 4:
# inv warps
tid_ = tid % 128
warp_id_ = warp_id % 4
def store_ab_abg(
Ai_f32, s_beta, s_beta_g, warp_id_, lane_id, tile_col, Ab_tmem_base
):
# compute Ab and Abg from Ai, then store to tmem
beta_col = cute.make_rmem_tensor((2, 2), Float32)
beta_g_col = cute.make_rmem_tensor((2, 2), Float32)
for i in cutlass.range_constexpr(2):
base = tile_col * 16 + i * 8 + (lane_id % 4) * 2
for j in cutlass.range_constexpr(2):
beta_col[j, i] = s_beta[base + j]
beta_g_col[j, i] = s_beta_g[base + j]
# without conversion to TensorSSA, cutlass.range(vectorize=True) fails
beta_col = beta_col.load()
beta_g_col = beta_g_col.load()
Ab_f32 = cute.make_rmem_tensor(8, Float32)
Abg_f32 = cute.make_rmem_tensor(8, Float32)
for i in cutlass.range(8, vectorize=True):
scale_idx = (i // 4) * 2 + (i % 2)
Ab_f32[i] = Ai_f32[i] * beta_col[scale_idx]
Abg_f32[i] = Ai_f32[i] * beta_g_col[scale_idx]
Ab = Ab_f32.load().to(BFloat16)
Abg = Abg_f32.load().to(BFloat16)
Ab_tmem = Ab_tmem_base + tile_col * 8
_tcgen05.st(warp_id_ * 32, Ab_tmem, "16x128b", 2, Ab)
_tcgen05.st(warp_id_ * 32 + 16, Ab_tmem, "16x128b", 2, Abg)
# clear the Ab/Abg tmem buffer once before mainloop
# this is to keep the upper triangular tiles zeros
cute.arch.barrier(barrier_id=TMEM_ALLOC_BAR, number_of_threads=160)
zeros_ = cute.make_rmem_tensor(BT // 2, Float32)
zeros_.fill(0.0)
_tcgen05.st(warp_id_ * 32, Ab_tmem_base, "32x32b", BT // 2, zeros_)
stage_id = 0
parity = 0
# for sA, we can avoid bank conflict without using swizzling because
# this is only used as tmp buffer between rmem<->smem, no gmem interactions.
# to do so, we can put (8,8) tile contiguous in memory. logically, we are
# partitioning (64,64) tile into 4x (16,16) tiles. to make indexing easier
# later, we view (16,16) tile as (32,8) tile here.
# (4,4) is (row_tile, col_tile).
sA_layout = cute.make_layout(((8, 32), (4, 4)))
sA = cute.make_tensor(sA_ptr, sA_layout)
sAi = cute.make_tensor(sAi_ptr, sA_layout)
# pre-compute ldmatrix addresses
sA_ldsm = sA[(None, lane_id), None]
sAi_ldsm = sAi[(None, lane_id), None]
# init Ai smem buffer with zeros (only the first 48 rows)
zeros_bf16 = cute.make_rmem_tensor(8, BFloat16)
zeros_bf16.fill(0.0)
for i in cutlass.range_constexpr(3):
cute.copy(stsm_atom, zeros_bf16, sAi_ldsm[None, (i, warp_id_)])
# indices for ldmatrix layout later
row_indices = cute.make_rmem_tensor((1, 2, 1), Int32)
row_indices[0, 0, 0] = warp_id_ * 16 + (lane_id // 4)
row_indices[0, 1, 0] = warp_id_ * 16 + (lane_id // 4) + 8
row_indices = row_indices.load()
col_indices = cute.make_rmem_tensor((2, 1, 2), Int32)
col_indices[0, 0, 0] = (lane_id % 4) * 2 + 0
col_indices[1, 0, 0] = (lane_id % 4) * 2 + 1
col_indices[0, 0, 1] = (lane_id % 4) * 2 + 8
col_indices[1, 0, 1] = (lane_id % 4) * 2 + 9
col_indices = col_indices.load()
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
off_t = bos + chunk_id * BT
##### Phase 1: A = strictLower(beta * kkt * Gamma) #####
# Ab/Abg share one tmem slot across stages. The MMA warp commits
# tcgen05 groups in program order: KKT_i, W_i, U_i, KKT_{i+1}.
# Waiting for KKT_i means the previous W/U commits have completed,
# so the INV warps can safely overwrite Ab/Abg for this iteration.
# Wait for prep warps to publish beta/gate and for KKT MMA.
if warp_id_ == 0:
cute.arch.mbarrier_wait(prep_mbar + stage_id, parity)
cute.arch.mbarrier_wait(mma_kkt_mbar + stage_id, parity)
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
# tmem 16x256b layout / ldmatrix layout
beta_row = cute.make_rmem_tensor(2, Float32)
g_cu_row = cute.make_rmem_tensor(2, Float32)
for i in cutlass.range_constexpr(2):
idx = warp_id_ * 16 + i * 8 + (lane_id // 4)
beta_row[i] = s_beta[idx, stage_id]
g_cu_row[i] = s_g_cu[idx, stage_id]
# mode0 is 8 rows together
# mode1 is top and bottom 8 rows
# mode2 is groups of 16 rows
beta_row = beta_row.load().reshape((1, 2, 1))
g_cu_row = g_cu_row.load().reshape((1, 2, 1))
# mode0 is 2 consecutive elems
# mode1 is top and bottom 8 rows
# mode2 is next 8 columns
# mode3 is repeating that 16x16 tile pattern
kkt = _tcgen05.ld(kkt_tmem, 0, "16x256b", BT // 8)
kkt = kkt.reshape((2, 2, 2, BT // 16))
for i in cutlass.range_constexpr(BT // 16):
# mode0 is 2 elems next to each other
# mode1 is 4 pairs of elems on 1 row
# mode2 is top and bottom 8 rows
# mode3 is next 16 columns
col_coord = (None, lane_id % 4, None, i)
s_g_cu_view = cute.make_tensor(
s_g_cu[None, stage_id].iterator, (2, 4, 2, BT // 16)
)
g_cu_col = s_g_cu_view[col_coord].load().reshape((2, 1, 2))
Gamma = cute.math.exp(g_cu_row - g_cu_col, fastmath=True)
A = kkt[None, None, None, i] * beta_row * Gamma
# strict lower mask
# NOTE: for OOB t position, s_beta is filled with zeros.
# hence, we don't need to apply bounds check for columns.
A_masked = cute.where(row_indices > col_indices + i * 16, A, 0.0)
# pack to BF16
# CuteDSL doesn't generate cvt.bf16x2.f32 here for some reasons
packed = cute.make_rmem_tensor(4, Uint32)
for j in cutlass.range_constexpr(4):
packed[j] = cvt.fp32x2_to_bf16x2(
A_masked[j * 2], A_masked[j * 2 + 1]
)
# store to smem
cute.copy(
stsm_atom,
cute.recast_tensor(packed, BFloat16),
sA_ldsm[None, (warp_id_, i)],
)
# use sync warp instead of bar.sync because for block-diagonal inverse,
# each warp reads its own private smem memory.
cute.arch.sync_warp()
##### Phase 2: matrix inverse #####
# we use Newton-Schulz iterations to compute the inverse
# of the four 16x16 diagonal blocks.
# Ai_new = 2 Ai - Ai @ M @ Ai
# where M = I + A
#
# we do this with 2 MMAs:
# 1. -AiM = Ai @ (-M)
# 2. Ai_new = 2 Ai + (-AiM) @ Ai
zeros_f32 = cute.make_rmem_tensor(4, Float32)
zeros_f32.fill(0.0)
Ai_bf16 = cute.make_rmem_tensor(8, BFloat16)
mma_B_bf16 = cute.make_rmem_tensor((4, 2), BFloat16)
M_bf16 = cute.make_rmem_tensor((4, 2), BFloat16)
acc = cute.make_rmem_tensor((4, 2), Float32)
# ldmatrix copies require rank-1 destination views.
mma_B_bf16_ldsm = cute.group_modes(mma_B_bf16, 0, 2)
# Packed aliases for BF16x2 arithmetic.
Ai = cute.recast_tensor(Ai_bf16, Uint32)
M = cute.recast_tensor(M_bf16, Uint32)
# construct rmem-backed identity matrix
eye = cute.make_rmem_tensor(4, Uint32)
eye[0] = Uint32(lane_id % 9 == 0) * Uint32(0x00003F80) + Uint32(
lane_id % 9 == 4
) * Uint32(0x3F800000)
eye[1] = 0
eye[2] = 0
eye[3] = eye[0]
# initial guess: Ai = I-A
cute.copy(ldsm_atom, sA_ldsm[None, (warp_id_, warp_id_)], Ai_bf16)
for i in cutlass.range_constexpr(4):
Ai[i] = _bf16x2_sub(eye[i], Ai[i])
# (4, 2)
Ai_f32 = cute.logical_divide(cvt.bf16x2_to_fp32x2(Ai), 4)
# M is holding -(I+A), stay constant throughout the iterations
cute.copy(
ldsm_trans_atom,
sA_ldsm[None, (warp_id_, warp_id_)],
cute.group_modes(M_bf16, 0, 2),
)
for i in cutlass.range_constexpr(4):
M[i] = _bf16x2_sub(_bf16x2_neg(eye[i]), M[i])
# The fourth round keeps correlated-key KKT blocks stable in BF16.
for _ in cutlass.range_constexpr(4):
# First MMA: -AiM = Ai @ (-M)
cute.copy(stsm_atom, Ai_bf16, sA_ldsm[None, (warp_id_, warp_id_)])
cute.arch.sync_warp()
acc[None, 0] = mma_sync(Ai_bf16, M_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, M_bf16[None, 1], zeros_f32)
Ai_bf16.store(acc.load().to(BFloat16))
# Second MMA: Ai_new = 2Ai + (-AiM) @ Ai
for j in cutlass.range(8, vectorize=True):
Ai_f32[j] *= 2.0
cute.copy(
ldsm_trans_atom,
sA_ldsm[None, (warp_id_, warp_id_)],
mma_B_bf16_ldsm,
)
Ai_f32[None, 0] = mma_sync(
Ai_bf16, mma_B_bf16[None, 0], Ai_f32[None, 0]
)
Ai_f32[None, 1] = mma_sync(
Ai_bf16, mma_B_bf16[None, 1], Ai_f32[None, 1]
)
Ai_bf16.store(Ai_f32.load().to(BFloat16))
cute.copy(stsm_atom, Ai_bf16, sAi_ldsm[None, (warp_id_, warp_id_)])
store_ab_abg(
Ai_f32,
s_beta[None, stage_id],
s_beta_g[None, stage_id],
warp_id_,
lane_id,
warp_id_,
Ab_tmem_base,
)
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
# off-diagonal by 1
# given
# [ Ai00 ]
# [ A10 Ai11 ]
# [ A20 A21 Ai22 ]
# [ A30 A31 A32 Ai33]
# warp1: Ai10 = -Ai11 @ A10 @ Ai00
# warp2: Ai21 = -Ai22 @ A21 @ Ai11
# warp3: Ai32 = -Ai33 @ A32 @ Ai22
if warp_id_ >= 1:
neg_Ai = cute.make_rmem_tensor(4, Uint32)
for i in cutlass.range_constexpr(4):
neg_Ai[i] = _bf16x2_neg(Ai[i])
neg_Ai_bf16 = cute.recast_tensor(neg_Ai, BFloat16)
cute.copy(
ldsm_trans_atom,
sA_ldsm[None, (warp_id_, warp_id_ - 1)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(neg_Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(neg_Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
Ai_bf16.store(acc.load().to(BFloat16))
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (warp_id_ - 1, warp_id_ - 1)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
Ai_bf16.store(acc.load().to(BFloat16))
store_ab_abg(
acc,
s_beta[None, stage_id],
s_beta_g[None, stage_id],
warp_id_,
lane_id,
warp_id_ - 1,
Ab_tmem_base,
)
cute.copy(
stsm_atom,
Ai_bf16,
sAi_ldsm[None, (warp_id_, warp_id_ - 1)],
)
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
# off-diagonal by 2
# warp2: Ai20 = -Ai22 @ (A20 @ Ai00 + A21 @ Ai10)
# warp3: Ai31 = -Ai33 @ (A31 @ Ai11 + A32 @ Ai21)
if warp_id_ >= 2:
tile_col = warp_id_ - 2
cute.copy(
ldsm_atom,
sA_ldsm[None, (warp_id_, tile_col)],
Ai_bf16,
)
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (tile_col, tile_col)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
cute.copy(
ldsm_atom, sA_ldsm[None, (warp_id_, tile_col + 1)], Ai_bf16
)
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (tile_col + 1, tile_col)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], acc[None, 0])
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], acc[None, 1])
tmp = cute.make_rmem_tensor(8, BFloat16)
tmp.store(acc.load().to(BFloat16))
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (warp_id_, tile_col)])
cute.arch.sync_warp()
cute.copy(ldsm_atom, sAi_ldsm[None, (warp_id_, warp_id_)], Ai_bf16)
for i in cutlass.range_constexpr(4):
Ai[i] = _bf16x2_neg(Ai[i])
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (warp_id_, tile_col)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
tmp.store(acc.load().to(BFloat16))
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (warp_id_, tile_col)])
store_ab_abg(
acc,
s_beta[None, stage_id],
s_beta_g[None, stage_id],
warp_id_,
lane_id,
tile_col,
Ab_tmem_base,
)
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
# off-diagonal by 3
# warp3: Ai30 = -Ai33 @ (A30 @ Ai00 + A31 @ Ai10 + A32 @ Ai20)
if warp_id_ == 3:
cute.copy(ldsm_atom, sA_ldsm[None, (3, 0)], Ai_bf16)
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (0, 0)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
for i in cutlass.range_constexpr(1, 3):
cute.copy(ldsm_atom, sA_ldsm[None, (3, i)], Ai_bf16)
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (i, 0)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(
Ai_bf16, mma_B_bf16[None, 0], acc[None, 0]
)
acc[None, 1] = mma_sync(
Ai_bf16, mma_B_bf16[None, 1], acc[None, 1]
)
tmp = cute.make_rmem_tensor(8, BFloat16)
tmp.store(acc.load().to(BFloat16))
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (3, 0)])
cute.arch.sync_warp()
cute.copy(ldsm_atom, sAi_ldsm[None, (3, 3)], Ai_bf16)
for i in cutlass.range_constexpr(4):
Ai[i] = _bf16x2_neg(Ai[i])
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (3, 0)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
tmp.store(acc.load().to(BFloat16))
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (3, 0)])
store_ab_abg(
acc,
s_beta[None, stage_id],
s_beta_g[None, stage_id],
warp_id_,
lane_id,
0,
Ab_tmem_base,
)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(inv_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
else:
# epi warps
stage_id = 0
parity = 0
# ((BT, num_global_chunks), V_dim)
gU_tiles = cute.logical_divide(
U_tma.tma_tensor[None, head_id, None], (BT, None)
)
gW_tiles = cute.logical_divide(
W_tma.tma_tensor[None, head_id, None], (BT, None)
)
# sW shape: [BT, (64, K_dim/64)]
# sW_view shape: [(8, 2), (4, K_dim/64)]
s_row = warp_id * 16 + lane_id % 16 # select the rows of [16,16] tile
sW_view = cute.zipped_divide(
sW[s_row, None],
tiler=cute.make_layout((8, 2)),
)
sU_view = cute.zipped_divide(
sU[s_row, None],
tiler=cute.make_layout((8, 2)),
)
# select the 8 columns within [16,16] tile
sW_view = sW_view[(None, lane_id // 16), None]
sU_view = sU_view[(None, lane_id // 16), None]
for global_chunk_id in range(bid, num_global_chunks, grid_x):
# wait for W MMA + previous TMA store to finish
U_tmem = U_tmem_base + V_dim * stage_id
if warp_id == 0:
cute.arch.mbarrier_wait(mma_w_mbar + stage_id, parity)
elif warp_id == 1:
with cute.arch.elect_one():
cute.arch.cp_async_bulk_wait_group(0, read=True)
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
w_f32 = _tcgen05.ld(warp_id * 32 + 16, U_tmem, "16x256b", K_dim // 8)
_tcgen05.wait_ld()
w_bf16 = cute.make_rmem_tensor((8, K_dim // 16), BFloat16)
w_bf16.store(w_f32.to(BFloat16))
cute.copy(stsm_atom, w_bf16, sW_view)
# wait for U MMA + issue W TMA store
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
fence_before_tma_store()
if warp_id == 0:
cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity)
elif warp_id == 1:
# don't need to commit
simple_tma_copy(
W_tma.atom, sW, gW_tiles[(None, global_chunk_id), None]
)
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
u_f32 = _tcgen05.ld(warp_id * 32, U_tmem, "16x256b", V_dim // 8)
_tcgen05.wait_ld()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(epi_mbar + stage_id)
u_bf16 = cute.make_rmem_tensor((8, V_dim // 16), BFloat16)
u_bf16.store(u_f32.to(BFloat16))
cute.copy(stsm_atom, u_bf16, sU_view)
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
fence_before_tma_store()
if warp_id == 1:
simple_tma_copy(
U_tma.atom, sU, gU_tiles[(None, global_chunk_id), None]
)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
@cache
@staticmethod
def compile(H: int, Hv: int, K_dim: int, V_dim: int, num_stages: int = 2):
total_t = cute.sym_int()
pad_t = cute.sym_int()
total_chunks_n = cute.sym_int()
num_sequences = cute.sym_int()
K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
V = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16)
U = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16)
g = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
beta = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
cu_seqlens = make_fake_tensor(Int32, (num_sequences,), divisibility=1)
chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2)
total_chunks = make_fake_tensor(Int32, (1,), divisibility=1)
kernel = Sm100ChunkUWKernel(H, Hv, K_dim, V_dim, num_stages)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
K,
V,
U,
W,
g,
beta,
g_cu,
cu_seqlens,
chunk_indices,
total_chunks,
Int32(148),
stream,
options="--enable-tvm-ffi",
)
def kkt_inv_uw_cutedsl(
K: torch.Tensor,
V: torch.Tensor,
U: torch.Tensor,
W: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
g_cu: torch.Tensor,
cu_seqlens: torch.Tensor,
chunk_indices: torch.Tensor,
total_chunks: torch.Tensor,
num_sms: int = 148,
) -> None:
_, Hv, V_dim = V.shape
_, H, K_dim = K.shape
Sm100ChunkUWKernel.compile(H, Hv, K_dim, V_dim)(
K,
V,
U,
W,
g,
beta,
g_cu,
cu_seqlens,
chunk_indices,
total_chunks,
num_sms,
)
@@ -1,632 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
from cutlass.cute.nvgpu import cpasync, warp
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import (
EVICT_FIRST,
_tcgen05,
cvt,
fence_before_tma_store,
simple_tma_copy,
)
class Sm100ChunkOKernel:
"""Compute per-token output from recurrent and intra-chunk terms.
Gamma[i,j] = exp(g_cu[i] - g_cu[j])
P = mask((Q @ K.T) * Gamma)
O = scale * (exp(g_cu) * (Q @ H.T) + P @ V)
"""
def __init__(
self,
H: int,
Hv: int,
K_dim: int,
V_dim: int,
BT: int = 64,
num_stages: int = 2,
) -> None:
assert Hv % H == 0
assert K_dim == 128
assert V_dim == 128
assert BT == 64
self.H = H
self.Hv = Hv
self.K_dim = K_dim
self.V_dim = V_dim
self.BT = BT
self.num_stages = num_stages
self.num_warps = 10
@cute.jit
def _make_bf16_tma_args(
self,
tensor: cute.Tensor,
dim: cutlass.Constexpr[int],
op: cpasync.TmaCopyOp,
stages: cutlass.Constexpr[int],
):
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(self.BT, 1, (64, dim // 64), stages),
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, 64)),
slayout,
cta_tiler=(self.BT, 1, dim),
)
@cute.jit
def _make_h_tma_args(
self,
tensor: cute.Tensor,
op: cpasync.TmaCopyOp,
stages: cutlass.Constexpr[int],
):
num_elems = 128 // (tensor.element_type.width // 8)
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(1, 1, self.V_dim, (num_elems, self.K_dim // num_elems), stages),
stride=(
0,
0,
num_elems,
(1, self.V_dim * num_elems),
self.V_dim * self.K_dim,
),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, None, num_elems)),
slayout,
cta_tiler=(1, 1, self.V_dim, self.K_dim),
)
@cute.jit
def __call__(
self,
q: cute.Tensor,
k: cute.Tensor,
v_new_chunks: cute.Tensor,
h: cute.Tensor,
g_cu: cute.Tensor,
o: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
total_chunks: cute.Tensor,
scale: Float32,
num_sms: Int32,
stream: CUstream,
):
grid = (num_sms // self.Hv, self.Hv, 1)
block = (self.num_warps * 32, 1, 1)
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
Q_tma = self._make_bf16_tma_args(q, self.K_dim, tma_g2s, self.num_stages)
K_tma = self._make_bf16_tma_args(k, self.K_dim, tma_g2s, self.num_stages)
V_tma = self._make_bf16_tma_args(
v_new_chunks, self.V_dim, tma_g2s, self.num_stages
)
H_tma = self._make_h_tma_args(h, tma_g2s, self.num_stages)
O_tma = self._make_bf16_tma_args(o, self.V_dim, tma_s2g, 1)
self.kernel(
Q_tma,
K_tma,
V_tma,
H_tma,
O_tma,
g_cu,
o,
cu_seqlens,
chunk_indices,
total_chunks,
scale,
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
@cute.kernel
def kernel(
self,
Q_tma: cpasync.TmaInfo,
K_tma: cpasync.TmaInfo,
V_tma: cpasync.TmaInfo,
H_tma: cpasync.TmaInfo,
O_tma: cpasync.TmaInfo,
g_cu: cute.Tensor,
o: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
total_chunks: cute.Tensor,
scale: Float32,
):
tid, _, _ = cute.arch.thread_idx()
bid, v_head_id, _ = cute.arch.block_idx()
grid_x, _, _ = cute.arch.grid_dim()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
BT = self.BT
K_dim = self.K_dim
V_dim = self.V_dim
num_stages = self.num_stages
heads_per_qk = self.Hv // self.H
k_head_id = v_head_id // heads_per_qk
num_global_chunks = total_chunks[0]
def allocate_tensor(smem, dtype, layout):
return smem.allocate_tensor(
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
)
smem = cutlass.utils.SmemAllocator()
sQ = allocate_tensor(smem, BFloat16, Q_tma.smem_layout)[None, 0, None, None]
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
sH = allocate_tensor(smem, BFloat16, H_tma.smem_layout)[0, 0, None, None, None]
sO = allocate_tensor(smem, BFloat16, O_tma.smem_layout)[None, 0, None, 0]
s_g_cu = smem.allocate_array(Float32, BT)
qk_full_mbar = smem.allocate_array(Int64, num_stages)
hv_full_mbar = smem.allocate_array(Int64, num_stages)
qk_empty_mbar = smem.allocate_array(Int64, num_stages)
pv_mma_mbar = smem.allocate_array(Int64, num_stages)
qk_mbar = smem.allocate_array(Int64, 1)
mask_mbar = smem.allocate_array(Int64, 1)
epi_mbar = smem.allocate_array(Int64, 1)
taddr = smem.allocate(Int32, 4)
qk_tmem = 0
p_tmem = 64
out_tmem = 128
qh_tmem = 256
if warp_id == 0:
with cute.arch.elect_one():
for i in cutlass.range_constexpr(num_stages):
cute.arch.mbarrier_init(qk_full_mbar + i, 1)
cute.arch.mbarrier_init(qk_empty_mbar + i, 1)
cute.arch.mbarrier_init(hv_full_mbar + i, 1)
cute.arch.mbarrier_init(pv_mma_mbar + i, 1)
cute.arch.mbarrier_init(qk_mbar, 1)
cute.arch.mbarrier_init(mask_mbar, 128)
cute.arch.mbarrier_init(epi_mbar, 128)
cute.arch.mbarrier_init_fence()
elif warp_id == 9:
cpasync.prefetch_descriptor(Q_tma.atom)
cpasync.prefetch_descriptor(K_tma.atom)
cpasync.prefetch_descriptor(V_tma.atom)
cpasync.prefetch_descriptor(H_tma.atom)
cute.arch.sync_threads()
if warp_id == 9:
# TMA warp
stage_id = 0
parity = 1
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
# copy Q and K
q_tile = cute.local_tile(
cute.domain_offset(
(bos, 0), Q_tma.tma_tensor[None, k_head_id, None]
),
tiler=(BT, K_dim),
coord=(chunk_id, 0),
)
k_tile = cute.local_tile(
cute.domain_offset(
(bos, 0), K_tma.tma_tensor[None, k_head_id, None]
),
tiler=(BT, K_dim),
coord=(chunk_id, 0),
)
mbar = qk_full_mbar + stage_id
cute.arch.mbarrier_wait(qk_empty_mbar + stage_id, parity)
with cute.arch.elect_one():
STAGE_SIZE = BT * (K_dim + K_dim) * 2
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
simple_tma_copy(Q_tma.atom, q_tile, sQ[None, None, stage_id], mbar)
simple_tma_copy(K_tma.atom, k_tile, sK[None, None, stage_id], mbar)
# copy H and V
gH = H_tma.tma_tensor[global_chunk_id, v_head_id, None, None]
gV = cute.local_tile(
V_tma.tma_tensor[None, v_head_id, None],
tiler=(BT, V_dim),
coord=(global_chunk_id, 0),
)
mbar = hv_full_mbar + stage_id
cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, parity)
with cute.arch.elect_one():
H_STAGE_SIZE = V_dim * K_dim * 2
V_STAGE_SIZE = BT * V_dim * 2
cute.arch.mbarrier_arrive_and_expect_tx(
mbar, H_STAGE_SIZE + V_STAGE_SIZE
)
simple_tma_copy(
H_tma.atom, gH, sH[None, None, stage_id], mbar, EVICT_FIRST
)
simple_tma_copy(
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id == 8:
# MMA warp
_tcgen05.alloc(taddr)
# LBO=BT*128 is ignored for K-major
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
qk_idesc = _tcgen05.make_bf16_idesc(BT, BT)
qh_idesc = _tcgen05.make_bf16_idesc(BT, V_dim)
pv_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True)
stage_id = 0
tma_parity = 0
mask_parity = 0
for global_chunk_id in range(bid, num_global_chunks, grid_x):
qaddr = sQ[None, None, stage_id].iterator.toint()
kaddr = sK[None, None, stage_id].iterator.toint()
haddr = sH[None, None, stage_id].iterator.toint()
vaddr = sV[None, None, stage_id].iterator.toint()
qdesc_base = sdesc_template | (qaddr >> 4)
kdesc_base = sdesc_template | (kaddr >> 4)
hdesc_base = sdesc_template | (haddr >> 4)
vdesc_base = sdesc_template | (vaddr >> 4)
##### 1st MMA: Q @ K.T #####
# do this first to unblock mask(QK)
cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1)
cute.arch.mbarrier_wait(qk_full_mbar + stage_id, tma_parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4)
kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4)
_tcgen05.mma_f16(
qk_tmem, qdesc, kdesc, qk_idesc, (i > 0) or (j > 0)
)
_tcgen05.commit(qk_mbar)
##### 2nd MMA: Q @ H.T #####
cute.arch.mbarrier_wait(hv_full_mbar + stage_id, tma_parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4)
hdesc = hdesc_base | ((i * V_dim * 128 + j * 32) >> 4)
_tcgen05.mma_f16(
qh_tmem, qdesc, hdesc, qh_idesc, (i > 0) or (j > 0)
)
_tcgen05.commit(qk_empty_mbar + stage_id)
##### 3rd MMA: P @ V #####
# stalled by mask(QK)
cute.arch.mbarrier_wait(mask_mbar, mask_parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(BT // 16):
vdesc = vdesc_base | ((i * 16 * 128) >> 4)
_tcgen05.mma_ts_f16(
out_tmem, p_tmem + i * 8, vdesc, pv_idesc, i > 0
)
_tcgen05.commit(pv_mma_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
tma_parity ^= 1
mask_parity ^= 1
# wait for epilogue to finish for deallocation
cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1)
_tcgen05.dealloc()
elif warp_id >= 4:
# masking warps
warp_id_ = warp_id % 4
tid_ = tid % 128
row0 = warp_id_ * 16 + lane_id // 4
row1 = row0 + 8
parity = 0
# for ldmatrix layout later
row_indices = cute.make_rmem_tensor(2, Int32)
row_indices[0] = warp_id_ * 16 + lane_id // 4
row_indices[1] = warp_id_ * 16 + lane_id // 4 + 8
row_indices = row_indices.load().reshape((1, 2))
col_indices = cute.make_rmem_tensor(2, Int32)
col_indices[0] = (lane_id % 4) * 2
col_indices[1] = (lane_id % 4) * 2 + 1
col_indices = col_indices.load().reshape((2, 1))
for global_chunk_id in range(bid, num_global_chunks, grid_x):
if tid_ < BT:
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
t_ = bos + chunk_id * BT + tid_
s_g_cu[tid_] = g_cu[t_, v_head_id] if t_ < eos else Float32(0.0)
# wait for QK MMA
if warp_id_ == 0:
cute.arch.mbarrier_wait(qk_mbar, parity)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
qk = _tcgen05.ld(warp_id_ * 32, qk_tmem, "16x256b", BT // 8)
qk = qk.reshape((2, 2, BT // 8))
_tcgen05.wait_ld()
g_cu_rows = cute.make_rmem_tensor(2, Float32)
g_cu_rows[0] = s_g_cu[row0]
g_cu_rows[1] = s_g_cu[row1]
g_cu_rows = g_cu_rows.load().reshape((1, 2))
for i in cutlass.range_constexpr(BT // 8):
col = i * 8 + (lane_id % 4) * 2
g_cu_cols = cute.make_rmem_tensor(2, Float32)
g_cu_cols[0] = s_g_cu[col]
g_cu_cols[1] = s_g_cu[col + 1]
g_cu_cols = g_cu_cols.load().reshape((2, 1))
# apply gamma and causal mask
Gamma = cute.math.exp(g_cu_rows - g_cu_cols, fastmath=True)
tmp = qk[None, None, i] * Gamma
tmp = cute.where(row_indices >= col_indices + i * 8, tmp, 0.0)
# CuteDSL can't emit cvt.bf16x2.f32 here
attn_lo = cute.make_rmem_tensor(2, Uint32)
attn_lo[0] = cvt.fp32x2_to_bf16x2(tmp[0, 0], tmp[1, 0])
attn_lo[1] = cvt.fp32x2_to_bf16x2(tmp[0, 1], tmp[1, 1])
_tcgen05.st(warp_id_ * 32, p_tmem + i * 4, "16x128b", 1, attn_lo)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(mask_mbar)
parity ^= 1
else:
# epilogue warps
# for ldmatrix layout later
row0 = warp_id * 16 + lane_id // 4
row1 = row0 + 8
stage_id = 0
mma_parity = 0
op = cute.nvgpu.CopyUniversalOp()
cp_4B = cute.make_copy_atom(op, BFloat16, num_bits_per_copy=32)
stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=False)
stsm_atom = cute.make_copy_atom(stsm_op, BFloat16)
# ldmatrix layout
# [total_seq_len, ((2, 4, WIDTH/8), V_DIM/WIDTH)]
WIDTH = 64
o_view = cute.logical_divide(
o[None, v_head_id, None],
(None, cute.make_layout((2, 4, WIDTH // 8))),
)
# select lane: [total_seq_len, 2, WIDTH/8, V_DIM/WIDTH]
o_view = o_view[None, ((None, lane_id % 4, None), None)]
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
chunk_start = bos + chunk_id * BT
full_chunk = chunk_start + BT <= eos
g_cu_rows = cute.make_rmem_tensor(2, Float32)
g_cu_rows.fill(0.0)
# load g_cu
if chunk_start + row0 < eos:
g_cu_rows[0] = cute.math.exp(
g_cu[chunk_start + row0, v_head_id], fastmath=True
)
if chunk_start + row1 < eos:
g_cu_rows[1] = cute.math.exp(
g_cu[chunk_start + row1, v_head_id], fastmath=True
)
g_cu_rows = g_cu_rows.load().reshape((1, 2, 1))
if warp_id == 0:
cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, mma_parity)
elif warp_id == 3 and full_chunk:
cute.arch.cp_async_bulk_wait_group(0, read=True)
cute.arch.barrier(barrier_id=2, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
if full_chunk:
# use TMA store: tmem->rmem->smem->gmem
for i in cutlass.range_constexpr(V_dim // WIDTH):
qh = _tcgen05.ld(
warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8
)
pv = _tcgen05.ld(
warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8
)
_tcgen05.wait_ld()
if i == V_dim // WIDTH - 1:
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(epi_mbar)
qh = qh.reshape((2, 2, WIDTH // 8))
pv = pv.reshape((2, 2, WIDTH // 8))
out_f32 = scale * (g_cu_rows * qh + pv)
out_bf16 = cute.make_rmem_tensor((8, WIDTH // 16), BFloat16)
out_bf16.store(out_f32.to(BFloat16).reshape((8, WIDTH // 16)))
# TODO: issue single cute.copy()
for j in cutlass.range_constexpr(WIDTH // 16):
s_row = warp_id * 16 + lane_id % 16
s_col = i * (WIDTH // 8) + j * 2 + lane_id // 16
sO_tile = cute.local_tile(sO[s_row, None], (8,), (s_col,))
cute.copy(stsm_atom, out_bf16[None, j], sO_tile)
cute.arch.barrier(barrier_id=2, number_of_threads=128)
fence_before_tma_store()
if warp_id == 3:
gO = cute.local_tile(
cute.domain_offset(
(bos, 0), O_tma.tma_tensor[None, v_head_id, None]
),
tiler=(BT, V_dim),
coord=(chunk_id, 0),
)
simple_tma_copy(O_tma.atom, sO, gO)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
else:
# direct gmem store
# TODO: explore doing multiple 1D TMAs
for i in cutlass.range_constexpr(V_dim // WIDTH):
qh = _tcgen05.ld(
warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8
)
pv = _tcgen05.ld(
warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8
)
_tcgen05.wait_ld()
if i == V_dim // WIDTH - 1:
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(epi_mbar)
qh = qh.reshape((2, 2, WIDTH // 8))
pv = pv.reshape((2, 2, WIDTH // 8))
out_f32 = scale * (g_cu_rows * qh + pv)
out_bf16 = cute.make_rmem_tensor((2, 2, WIDTH // 8), BFloat16)
out_bf16.store(out_f32.to(BFloat16))
if chunk_start + row0 < eos:
cute.copy(
cp_4B,
out_bf16[None, 0, None],
o_view[chunk_start + row0, None, None, i],
)
if chunk_start + row1 < eos:
cute.copy(
cp_4B,
out_bf16[None, 1, None],
o_view[chunk_start + row1, None, None, i],
)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
mma_parity ^= 1
@cache
@staticmethod
def compile(
H: int,
Hv: int,
K_dim: int,
V_dim: int,
BT: int = 64,
num_stages: int = 2,
):
total_t = cute.sym_int()
pad_t = cute.sym_int()
total_chunks_n = cute.sym_int()
cu_entries = cute.sym_int()
q = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
k = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
v_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
h = make_fake_tensor(
BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16
)
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
o = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16)
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2)
total_chunks = make_fake_tensor(Int32, (1,), divisibility=1)
kernel = Sm100ChunkOKernel(
H,
Hv,
K_dim,
V_dim,
BT,
num_stages,
)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
q,
k,
v_new,
h,
g_cu,
o,
cu_seqlens,
chunk_indices,
total_chunks,
Float32(1.0),
Int32(148),
stream,
options="--enable-tvm-ffi",
)
def o_cutedsl(
q: torch.Tensor,
k: torch.Tensor,
v_new: torch.Tensor,
h: torch.Tensor,
g_cu: torch.Tensor,
o: torch.Tensor,
cu_seqlens: torch.Tensor,
chunk_indices: torch.Tensor,
total_chunks: torch.Tensor,
scale: float,
num_sms: int = 148,
) -> None:
_, H, K_dim = q.shape
_, Hv, V_dim = o.shape
Sm100ChunkOKernel.compile(H, Hv, K_dim, V_dim)(
q,
k,
v_new,
h,
g_cu,
o,
cu_seqlens,
chunk_indices,
total_chunks,
float(scale),
num_sms,
)
-255
View File
@@ -1,255 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING
import torch
from vllm.config import CacheConfig
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
)
@dataclass
class MLAModules:
"""Modules used in MLA."""
kv_a_layernorm: torch.nn.Module
kv_b_proj: torch.nn.Module
rotary_emb: torch.nn.Module | None
o_proj: torch.nn.Module
fused_qkv_a_proj: torch.nn.Module | None
kv_a_proj_with_mqa: torch.nn.Module | None
q_a_layernorm: torch.nn.Module | None
q_b_proj: torch.nn.Module | None
q_proj: torch.nn.Module | None
indexer: torch.nn.Module | None
is_sparse: bool
topk_indices_buffer: torch.Tensor | None
indexer_rotary_emb: torch.nn.Module | None = None
g_proj: torch.nn.Module | None = None
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None
# --8<-- [start:multi_head_latent_attention]
@PluggableLayer.register("multi_head_latent_attention")
class MultiHeadLatentAttentionWrapper(PluggableLayer):
"""Pluggable MLA layer which allows OOT backends to add
custom implementations of the outer MLA layer (including rope & o_proj).
Note that currently oot platforms can still use CustomOp.register_oot to
replace MLA layer entirely, although we use PluggableLayer to register
this layer now.
This class takes positions and hidden_states as input.
The input tensors can either contain prefill tokens or decode tokens.
The class does the following:
1. MLA Preprocess.
2. Perform multi-head attention to prefill tokens and
multi-query attention to decode tokens separately.
3. Return the output tensor.
"""
# --8<-- [end:multi_head_latent_attention]
def __init__(
self,
hidden_size: int,
num_heads: int,
scale: float,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: int | None,
kv_lora_rank: int,
mla_modules: MLAModules,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
skip_topk: bool = False,
non_causal_multi_token_decode: bool = False,
allow_short_prefill_indexer_scoring_skip: bool = False,
fuse_qkv_rmsnorm: bool = False,
) -> None:
super().__init__()
self.hidden_size = hidden_size
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
self.v_head_dim = v_head_dim
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.num_heads = num_heads
self.fused_qkv_a_proj = mla_modules.fused_qkv_a_proj
self.kv_a_proj_with_mqa = mla_modules.kv_a_proj_with_mqa
self.q_a_layernorm = mla_modules.q_a_layernorm
self.q_b_proj = mla_modules.q_b_proj
self.q_proj = mla_modules.q_proj
self.kv_a_layernorm = mla_modules.kv_a_layernorm
self.kv_b_proj = mla_modules.kv_b_proj
self.rotary_emb = mla_modules.rotary_emb
self.o_proj = mla_modules.o_proj
self.indexer = mla_modules.indexer
self.indexer_rope_emb = mla_modules.indexer_rotary_emb
self.is_sparse = mla_modules.is_sparse
self.g_proj = mla_modules.g_proj
# Whether to skip top-k token selection computation in this layer.
# When True, the indexer will not be called, and the layer will reuse
# the topk_tokens buffer written by a previous layer in the same pass.
# Refer: https://arxiv.org/abs/2603.12201 for more details.
self.skip_topk = skip_topk
# When True, fuse the q_a and kv_a RMSNorms into a single kernel launch
# (MLA layers with q-LoRA). Opt-in; default False preserves other models.
self.fuse_qkv_rmsnorm = fuse_qkv_rmsnorm
# qrep is active when the query projection is a DCP-group-sharded layer
# that materializes the full group head set locally.
q_proj_layer = self.q_b_proj if self.q_lora_rank is not None else self.q_proj
self.dcp_q_replicate = getattr(q_proj_layer, "qrep_active", False)
if self.indexer is not None:
assert hasattr(self.indexer, "topk_tokens")
self.topk_tokens = self.indexer.topk_tokens
self.topk_indices_buffer = mla_modules.topk_indices_buffer
self.mla_attn = MLAAttention(
num_heads=self.num_heads,
scale=scale,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
q_lora_rank=self.q_lora_rank,
kv_lora_rank=self.kv_lora_rank,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
kv_b_proj=self.kv_b_proj,
dcp_q_replicate=self.dcp_q_replicate,
use_sparse=self.is_sparse,
indexer=self.indexer,
topk_indices_buffer=mla_modules.topk_indices_buffer,
index_group_builder=mla_modules.index_group_builder,
non_causal_multi_token_decode=non_causal_multi_token_decode,
)
indexer_op = getattr(self.indexer, "indexer_op", None)
if indexer_op is not None and hasattr(
indexer_op, "dense_mha_metadata_layer_name"
):
enable_short_prefill_scoring_skip = (
allow_short_prefill_indexer_scoring_skip
and not self.skip_topk
and not getattr(indexer_op, "use_pcp", False)
and current_platform.is_cuda()
)
# The indexer and main MLA use independent decode thresholds and
# may classify the same short extend differently. Bind the main
# MLA layer name so the eager indexer op can check whether the
# batch's top-k indices will be consumed.
# PCP is excluded because indexer cache/scoring ownership differs
# across ranks and the no-consumer invariant has not been
# established there.
indexer_op.dense_mha_metadata_layer_name = (
self.mla_attn.layer_name if enable_short_prefill_scoring_skip else ""
)
self.prefix = prefix
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
llama_4_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
q_c = None
kv_lora = None
if self.q_lora_rank is not None:
assert self.fused_qkv_a_proj is not None, (
"fused_qkv_a_proj is required when q_lora_rank is not None"
)
assert self.q_a_layernorm is not None, (
"q_a_layernorm is required when q_lora_rank is not None"
)
assert self.q_b_proj is not None, (
"q_b_proj is required when q_lora_rank is not None"
)
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
q_c, kv_lora = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
dim=-1,
)
q_proj_layer = self.q_b_proj
if not self.fuse_qkv_rmsnorm:
q_c = self.q_a_layernorm(q_c)
q_proj_input = q_c
else:
assert self.kv_a_proj_with_mqa is not None, (
"kv_a_proj_with_mqa is required when q_lora_rank is None"
)
assert self.q_proj is not None, (
"q_proj is required when q_lora_rank is None"
)
kv_lora = self.kv_a_proj_with_mqa(hidden_states)[0]
q_proj_layer = self.q_proj
q_proj_input = hidden_states
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
if self.fuse_qkv_rmsnorm and q_c is not None:
# Fuse q_a + kv_a RMSNorm into one kernel launch (q_c is still the
# pre-norm projection output here).
assert self.q_a_layernorm is not None
q_proj_input, kv_c_normed = fused_q_kv_rmsnorm(
q_c,
kv_c,
self.q_a_layernorm.weight.data,
self.kv_a_layernorm.weight.data,
self.q_a_layernorm.variance_epsilon,
)
q_c = q_proj_input
else:
kv_c_normed = self.kv_a_layernorm(kv_c)
# Add head dim of 1 to k_pe
k_pe = k_pe.unsqueeze(1)
q = q_proj_layer(q_proj_input)[0]
heads = self.num_heads
if self.dcp_q_replicate:
heads *= q_proj_layer.group_size
q = q.view(-1, heads, self.qk_head_dim)
if self.rotary_emb is not None:
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
positions, q[..., self.qk_nope_head_dim :], k_pe
)
if self.indexer and self.is_sparse and not self.skip_topk:
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
if self.is_sparse:
self.mla_attn.impl.record_logical_topk_ready() # type: ignore[attr-defined]
if llama_4_scaling is not None:
q *= llama_4_scaling
q_dcp_replicated = None
if self.dcp_q_replicate:
q_dcp_replicated, q = q, q_proj_layer._local_view(q)
attn_out = self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=(hidden_states.shape[0], self.num_heads * self.v_head_dim),
q_dcp_replicated=q_dcp_replicated,
)
if self.g_proj is not None:
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
return self.o_proj(attn_out)[0]
@@ -29,13 +29,10 @@ QuantizationMethods = Literal[
"humming",
"compressed-tensors",
"experts_int8",
"quark",
"moe_wna16",
"torchao",
"inc",
"mxfp4",
"gpt_oss_mxfp4",
"deepseek_v4_fp8",
"online",
# Below are online quant shorthand names (see vllm.config.quantization).
# Listed here as strings to avoid a circular import; kept in sync with
@@ -115,21 +112,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
# lazy import to avoid triggering `torch.compile` too early
from vllm.config.quantization import _ONLINE_SHORTHANDS
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
if current_platform.is_cuda() or current_platform.is_rocm():
# The v4.1 class is the v4 one extended to accept model_type
# "deepseek_v41" and its 32x32 MXFP8 linear layout. V4.1 has
# platform-specific implementations for both CUDA and ROCm.
from vllm.models.deepseek_v41 import (
DeepseekV4FP8Config as DeepseekV41FP8Config,
)
deepseek_config: type[QuantizationConfig] = DeepseekV41FP8Config
else:
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
deepseek_config = DeepseekV4FP8Config
from .auto_awq import AutoAWQConfig
from .auto_gptq import AutoGPTQConfig
@@ -141,7 +123,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
from .fp8 import Fp8Config
from .fp_quant import FPQuantConfig
from .humming import HummingConfig
from .inc import INCConfig
from .modelopt import (
ModelOptFp8Config,
ModelOptMixedPrecisionConfig,
@@ -169,13 +150,10 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"gptq_marlin": AutoGPTQConfig,
"compressed-tensors": CompressedTensorsConfig,
"experts_int8": ExpertsInt8Config,
"quark": QuarkConfig,
"moe_wna16": MoeWNA16Config,
"torchao": TorchAOConfig,
"inc": INCConfig,
"mxfp4": Mxfp4Config,
"gpt_oss_mxfp4": GptOssMxfp4Config,
"deepseek_v4_fp8": deepseek_config,
"humming": HummingConfig,
"online": OnlineQuantizationConfig,
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
@@ -1,6 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .inc import INCConfig
__all__ = ["INCConfig"]
@@ -1,237 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING
import regex as re
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
if TYPE_CHECKING:
import torch
from .inc import INCConfig
@dataclass(frozen=True)
class INCLayerConfig:
bits: int
group_size: int | tuple[int, int]
sym: bool
packing_format: str
backend: str
data_type: str
quantized: bool
@property
def is_gptq(self) -> bool:
return "gptq" in self.packing_format or "gptq" in self.backend
@property
def is_awq(self) -> bool:
return "awq" in self.packing_format or "awq" in self.backend
@property
def is_wna16_int(self) -> bool:
return self.data_type == "int" and self.quantized
@property
def is_mxfp4(self) -> bool:
return "mx_fp" in self.data_type and self.bits == 4
@property
def is_mxfp8(self) -> bool:
return "mx_fp" in self.data_type and self.bits == 8
@property
def is_fp8_block(self) -> bool:
return (
self.data_type == "fp"
and self.bits == 8
and self.sym
and self.quantized
and self.packing_format == "auto_round:fp8"
and isinstance(self.group_size, tuple)
and len(self.group_size) == 2
)
class INCConfigParser:
def __init__(self, config: "INCConfig") -> None:
self._config = config
def resolve(self, layer: "torch.nn.Module", layer_name: str) -> INCLayerConfig:
bits, group_size, sym = self._resolve_raw(layer, layer_name)
return INCLayerConfig(
bits=bits,
group_size=group_size,
sym=sym,
packing_format=self._config.packing_format,
backend=self._config.backend,
data_type=self._config.data_type,
quantized=bits < 16,
)
def get_layer_config(
self, layer: "torch.nn.Module", layer_name: str
) -> tuple[int, int | tuple[int, int], bool]:
layer_config = self.resolve(layer, layer_name)
return layer_config.bits, layer_config.group_size, layer_config.sym
def _resolve_raw(
self, layer: "torch.nn.Module", layer_name: str
) -> tuple[int, int | tuple[int, int], bool]:
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
def is_explicitly_configured(name: str) -> bool:
"""Return True if *name* has an explicit entry in extra_config,
either via exact key match or via a regex pattern key."""
if not self._config.extra_config:
return False
if name in self._config.extra_config:
return True
for pattern in self._config.extra_config:
if not isinstance(pattern, str) or not any(
c in REGEX_SPECIAL_CHARS for c in pattern
):
continue
try:
if re.search(re.compile(pattern), name) is not None:
return True
except re.error:
continue
return False
def get_config(
name: str, quantized: bool = True
) -> tuple[int, int | tuple[int, int], bool]:
if not self._config.extra_config:
return (
self._config.weight_bits if quantized else 16,
self._config.group_size if quantized else -1,
self._config.sym if quantized else True,
)
if name in self._config.extra_config:
cfg = self._config.extra_config[name]
return (
cfg.get("bits", self._config.weight_bits if quantized else 16),
self._normalize_group_size(
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
)
),
cfg.get("sym", self._config.sym if quantized else True),
)
regex_special_chars = set(r"*+?^$()[]{}|\\")
for pattern, cfg in self._config.extra_config.items():
if not isinstance(pattern, str) or not any(
c in regex_special_chars for c in pattern
):
continue
try:
if re.search(re.compile(pattern), name) is not None:
return (
cfg.get(
"bits",
self._config.weight_bits if quantized else 16,
),
self._normalize_group_size(
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
)
),
cfg.get("sym", self._config.sym if quantized else True),
)
except re.error:
continue
return (
self._config.weight_bits if quantized else 16,
self._config.group_size if quantized else -1,
self._config.sym if quantized else True,
)
if self._config.extra_config and layer_name in self._config.extra_config:
return get_config(layer_name)
# Suffix match: handle cases where extra_config keys use short names
# (e.g. "lm_head") but the layer_name is fully qualified
# (e.g. "model.language_model.lm_head") due to model nesting.
if self._config.extra_config:
for cfg_key in self._config.extra_config:
if layer_name.endswith(f".{cfg_key}"):
return get_config(cfg_key)
quantized = not isinstance(layer, ParallelLMHead)
if self._config.block_name_to_quantize:
quantized = any(
layer_name.startswith(name)
for name in self._config.block_name_to_quantize
)
if self._config.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
moe_configs = [
get_config(name, quantized)
for name in self._config.extra_config
if name.startswith(layer_name)
]
if moe_configs:
if len(set(moe_configs)) == 1:
return moe_configs[0]
raise ValueError(
f"Fused MoE layer '{layer_name}' requires "
f"consistent quant config for all sub-layers"
)
if self._config.extra_config:
for fusion_key, sub_keys in self._config.packed_modules_mapping.items():
if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
sub_names = [
layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
]
# Only trigger if at least one sub_name is explicitly
# configured in extra_config (via exact match or regex).
# This prevents false matches when a short fusion_key
# (e.g. "qkv") is merely a substring of a longer layer
# name (e.g. "in_proj_qkvz") and none of the generated
# sub_names are actually configured.
if not any(is_explicitly_configured(n) for n in sub_names):
continue
sub_configs = [get_config(name, quantized) for name in sub_names]
if len(set(sub_configs)) == 1:
return sub_configs[0]
raise ValueError(
f"Fused module '{layer_name}' requires "
f"consistent quant config for {sub_names}"
)
return get_config(layer_name, quantized)
@staticmethod
def _normalize_group_size(
group_size: int | list[int] | tuple[int, int],
) -> int | tuple[int, int]:
"""Normalize INC group_size into either an int or a 2-D int tuple."""
if isinstance(group_size, (list, tuple)):
if len(group_size) != 2 or not all(
isinstance(value, int) for value in group_size
):
raise ValueError(
"INC block-wise FP8 requires group_size to be a 2-D "
f"integer sequence, but found {group_size!r}."
)
return (group_size[0], group_size[1])
if not isinstance(group_size, int):
raise ValueError(
"INC group_size must be an int or a 2-D integer sequence, "
f"but found {type(group_size).__name__}."
)
return group_size
@@ -1,368 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fractions import Fraction
from typing import TYPE_CHECKING, Any
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
QuantizationMethods,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from .config_parser import INCConfigParser
if TYPE_CHECKING:
from vllm.model_executor.models.utils import WeightsMapper
logger = init_logger(__name__)
class INCConfig(QuantizationConfig):
"""Config class for Intel Neural Compressor (INC).
Repo: https://github.com/intel/neural-compressor
"""
SUPPORTED_BITS = {2, 3, 4, 5, 6, 7, 8}
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
SUPPORTED_FORMATS = {
"auto_round:auto_gptq",
"auto_round:auto_awq",
"auto_round:fp8",
"auto_round:llm_compressor",
}
SUPPORTED_BACKENDS = {
"auto",
"gptq",
"gptq:marlin",
"awq",
"awq:marlin",
"marlin",
}
FP8_BLOCK_BITS = 8
FP8_BLOCK_DATA_TYPE = "fp"
FP8_BLOCK_PACKING_FORMAT = "auto_round:fp8"
FP8_BLOCK_FMT = "e4m3"
FP8_BLOCK_ACTIVATION_SCHEME = "dynamic"
MXFP8_BITS = 8
MXFP8_GROUP_SIZE = 32
MXFP8_DATA_TYPE = "mx_fp"
MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
@staticmethod
def _normalize_data_type(data_type: str) -> str:
# Keep a small alias for branches that already experimented with "fp8".
return "fp" if data_type == "fp8" else data_type
def __init__(
self,
weight_bits: int,
group_size: int | tuple[int, int],
sym: bool = True,
packing_format: str = DEFAULT_INT_PACKING_FORMAT,
block_name_to_quantize: str | list[str] | None = None,
extra_config: dict[str, Any] | None = None,
data_type: str = "int",
backend: str = "auto",
) -> None:
super().__init__()
data_type = self._normalize_data_type(data_type)
group_size = INCConfigParser._normalize_group_size(group_size)
if weight_bits not in self.SUPPORTED_BITS:
raise ValueError(
f"Unsupported weight_bits: {weight_bits}, "
f"currently only support {self.SUPPORTED_BITS}."
)
# auto-round mxfp data_type is e.g. "mx_fp4" / "mx_fp4e2m1"; match the
# "mx_fp" family by substring like auto_round.compressors.is_mx_fp.
is_mxfp = "mx_fp" in data_type
if data_type not in self.SUPPORTED_DTYPES and not is_mxfp:
raise ValueError(
f"Unsupported data_type: {data_type}, "
f"currently only support {self.SUPPORTED_DTYPES}."
)
if packing_format not in self.SUPPORTED_FORMATS:
raise ValueError(
f"Unsupported packing_format: {packing_format}, "
f"currently only support {self.SUPPORTED_FORMATS}."
)
if backend not in self.SUPPORTED_BACKENDS:
raise ValueError(
f"Unsupported backend: {backend}, "
f"currently only support {self.SUPPORTED_BACKENDS}."
)
self.weight_bits = weight_bits
self.group_size = group_size
self.sym = sym
self.packing_format = packing_format
self.block_name_to_quantize = (
block_name_to_quantize.split(",")
if isinstance(block_name_to_quantize, str)
else block_name_to_quantize
)
self.extra_config = extra_config
self.data_type = data_type
self.backend = backend
self.pack_factor = Fraction(32, weight_bits)
self.config_parser = INCConfigParser(self)
self._validate_supported_quantization()
def __repr__(self) -> str:
return (
f"INCConfig(weight_bits={self.weight_bits}, "
f"group_size={self.group_size}, sym={self.sym})"
)
@property
def is_mxfp(self) -> bool:
# auto-round mxfp data_type is the "mx_fp" family (e.g. "mx_fp",
# "mx_fp4e2m1"); match by substring like auto_round.compressors.is_mx_fp.
return "mx_fp" in self.data_type
@property
def is_mxfp8(self) -> bool:
# MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width.
return self.is_mxfp and self.weight_bits == self.MXFP8_BITS
@classmethod
def _check_expected(
cls,
values: dict[str, Any],
expected_fields: dict[str, Any],
quantization_name: str,
) -> None:
for field_name, expected_value in expected_fields.items():
actual_value = cls.get_from_keys_or(values, [field_name], expected_value)
assert actual_value == expected_value, (
f"{quantization_name} only supports "
f"{field_name}={expected_value!r}, "
f"but found {field_name}={actual_value!r}."
)
def _validate_int_quantization(self) -> None:
int_packing_formats = {
self.DEFAULT_INT_PACKING_FORMAT,
"auto_round:auto_awq",
}
assert isinstance(self.group_size, int), (
"INC int quantization requires scalar group_size, "
f"but found group_size={self.group_size!r}."
)
assert self.packing_format in int_packing_formats, (
"INC int quantization only supports "
f"packing_format in {sorted(int_packing_formats)!r}, "
f"but found packing_format={self.packing_format!r}."
)
def _validate_fp8_block_quantization(self) -> None:
assert self.weight_bits == self.FP8_BLOCK_BITS, (
"INC block-wise FP8 only supports bits=8, "
f"but found bits={self.weight_bits}."
)
assert isinstance(self.group_size, tuple), (
"INC block-wise FP8 requires a 2-D group_size, "
f"but found group_size={self.group_size!r}."
)
assert self.sym, "INC block-wise FP8 only supports symmetric weights."
assert self.packing_format == self.FP8_BLOCK_PACKING_FORMAT, (
"INC block-wise FP8 only supports "
f"packing_format={self.FP8_BLOCK_PACKING_FORMAT!r}, "
f"but found {self.packing_format!r}."
)
assert self.backend == "auto", (
"INC block-wise FP8 only supports backend='auto', "
f"but found backend={self.backend!r}."
)
def _validate_mxfp_quantization(self) -> None:
assert self.weight_bits in {4, self.MXFP8_BITS}, (
"INC MXFP only supports bits=4 or bits=8, "
f"but found bits={self.weight_bits}."
)
assert self.group_size == self.MXFP8_GROUP_SIZE, (
"INC MXFP only supports group_size=32, "
f"but found group_size={self.group_size}."
)
assert self.sym, "INC MXFP only supports symmetric weights."
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
"INC MXFP only supports "
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
f"but found {self.packing_format!r}."
)
assert self.backend == "auto", (
"INC MXFP only supports backend='auto', "
f"but found backend={self.backend!r}."
)
def _validate_supported_quantization(self) -> None:
if self.data_type == "int":
self._validate_int_quantization()
elif self.data_type == self.FP8_BLOCK_DATA_TYPE:
self._validate_fp8_block_quantization()
elif self.data_type == self.MXFP8_DATA_TYPE:
self._validate_mxfp_quantization()
def _validate_raw_config(self, config: dict[str, Any]) -> None:
if self.data_type == self.FP8_BLOCK_DATA_TYPE:
assert isinstance(self.group_size, tuple), (
"INC block-wise FP8 group_size must be a tuple."
)
expected_fields = {
"act_bits": self.FP8_BLOCK_BITS,
"act_group_size": self.group_size[0],
"act_data_type": self.FP8_BLOCK_DATA_TYPE,
"act_sym": True,
"act_dynamic": True,
"activation_scheme": self.FP8_BLOCK_ACTIVATION_SCHEME,
"fmt": self.FP8_BLOCK_FMT,
}
self._check_expected(
config,
expected_fields,
"INC block-wise FP8",
)
elif self.is_mxfp8:
expected_fields = {
"act_bits": self.MXFP8_BITS,
"act_data_type": self.MXFP8_DATA_TYPE,
"act_group_size": self.MXFP8_GROUP_SIZE,
"act_sym": True,
"act_dynamic": True,
}
self._check_expected(
config,
expected_fields,
"INC MXFP8",
)
@classmethod
def get_name(cls) -> QuantizationMethods:
return "inc"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.half, torch.bfloat16]
@classmethod
def get_min_capability(cls) -> int:
return 60
@classmethod
def get_config_filenames(cls) -> list[str]:
return ["quantization_config.json"]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
data_type = cls._normalize_data_type(
cls.get_from_keys_or(config, ["data_type"], "int")
)
weight_bits = cls.get_from_keys(config, ["bits"])
group_size = INCConfigParser._normalize_group_size(
cls.get_from_keys(config, ["group_size"])
)
packing_format = cls.get_from_keys_or(config, ["packing_format"], None)
if packing_format is None:
if (
data_type == cls.FP8_BLOCK_DATA_TYPE
and weight_bits == cls.FP8_BLOCK_BITS
and isinstance(group_size, tuple)
):
packing_format = cls.FP8_BLOCK_PACKING_FORMAT
elif "mx_fp" in data_type:
packing_format = cls.MXFP8_PACKING_FORMAT
else:
packing_format = cls.DEFAULT_INT_PACKING_FORMAT
quant_config = cls(
weight_bits=weight_bits,
group_size=group_size,
sym=cls.get_from_keys_or(config, ["sym"], True),
packing_format=packing_format,
block_name_to_quantize=cls.get_from_keys_or(
config, ["block_name_to_quantize", "to_quant_block_names"], None
),
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
data_type=data_type,
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
)
quant_config._validate_raw_config(config)
return quant_config
def get_layer_config(self, layer, layer_name: str):
return self.config_parser.get_layer_config(layer, layer_name)
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
if self.block_name_to_quantize is not None:
self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
self.block_name_to_quantize
)
if self.extra_config is not None:
self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
from .schemes.factory import resolve_scheme
if prefix and self.extra_config:
for layer_name in self.extra_config:
if (
layer_name == prefix or layer_name == f"model.{prefix}"
) and self.extra_config[layer_name].get("bits", 16) >= 16:
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
layer_config = self.config_parser.resolve(layer, prefix)
if not layer_config.quantized:
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
logger.debug(
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
prefix,
layer.__class__.__name__,
layer_config.bits,
layer_config.group_size,
layer_config.sym,
)
scheme = resolve_scheme(layer_config)
if isinstance(layer, (LinearBase, ParallelLMHead)):
return scheme.get_linear_method(self, layer, prefix, layer_config)
if isinstance(layer, RoutedExperts):
return scheme.get_moe_method(self, layer, prefix, layer_config)
return None
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> "QuantizationMethods | None":
"""Override the `auto-round` method to `inc`."""
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
if is_auto_round_format:
return cls.get_name()
return None
@@ -1,47 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import torch
from vllm.model_executor.layers.linear import LinearMethodBase
if TYPE_CHECKING:
from .schemes.inc_scheme import INCLinearScheme
class INCLinearMethod(LinearMethodBase):
def __init__(self, scheme: "INCLinearScheme") -> None:
self.scheme = scheme
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
return self.scheme.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
return self.scheme.process_weights_after_loading(layer)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.scheme.apply_weights(layer, x, bias)
@@ -1,19 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .factory import resolve_scheme
from .inc_fp8_scheme import INCFp8Scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_scheme import INCLinearScheme, INCScheme
from .inc_wna16_scheme import INCWna16Scheme
__all__ = [
"INCScheme",
"INCLinearScheme",
"INCMxfp8Scheme",
"INCFp8Scheme",
"INCWna16Scheme",
"INCMxfp4Scheme",
"resolve_scheme",
]
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
from .inc_scheme import INCScheme
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
from .inc_fp8_scheme import INCFp8Scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_wna16_scheme import INCWna16Scheme
scheme_list: list[type[INCScheme]] = [
INCFp8Scheme,
INCMxfp8Scheme,
INCWna16Scheme,
INCMxfp4Scheme,
]
for scheme_cls in scheme_list:
if scheme_cls.can_handle(layer_config):
return scheme_cls()
raise NotImplementedError(f"No INC scheme found for layer config: {layer_config}")
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import lru_cache
from typing import Any
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
_OPS_REGISTERED = False
@lru_cache(maxsize=1)
def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
"""Return ARK availability, error details, cached module, and QuantLinear."""
try:
import auto_round_kernel as ark
from auto_round_kernel.qlinear import QuantLinear
logger.info("Successfully imported auto_round_kernel.")
except ImportError as error:
return False, str(error), None, None
if getattr(ark, "cpu_lib", None) is None and getattr(ark, "xpu_lib", None) is None:
return (
False,
"No ARK backend library is available.",
None,
None,
)
logger.info("Successfully loaded auto_round_kernel backend library.")
return True, None, ark, QuantLinear
def _inc_ark_woq_linear_impl(
x: torch.Tensor,
qweight: torch.Tensor,
bias: torch.Tensor | None,
out_features: int,
in_features: int,
group_size: int,
compute_type: str,
weight_type: str,
scale_type: str,
asym: bool,
) -> torch.Tensor:
ark = get_ark_state()[2]
assert ark is not None
return ark.woqgemm_linear(
x,
qweight,
bias,
out_features,
in_features,
group_size,
compute_type,
weight_type,
scale_type,
asym,
)
def _inc_ark_woq_linear_fake(
x: torch.Tensor,
qweight: torch.Tensor,
bias: torch.Tensor | None,
out_features: int,
in_features: int,
group_size: int,
compute_type: str,
weight_type: str,
scale_type: str,
asym: bool,
) -> torch.Tensor:
del qweight
del bias
del in_features
del group_size
del compute_type
del weight_type
del scale_type
del asym
return torch.empty(
(*x.shape[:-1], out_features),
dtype=x.dtype,
device=x.device,
)
class ark_ops:
@staticmethod
def register_ops_once() -> None:
global _OPS_REGISTERED
if _OPS_REGISTERED:
return
is_available, error_str, _, _ = get_ark_state()
if not is_available:
logger.debug(
"Skip registering ark op because ARK is unavailable: %s",
error_str or "unknown error",
)
return
direct_register_custom_op(
op_name="inc_ark_woq_linear",
op_func=_inc_ark_woq_linear_impl,
fake_impl=_inc_ark_woq_linear_fake,
dispatch_key=current_platform.dispatch_key,
)
_OPS_REGISTERED = True
ark_ops.register_ops_once()
__all__ = ["get_ark_state"]
@@ -1,57 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
)
from .inc_scheme import INCLinearScheme
class INCFp8LinearScheme(INCLinearScheme):
def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None:
self.quant_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme="dynamic",
weight_block_size=list(weight_block_size),
)
self.linear_method = Fp8LinearMethod(self.quant_config)
self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
@classmethod
def get_min_capability(cls) -> int:
return Fp8Config.get_min_capability()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
self.linear_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.linear_method.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.linear_method.apply(layer, x, bias)
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCFp8Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_fp8_block
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer
assert isinstance(layer_config.group_size, tuple)
from .inc_fp8_linear import INCFp8LinearScheme
return INCLinearMethod(
INCFp8LinearScheme(
prefix=prefix,
weight_block_size=layer_config.group_size,
)
)
@@ -1,97 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
import torch
from torch.nn.parameter import Parameter
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
)
from .inc_scheme import INCLinearScheme
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
class INCMxfp4LinearMethod(INCLinearScheme):
"""MXFP4 (W4A4) linear method for AutoRound checkpoints.
E2M1 weights packed two per byte with per-group E8M0 scales
(group_size=32, no global scale). The platform kernel is selected by
``init_mxfp4_linear_kernel`` (FlashInfer / Marlin on CUDA, ``fp4_gemm``
on XPU).
"""
def __init__(self, layer_config: "INCLayerConfig") -> None:
if not isinstance(layer_config.group_size, int):
raise ValueError(
"INC MXFP4 requires scalar group_size, "
f"but found group_size={layer_config.group_size!r}."
)
self.group_size = layer_config.group_size or 32
self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
@classmethod
def get_min_capability(cls) -> int:
return 80
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs: Any,
) -> None:
del input_size, output_size
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
weight_loader = extra_weight_attrs.get("weight_loader")
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // 2,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_packed", weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.group_size,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = Parameter(layer.weight_packed.data, requires_grad=False)
del layer.weight_packed
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
@@ -1,252 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
MarlinExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
Mxfp4MoeBackend,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
logger = init_logger(__name__)
class INCMxfp4MoEMethod(FusedMoEMethodBase):
"""W4A4 MXFP4 group MoE for AutoRound ``auto_round:llm_compressor`` exports.
Registers the packed MXFP4 layout (uint8 ``weight_packed`` + uint8 E8M0
``weight_scale``, ``group_size=32``) and dispatches the fused MoE to b12x
when requested; otherwise it uses CUTLASS W4A4, XPU W4A4, or Marlin W4A16.
The per-expert ``gate_proj`` / ``up_proj`` / ``down_proj`` tensors are
folded into the stacked ``w13`` / ``w2`` parameters by
``make_expert_params_mapping``.
"""
def __init__(self, moe) -> None:
super().__init__(moe)
if moe.moe_backend in FLASHINFER_MOE_EP_BACKENDS:
raise ValueError(
f"moe_backend={moe.moe_backend!r} is not supported for AutoRound "
"(INC) MXFP4 MoE checkpoints"
)
self.group_size = 32
# Backend selection must stay consistent with the weight preparation in
# process_weights_after_loading / get_fused_moe_quant_config, which use
# three layouts: CUTLASS swizzle (true W4A4), b12x and XPU packed
# passthrough, and Marlin weight-only. XPU dispatch is deferred to the
# shared oracle; b12x dispatch is explicit; every remaining non-CUTLASS
# device falls back to Marlin.
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
self.experts_cls: type[mk.FusedMoEExperts] | None = None
if moe.moe_backend == "b12x":
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
self.use_cutlass_mxfp4 = False
elif self.use_cutlass_mxfp4:
self.experts_cls = CutlassExpertsMxfp4
logger.info_once("Using CutlassExpertsMxfp4 for AutoRound MXFP4 MoE")
elif current_platform.is_xpu():
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
else:
self.experts_cls = MarlinExperts
logger.info_once(
"Using MarlinExperts (weight-only FP4) for AutoRound MXFP4 MoE"
)
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
layer.num_experts = num_experts
layer.params_dtype = params_dtype
# gate + up fused on the output dim; two FP4 packed per input byte.
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_packed", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
# Per-group E8M0 block scales (group_size=32), stored as uint8.
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
extra_weight_attrs.update(
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
if self.use_cutlass_mxfp4:
# W4A4: both weights and activations quantized to MXFP4.
return mxfp4_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
return make_mxfp4_moe_quant_config(
mxfp4_backend=self.mxfp4_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.w13_weight = torch.nn.Parameter(
layer.w13_weight_packed.data, requires_grad=False
)
delattr(layer, "w13_weight_packed")
layer.w2_weight = torch.nn.Parameter(
layer.w2_weight_packed.data, requires_grad=False
)
delattr(layer, "w2_weight_packed")
if self.use_cutlass_mxfp4:
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
# to the CUTLASS tiled layout.
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
swizzle_mxfp4_scales,
)
E = layer.w13_weight_scale.shape[0]
w13_N = layer.w13_weight_scale.shape[1]
w13_scale_K = layer.w13_weight_scale.shape[2]
w13_K = w13_scale_K * 32
w2_M = layer.w2_weight_scale.shape[1]
w2_scale_N = layer.w2_weight_scale.shape[2]
w2_N = w2_scale_N * 32
swizzled_w13 = []
swizzled_w2 = []
for e_idx in range(E):
s13 = layer.w13_weight_scale[e_idx]
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
s2 = layer.w2_weight_scale[e_idx]
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
layer.w13_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w13), requires_grad=False
)
layer.w2_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w2), requires_grad=False
)
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
pass
else:
logger.warning_once(
"This device lacks native FP4 compute; using weight-only FP4 "
"via the Marlin kernel, which may reduce performance for "
"compute-heavy workloads."
)
prepare_moe_fp4_layer_for_marlin(layer)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
mxfp4_backend=self.mxfp4_backend,
routing_tables=layer._expert_routing_tables(),
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -1,54 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
logger = init_logger(__name__)
class INCMxfp4Scheme(INCScheme):
"""MXFP4 (W4A4) scheme for AutoRound checkpoints.
Dispatches to :class:`INCMxfp4LinearMethod` for linear layers and
:class:`INCMxfp4MoEMethod` for fused MoE layers; see those classes for the
per-module weight layout and kernel-selection details.
"""
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_mxfp4
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer, prefix
from .inc_mxfp4_linear import INCMxfp4LinearMethod
return INCLinearMethod(INCMxfp4LinearMethod(layer_config))
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix, layer_config
from .inc_mxfp4_moe import INCMxfp4MoEMethod
return INCMxfp4MoEMethod(layer.moe_config)
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
)
from .inc_scheme import INCLinearScheme
class INCMxfp8LinearScheme(INCLinearScheme):
@classmethod
def get_min_capability(cls) -> int:
return 75
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size
if input_size_per_partition % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
"INC MXFP8 requires input_size_per_partition "
f"({input_size_per_partition}) to be divisible by "
f"{MXFP8_BLOCK_SIZE}."
)
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
layer.register_parameter("weight", weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
layer.register_parameter("weight_scale", weight_scale)
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
@@ -1,197 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fused_moe import (
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.parameter import ModelWeightParameter
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
class INCMxfp8MoEMethod(FusedMoEMethodBase):
"""W8A8 MXFP8 MoE method for serialized AutoRound checkpoints."""
def __init__(self, moe) -> None:
super().__init__(moe)
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
self.mxfp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
def create_weights(
self,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
assert layer.intermediate_size_per_partition == intermediate_size_per_partition
assert layer.hidden_size == hidden_size
layer.orig_dtype = params_dtype
if hidden_size % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
f"MXFP8 MoE requires hidden_size divisible by {MXFP8_BLOCK_SIZE}, "
f"got {hidden_size}."
)
if intermediate_size_per_partition % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
"MXFP8 MoE requires intermediate_size_per_partition divisible by "
f"{MXFP8_BLOCK_SIZE}, got {intermediate_size_per_partition}."
)
layer.num_experts = num_experts
weight_loader = extra_weight_attrs.get("weight_loader")
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_weight = ModelWeightParameter(
data=torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w13_weight", w13_weight)
w2_weight = ModelWeightParameter(
data=torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w2_weight", w2_weight)
w13_weight_scale = ModelWeightParameter(
data=torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
w2_weight_scale = ModelWeightParameter(
data=torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(
layer.w13_weight_scale,
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
)
set_weight_attrs(
layer.w2_weight_scale,
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.weight_block_size = self.weight_block_size
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
fp8_backend=self.mxfp8_backend,
layer=layer,
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_input_scale=None,
w2_input_scale=None,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
self.moe_kernel = make_fp8_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
fp8_backend=self.mxfp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
return make_fp8_moe_quant_config(
fp8_backend=self.mxfp8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a1_scale=None,
a2_scale=None,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCMxfp8Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_mxfp8
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer, prefix, layer_config
from .inc_mxfp8_linear import INCMxfp8LinearScheme
return INCLinearMethod(INCMxfp8LinearScheme())
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix, layer_config
from .inc_mxfp8_moe import INCMxfp8MoEMethod
return INCMxfp8MoEMethod(layer.moe_config)
@@ -1,88 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
from vllm.model_executor.layers.linear import LinearMethodBase
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCScheme(ABC):
"""One class per quant type. Single registration point for the factory.
Each subclass defines:
- can_handle(): when does this scheme apply?
- get_linear_method(): required — how to quantize Linear layers
- get_moe_method(): optional — how to quantize MoE layers
Schemes that don't support MoE/KVCache inherit the default raise.
"""
@staticmethod
@abstractmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
raise NotImplementedError
@abstractmethod
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
) -> "LinearMethodBase":
raise NotImplementedError
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
) -> "FusedMoEMethodBase | None":
"""Optional. Override if this scheme supports MoE.
Default raises NotImplementedError."""
raise NotImplementedError(
f"{type(self).__name__} does not support MoE layers. "
f"Layer config: {layer_config}"
)
class INCLinearScheme(ABC):
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
raise NotImplementedError
@abstractmethod
def create_weights(
self,
layer: "torch.nn.Module",
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: "torch.dtype",
**extra_weight_attrs,
) -> None:
raise NotImplementedError
@abstractmethod
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
raise NotImplementedError
@abstractmethod
def apply_weights(
self,
layer: "torch.nn.Module",
x: "torch.Tensor",
bias: "torch.Tensor | None" = None,
) -> "torch.Tensor":
raise NotImplementedError
@@ -1,112 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.nn.parameter import Parameter
from .inc_wna16_linear import INCXPULinearMethod
class INCXPUW4A8LinearMethod(INCXPULinearMethod):
"""XPU linear method for INC int4 weights with dynamic int8 activations.
Uses the same GPTQ-packed "NT" qweight layout as ``INCXPULinearMethod`` —
``int4_gemm_w4a8`` and ``int4_gemm_w4a16`` accept bit-identical weights, so
no extra repacking is needed. Activations are dynamically quantized
per-token to symmetric int8, which keeps the GEMM on the int8 datapath
instead of upconverting the weights to the activation dtype. That is a win
for compute-bound shapes (large token counts, e.g. diffusion), where w4a16
is dominated by dequantization rather than by weight bandwidth.
Both scale tensors must be float16: the kernel reads them as fp16
regardless of the activation dtype, so bf16 scales are silently
misinterpreted. The activations themselves may be bf16 or fp16.
Opt in with ``VLLM_XPU_INC_WNA16_BACKEND=w4a8``. Whether this beats ARK is
device-dependent, so it is not the default: it was measured faster on B70
(Xe2), where ARK cannot use its XMX int8 path and falls back to fp16.
"""
# int8 activations only pay off once the GEMM is compute-bound. Below this
# token count the per-token quantization overhead dominates and w4a16 is
# faster, so fall back to it per-call.
_MIN_TOKENS_FOR_INT8 = 512
# int4_gemm_w4a8 requires both GEMM dims to be multiples of 8. Tensor
# parallelism shards these, so an aligned model dimension can still yield an
# unaligned partition.
_DIM_ALIGNMENT = 8
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
unaligned = [
(name, size)
for name, size in (
("input", input_size_per_partition),
("output", output_size_per_partition),
)
if size % self._DIM_ALIGNMENT != 0
]
if unaligned:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires partitioned in/out "
f"sizes that are multiples of {self._DIM_ALIGNMENT}, got "
+ ", ".join(f"{name}={size}" for name, size in unaligned)
+ f". Partition shape: ({input_size_per_partition}, "
f"{output_size_per_partition})."
)
super().create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
# The kernel reads both scale tensors as fp16; keep an fp16 copy for the
# w4a8 path and leave ``scales`` untouched for the w4a16 fallback.
layer.scales_fp16 = Parameter(
layer.scales.data.to(torch.float16), requires_grad=False
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
if reshaped_x.shape[0] < self._MIN_TOKENS_FOR_INT8:
return super().apply_weights(layer, x, bias)
from vllm._xpu_ops import xpu_ops as ops
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
reshaped_x, True, 8
)
out = torch.ops._xpu_C.int4_gemm_w4a8(
quant_x,
x_scale.to(torch.float16),
x_zero,
layer.qweight,
layer.scales_fp16,
layer.qzeros,
self.group_size,
None, # Retained by the external XPU op ABI.
bias,
)
return out.to(x.dtype).reshape(out_shape)
@@ -1,460 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
import torch
from torch.nn.parameter import Parameter
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.scalar_type import scalar_types
from .inc_scheme import INCLinearScheme
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
class INCWNA16LinearScheme(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.layer_config = layer_config
self.linear_method = self._build_inner_method()
@classmethod
def get_min_capability(cls) -> int:
return 60
def _build_inner_method(self):
if self.layer_config.is_gptq:
return self._build_gptq_method()
if self.layer_config.is_awq:
return self._build_awq_method()
raise NotImplementedError(
f"WNA16 linear scheme does not support {self.layer_config}"
)
def _build_gptq_method(self):
assert isinstance(self.layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
gptq_type_map = {
(4, True): scalar_types.uint4b8,
(8, True): scalar_types.uint8b128,
}
use_marlin = (
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
) and (self.layer_config.bits, self.layer_config.sym) in gptq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
gptq_type_map[(self.layer_config.bits, self.layer_config.sym)],
self.layer_config.group_size,
has_zp=not self.layer_config.sym,
)
if use_marlin:
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQLinearMethod,
)
return AutoGPTQLinearMethod(
AutoGPTQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
desc_act=False,
is_sym=self.layer_config.sym,
lm_head_quantized=False,
dynamic={},
full_config={},
)
)
raise NotImplementedError(
f"INC quantization with bits={self.layer_config.bits}, "
f"sym={self.layer_config.sym} is not supported. "
"Only 4-bit and 8-bit symmetric quantization is supported "
"with Marlin kernels."
)
def _build_awq_method(self):
assert isinstance(self.layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
awq_type_map = {
4: scalar_types.uint4,
8: scalar_types.uint8,
}
use_marlin = (
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
) and self.layer_config.bits in awq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
awq_type_map[self.layer_config.bits],
self.layer_config.group_size,
not self.layer_config.sym,
)
if use_marlin:
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQMarlinLinearMethod,
)
return AutoAWQMarlinLinearMethod(
AutoAWQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
zero_point=not self.layer_config.sym,
lm_head_quantized=False,
modules_to_not_convert=[],
full_config={},
)
)
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQLinearMethod,
)
return AutoAWQLinearMethod(
AutoAWQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
zero_point=not self.layer_config.sym,
lm_head_quantized=False,
)
)
def create_weights(
self,
layer: "torch.nn.Module",
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: "torch.dtype",
**extra_weight_attrs,
) -> None:
return self.linear_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
return self.linear_method.process_weights_after_loading(layer)
def apply_weights(
self,
layer: "torch.nn.Module",
x: "torch.Tensor",
bias: "torch.Tensor | None" = None,
) -> "torch.Tensor":
return self.linear_method.apply(layer, x, bias)
class INCXPULinearBase(INCLinearScheme):
# AWQ packs nibbles within each int32 in the order [0, 2, 4, 6, 1, 3, 5, 7];
# this permutation undoes that ordering so values can be repacked in
# standard sequential (GPTQ) order.
_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.weight_bits = layer_config.bits
assert isinstance(layer_config.group_size, int), (
"INCXPULinearBase requires integer group_size."
)
self.group_size = layer_config.group_size
self.sym = layer_config.sym
self.pack_factor = 32 // self.weight_bits
self.is_awq_packed = layer_config.is_awq
@classmethod
def get_min_capability(cls) -> int:
return 0
def _create_inc_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
params_dtype: torch.dtype,
weight_loader: Any,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
scales_and_zp_size = input_size_per_partition // self.group_size
if self.is_awq_packed:
# AWQ: qweight [in, out // pack_factor] packed along output dim
qweight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition,
output_size_per_partition // self.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
else:
# GPTQ: qweight [in // pack_factor, out] packed along input dim
qweight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition // self.pack_factor,
output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
scales = GroupQuantScaleParameter(
data=torch.empty(
scales_and_zp_size,
output_size_per_partition,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_loader,
)
# Both AWQ and GPTQ checkpoints store qzeros with this shape; for
# symmetric quantization the values are ignored downstream.
qzeros = PackedvLLMParameter(
data=torch.empty(
scales_and_zp_size,
output_size_per_partition // self.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
layer.register_parameter("qweight", qweight)
layer.register_parameter("scales", scales)
layer.register_parameter("qzeros", qzeros)
def _convert_awq_qweight_to_gptq(self, qw: torch.Tensor) -> torch.Tensor:
"""Convert AWQ qweight [K, N // pf] to GPTQ qweight [K // pf, N].
AWQ packs along the output dim with a non-standard nibble order; GPTQ
packs along the input dim with sequential nibble order. The conversion
is lossless — it only reshuffles bits.
"""
size_bits = self.weight_bits
pack_factor = self.pack_factor
mask = (1 << size_bits) - 1
device = qw.device
reverse_order = torch.tensor(
self._REVERSE_AWQ_PACK_ORDER, dtype=torch.long, device=device
)
shifts = torch.arange(0, 32, size_bits, dtype=torch.int32, device=device)
K, N_packed = qw.shape
N = N_packed * pack_factor
# Unpack int32 → individual values, fix AWQ nibble ordering
unpacked = (qw.unsqueeze(-1) >> shifts) & mask # (K, N_packed, pf)
unpacked = unpacked[:, :, reverse_order]
unpacked = unpacked.reshape(K, N) # (K, N)
# Repack along input dim (dim 0) in sequential nibble order
unpacked = unpacked.reshape(K // pack_factor, pack_factor, N)
new_qw = (unpacked.to(torch.int32) << shifts[None, :, None]).sum(
dim=1, dtype=torch.int32
)
return new_qw.contiguous()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size
self._create_inc_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
params_dtype=params_dtype,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
class INCXPULinearMethod(INCXPULinearBase):
"""XPU linear method for INC w4a16 quantization (symmetric only).
Supports both GPTQ-packed (``auto_round:auto_gptq``) and AWQ-packed
(``auto_round:auto_awq``) AutoRound checkpoints. AWQ-packed qweights are
losslessly repacked into the GPTQ-style nibble layout during
``process_weights_after_loading``, before the final oneDNN "NT" transpose
that ``torch.ops._xpu_C.int4_gemm_w4a16`` expects.
"""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
device = layer.qweight.data.device
qweight_data = layer.qweight.data
if self.is_awq_packed:
# Lossless repack: AWQ [K, N // pf] → GPTQ [K // pf, N]
qweight_data = self._convert_awq_qweight_to_gptq(qweight_data)
qweight_ct = qweight_data.t().contiguous()
layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
layer.scales = Parameter(layer.scales.data, requires_grad=False)
layer.qzeros = Parameter(
torch.tensor([8], dtype=torch.int8, device=device),
requires_grad=False,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
reshaped_x = x.reshape(-1, x.shape[-1])
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
layer.qweight,
bias,
layer.scales,
layer.qzeros,
self.group_size,
None, # Retained by the external XPU op ABI.
)
return out.reshape(out_shape)
class INCARKLinearMethod(INCXPULinearBase):
def __init__(self, layer_config: "INCLayerConfig") -> None:
super().__init__(layer_config)
from .inc_ark_ops import get_ark_state
is_available, error_str, _, quant_linear_cls = get_ark_state()
if not is_available or quant_linear_cls is None:
reason = error_str or "unknown error"
raise ImportError(f"Failed to import auto_round_kernel. {reason}")
self.quant_linear_cls = quant_linear_cls
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
super().create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
layer.in_features = input_size_per_partition
layer.out_features = sum(output_partition_sizes)
layer.params_dtype = params_dtype
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if hasattr(layer, "input_size_per_partition"):
in_features = layer.input_size_per_partition
elif hasattr(layer, "input_size"):
in_features = layer.input_size
else:
raise AttributeError("Cannot determine in_features for layer.")
if hasattr(layer, "output_partition_sizes"):
out_features = sum(layer.output_partition_sizes)
elif hasattr(layer, "output_size_per_partition"):
out_features = layer.output_size_per_partition
elif hasattr(layer, "output_size"):
out_features = layer.output_size
else:
out_features = layer.scales.shape[-1]
ark_linear = self.quant_linear_cls(
bits=self.weight_bits,
group_size=self.group_size,
sym=self.sym,
in_features=in_features,
out_features=out_features,
bias=layer.bias is not None,
weight_dtype=layer.params_dtype,
)
ark_linear.to(layer.qweight.device)
with torch.no_grad():
qweight_src = layer.qweight.detach()
if self.is_awq_packed:
# ARK consumes GPTQ-style packed nibbles; convert AWQ losslessly.
qweight_src = self._convert_awq_qweight_to_gptq(qweight_src)
ark_linear.qweight.copy_(qweight_src)
if hasattr(layer, "qzeros") and layer.qzeros is not None:
ark_linear.qzeros.copy_(layer.qzeros.detach())
else:
ark_linear.qzeros = None
ark_linear.scales.copy_(layer.scales.detach())
if hasattr(layer, "bias") and layer.bias is not None:
ark_linear.bias.copy_(layer.bias.detach())
ark_linear.post_init()
layer.qweight = Parameter(ark_linear.qweight.detach(), requires_grad=False)
layer.ark_bias = ark_linear.bias
layer.ark_compute_type = ark_linear.cdt
layer.ark_weight_type = ark_linear.wdt
layer.ark_scale_type = ark_linear.sdt
if hasattr(layer, "qzeros"):
del layer.qzeros
del layer.scales
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return torch.ops.vllm.inc_ark_woq_linear.default(
x,
layer.qweight,
layer.ark_bias,
layer.out_features,
layer.in_features,
self.group_size,
layer.ark_compute_type,
layer.ark_weight_type,
layer.ark_scale_type,
not self.sym,
)
@@ -1,318 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.platforms import current_platform
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
logger = init_logger(__name__)
XPU_WNA16_SUPPORTED_BITS = {2, 4}
# On CUDA the Marlin/GPTQ/AWQ kernels only cover 4/8-bit. The remaining
# widths (2/3/5/6/7) have no dedicated CUDA kernel, so they are dispatched to
# the humming kernel instead. This mirrors compressed-tensors WNA16, whose
# 5/6/7-bit biased scalar types also fall back to humming at kernel selection.
CUDA_HUMMING_SUPPORTED_BITS = {2, 3, 5, 6, 7}
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
XPU_ONEDNN_BACKENDS = ("w4a16", "w4a8")
def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> None:
"""Raise unless ``int4_gemm_w4a8`` can serve this layer.
The backend is requested explicitly, so an unusable configuration is an
error rather than something to silently fall back from.
"""
import torch
if not hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires the int4_gemm_w4a8 op, "
"which this build of vllm-xpu-kernels does not provide. "
f"Layer: {prefix}."
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
f"positive multiple of 32, got {layer_config.group_size}. "
f"Layer: {prefix}."
)
class INCWna16Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_wna16_int
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer
if current_platform.is_xpu():
if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym:
from .inc_ark_ops import get_ark_state
from .inc_w4a8_linear import INCXPUW4A8LinearMethod
from .inc_wna16_linear import (
INCARKLinearMethod,
INCXPULinearMethod,
)
backend = envs.VLLM_XPU_INC_WNA16_BACKEND
if backend in XPU_ONEDNN_BACKENDS:
if layer_config.bits != 4:
raise NotImplementedError(
f"VLLM_XPU_INC_WNA16_BACKEND={backend} only supports "
f"int4, got int{layer_config.bits}. Layer: {prefix}."
)
if backend == "w4a8":
_check_xpu_w4a8_supported(layer_config, prefix)
return INCLinearMethod(INCXPUW4A8LinearMethod(layer_config))
return INCLinearMethod(INCXPULinearMethod(layer_config))
is_ark_available, ark_error, _, _ = get_ark_state()
if backend == "ark" and not is_ark_available:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=ark was requested but "
f"auto_round_kernel is unavailable: "
f"{ark_error or 'unknown error'}. Layer: {prefix}."
)
if is_ark_available:
return INCLinearMethod(INCARKLinearMethod(layer_config))
elif layer_config.bits == 2:
raise NotImplementedError(
"INC int2 on XPU requires the ARK backend. "
f"Layer: {prefix}. "
f"auto_round_kernel unavailable: "
f"{ark_error or 'unknown error'}"
)
logger.debug(
"ARK backend is unavailable for layer %s; "
"falling back to the default XPU INC path. Error: %s",
prefix,
ark_error or "unknown error",
)
return INCLinearMethod(INCXPULinearMethod(layer_config))
raise NotImplementedError(f"INC on XPU: unsupported config {layer_config}")
if current_platform.is_cpu() and layer_config.is_gptq:
if layer_config.bits == 4 and layer_config.sym:
from .inc_ark_ops import get_ark_state
from .inc_wna16_linear import (
INCARKLinearMethod,
INCWNA16LinearScheme,
)
is_ark_available, ark_error, _, _ = get_ark_state()
if is_ark_available:
return INCLinearMethod(INCARKLinearMethod(layer_config))
logger.debug(
"ARK backend is unavailable for layer %s; "
"falling back to the default CPU INC path. Error: %s",
prefix,
ark_error or "unknown error",
)
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
# CUDA low-bit (2/3/5/6/7): no Marlin/GPTQ/AWQ kernel, route to humming
# so a single model can mix 4/8-bit (Marlin) and 2/3/5/6/7-bit (humming)
# layers.
if (
current_platform.is_cuda()
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
):
return _build_humming_linear_method(layer_config)
from .inc_wna16_linear import INCWNA16LinearScheme
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix
# CPU does not support quantized MoE yet.
if current_platform.is_cpu():
from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
return UnquantizedFusedMoEMethod(layer.moe_config)
# CUDA low-bit (2/3/5/6/7): route to the humming MoE kernel (see above).
if (
current_platform.is_cuda()
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
):
return _build_humming_moe_method(layer, layer_config)
if layer_config.is_gptq:
return _resolve_gptq_moe(layer, layer_config)
if layer_config.is_awq:
return _resolve_awq_moe(layer, layer_config)
raise NotImplementedError(f"WNA16 MoE does not support config {layer_config}")
def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQMoEMethod,
)
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_layer,
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
# check like compressed-tensors does; the capability-based
# check_marlin_supported is skipped so the XPU path is reachable.
use_marlin = (layer_config.bits, layer_config.sym) in {
(4, True),
(8, True),
} and check_moe_marlin_supports_layer(layer, layer_config.group_size)
if use_marlin:
return AutoGPTQMoEMethod(
AutoGPTQConfig(
weight_bits=layer_config.bits,
group_size=layer_config.group_size,
desc_act=False,
is_sym=layer_config.sym,
lm_head_quantized=False,
dynamic={},
full_config={},
),
layer.moe_config,
)
moe_config = MoeWNA16Config.from_config(
{
"quant_method": "gptq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"sym": layer_config.sym,
"lm_head": False,
}
)
return MoeWNA16Method(moe_config, layer.moe_config)
def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQMoEMethod
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_layer,
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
layer, layer_config.group_size
)
if use_marlin:
return AutoAWQMoEMethod(
AutoAWQConfig(
weight_bits=layer_config.bits,
group_size=layer_config.group_size,
zero_point=not layer_config.sym,
lm_head_quantized=False,
modules_to_not_convert=[],
full_config={},
),
layer.moe_config,
)
moe_config = MoeWNA16Config.from_config(
{
"quant_method": "awq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"zero_point": not layer_config.sym,
"lm_head": False,
}
)
return MoeWNA16Method(moe_config, layer.moe_config)
def _humming_weight_config(layer_config: "INCLayerConfig") -> dict:
"""Build the humming weight-schema config for a WNA16 int checkpoint."""
if layer_config.is_gptq:
return {
"quant_method": "gptq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"desc_act": False,
"sym": layer_config.sym,
}
if layer_config.is_awq:
return {
"quant_method": "awq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"zero_point": not layer_config.sym,
}
raise NotImplementedError(
"INC humming dispatch only supports gptq/awq packed int checkpoints, "
f"but found {layer_config}."
)
def _build_humming_quant_config(layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import (
HummingLayerQuantizationConfig,
)
from vllm.utils.humming import BaseWeightSchema
weight_schema = BaseWeightSchema.from_config(_humming_weight_config(layer_config))
return HummingLayerQuantizationConfig(weight_schema=weight_schema)
def _build_humming_linear_method(layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import HummingLinearMethod
return HummingLinearMethod(_build_humming_quant_config(layer_config))
def _build_humming_moe_method(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
return HummingMoEMethod(_build_humming_quant_config(layer_config), layer.moe_config)
@@ -19,7 +19,7 @@ from vllm.model_executor.kernels.linear import (
init_mxfp8_linear_kernel,
init_nvfp4_linear_kernel,
)
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEMethodBase,
@@ -228,6 +228,8 @@ class ModelOptQuantConfigBase(QuantizationConfig):
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# handle kv-cache first so we can focus only on weight quantization thereafter
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
if isinstance(layer, (Attention, MLAAttention)):
return self.KVCacheMethodCls(self)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,21 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .quark_nvfp4 import QuarkNVFP4
from .quark_ocp_mx import QuarkOCP_MX
from .quark_scheme import QuarkScheme
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
from .quark_w4a16_int4 import QuarkW4A16Int4
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
from .quark_w8a8_int8 import QuarkW8A8Int8
__all__ = [
"QuarkScheme",
"QuarkW8A8Fp8",
"QuarkW8A8Fp8PerBlock",
"QuarkW8A8Int8",
"QuarkOCP_MX",
"QuarkW4A16Int4",
"QuarkW4A8_MXFP4_FP8",
"QuarkNVFP4",
]
@@ -1,163 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
from vllm.model_executor.kernels.linear.nvfp4.emulation import (
EmulationNvFp4LinearKernel,
)
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
QuarkScheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
PerTensorScaleParameter,
)
__all__ = ["QuarkNVFP4"]
logger = init_logger(__name__)
class QuarkNVFP4(QuarkScheme):
"""Quark NVFP4 quantization scheme.
Supports loading NVFP4 checkpoints with the following structure:
- weight: uint8, shape [out_features, in_features // 2] (packed FP4)
- weight_scale: float8_e4m3fn, shape [out_features, in_features // group_size]
- weight_scale_2: bfloat16/float32, scalar (global weight scale)
- input_scale_2: bfloat16/float32, scalar (global input scale)
"""
supported_activation_quant_keys = [kNvfp4Dynamic]
supported_weight_quant_keys = [kNvfp4Static]
def __init__(
self,
activation_quant_key: QuantKey | None,
):
super().__init__(kNvfp4Static, activation_quant_key)
self.kernel = init_nvfp4_linear_kernel()
self.group_size = 16
if not isinstance(self.kernel, EmulationNvFp4LinearKernel):
logger.warning_once(
"Only EmulationNvFp4LinearKernel NVFP4 dense implementation is "
"tested with QuarkNVFP4, got kernel=%s. Correctness is not validated.",
type(self.kernel).__name__,
)
@classmethod
def get_min_capability(cls) -> int:
# FP4 requires Turing (75) or newer
return 75
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
if input_size_per_partition % self.group_size != 0:
raise ValueError(
f"Input size per partition ({input_size_per_partition}) must be "
f"divisible by group size ({self.group_size})"
)
# Weight: FP4 packed as uint8 (2 FP4 values per uint8)
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // 2,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# Per-group weight scale (FP8 E4M3)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.group_size,
dtype=torch.float8_e4m3fn,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
# Global weight scale (scalar, per partition)
weight_scale_2 = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale_2", weight_scale_2)
# Global input scale (scalar, per partition)
input_scale_2 = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
layer.register_parameter("input_scale_2", input_scale_2)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
input_global_scale = layer.input_scale_2.max().to(torch.float32)
layer.input_global_scale = Parameter(input_global_scale, requires_grad=False)
del layer.input_scale_2
weight_global_scale = layer.weight_scale_2.to(torch.float32)
if torch.unique(weight_global_scale).numel() != 1:
logger.warning_once(
"In NVFP4 linear, the global scale for weight are different"
" for parallel layers (e.g. q_proj, k_proj, v_proj). This"
" will likely result in reduced accuracy. Please verify the"
" model accuracy. Consider using a checkpoint with a shared"
" global NVFP4 scale for fused layers."
)
weight_global_scale = weight_global_scale.max()
layer.weight_global_scale = Parameter(weight_global_scale, requires_grad=False)
del layer.weight_scale_2
layer.alpha = Parameter(
layer.input_global_scale * layer.weight_global_scale, requires_grad=False
)
layer.input_global_scale_inv = Parameter(
(1.0 / layer.input_global_scale).to(torch.float32), requires_grad=False
)
# Convert layer to NVFP4 linear kernel format
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer=layer, x=x, bias=bias)
@@ -1,264 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from fractions import Fraction
import torch
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
MxFp4LinearKernel,
MxFp6LinearKernel,
Mxfp8LinearKernel,
init_mxfp4_linear_kernel,
init_mxfp6_linear_kernel,
init_mxfp8_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
OCP_MX_BLOCK_SIZE,
OCP_MX_UNPACKED_DTYPES,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kMxfp6E2M3Static,
kMxfp6E3M2Static,
kMxfp8Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
PackedvLLMParameter,
)
from vllm.platforms import current_platform
from .quark_scheme import QuarkScheme
logger = init_logger(__name__)
class QuarkOCP_MX(QuarkScheme):
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel | Mxfp8LinearKernel
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
scale_block_rows: int = 1,
):
super().__init__(weight_quant_key, activation_quant_key)
self.scale_block_rows = scale_block_rows
self.weight_dtype = next(
dtype
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
if quant_key == weight_quant_key
)
self.input_dtype = (
next(
dtype
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
if quant_key == activation_quant_key
)
if activation_quant_key is not None
else None
)
self.is_unpacked = self.weight_dtype in OCP_MX_UNPACKED_DTYPES
if self.is_unpacked:
self.packed_factor: int | Fraction = 1
elif self.weight_dtype == "mxfp4":
self.packed_factor = 2
else:
self.packed_factor = Fraction(numerator=8, denominator=6)
if self.is_unpacked:
# MXFP8 runs on the shared kernel abstraction, which picks a native
# backend where one exists and emulates otherwise. The MXFP4/MXFP6
# warnings below do not apply.
return
if not current_platform.supports_mx():
logger.warning_once(
"The current platform does not support native MXFP4/MXFP6 "
"computation. Simulated weight dequantization and activation "
"QDQ (quantize and dequantize) will be used, with the linear "
"layers computed in high precision."
)
if current_platform.supports_mx() and (
self.input_dtype != "mxfp4" or self.weight_dtype != "mxfp4"
):
logger.warning_once(
"The current platform supports native MXFP4/MXFP6 "
f"computation, but kernels for input_dtype={self.input_dtype} "
f"and weight_dtype={self.weight_dtype} are not yet integrated "
"in vLLM. Simulated weight dequantization and activation "
"QDQ (quantize and dequantize) will be used, with the linear "
"layers computed in high precision."
)
def get_packed_dim(self, dim: int, quant_dtype: str):
if quant_dtype in OCP_MX_UNPACKED_DTYPES:
return dim
elif quant_dtype == "mxfp4":
assert dim % 2 == 0
return dim // 2
elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}:
# FP6 packs 4 * 6 = 24 bits on 3 bytes.
assert (dim * 3) % 4 == 0
return (dim * 3) // 4
else:
raise NotImplementedError(
"Unsupported quant_dtype in QuarkOCP_MX.get_packed_dim, "
f"got quant_dtype={quant_dtype}. Something is wrong, please "
"open an issue."
)
def _expanding_scale_loader(self, weight_loader: Callable) -> Callable:
"""Expand a 2-D block scale to one row per weight row.
A ``block_size=[R, 32]`` export carries one e8m0 value per R weight
rows; the kernels want one per row. At ``R == 1`` the layouts already
coincide, so the loader is passed through untouched.
"""
if self.scale_block_rows == 1:
return weight_loader
rows = self.scale_block_rows
def expanding_loader(param, loaded_weight, *args, **kwargs):
loaded_weight = loaded_weight.view(torch.uint8).repeat_interleave(
rows, dim=0
)
return weight_loader(param, loaded_weight, *args, **kwargs)
return expanding_loader
@classmethod
def get_min_capability(cls) -> int:
return 70
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
self.ocp_mx_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
raise ValueError(
f"OCP MX linear layer {layer_name!r} has an input size per "
f"partition of {input_size_per_partition}, which must be "
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
"Choose a compatible tensor-parallel size or avoid "
"tensor-parallel sharding for this layer."
)
if self.is_unpacked:
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=self._expanding_scale_loader(weight_loader),
)
layer.register_parameter("weight_scale", weight_scale)
layer.weight_block_size = [1, OCP_MX_BLOCK_SIZE]
else:
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
# WEIGHT
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
if self.weight_quant_key == kMxfp4Static:
self.ocp_mx_linear = init_mxfp4_linear_kernel(
activation_quant_key=self.activation_quant_key,
)
elif self.weight_quant_key in [kMxfp6E2M3Static, kMxfp6E3M2Static]:
self.ocp_mx_linear = init_mxfp6_linear_kernel(
weight_quant_key=self.weight_quant_key,
activation_quant_key=self.activation_quant_key,
)
elif self.weight_quant_key == kMxfp8Static:
self.ocp_mx_linear = init_mxfp8_linear_kernel(
weight_shape=layer.weight.shape
)
else:
raise NotImplementedError(
"No OCP MX linear kernel for weight_quant_key="
f"{self.weight_quant_key}. Something is wrong, please open an "
"issue."
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.ocp_mx_linear.apply_weights(layer, x, bias)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
__all__ = ["QuarkScheme"]
class QuarkScheme(ABC):
"""Abstract class used to describe the weight creation and forward pass
of different quantization schemes supported by Quark.
"""
supported_activation_quant_keys: list[QuantKey | None] = []
supported_weight_quant_keys: list[QuantKey] = []
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
if activation_quant_key not in self.supported_activation_quant_keys:
raise ValueError(
f"Unsupported activation quant key: {activation_quant_key}"
)
if weight_quant_key not in self.supported_weight_quant_keys:
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
self.activation_quant_key = activation_quant_key
self.weight_quant_key = weight_quant_key
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
"""Get minimum device capability."""
raise NotImplementedError
@abstractmethod
def create_weights(self, *args, **kwargs):
"""Weight creation for the particular scheme. Inputs to this function."""
raise NotImplementedError
@abstractmethod
def apply_weights(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
):
"""Run the forward pass for the particular scheme. This is where
scheme-specific dequant/quant steps/kernels should be applied.
Args:
layer: torch.nn.Module with the registered weights and
other parameters relevant to the particular scheme.
x: input to the layer
bias: bias parameter
"""
raise NotImplementedError
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module):
"""Called after weight loading is complete for any cleanup that
needs to occur.
"""
raise NotImplementedError
@@ -1,212 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Callable
from typing import Any
import torch
from vllm.model_executor.kernels.linear import (
MPLinearKernel,
MPLinearLayerConfig,
choose_mp_linear_kernel,
)
from vllm.model_executor.layers.quantization.auto_awq import (
_convert_awq_to_standard_format,
)
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
QuarkScheme,
)
from vllm.model_executor.layers.quantization.quark.utils import (
canonicalize_quark_packed_int4,
parse_w4a16_int4_weight_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.scalar_type import scalar_types
class QuarkW4A16Int4(QuarkScheme):
"""Quark packed INT4 weight-only linear scheme via MPLinearKernel."""
supported_activation_quant_keys: list[QuantKey | None] = [None]
supported_weight_quant_keys: list[QuantKey] = [
kInt4Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
pack_method: str,
weight_config: dict[str, Any],
):
super().__init__(weight_quant_key, activation_quant_key)
# group_size is read from the Quark config rather than weight_quant_key:
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
# checkpoints commonly use group_size 128, which the QuantKey cannot
# represent. Symmetry is unambiguous, so it comes from the key.
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
self.pack_factor = 8
self.pack_reorder = pack_method == "reorder"
self.is_symmetric = weight_quant_key.symmetric
self.quant_type = (
scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4
)
self.kernel: MPLinearKernel | None = None
@classmethod
def get_min_capability(cls) -> int:
return 70
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
input_size = kwargs["input_size"]
output_size = kwargs["output_size"]
group_size = (
self.group_size if self.group_size != -1 else input_size_per_partition
)
if input_size_per_partition % group_size != 0:
raise ValueError(
"The input size is not aligned with the quantized weight shape. "
"This can be caused by too large tensor parallel size. "
f"input_size_per_partition={input_size_per_partition}, "
f"group_size={group_size}."
)
output_size_per_partition = sum(output_partition_sizes)
packed_output_size_per_partition = math.ceil(
output_size_per_partition / self.pack_factor
)
layer.output_size_per_partition = output_size_per_partition
layer.packed_output_size_per_partition = packed_output_size_per_partition
mp_linear_kernel_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
partition_weight_shape=(
input_size_per_partition,
output_size_per_partition,
),
weight_type=self.quant_type,
act_type=params_dtype,
group_size=group_size,
zero_points=not self.is_symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
def weight_scale_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
*args,
**loader_kwargs,
) -> None:
if loaded_weight.shape[1] < param.data.shape[1]:
padded_weight = loaded_weight.new_zeros(param.data.shape)
padded_weight[:, : loaded_weight.shape[1]] = loaded_weight
loaded_weight = padded_weight
weight_loader(param, loaded_weight, *args, **loader_kwargs)
weight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition,
packed_output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
num_groups = input_size_per_partition // group_size
weight_zero_point = PackedvLLMParameter(
data=torch.zeros(
num_groups,
packed_output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
num_groups,
packed_output_size_per_partition * self.pack_factor,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_scale_loader,
)
layer.register_parameter("weight", weight)
layer.register_parameter("weight_zero_point", weight_zero_point)
layer.register_parameter("weight_scale", weight_scale)
self.kernel = kernel_type(
mp_linear_kernel_config,
w_q_param_name="weight",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point" if not self.is_symmetric else None,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight.data = canonicalize_quark_packed_int4(
layer.weight.data,
pack_reorder=self.pack_reorder,
is_symmetric=self.is_symmetric,
pack_factor=self.pack_factor,
)
if not self.is_symmetric:
layer.weight_zero_point.data = canonicalize_quark_packed_int4(
layer.weight_zero_point.data,
pack_reorder=self.pack_reorder,
is_symmetric=self.is_symmetric,
pack_factor=self.pack_factor,
)
output_size = layer.output_size_per_partition
packed_output_size = layer.packed_output_size_per_partition * self.pack_factor
if output_size < packed_output_size:
layer.weight_scale.data[:, output_size:].zero_()
_convert_awq_to_standard_format(
layer,
"weight",
"weight_zero_point" if not self.is_symmetric else None,
self.quant_type.size_bits,
)
assert self.kernel is not None
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
):
assert self.kernel is not None
return self.kernel.apply_weights(layer, x, bias)
@@ -1,204 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from fractions import Fraction
import torch
import torch.nn.functional as F
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
get_fp8_min_max,
kFp8StaticTensorSym,
kMxfp4Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
PerTensorScaleParameter,
)
from vllm.platforms import current_platform
from .quark_scheme import QuarkScheme
logger = init_logger(__name__)
__all__ = ["QuarkW4A8_MXFP4_FP8"]
OCP_MX_BLOCK_SIZE = 32
class QuarkW4A8_MXFP4_FP8(QuarkScheme):
"""- Weights: MXFP4 with E8M0 scales per block of 32
- Activations: FP8 E4M3 (static per-tensor quantization)
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
"""
supported_activation_quant_keys = [kFp8StaticTensorSym]
supported_weight_quant_keys = [kMxfp4Static]
def __init__(
self,
activation_quant_key: QuantKey | None,
):
super().__init__(kMxfp4Static, activation_quant_key)
self.out_dtype = None
self.weight_dtype = "mxfp4"
self.packed_factor: Fraction = Fraction(2, 1) # 2 FP4 values per byte
self.weight_block_size = OCP_MX_BLOCK_SIZE
self.fp8_min, self.fp8_max = get_fp8_min_max()
self.fp8_dtype = current_platform.fp8_dtype()
kernel_supported_gpu = False
if current_platform.is_rocm():
kernel_supported_gpu = current_platform.supports_mx()
self.use_aiter_kernel = is_aiter_found_and_supported() and kernel_supported_gpu
if not self.use_aiter_kernel:
logger.warning_once(
"[W4A8 MXFP4+FP8] Aiter Triton kernel not found. Using emulation mode."
)
@classmethod
def get_min_capability(cls) -> int:
return 70
def get_packed_dim(self, dim: int) -> int:
assert dim % 2 == 0, f"Dimension {dim} must be even for MXFP4 packing"
return dim // 2
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
# MXFP4 WEIGHT (packed, 2 values per byte)
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE (E8M0 format, per block of 32)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.weight_block_size,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
# INPUT SCALE (FP8 per-tensor static scale)
input_scale = PerTensorScaleParameter(
data=torch.empty(
len(output_partition_sizes),
dtype=torch.float32,
),
weight_loader=weight_loader,
)
# Initialize to avoid NaN
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Ensuring weights & scales are non-trainable
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(
layer.weight_scale.data, requires_grad=False
)
input_scale = layer.input_scale.data
# For fused modules (QKV), take the max scale
if input_scale.numel() != 1:
input_scale = input_scale.max()
layer.input_scale = torch.nn.Parameter(
torch.tensor(input_scale, dtype=torch.float32),
requires_grad=False,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if self.use_aiter_kernel:
return self._apply_aiter_kernel(layer, x, bias)
else:
return self._apply_emulation(layer, x, bias)
def _apply_aiter_kernel(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
M = x.shape[0]
out_dtype = x.dtype if self.out_dtype is None else self.out_dtype
input_scale = layer.input_scale
x_fp8 = (x / input_scale).clamp(self.fp8_min, self.fp8_max).to(self.fp8_dtype)
# Broadcast per-tensor scale to per-row (M, 1) for Aiter kernel
x_scales = input_scale.expand(M, 1).to(dtype=torch.float32, device=x.device)
y = rocm_aiter_ops.gemm_a8wfp4(
x_fp8, layer.weight, x_scales, layer.weight_scale, out_dtype
)
if bias is not None:
y = y + bias
return y
def _apply_emulation(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
dequant_mxfp4,
)
weight_dq = dequant_mxfp4(
layer.weight,
layer.weight_scale,
x.dtype,
)
input_scale = layer.input_scale
x_fp8 = (x / input_scale).clamp(self.fp8_min, self.fp8_max).to(self.fp8_dtype)
x_dq = (x_fp8.to(x.dtype) * input_scale).to(x.dtype)
return F.linear(x_dq, weight_dq, bias)
@@ -1,317 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from torch.nn import Parameter
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
expose_input_quant_key,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
create_fp8_scale_parameter,
create_fp8_weight_parameter,
validate_fp8_block_shape,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockE8M0Sym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
requantize_with_max_scale,
)
from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
ChannelQuantScaleParameter,
ModelWeightParameter,
PerTensorScaleParameter,
)
from vllm.platforms import current_platform
__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"]
logger = init_logger(__name__)
class QuarkW8A8Fp8(QuarkScheme):
supported_activation_quant_keys: list[QuantKey | None] = [
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
]
supported_weight_quant_keys: list[QuantKey] = [
kFp8StaticChannelSym,
kFp8StaticTensorSym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(weight_quant_key, activation_quant_key)
self.weight_qscheme = (
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
)
self.is_static_input_scheme = activation_quant_key == kFp8StaticTensorSym
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@classmethod
def get_min_capability(cls) -> int:
# lovelace and up
return 89
def process_weights_after_loading(self, layer) -> None:
# If per tensor, when we have a fused module (e.g. QKV) with per
# tensor scales (thus N scales being passed to the kernel),
# requantize so we can always run per tensor
if self.weight_qscheme == "per_tensor":
if current_platform.is_fp8_fnuz():
input_scale = getattr(layer, "input_scale", None)
weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
weight=layer.weight,
weight_scale=layer.weight_scale,
input_scale=input_scale,
)
if input_scale is not None:
layer.input_scale = Parameter(input_scale, requires_grad=False)
else:
max_w_scale = layer.weight_scale
weight = layer.weight
max_w_scale, weight = requantize_with_max_scale(
weight=weight,
weight_scale=max_w_scale,
logical_widths=layer.logical_widths,
)
layer.weight = Parameter(weight.t(), requires_grad=False)
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
# If channelwise, scales are already lined up, so just transpose.
elif self.weight_qscheme == "per_channel":
weight = layer.weight
if current_platform.is_fp8_fnuz():
input_scale = getattr(layer, "input_scale", None)
weight, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
weight=weight,
weight_scale=layer.weight_scale,
input_scale=input_scale,
)
if input_scale is not None:
layer.input_scale = Parameter(input_scale, requires_grad=False)
else:
weight_scale = layer.weight_scale.data
assert self.activation_quant_key is not None
if self.activation_quant_key.scale.group_shape == GroupShape.PER_TOKEN:
weight_scale = weight_scale.view(-1, 1)
layer.weight = Parameter(weight.t(), requires_grad=False)
# required by torch.compile to be torch.nn.Parameter
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
else:
raise ValueError(f"Unknown quantization scheme {self.weight_qscheme}")
# INPUT SCALE
if self.is_static_input_scheme:
layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False)
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
# WEIGHT
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=torch.float8_e4m3fn,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
# TODO: update create_xxx_parameter functions to return
# the newly added parameters
if self.weight_qscheme == "per_channel":
weight_scale = ChannelQuantScaleParameter(
data=torch.empty((sum(output_partition_sizes)), dtype=torch.float32),
output_dim=0,
weight_loader=weight_loader,
)
else:
assert self.weight_qscheme == "per_tensor"
weight_scale = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
# min requirement for fp8 kernels
weight_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("weight_scale", weight_scale)
# INPUT SCALE
if self.is_static_input_scheme:
input_scale = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
weight_shape=layer.weight.shape,
input_dtype=self.input_dtype,
out_dtype=self.out_dtype,
module_name=self.__class__.__name__,
)
expose_input_quant_key(layer, self.fp8_linear)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor | QuantizedActivation,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)
class QuarkW8A8Fp8PerBlock(QuarkScheme):
supported_activation_quant_keys = [kFp8Dynamic128Sym]
supported_weight_quant_keys = [
kFp8Static128BlockSym,
kFp8Static128BlockE8M0Sym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(weight_quant_key, activation_quant_key)
self.weight_scale_dtype = weight_quant_key.scale.dtype
if self.weight_quant_key == kFp8Static128BlockE8M0Sym:
# TODO: oracle needs to support kFp8Static128BlockE8M0Sym.
self.weight_quant_key = kFp8Static128BlockSym
self.weight_block_size = list(weight_quant_key.scale.group_shape)
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@classmethod
def get_min_capability(cls) -> int:
return QuarkW8A8Fp8.get_min_capability()
def process_weights_after_loading(self, layer) -> None:
# Quark exports the dequant multiplier as ``weight_scale`` (the same
# numerical convention as DeepSeek's ``weight_scale_inv``). Kernels
# multiply ``weight * scale``; do not invert here.
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
input_size = kwargs.get("input_size", input_size_per_partition)
output_size = kwargs.get("output_size", sum(output_partition_sizes))
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.orig_dtype = params_dtype
layer.weight_block_size = self.weight_block_size
validate_fp8_block_shape(
layer,
input_size,
output_size,
input_size_per_partition,
output_partition_sizes,
self.weight_block_size,
)
weight = create_fp8_weight_parameter(
output_size_per_partition,
input_size_per_partition,
weight_loader,
)
layer.register_parameter("weight", weight)
scale_dtype = (
self.weight_scale_dtype
if self.weight_scale_dtype != torch.float32
else None
)
weight_scale = create_fp8_scale_parameter(
BlockQuantScaleParameter,
output_partition_sizes,
input_size_per_partition,
self.weight_block_size,
weight_loader,
scale_dtype=scale_dtype,
)
# Match Quark-exported checkpoints (``.weight_scale``).
# ``Fp8LinearMethod.create_weights`` registers weight_scale_inv, which has the
# same semantic meaning as Quark's `weight_scale`.
layer.register_parameter("weight_scale", weight_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
weight_shape=layer.weight.shape,
input_dtype=self.input_dtype,
out_dtype=self.out_dtype,
module_name=self.__class__.__name__,
)
logger.info_once(
"Selected %s for QuarkW8A8Fp8PerBlock",
type(self.fp8_linear).__name__,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)
@@ -1,165 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_int8_linear_kernel,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTokenAsym,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kInt8StaticTensorAsym,
kInt8StaticTensorSym,
)
from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
ModelWeightParameter,
PerTensorScaleParameter,
)
logger = init_logger(__name__)
class QuarkW8A8Int8(QuarkScheme):
supported_activation_quant_keys = [
kInt8StaticTensorSym,
kInt8StaticTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTensorAsym,
kInt8DynamicTokenSym,
kInt8DynamicTokenAsym,
]
supported_weight_quant_keys = [
kInt8StaticChannelSym,
kInt8StaticTensorSym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(weight_quant_key, activation_quant_key)
self.qscheme = (
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
)
assert activation_quant_key is not None
self.is_static_input_scheme = activation_quant_key.scale.static
self.input_symmetric = activation_quant_key.symmetric
@classmethod
def get_min_capability(cls) -> int:
# turing and up
return 75
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
layer.logical_widths = output_partition_sizes
# Quark stores per-channel weight_scale as 1D [N]; reshape to [N, 1].
def _scale_weight_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
*args,
**kwargs,
):
if loaded_weight.dim() == 1:
loaded_weight = loaded_weight.unsqueeze(-1)
return weight_loader(param, loaded_weight, *args, **kwargs)
self.kernel = init_int8_linear_kernel(
is_channelwise=(self.qscheme == "per_channel"),
is_static_input_scheme=(self.is_static_input_scheme is True),
input_symmetric=(self.input_symmetric is True),
module_name=self.__class__.__name__,
)
# WEIGHT
weight = ModelWeightParameter(
data=torch.empty(
sum(output_partition_sizes), input_size_per_partition, dtype=torch.int8
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
if self.qscheme == "per_channel":
weight_scale = ChannelQuantScaleParameter(
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32),
output_dim=0,
weight_loader=_scale_weight_loader,
)
ChannelQuantZPParameter = ChannelQuantScaleParameter
weight_zero_point = ChannelQuantZPParameter(
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.int8),
output_dim=0,
weight_loader=_scale_weight_loader,
)
else:
assert self.qscheme == "per_tensor"
weight_scale = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
PerTensorZPParameter = PerTensorScaleParameter
weight_zero_point = PerTensorZPParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.int8),
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
layer.register_parameter("weight_zero_point", weight_zero_point)
# INPUT SCALE
input_zero_point = None
input_scale = None
if self.is_static_input_scheme:
input_scale = BasevLLMParameter(
data=torch.empty(1, dtype=torch.float32), weight_loader=weight_loader
)
input_zero_point = BasevLLMParameter(
data=torch.empty(1, dtype=torch.int8), weight_loader=weight_loader
)
layer.register_parameter("input_scale", input_scale)
layer.register_parameter("input_zero_point", input_zero_point)
if not hasattr(layer, "azp_adj"):
layer.register_parameter("azp_adj", None)
# Checkpoints are serialized in quark format, which is
# different from the format the kernel may want. Handle repacking here.
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.register_parameter("weight_zero_point", None)
delattr(layer, "weight_zero_point")
if self.input_symmetric:
layer.register_parameter("input_zero_point", None)
delattr(layer, "input_zero_point")
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
@@ -1,136 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping
from types import MappingProxyType
from typing import Any
import torch
from vllm.model_executor.layers.quantization.utils.config_utils import (
find_matching_patterns,
)
QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None
def deep_compare(dict1: Any, dict2: Any) -> bool:
if type(dict1) is not type(dict2):
return False
if isinstance(dict1, dict):
if dict1.keys() != dict2.keys():
return False
return all(deep_compare(dict1[k], dict2[k]) for k in dict1)
elif isinstance(dict1, list):
# `dict1` may be a list of dict.
return all(deep_compare(dict1[i], dict2[i]) for i in range(len(dict1)))
else:
return dict1 == dict2
def should_ignore_layer(
layer_name: str | None,
ignore: Iterable[str],
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
*,
check_children: bool = False,
) -> bool:
if layer_name is None:
return False
# MoE layers are currently all-or-nothing: if any child is ignored,
# the parent layer must be ignored as well. For example, the
# amd/GLM-5.2-MXFP4 config ignores children like
# model.layers.78.mlp.experts.*.down_proj, while the layer checked
# here is the parent model.layers.N.mlp.experts.
# See:
# https://huggingface.co/amd/GLM-5.2-MXFP4/blob/main/config.json#L793-L795
if check_children and any(
target == layer_name or target.startswith(layer_name + ".")
for target in ignore
if not target.startswith("re:")
):
return True
# A direct fused-layer pattern takes precedence over expansion. For
# model.layers.0.self_attn.qkv_proj,
# ignore=["re:.*qkv_proj.*"] yields [{"re:.*qkv_proj.*"}]. In contrast,
# ignore=["re:.*[qkv]_proj"] yields one matching set per expanded shard.
per_shard_matches = find_matching_patterns(layer_name, ignore, fused_mapping)
shards_ignored = [len(matches) > 0 for matches in per_shard_matches]
if any(shards_ignored) and not all(shards_ignored):
raise ValueError(
f"Found different quantization schemes for the shards of "
f"{layer_name}. vLLM requires all to use the same scheme."
)
return all(shards_ignored)
def parse_w4a16_int4_weight_config(
weight_config: Mapping[str, Any],
) -> tuple[int, bool]:
"""Parse required W4A16 INT4/UINT4 weight fields from Quark config."""
if "group_size" not in weight_config:
raise ValueError(
"Quark W4A16 INT4/UINT4 configs must specify weight.group_size"
)
if "symmetric" not in weight_config:
raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric")
group_size = weight_config["group_size"]
is_symmetric = weight_config["symmetric"]
if not isinstance(group_size, int) or group_size <= 0:
raise ValueError(
f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}"
)
if not isinstance(is_symmetric, bool):
raise ValueError(
f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}"
)
return group_size, is_symmetric
def canonicalize_quark_packed_int4(
packed_weight: torch.Tensor,
*,
pack_reorder: bool,
is_symmetric: bool,
pack_factor: int = 8,
) -> torch.Tensor:
"""Convert Quark export nibble layout to AWQ checkpoint layout."""
from vllm.model_executor.layers.quantization.auto_awq import (
_REVERSE_AWQ_PACK_ORDER,
)
if pack_reorder:
source_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
else:
source_order = torch.arange(
pack_factor, device=packed_weight.device, dtype=torch.int32
)
target_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
source_shifts = source_order * 4
target_shifts = target_order * 4
values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF
if is_symmetric:
values = values ^ 0x8
packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1)
return packed.to(torch.int32)
# utility for tensor dims > 2 cases
def quark_quantize_weight_to_mxfp4(w: torch.Tensor):
assert w.dtype == torch.bfloat16, (
"Quark dynamic quantization is supported only for fp16 weights and only to MXF4"
)
from aiter.ops.triton.quant import dynamic_mxfp4_quant
*dims, d = w.shape
w, w_scales = dynamic_mxfp4_quant(w.reshape(-1, d))
return w.view(*dims, d // 2), w_scales.view(*dims, d // 32)
@@ -20,40 +20,8 @@ def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
layer_name: str,
) -> int | None:
"""Return the OCP MX group size for a quantized Quark linear layer."""
if quant_config is None:
return None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
if not isinstance(quant_config, QuarkConfig):
return None
if should_ignore_layer(
layer_name,
ignore=quant_config.quant_config.get("exclude") or [],
fused_mapping=quant_config.packed_modules_mapping,
):
return None
layer_quant_config = (
quant_config.get_layer_quant_config_from_name(layer_name)
or quant_config.quant_config.get("global_quant_config")
or {}
)
weight_quant = layer_quant_config.get("weight")
input_quant = layer_quant_config.get("input_tensors")
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
return None
assert weight_quant is not None
group_size = weight_quant.get("group_size")
if group_size is None:
block_size = weight_quant.get("block_size")
if isinstance(block_size, list | tuple) and len(block_size) >= 2:
group_size = block_size[1]
return int(group_size) if group_size is not None else None
"""Quark support was removed; kept for callers that probe MX group size."""
return None
def is_shared_expert_quant_fse_compatible(
@@ -75,12 +43,9 @@ def is_shared_expert_quant_fse_compatible(
"""
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
@@ -104,191 +69,12 @@ def is_shared_expert_quant_fse_compatible(
shared_expert_targets = None
if shared_expert_targets is not None:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.base import (
ONLINE_SHARED_EXPERT_QUANTIZERS,
)
# NOTE: online shared experts quantization check is only implemented for quark
# quant method at the moment. This can be extended here for other quant methods.
# Requirement is:
# - Retrieve routed weight quant key
# - Retrieve routed activation quant key
# and ensure the online quantization set on shared expert match them.
if isinstance(quant_config, QuarkConfig):
routed_weight_key, routed_activation_key, _ = (
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
)
else:
reason = (
"online quantization FSE compatibility check is not implemented for "
f"{type(quant_config).__name__}"
)
logger.warning_once(reason)
return False, reason
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
return (
False,
"online shared-expert quantization is not supported for routed "
f"expert weight key {routed_weight_key}",
)
for shared_expert_target in shared_expert_targets:
if shared_expert_target is None:
return (
False,
"online quantization targets only part of the shared expert at "
f"{shared_expert_prefix}",
)
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
if shared_method_cls in (None, UnquantizedLinearMethod):
return (
False,
"online quantization excludes FSE weights at "
f"{shared_expert_prefix}",
)
assert issubclass(shared_method_cls, OnlineLinearBase)
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
# does not define the default activation quant key.
# `online_quant_config.resolve_quant_method_cls` should return the
# activation quant key as well. Only some online methods define it
# on the class, so compare the weight key first.
if (
shared_quant_spec.weight != routed_weight_key
or shared_method_cls.activation_quant_key != routed_activation_key
):
return (
False,
"online shared-expert quantization keys do not match the "
"routed expert keys",
)
return True, None
if isinstance(quant_config, DeepseekV4FP8Config):
from vllm.config import get_current_vllm_config
from vllm.model_executor.models.utils import extract_layer_index
if quant_config.expert_dtype != "fp4":
return False, "DeepSeek-V4 routed experts are not MXFP4"
hf_config = get_current_vllm_config().model_config.hf_config
# TODO: This is adapted from former `_shared_experts_are_fp4`, and
# needs to be cleaned up this . There should not be Quark-specific
# logic in DeepseekV4FP8Config.
quantization_config = getattr(hf_config, "quantization_config", None)
if quantization_config is None:
return False, "DeepSeek-V4 has no quantization configuration"
if not quant_config._is_quark_mxfp4_ocp(quantization_config):
return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4"
layer_idx = extract_layer_index(shared_expert_prefix)
if layer_idx >= hf_config.num_hidden_layers:
shared_expert_prefix = (
f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts"
)
else:
shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts"
if any(
entry.startswith(shared_expert_prefix)
for entry in quantization_config.get("exclude") or []
if isinstance(entry, str)
):
return (
False,
f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}",
)
shared_expert_weight_name = f"{shared_expert_prefix}.w1"
layer_quant_config = quantization_config.get("layer_quant_config") or {}
layer_config = layer_quant_config.get(shared_expert_weight_name)
if layer_config is None:
layer_config = next(
(
config
for pattern, config in layer_quant_config.items()
if isinstance(pattern, str)
and pattern.startswith("re:")
and re.fullmatch(
pattern.removeprefix("re:"), shared_expert_weight_name
)
),
None,
)
shared_weight_config = (
layer_config or quantization_config.get("global_quant_config") or {}
).get("weight") or {}
if shared_weight_config.get("dtype") == "fp4":
return True, None
return (
False,
f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4",
)
if isinstance(quant_config, QuarkConfig):
from vllm.model_executor.layers.quantization.quark.utils import (
should_ignore_layer,
)
# TODO: layer_type_quant_config is not taken into account here.
assert "exclude" in quant_config.quant_config
assert "global_quant_config" in quant_config.quant_config
exclude = quant_config.quant_config["exclude"]
# should_ignore_layer raises a rightful ValueError in case different shards
# have a different ignore policy, as unsupported.
is_excluded = any(
should_ignore_layer(
f"{shared_expert_prefix}.{projection_name}",
ignore=exclude,
fused_mapping=quant_config.packed_modules_mapping,
)
for projection_name in projection_names
)
if is_excluded:
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
global_quant_config = quant_config.quant_config["global_quant_config"]
def get_projection_quant_configs(layer_name: str) -> list[object]:
module_prefix, _, projection_name = layer_name.rpartition(".")
packed_projection_names = quant_config.packed_modules_mapping.get(
projection_name, [projection_name]
)
return [
quant_config.get_layer_quant_config_from_name(
f"{module_prefix}.{packed_projection_name}"
)
or global_quant_config
for packed_projection_name in packed_projection_names
]
expert_quant_config = (
quant_config.get_layer_quant_config_from_name(expert_prefix)
or global_quant_config
)
shared_expert_quant_configs = [
config
for projection_name in projection_names
for config in get_projection_quant_configs(
f"{shared_expert_prefix}.{projection_name}"
)
]
if all(config == expert_quant_config for config in shared_expert_quant_configs):
return True, None
return (
False,
"Quark uses different quantization configurations for routed and "
f"shared experts at {shared_expert_prefix}",
reason = (
"online quantization FSE compatibility check is not implemented for "
f"{type(quant_config).__name__}"
)
logger.warning_once(reason)
return False, reason
if isinstance(quant_config, Fp8Config):
if quant_config.store_dtype is not None:
@@ -52,16 +52,6 @@ def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
def get_fp8_block_weight_scale(layer: torch.nn.Module) -> torch.Tensor | None:
"""Return the block-FP8 weight scale for supported quant methods."""
# Local import avoids a circular import: quark_w8a8_fp8 imports this module.
from vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8 import (
QuarkW8A8Fp8PerBlock,
)
if isinstance(
getattr(layer, "scheme", None),
QuarkW8A8Fp8PerBlock,
):
return getattr(layer, "weight_scale", None)
return getattr(layer, "weight_scale_inv", None)
@@ -549,15 +549,6 @@ def mxfp4_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if current_platform.is_xpu():
return xpu_mxfp4_quantize(x)
from vllm._aiter_ops import is_aiter_found_and_supported
if is_aiter_found_and_supported() and x.dtype == torch.bfloat16:
from vllm.model_executor.layers.quantization.quark.utils import (
quark_quantize_weight_to_mxfp4,
)
return quark_quantize_weight_to_mxfp4(x)
quant_tensor, scale, _ = downcast_to_mxfp(x, axis=-1)
return quant_tensor, scale
@@ -1,116 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse attention indexer layers on aiter's paged MXFP4 kernels, for ROCm.
With ``indexer_kv_dtype="mxfp4"`` on gfx950, the model writes the indexer K
cache in the order aiter's MQA-logits kernel reads (see
`rocm_paged_mxfp4_cache_layout`), and the kernel scores the cache in place.
Requires `RocmMxfp4IndexerMetadataBuilder` metadata.
"""
import torch
from torch import nn
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import (
rocm_mxfp4_sparse_attn_indexer,
rocm_mxfp4_sparse_mqa_indexer,
)
class RocmSparseAttnIndexer(SparseAttnIndexer):
"""`SparseAttnIndexer` whose HIP path walks the paged MXFP4 cache with
aiter's kernel. A two-level indexer's source layer also publishes its
candidate pool here."""
def forward_hip(
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
):
assert isinstance(q_quant, tuple) and self.skip_k_cache_insert, (
"the ROCm MXFP4 indexer takes (values, scales) Q and a K cache "
"the model already wrote"
)
q_values, q_scale = q_quant
return rocm_mxfp4_sparse_attn_indexer(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_values,
q_scale,
weights,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.topk_indices_buffer,
compress_ratio=self.compress_ratio,
candidate_blocks=self.candidate_blocks,
candidate_block_size=self.candidate_block_size,
candidate_write=self.candidate_write,
)
class RocmSparseMQAIndexer(nn.Module):
"""DeepSeek-V4.1's candidate-consuming indexer on aiter's paged MXFP4
MQA-logits kernel: the consumers score only the candidate pool the source
layer published, read straight from the paged cache.
Built and called as `SparseMQAIndexer` is, so the model can take either.
"""
weights_dtype = torch.float32
"""The scorer's weights and logits stay fp32: gfx950 has no packed bf16
arithmetic, so its head reduce costs the same in either dtype."""
def __init__(
self,
k_cache,
topk_tokens: int,
head_dim: int,
max_total_seq_len: int,
topk_indices_buffer: torch.Tensor,
candidate_blocks: torch.Tensor,
candidate_block_size: int,
):
super().__init__()
self.k_cache = k_cache
self.topk_tokens = topk_tokens
self.head_dim = head_dim
# aiter's kernel bounds a row by the model length in compressed
# positions, not by the prefill buffer SparseMQAIndexer sizes.
model_config = get_current_vllm_config().model_config
self.max_model_len = model_config.max_model_len // k_cache.compress_ratio
self.topk_indices_buffer = topk_indices_buffer
self.candidate_blocks = candidate_blocks
self.candidate_block_size = candidate_block_size
self.num_candidate_cols = candidate_blocks.shape[1] * candidate_block_size
def forward(
self,
hidden_states: torch.Tensor,
q_quant: tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
) -> torch.Tensor:
assert k is None, "the model writes the indexer K cache"
q_values, q_scale = q_quant
return rocm_mxfp4_sparse_mqa_indexer(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_values,
q_scale,
weights,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.topk_indices_buffer,
self.k_cache.compress_ratio,
self.candidate_blocks,
self.candidate_block_size,
self.num_candidate_cols,
)
File diff suppressed because it is too large Load Diff
@@ -1,209 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse Attention Indexer that scores only the candidate blocks.
DeepSeek V4.1 two-level selection: the candidate-source indexer publishes
the top candidate blocks and later indexers pick their top-k inside them.
`SparseAttnIndexer` does that by computing dense logits over the whole
context and masking; this layer instead calls DeepGEMM's sparse MQA-logits
kernels on the candidate blocks only, so the work is O(candidate blocks)
instead of O(context). It requires the `DeepseekV41SparseIndexerBackend`
metadata (see ``AttentionConfig.indexer_sparse_logits``).
"""
import torch
from torch import nn
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.forward_context import get_forward_context
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
has_deep_select,
sparse_mqa_logits_paged_decode,
sparse_mqa_logits_prefill_chunk,
)
from vllm.model_executor.layers.sparse_attn_indexer import (
_gather_workspace_shapes,
kv_cache_as_quant_view,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import has_deep_gemm_sparse_mqa
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerPrefillChunkMetadata,
)
from vllm.v1.attention.backends.mla.sparse_indexer import (
DeepseekV41SparseIndexerMetadata,
)
from vllm.v1.worker.workspace import current_workspace_manager
def _prefill_k_workspaces(
total_seq_lens: int, head_dim: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""The packed MXFP4 K-gather workspace ``(values, scales)``, shared with
the dense indexer layers of the same model."""
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, current_platform.fp8_dtype(), use_fp4_cache=True
)
k_quant, k_scale = current_workspace_manager().get_simultaneous(
values_spec, scales_spec
)
return k_quant, k_scale
def _gather_prefill_chunk_k(
kv_cache: torch.Tensor,
k_quant_full: torch.Tensor,
k_scale_full: torch.Tensor,
chunk: DeepseekV32IndexerPrefillChunkMetadata,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Gather one prefill chunk's paged K into the packed workspace."""
assert chunk.local_cu_seq_lens is not None
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
ops.cp_gather_indexer_k_quant_cache(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.local_cu_seq_lens,
)
return k_quant, k_scale
class SparseMQAIndexer(nn.Module):
"""Candidate-consuming indexer on DeepGEMM's sparse MQA-logits kernels.
Only valid for indexer layers that read candidate blocks with the MXFP4
indexer cache on SM100. The K cache is written by the model before this
runs; ``forward`` takes the same arguments as `SparseAttnIndexer` so the
attention layer can call either.
"""
weights_dtype = torch.bfloat16
"""Per-head weights dtype the sparse kernels take. The fused Q RoPE-quant
kernel writes it directly so no cast runs per step."""
def __init__(
self,
k_cache,
topk_tokens: int,
head_dim: int,
max_total_seq_len: int,
topk_indices_buffer: torch.Tensor,
candidate_blocks: torch.Tensor,
candidate_block_size: int,
):
super().__init__()
if not (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and has_deep_gemm_sparse_mqa()
):
raise ValueError(
"SparseMQAIndexer requires an SM100-class GPU and DeepGEMM >= 2.8."
)
if not has_deep_select():
raise ValueError(
"SparseMQAIndexer requires the DeepSelect top-k extension "
"(vllm._deepselect_C)."
)
self.k_cache = k_cache
self.topk_tokens = topk_tokens
self.head_dim = head_dim
self.max_total_seq_len = max_total_seq_len
self.topk_indices_buffer = topk_indices_buffer
self.candidate_blocks = candidate_blocks
self.candidate_block_size = candidate_block_size
def _reserve_workspaces(self, device: torch.device) -> None:
"""Profiling run: claim the K-gather workspace and the peak sparse
logits allocation so the memory estimate covers them."""
_prefill_k_workspaces(self.max_total_seq_len, self.head_dim)
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
torch.empty(max_logits_bytes, dtype=torch.uint8, device=device)
def forward(
self,
hidden_states: torch.Tensor,
q_quant: tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
) -> torch.Tensor:
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
self._reserve_workspaces(hidden_states.device)
return self.topk_indices_buffer
metadata = attn_metadata[self.k_cache.prefix]
assert isinstance(metadata, DeepseekV41SparseIndexerMetadata), (
"SparseMQAIndexer needs DeepseekV41SparseIndexerBackend metadata"
)
assert k is None, "the model writes the indexer K cache"
q_values, q_scale = q_quant
kv_cache = self.k_cache.kv_cache
topk_tokens = self.topk_tokens
topk_indices_buffer = self.topk_indices_buffer
topk_indices_buffer[: hidden_states.shape[0]] = -1
sparse_block_kv = metadata.sparse_block_kv
if metadata.num_prefills > 0:
prefill = metadata.prefill
sparse_chunks = metadata.sparse_prefill
assert prefill is not None and sparse_chunks is not None
k_quant_full, k_scale_full = _prefill_k_workspaces(
self.max_total_seq_len, self.head_dim
)
for chunk, rows in zip(prefill.chunks, sparse_chunks):
k_quant, k_scale = _gather_prefill_chunk_k(
kv_cache, k_quant_full, k_scale_full, chunk
)
if chunk.local_total_seq_lens == 0:
continue # buffer already holds -1
start, end = chunk.token_start, chunk.token_end
rows.kernel_metadata = sparse_mqa_logits_prefill_chunk(
q_values[start:end].view(torch.int8),
q_scale[start:end],
k_quant.view(torch.int8),
k_scale.view(torch.int32).squeeze(-1),
weights[start:end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
self.candidate_blocks[start:end],
self.candidate_block_size,
sparse_block_kv,
topk_tokens,
topk_indices_buffer[start:end, :topk_tokens],
sparse_indices=rows.sparse_indices,
end=rows.end,
col_indices=rows.col_indices,
kernel_metadata=rows.kernel_metadata,
)
if metadata.num_decodes > 0:
decode_rows = metadata.sparse_decode
assert decode_rows is not None and decode_rows.block_table is not None
assert decode_rows.row_indices is not None
num_rows = decode_rows.row_ke.shape[0]
if num_rows > 0:
decode_rows.kernel_metadata = sparse_mqa_logits_paged_decode(
q_values[:num_rows].view(torch.int8).unsqueeze(1),
q_scale[:num_rows].unsqueeze(1),
kv_cache_as_quant_view(kv_cache, self.head_dim, use_fp4_cache=True),
weights[:num_rows],
decode_rows.row_ke,
decode_rows.block_table,
decode_rows.row_indices,
self.candidate_blocks[:num_rows],
self.candidate_block_size,
sparse_block_kv,
topk_tokens,
topk_indices_buffer[:num_rows, :topk_tokens],
row_ks=decode_rows.row_ks,
sparse_indices=decode_rows.sparse_indices,
end=decode_rows.end,
col_indices=decode_rows.col_indices,
kernel_metadata=decode_rows.kernel_metadata,
)
return topk_indices_buffer