[Perf][GLM-5.2] Blackwell decode optimizations (#48597)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local>
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local>
Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
xiaozhoupy
2026-07-23 21:36:27 -07:00
committed by GitHub
co-authored by Claude Giancarlo Delfin Thien Tran OpenAI Codex zhou stefankoncarevic
parent 0d77325b10
commit dd72658e7d
29 changed files with 2139 additions and 138 deletions
+18 -4
View File
@@ -6,17 +6,31 @@ DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
attend. The same model code serves any DSA checkpoint, including GLM-5.2
(``glm_moe_dsa``), which reuses this architecture.
The optimized kernels under ``nvidia/`` target the Blackwell (SM100) family.
Every other platform — ROCm, XPU, pre-SM100 CUDA (e.g. H100), CPU — falls back
to the generic ``deepseek_v2`` implementation, which already handles the DSA
(index_topk) architecture and is ``torch.compile``-friendly there. This matches
main's behavior on those platforms (no hard failure).
"""
from vllm.platforms import current_platform
if current_platform.is_rocm() or current_platform.is_xpu():
raise NotImplementedError("deepseek_v32 currently supports NVIDIA SM100 only.")
if current_platform.is_cuda() and current_platform.is_device_capability_family(100):
from .nvidia.model import DeepseekV32ForCausalLM
from .nvidia.mtp import DeepseekV32MTP
from .nvidia.model import DeepseekV32ForCausalLM
from .nvidia.mtp import DeepseekV32MTP
# GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100.
GlmMoeDsaForCausalLM = DeepseekV32ForCausalLM
else:
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
)
from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
__all__ = [
"DeepseekV32ForCausalLM",
"DeepseekV32MTP",
"GlmMoeDsaForCausalLM",
]
+72 -12
View File
@@ -4,6 +4,7 @@ import torch
import torch.nn as nn
from transformers import DeepseekV2Config, DeepseekV3Config
import vllm._custom_ops as ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
@@ -21,17 +22,15 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.sparse_attn_indexer import (
SparseAttnIndexer,
sparse_attn_indexer,
)
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.models.deepseek_v2 import (
DeepSeekV2FusedQkvAProjLinear,
DeepseekV32IndexerCache,
yarn_get_mscale,
)
from vllm.model_executor.models.utils import extract_layer_index
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _encode_layer_name, is_quantized_kv_cache
from .kernels import fused_norm_rope, fused_q
@@ -171,6 +170,11 @@ class DeepseekV32Attention(MLAAttention):
) -> None:
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config is not None and cache_config.cache_dtype == "auto":
# This implementation requires an FP8 sparse cache. Start with the
# generic FP8 format; MLAAttention will canonicalize it for the
# selected sparse backend.
cache_config.cache_dtype = "fp8"
hidden_size = config.hidden_size
qk_nope_head_dim = config.qk_nope_head_dim
@@ -331,6 +335,37 @@ class DeepseekV32Attention(MLAAttention):
rope_parameters=config.rope_parameters,
is_neox_style=False,
)
# Decode-M GEMM dispatch for the bf16 A/B projections, measured on
# B300 and B200 (graph replay + rotating weights vs fp64):
# bf16_skinny_gemm for M <= skinny_max, dsv3_fused_a_gemm up to
# M = 16, cuBLAS above. Blackwell-only: the boundaries and the
# fused_a tile choices are tied to the 148/152-SM single-wave
# geometry. DSv3.2 q_b has no skinny tier (K=1536 does not fit the
# GEMV's 128x8 K-step; fused_a still wins 1.6-1.8x there).
# Quantized linear methods may not register .weight until
# process_weights_after_loading (e.g. compressed-tensors NVFP4
# registers weight_packed at init) — probe with getattr.
# (N, K) -> skinny_max
qkv_a_dispatch = {(2624, 6144): 2, (2112, 7168): 2}
q_b_dispatch = {(2048, 2048): 2, (3072, 1536): 0}
ok = (
current_platform.is_device_capability_family(100)
and hasattr(torch.ops._C, "bf16_skinny_gemm")
and hasattr(torch.ops._C, "dsv3_fused_a_gemm")
)
qkv_a_weight = getattr(self.fused_qkv_a_proj, "weight", None)
q_b_weight = getattr(self.q_b_proj, "weight", None)
self._qkv_a_skinny_max = (
qkv_a_dispatch.get(tuple(qkv_a_weight.shape))
if ok and qkv_a_weight is not None and qkv_a_weight.dtype == torch.bfloat16
else None
)
self._q_b_skinny_max = (
q_b_dispatch.get(tuple(q_b_weight.shape))
if ok and q_b_weight is not None and q_b_weight.dtype == torch.bfloat16
else None
)
# Lightning indexer uses its own RoPE; interleave maps to non-NeoX.
self.indexer_rope_emb = get_rope(
qk_rope_head_dim,
@@ -339,13 +374,32 @@ class DeepseekV32Attention(MLAAttention):
is_neox_style=not getattr(config, "indexer_rope_interleave", False),
)
def _decode_m_gemm(
self, x: torch.Tensor, weight: torch.Tensor, skinny_max: int
) -> torch.Tensor:
"""x @ weight.T for decode M <= 16."""
if x.shape[0] == 0:
return torch.empty((0, weight.shape[0]), dtype=x.dtype, device=x.device)
if x.shape[0] <= skinny_max:
return ops.bf16_skinny_gemm(x, weight)
out = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device)
ops.dsv3_fused_a_gemm(out, x, weight.t())
return out
def forward( # type: ignore[override]
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
# Captured: A-projections (+ indexer A-GEMM on indexer layers).
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
if self._qkv_a_skinny_max is not None and hidden_states.shape[0] <= 16:
qkv_lora = self._decode_m_gemm(
hidden_states,
self.fused_qkv_a_proj.weight,
self._qkv_a_skinny_max,
)
else:
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
q_c, kv_c, k_pe = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
)
@@ -398,7 +452,9 @@ class DeepseekV32Attention(MLAAttention):
assert isinstance(slot_mapping, dict)
mla_slot = slot_mapping.get(self.layer_name)
if self.indexer is not None:
# skip_topk (index_share_for_mtp_iteration reuse mode): don't run the
# indexer this step; keep the top-k already in the shared buffer.
if self.indexer is not None and not self.skip_topk:
has_indexer = True
indexer_k_norm_w = self.indexer.k_norm.weight
indexer_k_norm_bias = self.indexer.k_norm.bias
@@ -451,12 +507,16 @@ class DeepseekV32Attention(MLAAttention):
index_rope_interleave=self._index_rope_interleave,
)
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
if self._q_b_skinny_max is not None and q_c.shape[0] <= 16:
q = self._decode_m_gemm(q_c, self.q_b_proj.weight, self._q_b_skinny_max)
else:
q = self.q_b_proj(q_c)[0]
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
q_nope = q_nope.transpose(0, 1)
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
if self.indexer is not None:
if self.indexer is not None and not self.skip_topk:
index_q = self.indexer.wq_b(q_c)[0]
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
else:
@@ -478,10 +538,10 @@ class DeepseekV32Attention(MLAAttention):
quantize_mqa=self._fp8_query,
)
if self.indexer is not None:
sparse_attn_indexer(
if self.indexer is not None and not self.skip_topk:
torch.ops.vllm.sparse_attn_indexer(
q_c,
self.indexer.k_cache.prefix,
_encode_layer_name(self.indexer.k_cache.prefix),
self.indexer.k_cache.kv_cache,
index_q_fp8,
None, # q_scale folded into weights on the fp8 path
+284 -16
View File
@@ -2,13 +2,80 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.torch_utils import direct_register_custom_op
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
# has_indexer=False path so the indexer args don't allocate every call.
_DUMMY_CACHE: dict[tuple, torch.Tensor] = {}
@torch.compiler.assume_constant_result
def _can_use_fused_q_cutedsl() -> bool:
return current_platform.has_device_capability(100) and has_cutedsl()
@torch.compiler.assume_constant_result
def _is_arch_support_pdl() -> bool:
return (
current_platform.has_device_capability(100)
and current_platform.is_arch_support_pdl()
)
def _fused_q_cutedsl_impl(
positions: torch.Tensor,
q_pe: torch.Tensor,
rope_cache: torch.Tensor,
ql_nope: torch.Tensor,
q_scale: torch.Tensor,
mqa_output: torch.Tensor,
idx_q: torch.Tensor,
idx_rope_cache: torch.Tensor,
idx_weights: torch.Tensor,
idx_weights_softmax_scale: float,
idx_weights_head_scale: float,
idx_q_fp8: torch.Tensor,
idx_weights_out: torch.Tensor,
has_indexer: bool,
index_rope_interleave: bool,
) -> None:
from .ops.fused_q_cutedsl import fused_q_cutedsl
fused_q_cutedsl(
positions,
q_pe,
rope_cache,
ql_nope,
q_scale,
mqa_output,
idx_q,
idx_rope_cache,
idx_weights,
idx_weights_softmax_scale,
idx_weights_head_scale,
idx_q_fp8,
idx_weights_out,
has_indexer=has_indexer,
index_rope_interleave=index_rope_interleave,
)
def _fused_q_cutedsl_fake(*args, **kwargs) -> None:
pass
direct_register_custom_op(
op_name="fused_q_cutedsl",
op_func=_fused_q_cutedsl_impl,
mutates_args=["mqa_output", "idx_q_fp8", "idx_weights_out"],
fake_impl=_fused_q_cutedsl_fake,
dispatch_key="CUDA",
)
def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
key = (shape, dtype, device)
t = _DUMMY_CACHE.get(key)
@@ -147,9 +214,14 @@ def _fused_norm_rope_kernel(
TOPK_BLOCK_SIZE: tl.constexpr,
HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr,
USE_PDL: tl.constexpr,
):
pid = tl.program_id(0)
tok_idx = tl.program_id(1)
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
if pid == 3:
if not HAS_INDEXER:
# Shared layer: reuse the previous indexer layer's top-k; do not
@@ -166,6 +238,17 @@ def _fused_norm_rope_kernel(
)
return
if pid == 2:
# Q RMS norm does not depend on cache availability. In particular,
# profiling uses a negative dummy slot mapping but still consumes Q.
q_block = tl.arange(0, Q_BLOCK_SIZE)
q_mask = q_block < Q_DIM
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
return
if slot_mapping_ptr is None:
# Memory profiling run.
return
@@ -174,15 +257,7 @@ def _fused_norm_rope_kernel(
# Padding
return
if pid == 2:
# Q RMS norm
q_block = tl.arange(0, Q_BLOCK_SIZE)
q_mask = q_block < Q_DIM
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
elif pid == 1:
if pid == 1:
# KV RMS Norm + KV RoPE + MLA concat_and_cache.
# Merged so the normed kv_c and RoPE'd k_pe can be written
# to the MLA KV cache directly without a separate kernel.
@@ -211,9 +286,6 @@ def _fused_norm_rope_kernel(
r2 = x2 * cos + x1 * sin
# MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache.
if mla_cache_entry_stride == 0:
return
mla_block_size = mla_cache_block_stride // mla_cache_entry_stride
mla_block_idx = slot_idx // mla_block_size
mla_block_off = slot_idx % mla_block_size
@@ -363,7 +435,7 @@ def _fused_norm_rope_kernel(
)
def fused_norm_rope(
def _fused_norm_rope_impl(
positions: torch.Tensor,
q_c: torch.Tensor,
q_rms_norm_w: torch.Tensor,
@@ -465,12 +537,17 @@ def fused_norm_rope(
# Dummy values — pid 2 will skip the MLA cache write because
# slot_mapping is all -1.
mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device)
mla_block_stride = 0
mla_entry_stride = 0
# Torch's Triton HOP mutation analysis compiles every program-id
# branch even though the dummy slot mapping returns before cache
# access. Keep the unused strides nonzero so that analysis can lower
# the block-size calculation safely.
mla_block_stride = 1
mla_entry_stride = 1
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
if q_c_out is None:
q_c_out = torch.empty_like(q_c)
use_pdl = _is_arch_support_pdl()
_fused_norm_rope_kernel[(4, num_tokens)](
positions,
# Q RMS norm
@@ -529,6 +606,157 @@ def fused_norm_rope(
TOPK_BLOCK_SIZE=1024,
HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
)
return q_c_out
def _fused_norm_rope_op(
positions: torch.Tensor,
q_c: torch.Tensor,
q_rms_norm_w: torch.Tensor,
q_rms_eps: float,
kv_c: torch.Tensor,
kv_rms_norm_w: torch.Tensor,
kv_rms_eps: float,
k_pe: torch.Tensor,
k_rope_cos_sin_cache: torch.Tensor,
index_k: torch.Tensor | None,
index_k_layer_norm_w: torch.Tensor | None,
index_k_layer_norm_bias: torch.Tensor | None,
index_k_layer_norm_eps: float,
index_k_rope_cos_sin_cache: torch.Tensor | None,
topk_indices_buffer: torch.Tensor,
slot_mapping: torch.Tensor | None,
indexer_k_cache: torch.Tensor | None,
mla_kv_cache: torch.Tensor | None,
mla_kv_cache_dtype: str,
mla_k_scale: torch.Tensor | None,
has_indexer: bool,
index_rope_interleave: bool,
q_c_out: torch.Tensor,
) -> None:
_fused_norm_rope_impl(
positions,
q_c,
q_rms_norm_w,
q_rms_eps,
kv_c,
kv_rms_norm_w,
kv_rms_eps,
k_pe,
k_rope_cos_sin_cache,
index_k,
index_k_layer_norm_w,
index_k_layer_norm_bias,
index_k_layer_norm_eps,
index_k_rope_cos_sin_cache,
topk_indices_buffer,
slot_mapping=slot_mapping,
indexer_k_cache=indexer_k_cache,
mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=mla_kv_cache_dtype,
mla_k_scale=mla_k_scale,
has_indexer=has_indexer,
index_rope_interleave=index_rope_interleave,
q_c_out=q_c_out,
)
def _fused_norm_rope_fake(
positions: torch.Tensor,
q_c: torch.Tensor,
q_rms_norm_w: torch.Tensor,
q_rms_eps: float,
kv_c: torch.Tensor,
kv_rms_norm_w: torch.Tensor,
kv_rms_eps: float,
k_pe: torch.Tensor,
k_rope_cos_sin_cache: torch.Tensor,
index_k: torch.Tensor | None,
index_k_layer_norm_w: torch.Tensor | None,
index_k_layer_norm_bias: torch.Tensor | None,
index_k_layer_norm_eps: float,
index_k_rope_cos_sin_cache: torch.Tensor | None,
topk_indices_buffer: torch.Tensor,
slot_mapping: torch.Tensor | None,
indexer_k_cache: torch.Tensor | None,
mla_kv_cache: torch.Tensor | None,
mla_kv_cache_dtype: str,
mla_k_scale: torch.Tensor | None,
has_indexer: bool,
index_rope_interleave: bool,
q_c_out: torch.Tensor,
) -> None:
return
direct_register_custom_op(
op_name="fused_norm_rope_deepseek_v32",
op_func=_fused_norm_rope_op,
mutates_args=[
"topk_indices_buffer",
"indexer_k_cache",
"mla_kv_cache",
"q_c_out",
],
fake_impl=_fused_norm_rope_fake,
dispatch_key="CUDA",
)
def fused_norm_rope(
positions: torch.Tensor,
q_c: torch.Tensor,
q_rms_norm_w: torch.Tensor,
q_rms_eps: float,
kv_c: torch.Tensor,
kv_rms_norm_w: torch.Tensor,
kv_rms_eps: float,
k_pe: torch.Tensor,
k_rope_cos_sin_cache: torch.Tensor,
index_k: torch.Tensor | None,
index_k_layer_norm_w: torch.Tensor | None,
index_k_layer_norm_bias: torch.Tensor | None,
index_k_layer_norm_eps: float,
index_k_rope_cos_sin_cache: torch.Tensor | None,
topk_indices_buffer: torch.Tensor,
slot_mapping: torch.Tensor | None = None,
indexer_k_cache: torch.Tensor | None = None,
mla_kv_cache: torch.Tensor | None = None,
mla_kv_cache_dtype: str = "auto",
mla_k_scale: torch.Tensor | None = None,
has_indexer: bool = True,
index_rope_interleave: bool = False,
q_c_out: torch.Tensor | None = None,
) -> torch.Tensor:
if q_c_out is None:
q_c_out = torch.empty_like(q_c)
torch.ops.vllm.fused_norm_rope_deepseek_v32(
positions,
q_c,
q_rms_norm_w,
q_rms_eps,
kv_c,
kv_rms_norm_w,
kv_rms_eps,
k_pe,
k_rope_cos_sin_cache,
index_k,
index_k_layer_norm_w,
index_k_layer_norm_bias,
index_k_layer_norm_eps,
index_k_rope_cos_sin_cache,
topk_indices_buffer,
slot_mapping,
indexer_k_cache,
mla_kv_cache,
mla_kv_cache_dtype,
mla_k_scale,
has_indexer,
index_rope_interleave,
q_c_out,
)
return q_c_out
@@ -582,10 +810,14 @@ def _fused_q_kernel(
HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr,
QUANTIZE_MQA: tl.constexpr,
USE_PDL: tl.constexpr,
):
pid = tl.program_id(0)
tok_idx = tl.program_id(1)
head_idx = tl.program_id(2)
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
if pid == 2:
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
@@ -773,10 +1005,12 @@ def fused_q(
``(ql_nope, q_pe)`` tuple the backend expects.
"""
assert positions.ndim == 1
assert positions.dtype == torch.int64
assert q_pe.ndim == 3
assert q_pe_cos_sin_cache.ndim == 2
assert ql_nope.ndim == 3
assert ql_nope.shape[:2] == q_pe.shape[:2]
assert q_scale.dtype == torch.float32 and q_scale.numel() == 1
num_tokens = positions.shape[0]
num_q_heads = q_pe.shape[1]
@@ -794,7 +1028,18 @@ def fused_q(
assert index_weights is not None
num_index_q_heads = index_q.shape[1]
index_q_head_dim = index_q.shape[2]
grid_heads = max(mqa_grid_heads, num_index_q_heads)
use_cutedsl = False
if _can_use_fused_q_cutedsl():
from .ops.fused_q_cutedsl import is_fused_q_cutedsl_supported
use_cutedsl = is_fused_q_cutedsl_supported(
q_pe,
index_q,
ql_nope,
has_indexer=has_indexer,
quantize_mqa=quantize_mqa,
)
grid_heads = max(mqa_grid_heads, num_index_q_heads if has_indexer else 1)
if quantize_mqa:
# fp8 path: pack [ql_nope; q_pe] into a single fp8 tensor.
mqa_q_fp8 = torch.empty(
@@ -815,6 +1060,27 @@ def fused_q(
index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn)
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
if use_cutedsl:
torch.ops.vllm.fused_q_cutedsl(
positions,
q_pe,
q_pe_cos_sin_cache,
ql_nope,
q_scale,
mqa_q,
index_q,
index_q_cos_sin_cache,
index_weights,
index_weights_softmax_scale,
index_weights_head_scale,
index_q_fp8,
index_weights_out,
has_indexer=has_indexer,
index_rope_interleave=index_rope_interleave,
)
return index_q_fp8, index_weights_out, mqa_q
use_pdl = _is_arch_support_pdl()
_fused_q_kernel[(3, num_tokens, grid_heads)](
positions,
q_pe,
@@ -856,6 +1122,8 @@ def fused_q(
HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
QUANTIZE_MQA=quantize_mqa,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
# num_warps=1 is optimal here: each program is a single 128-element
# rope+quant, so the kernel is program-count/occupancy bound, not
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).
+4
View File
@@ -6,6 +6,7 @@ from itertools import islice
import torch
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
@@ -39,6 +40,7 @@ from vllm.model_executor.models.utils import (
sequence_parallel_chunk,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.sparse_utils import register_phys_shadow
from .attention import DeepseekV32Attention
from .fused_ops import fused_allreduce_rms_norm
@@ -177,6 +179,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
return hidden_states, residual
@support_torch_compile
class DeepseekV32Model(torch.nn.Module):
fall_back_to_pt_during_load = False
@@ -200,6 +203,7 @@ class DeepseekV32Model(torch.nn.Module):
dtype=torch.int32,
device=self.device,
)
register_phys_shadow(topk_indices_buffer)
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
+76 -6
View File
@@ -6,9 +6,9 @@ from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
import vllm._custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
@@ -40,7 +40,12 @@ from vllm.model_executor.models.utils import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.sparse_utils import (
phys_shadow,
register_phys_shadow,
)
from .fused_ops import fused_allreduce_rms_norm
from .kernels import fused_eh_norm
from .model import DeepseekV32DecoderLayer
@@ -56,6 +61,19 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
# bf16 skinny GEMM for the eh_proj, B300+B200 measured: GLM-5.2
# (6144, 12288) wins at M <= 3, DSv3.2 (7168, 14336) at M <= 2;
# cuBLAS holds above (the 151/205MB weights stream at ~5.7TB/s).
eh_dispatch = {(6144, 12288): 3, (7168, 14336): 2}
eh_weight = getattr(self.eh_proj, "weight", None)
self._eh_skinny_max = (
eh_dispatch.get(tuple(eh_weight.shape), 0)
if current_platform.is_device_capability_family(100)
and hasattr(torch.ops._C, "bf16_skinny_gemm")
and eh_weight is not None
and eh_weight.dtype == torch.bfloat16
else 0
)
topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
@@ -63,6 +81,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
dtype=torch.int32,
device=current_platform.device_type,
)
register_phys_shadow(topk_indices_buffer)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
@@ -91,7 +110,13 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
self.hnorm.weight,
self.enorm.variance_epsilon,
)
hidden_states = self.eh_proj(eh_input)
if (
eh_input.shape[0] <= self._eh_skinny_max
and eh_input.dtype == torch.bfloat16
):
hidden_states = ops.bf16_skinny_gemm(eh_input, self.eh_proj.weight)
else:
hidden_states = self.eh_proj(eh_input)
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
)
@@ -101,9 +126,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
positions.shape[0],
is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe,
)
if not self.mtp_block.use_sequence_parallel_moe:
# Without sequence parallelism, the MoE output is left un-reduced.
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
# Recycle the POST-final-norm hidden into the next draft step. The
# residual-add is fused into the final RMSNorm so it is computed
# exactly once, and the result is returned for both tuple positions:
@@ -114,7 +136,14 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
# is understood by both the V2 speculator (isinstance-tuple check) and
# the legacy proposer (model_returns_tuple is True for the
# DeepSeekMTPModel architecture).
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
if self.mtp_block.use_sequence_parallel_moe:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
else:
# The MoE output is left un-reduced; fuse its all-reduce into the
# final norm, as the main model does at layer boundaries.
hidden_states, _ = fused_allreduce_rms_norm(
hidden_states, residual, self.shared_head.norm
)
return hidden_states, hidden_states
@@ -157,6 +186,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
topk_indices_buffer = self_attn.topk_indices_buffer
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
# The physical-index shadow mirrors this buffer row-wise
# (shadow[j] == convert(logical[j])); re-arrange it the same
# way or skip_topk draft steps read the step-0 multi-token
# layout's rows for the wrong tokens/requests.
shadow = phys_shadow(topk_indices_buffer)
if shadow is not None:
phys, seq = shadow
phys[:num_slots] = phys[slot_ids]
seq[:num_slots] = seq[slot_ids]
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@@ -192,6 +230,19 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
# second RMSNorm.
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
# Greedy draft token ids via vocab-parallel local argmax: skips the
# full-vocab logits all-gather (use_local_argmax_reduction).
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
return self.logits_processor.get_top_tokens(
mtp_layer.shared_head.head, hidden_states
)
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -240,6 +291,25 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int | None = None,
) -> torch.Tensor:
# The V2 speculator calls this without spec_step_idx; that is only
# correct while a single MTP layer is cycled for every draft step
# (num_nextn_predict_layers == 1, as in GLM-5.2/DSV3.2). Guard the
# assumption so a future multi-layer MTP fails loudly instead of
# silently using step 0's head for every step.
if spec_step_idx is None:
assert self.model.num_mtp_layers == 1, (
"get_top_tokens called without spec_step_idx on a "
"multi-layer MTP; thread the draft step index through "
"the speculator."
)
spec_step_idx = 0
return self.model.get_top_tokens(hidden_states, spec_step_idx)
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
spec_layer_weight_names = [
"embed_tokens",
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,532 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float8E4M3FN, Float32, Int64, Uint8, Uint16, Uint32
from vllm.cute_utils import TORCH_TO_CUTE_DTYPE, cvt
def _make_fake_tensor(dtype, shape, divisibility):
stride = tuple(
1 if i == len(shape) - 1 else cute.sym_int64(divisibility=divisibility)
for i in range(len(shape))
)
return cute.runtime.make_fake_tensor(
dtype,
shape,
stride,
assumed_align=divisibility * dtype.width // 8,
)
def is_fused_q_cutedsl_supported(
q_pe: torch.Tensor,
index_q: torch.Tensor | None,
ql_nope: torch.Tensor,
*,
has_indexer: bool,
quantize_mqa: bool,
) -> bool:
if not (
quantize_mqa
and q_pe.dtype == ql_nope.dtype == torch.bfloat16
and q_pe.shape[-1] == 64
and ql_nope.shape[-1] == 512
):
return False
return not has_indexer or (
index_q is not None
and index_q.dtype == torch.bfloat16
and index_q.shape[1] % 16 == 0
and index_q.shape[-1] == 128
)
def fused_q_cutedsl(
positions: torch.Tensor,
q_pe: torch.Tensor,
rope_cache: torch.Tensor,
ql_nope: torch.Tensor,
q_scale: torch.Tensor,
mqa_output: torch.Tensor,
idx_q: torch.Tensor,
idx_rope_cache: torch.Tensor,
idx_weights: torch.Tensor,
idx_weights_softmax_scale: float,
idx_weights_head_scale: float,
idx_q_fp8: torch.Tensor,
idx_weights_out: torch.Tensor,
has_indexer: bool = True,
index_rope_interleave: bool = True,
) -> None:
_, num_heads, rope_dim = q_pe.shape
_, _, nope_dim = ql_nope.shape
_, num_idx_heads, idx_dim = idx_q.shape
if has_indexer:
idx_rope_type = TORCH_TO_CUTE_DTYPE[idx_rope_cache.dtype]
idx_weights_type = TORCH_TO_CUTE_DTYPE[idx_weights.dtype]
else:
idx_dim = num_idx_heads = 0
idx_q = idx_rope_cache = idx_q_fp8 = None
idx_weights = idx_weights_out = None
idx_rope_type = idx_weights_type = None
rope_type = TORCH_TO_CUTE_DTYPE[rope_cache.dtype]
compiled = FusedQKernel.compile(
rope_dim,
nope_dim,
num_heads,
rope_type,
idx_dim,
num_idx_heads,
idx_rope_type,
idx_weights_type,
index_rope_interleave,
)
compiled(
positions,
q_pe,
rope_cache,
ql_nope,
q_scale.view(1),
mqa_output,
idx_q,
idx_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
float(idx_weights_softmax_scale * idx_weights_head_scale),
)
class FusedQKernel:
def __init__(
self,
rope_dim: int,
nope_dim: int,
num_heads: int,
idx_dim: int,
num_idx_heads: int,
index_rope_interleave: bool,
) -> None:
assert rope_dim == 64
assert nope_dim == 512
assert idx_dim in (128, 0)
self.rope_dim = rope_dim
self.nope_dim = nope_dim
self.num_heads = num_heads
self.idx_dim = idx_dim
self.num_idx_heads = num_idx_heads
self.index_rope_interleave = index_rope_interleave
# mqa: rope_dim=64, nope_dim=512, num_heads=64/TP
# indexer: rope_dim=64, nope_dim=64, num_heads=32
self.num_warps = 4
assert num_heads % self.num_warps == 0
assert num_idx_heads % (4 * self.num_warps) == 0
self.num_ctas_per_tok = num_heads // self.num_warps
self.num_ctas_per_idx_tok = num_idx_heads // (4 * self.num_warps)
@cute.jit
def __call__(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
idx_q: cute.Tensor,
idx_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
stream: CUstream,
):
num_tokens = positions.shape[0]
if cutlass.const_expr(self.idx_dim == 0):
grid = (num_tokens, self.num_ctas_per_tok, 1)
else:
num_mqa_ctas = num_tokens * self.num_ctas_per_tok
num_idx_ctas = num_tokens * self.num_ctas_per_idx_tok
grid = (num_mqa_ctas + num_idx_ctas, 1, 1)
self.kernel(
positions,
q_pe,
rope_cache,
ql_nope,
q_scale,
mqa_output,
idx_q,
idx_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
weight_scale,
).launch(
grid=grid,
block=(self.num_warps * 32, 1, 1),
stream=stream,
use_pdl=True,
)
@cute.kernel
def kernel(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
q_pe_rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
idx_q: cute.Tensor,
idx_q_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
):
if cutlass.const_expr(self.idx_dim == 0):
token_id, group_id, _ = cute.arch.block_idx()
self.mqa(
positions,
q_pe,
q_pe_rope_cache,
ql_nope,
q_scale,
mqa_output,
token_id,
group_id,
)
else:
# CTA-specialization
bid, _, _ = cute.arch.block_idx()
num_mqa_ctas = positions.shape[0] * self.num_ctas_per_tok
if bid < num_mqa_ctas:
self.mqa(
positions,
q_pe,
q_pe_rope_cache,
ql_nope,
q_scale,
mqa_output,
bid // self.num_ctas_per_tok,
bid % self.num_ctas_per_tok,
)
else:
bid -= num_mqa_ctas
self.indexer(
positions,
idx_q,
idx_q_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
weight_scale,
bid // self.num_ctas_per_idx_tok,
bid % self.num_ctas_per_idx_tok,
)
@cute.jit
def mqa(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
q_pe_rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
token_id,
group_id,
):
tid, _, _ = cute.arch.thread_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
head_id = group_id * self.num_warps + warp_id
cute.arch.griddepcontrol_wait()
pos = positions[token_id]
inv_scale = 1.0 / q_scale[0]
cp_op = cute.nvgpu.CopyUniversalOp()
cp_32B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256)
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
cp_2B = cute.make_copy_atom(cp_op, Uint8, num_bits_per_copy=16)
##### issue all loads asap #####
rQ_nope_bf16 = cute.make_rmem_tensor(16, BFloat16)
rQ_rope_bf16 = cute.make_rmem_tensor(2, BFloat16)
src_ql_nope = cute.local_tile(
ql_nope[token_id, head_id, None], (16,), (lane_id,)
)
src_q_rope = cute.local_tile(q_pe[token_id, head_id, None], (2,), (lane_id,))
cute.copy(cp_32B, src_ql_nope, rQ_nope_bf16)
cute.copy(cp_4B, src_q_rope, rQ_rope_bf16)
rCos_raw = q_pe_rope_cache[pos, 0 + lane_id]
rSin_raw = q_pe_rope_cache[pos, 32 + lane_id]
##### process NoPE #####
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16).load() * inv_scale
rQ_nope_f8 = cute.make_rmem_tensor(16, Float8E4M3FN)
rQ_nope_f8.store(rQ_nope_f32.to(Float8E4M3FN))
dst_Q_nope = cute.local_tile(
mqa_output[token_id, head_id, None], (16,), (lane_id,)
)
cute.copy(cp_16B, rQ_nope_f8, dst_Q_nope)
##### process RoPE ######
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
rCos = rCos_raw.to(Float32)
rSin = rSin_raw.to(Float32)
r0 = (rQ_rope_f32[0] * rCos - rQ_rope_f32[1] * rSin) * inv_scale
r1 = (rQ_rope_f32[1] * rCos + rQ_rope_f32[0] * rSin) * inv_scale
cute.arch.griddepcontrol_launch_dependents()
# TensorSSA fp32->fp8 cvt has a bug. rely on direct PTX
rQ_rope_f8 = cute.make_rmem_tensor(2, Float8E4M3FN)
cute.recast_tensor(rQ_rope_f8, Uint16)[0] = cvt.fp32x2_to_fp8x2(r0, r1)
dst_Q_rope = cute.local_tile(
mqa_output[token_id, head_id, None], (2,), (256 + lane_id,)
)
cute.copy(cp_2B, rQ_rope_f8, dst_Q_rope)
@cute.jit
def indexer(
self,
positions: cute.Tensor,
idx_q: cute.Tensor,
idx_q_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
token_id,
group_id,
):
tid, _, _ = cute.arch.thread_idx()
subwarp_id = tid // 8
sublane_id = tid % 8
head_id = group_id * (4 * self.num_warps) + subwarp_id
cute.arch.griddepcontrol_wait()
pos = positions[token_id]
cp_op = cute.nvgpu.CopyUniversalOp()
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
cp_8B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
##### issue all loads first #####
rQ_rope_bf16 = cute.make_rmem_tensor(8, BFloat16)
if cutlass.const_expr(self.index_rope_interleave):
src_idx_q_rope = cute.local_tile(
idx_q[token_id, head_id, None], (8,), (sublane_id,)
)
cute.copy(cp_16B, src_idx_q_rope, rQ_rope_bf16)
else:
src_idx_q_rope = cute.zipped_divide(
idx_q[token_id, head_id, None], (4,)
) # (4,32)
cute.copy(
cp_8B,
src_idx_q_rope[None, 0 + sublane_id],
cute.local_tile(rQ_rope_bf16, (4,), (0,)),
)
cute.copy(
cp_8B,
src_idx_q_rope[None, 8 + sublane_id],
cute.local_tile(rQ_rope_bf16, (4,), (1,)),
)
rQ_nope_bf16 = cute.make_rmem_tensor(8, BFloat16)
src_idx_q_nope = cute.local_tile(
idx_q[token_id, head_id, None], (8,), (8 + sublane_id,)
)
cute.copy(cp_16B, src_idx_q_nope, rQ_nope_bf16)
rCos_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
rSin_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
rope_cache_view = cute.zipped_divide(idx_q_rope_cache[pos, None], (4,))
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
cute.copy(cp_16B, rope_cache_view[None, sublane_id], rCos_raw)
cute.copy(cp_16B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
cute.copy(cp_8B, rope_cache_view[None, sublane_id], rCos_raw)
cute.copy(cp_8B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
# unpack to FP32
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16)
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
rCos = rCos_raw
rSin = rSin_raw
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
rCos = cvt.bf16x2_to_fp32x2(rCos_raw)
rSin = cvt.bf16x2_to_fp32x2(rSin_raw)
# apply rope
for i in cutlass.range_constexpr(4):
if cutlass.const_expr(self.index_rope_interleave):
r0 = rQ_rope_f32[i * 2 + 0] * rCos[i] - rQ_rope_f32[i * 2 + 1] * rSin[i]
r1 = rQ_rope_f32[i * 2 + 1] * rCos[i] + rQ_rope_f32[i * 2 + 0] * rSin[i]
rQ_rope_f32[i * 2 + 0] = r0
rQ_rope_f32[i * 2 + 1] = r1
else:
r0 = rQ_rope_f32[0 + i] * rCos[i] - rQ_rope_f32[4 + i] * rSin[i]
r1 = rQ_rope_f32[4 + i] * rCos[i] + rQ_rope_f32[0 + i] * rSin[i]
rQ_rope_f32[0 + i] = r0
rQ_rope_f32[4 + i] = r1
# amax
amax = Float32(1e-4)
for i in cutlass.range_constexpr(8):
amax = cute.arch.fmax(amax, cute.math.absf(rQ_rope_f32[i]))
amax = cute.arch.fmax(amax, cute.math.absf(rQ_nope_f32[i]))
# warp reduction among 8 lanes
for i in cutlass.range_constexpr(3):
other = cute.arch.shuffle_sync_bfly(amax, 1 << i)
amax = cute.arch.fmax(amax, other)
# compute scale from amax
# exp2(ceil(log2(scale))) via bit manipulation
scale = amax * (1.0 / 448.0)
bits = scale.bitcast(Uint32)
exp_bits = (bits + Uint32(0x007FFFFF)) & Uint32(0x7F800000)
scale = exp_bits.bitcast(Float32)
inv_scale = (Uint32(0x7F000000) - exp_bits).bitcast(Float32)
for i in cutlass.range_constexpr(8):
rQ_nope_f32[i] *= inv_scale
rQ_rope_f32[i] *= inv_scale
cute.arch.griddepcontrol_launch_dependents()
# quantize and store
rQ_nope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
rQ_nope_f8.store(rQ_nope_f32.load().to(Float8E4M3FN))
dst_idx_q_nope = cute.local_tile(
idx_q_fp8[token_id, head_id, None], (8,), (8 + sublane_id,)
)
cute.copy(cp_8B, rQ_nope_f8, dst_idx_q_nope)
rQ_rope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
rQ_rope_f8.store(rQ_rope_f32.load().to(Float8E4M3FN))
if cutlass.const_expr(self.index_rope_interleave):
dst_idx_q_rope = cute.local_tile(
idx_q_fp8[token_id, head_id, None], (8,), (sublane_id,)
)
cute.copy(cp_8B, rQ_rope_f8, dst_idx_q_rope)
else:
dst_idx_q_rope = cute.zipped_divide(
idx_q_fp8[token_id, head_id, None], (4,)
) # (4,32)
cute.copy(
cp_4B,
cute.local_tile(rQ_rope_f8, (4,), (0,)),
dst_idx_q_rope[None, 0 + sublane_id],
)
cute.copy(
cp_4B,
cute.local_tile(rQ_rope_f8, (4,), (1,)),
dst_idx_q_rope[None, 8 + sublane_id],
)
# scale indexer weights
if sublane_id == 0:
w = idx_weights[token_id, head_id].to(Float32)
idx_weights_out[token_id, head_id] = w * scale * weight_scale
@cache
@staticmethod
def compile(
rope_dim: int,
nope_dim: int,
num_heads: int,
rope_type: type[cutlass.Numeric],
idx_dim: int,
num_idx_heads: int,
idx_rope_type: type[cutlass.Numeric] | None,
idx_weights_type: type[cutlass.Numeric] | None,
index_rope_interleave: bool,
):
num_tokens = cute.sym_int()
max_pos = cute.sym_int()
positions = _make_fake_tensor(Int64, (num_tokens,), divisibility=1)
q_pe = _make_fake_tensor(
BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16
)
rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8)
ql_nope = _make_fake_tensor(
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
)
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4)
mqa_output = _make_fake_tensor(
Float8E4M3FN,
(num_tokens, num_heads, nope_dim + rope_dim),
divisibility=16,
)
if idx_rope_type is not None:
index_q = _make_fake_tensor(
BFloat16, (num_tokens, num_idx_heads, idx_dim), divisibility=16
)
index_rope_cache = _make_fake_tensor(
idx_rope_type, (max_pos, rope_dim), divisibility=8
)
index_weights = _make_fake_tensor(
idx_weights_type, (num_tokens, num_idx_heads), divisibility=8
)
index_q_fp8 = _make_fake_tensor(
Float8E4M3FN, (num_tokens, num_idx_heads, idx_dim), divisibility=16
)
index_weights_out = _make_fake_tensor(
Float32, (num_tokens, num_idx_heads), divisibility=4
)
else:
index_q = index_rope_cache = index_q_fp8 = None
index_weights = index_weights_out = None
kernel = FusedQKernel(
rope_dim,
nope_dim,
num_heads,
idx_dim,
num_idx_heads,
index_rope_interleave,
)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
positions,
q_pe,
rope_cache,
ql_nope,
q_scale,
mqa_output,
index_q,
index_rope_cache,
index_weights,
index_q_fp8,
index_weights_out,
Float32(0.0),
stream,
options="--enable-tvm-ffi",
)