[Perf][GLM-5.2] Blackwell decode optimizations (#48597)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local> Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local> Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
co-authored by
Claude
Giancarlo Delfin
Thien Tran
OpenAI Codex
zhou
stefankoncarevic
parent
0d77325b10
commit
dd72658e7d
@@ -6,17 +6,31 @@ DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
|
||||
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
|
||||
attend. The same model code serves any DSA checkpoint, including GLM-5.2
|
||||
(``glm_moe_dsa``), which reuses this architecture.
|
||||
|
||||
The optimized kernels under ``nvidia/`` target the Blackwell (SM100) family.
|
||||
Every other platform — ROCm, XPU, pre-SM100 CUDA (e.g. H100), CPU — falls back
|
||||
to the generic ``deepseek_v2`` implementation, which already handles the DSA
|
||||
(index_topk) architecture and is ``torch.compile``-friendly there. This matches
|
||||
main's behavior on those platforms (no hard failure).
|
||||
"""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
raise NotImplementedError("deepseek_v32 currently supports NVIDIA SM100 only.")
|
||||
if current_platform.is_cuda() and current_platform.is_device_capability_family(100):
|
||||
from .nvidia.model import DeepseekV32ForCausalLM
|
||||
from .nvidia.mtp import DeepseekV32MTP
|
||||
|
||||
from .nvidia.model import DeepseekV32ForCausalLM
|
||||
from .nvidia.mtp import DeepseekV32MTP
|
||||
# GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100.
|
||||
GlmMoeDsaForCausalLM = DeepseekV32ForCausalLM
|
||||
else:
|
||||
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
|
||||
|
||||
__all__ = [
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV32MTP",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
]
|
||||
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
@@ -21,17 +22,15 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
SparseAttnIndexer,
|
||||
sparse_attn_indexer,
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepSeekV2FusedQkvAProjLinear,
|
||||
DeepseekV32IndexerCache,
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import _encode_layer_name, is_quantized_kv_cache
|
||||
|
||||
from .kernels import fused_norm_rope, fused_q
|
||||
|
||||
@@ -171,6 +170,11 @@ class DeepseekV32Attention(MLAAttention):
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config is not None and cache_config.cache_dtype == "auto":
|
||||
# This implementation requires an FP8 sparse cache. Start with the
|
||||
# generic FP8 format; MLAAttention will canonicalize it for the
|
||||
# selected sparse backend.
|
||||
cache_config.cache_dtype = "fp8"
|
||||
|
||||
hidden_size = config.hidden_size
|
||||
qk_nope_head_dim = config.qk_nope_head_dim
|
||||
@@ -331,6 +335,37 @@ class DeepseekV32Attention(MLAAttention):
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
# Decode-M GEMM dispatch for the bf16 A/B projections, measured on
|
||||
# B300 and B200 (graph replay + rotating weights vs fp64):
|
||||
# bf16_skinny_gemm for M <= skinny_max, dsv3_fused_a_gemm up to
|
||||
# M = 16, cuBLAS above. Blackwell-only: the boundaries and the
|
||||
# fused_a tile choices are tied to the 148/152-SM single-wave
|
||||
# geometry. DSv3.2 q_b has no skinny tier (K=1536 does not fit the
|
||||
# GEMV's 128x8 K-step; fused_a still wins 1.6-1.8x there).
|
||||
# Quantized linear methods may not register .weight until
|
||||
# process_weights_after_loading (e.g. compressed-tensors NVFP4
|
||||
# registers weight_packed at init) — probe with getattr.
|
||||
# (N, K) -> skinny_max
|
||||
qkv_a_dispatch = {(2624, 6144): 2, (2112, 7168): 2}
|
||||
q_b_dispatch = {(2048, 2048): 2, (3072, 1536): 0}
|
||||
ok = (
|
||||
current_platform.is_device_capability_family(100)
|
||||
and hasattr(torch.ops._C, "bf16_skinny_gemm")
|
||||
and hasattr(torch.ops._C, "dsv3_fused_a_gemm")
|
||||
)
|
||||
qkv_a_weight = getattr(self.fused_qkv_a_proj, "weight", None)
|
||||
q_b_weight = getattr(self.q_b_proj, "weight", None)
|
||||
self._qkv_a_skinny_max = (
|
||||
qkv_a_dispatch.get(tuple(qkv_a_weight.shape))
|
||||
if ok and qkv_a_weight is not None and qkv_a_weight.dtype == torch.bfloat16
|
||||
else None
|
||||
)
|
||||
self._q_b_skinny_max = (
|
||||
q_b_dispatch.get(tuple(q_b_weight.shape))
|
||||
if ok and q_b_weight is not None and q_b_weight.dtype == torch.bfloat16
|
||||
else None
|
||||
)
|
||||
|
||||
# Lightning indexer uses its own RoPE; interleave maps to non-NeoX.
|
||||
self.indexer_rope_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
@@ -339,13 +374,32 @@ class DeepseekV32Attention(MLAAttention):
|
||||
is_neox_style=not getattr(config, "indexer_rope_interleave", False),
|
||||
)
|
||||
|
||||
def _decode_m_gemm(
|
||||
self, x: torch.Tensor, weight: torch.Tensor, skinny_max: int
|
||||
) -> torch.Tensor:
|
||||
"""x @ weight.T for decode M <= 16."""
|
||||
if x.shape[0] == 0:
|
||||
return torch.empty((0, weight.shape[0]), dtype=x.dtype, device=x.device)
|
||||
if x.shape[0] <= skinny_max:
|
||||
return ops.bf16_skinny_gemm(x, weight)
|
||||
out = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device)
|
||||
ops.dsv3_fused_a_gemm(out, x, weight.t())
|
||||
return out
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# Captured: A-projections (+ indexer A-GEMM on indexer layers).
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
if self._qkv_a_skinny_max is not None and hidden_states.shape[0] <= 16:
|
||||
qkv_lora = self._decode_m_gemm(
|
||||
hidden_states,
|
||||
self.fused_qkv_a_proj.weight,
|
||||
self._qkv_a_skinny_max,
|
||||
)
|
||||
else:
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
q_c, kv_c, k_pe = qkv_lora.split(
|
||||
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
@@ -398,7 +452,9 @@ class DeepseekV32Attention(MLAAttention):
|
||||
assert isinstance(slot_mapping, dict)
|
||||
mla_slot = slot_mapping.get(self.layer_name)
|
||||
|
||||
if self.indexer is not None:
|
||||
# skip_topk (index_share_for_mtp_iteration reuse mode): don't run the
|
||||
# indexer this step; keep the top-k already in the shared buffer.
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
has_indexer = True
|
||||
indexer_k_norm_w = self.indexer.k_norm.weight
|
||||
indexer_k_norm_bias = self.indexer.k_norm.bias
|
||||
@@ -451,12 +507,16 @@ class DeepseekV32Attention(MLAAttention):
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
)
|
||||
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
if self._q_b_skinny_max is not None and q_c.shape[0] <= 16:
|
||||
q = self._decode_m_gemm(q_c, self.q_b_proj.weight, self._q_b_skinny_max)
|
||||
else:
|
||||
q = self.q_b_proj(q_c)[0]
|
||||
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
|
||||
q_nope = q_nope.transpose(0, 1)
|
||||
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
|
||||
|
||||
if self.indexer is not None:
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
index_q = self.indexer.wq_b(q_c)[0]
|
||||
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
|
||||
else:
|
||||
@@ -478,10 +538,10 @@ class DeepseekV32Attention(MLAAttention):
|
||||
quantize_mqa=self._fp8_query,
|
||||
)
|
||||
|
||||
if self.indexer is not None:
|
||||
sparse_attn_indexer(
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
torch.ops.vllm.sparse_attn_indexer(
|
||||
q_c,
|
||||
self.indexer.k_cache.prefix,
|
||||
_encode_layer_name(self.indexer.k_cache.prefix),
|
||||
self.indexer.k_cache.kv_cache,
|
||||
index_q_fp8,
|
||||
None, # q_scale folded into weights on the fp8 path
|
||||
|
||||
@@ -2,13 +2,80 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
|
||||
# has_indexer=False path so the indexer args don't allocate every call.
|
||||
_DUMMY_CACHE: dict[tuple, torch.Tensor] = {}
|
||||
|
||||
|
||||
@torch.compiler.assume_constant_result
|
||||
def _can_use_fused_q_cutedsl() -> bool:
|
||||
return current_platform.has_device_capability(100) and has_cutedsl()
|
||||
|
||||
|
||||
@torch.compiler.assume_constant_result
|
||||
def _is_arch_support_pdl() -> bool:
|
||||
return (
|
||||
current_platform.has_device_capability(100)
|
||||
and current_platform.is_arch_support_pdl()
|
||||
)
|
||||
|
||||
|
||||
def _fused_q_cutedsl_impl(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
rope_cache: torch.Tensor,
|
||||
ql_nope: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
mqa_output: torch.Tensor,
|
||||
idx_q: torch.Tensor,
|
||||
idx_rope_cache: torch.Tensor,
|
||||
idx_weights: torch.Tensor,
|
||||
idx_weights_softmax_scale: float,
|
||||
idx_weights_head_scale: float,
|
||||
idx_q_fp8: torch.Tensor,
|
||||
idx_weights_out: torch.Tensor,
|
||||
has_indexer: bool,
|
||||
index_rope_interleave: bool,
|
||||
) -> None:
|
||||
from .ops.fused_q_cutedsl import fused_q_cutedsl
|
||||
|
||||
fused_q_cutedsl(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
idx_weights,
|
||||
idx_weights_softmax_scale,
|
||||
idx_weights_head_scale,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=index_rope_interleave,
|
||||
)
|
||||
|
||||
|
||||
def _fused_q_cutedsl_fake(*args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_q_cutedsl",
|
||||
op_func=_fused_q_cutedsl_impl,
|
||||
mutates_args=["mqa_output", "idx_q_fp8", "idx_weights_out"],
|
||||
fake_impl=_fused_q_cutedsl_fake,
|
||||
dispatch_key="CUDA",
|
||||
)
|
||||
|
||||
|
||||
def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
|
||||
key = (shape, dtype, device)
|
||||
t = _DUMMY_CACHE.get(key)
|
||||
@@ -147,9 +214,14 @@ def _fused_norm_rope_kernel(
|
||||
TOPK_BLOCK_SIZE: tl.constexpr,
|
||||
HAS_INDEXER: tl.constexpr,
|
||||
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
tok_idx = tl.program_id(1)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
if pid == 3:
|
||||
if not HAS_INDEXER:
|
||||
# Shared layer: reuse the previous indexer layer's top-k; do not
|
||||
@@ -166,6 +238,17 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
return
|
||||
|
||||
if pid == 2:
|
||||
# Q RMS norm does not depend on cache availability. In particular,
|
||||
# profiling uses a negative dummy slot mapping but still consumes Q.
|
||||
q_block = tl.arange(0, Q_BLOCK_SIZE)
|
||||
q_mask = q_block < Q_DIM
|
||||
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
|
||||
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
|
||||
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
|
||||
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
|
||||
return
|
||||
|
||||
if slot_mapping_ptr is None:
|
||||
# Memory profiling run.
|
||||
return
|
||||
@@ -174,15 +257,7 @@ def _fused_norm_rope_kernel(
|
||||
# Padding
|
||||
return
|
||||
|
||||
if pid == 2:
|
||||
# Q RMS norm
|
||||
q_block = tl.arange(0, Q_BLOCK_SIZE)
|
||||
q_mask = q_block < Q_DIM
|
||||
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
|
||||
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
|
||||
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
|
||||
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
|
||||
elif pid == 1:
|
||||
if pid == 1:
|
||||
# KV RMS Norm + KV RoPE + MLA concat_and_cache.
|
||||
# Merged so the normed kv_c and RoPE'd k_pe can be written
|
||||
# to the MLA KV cache directly without a separate kernel.
|
||||
@@ -211,9 +286,6 @@ def _fused_norm_rope_kernel(
|
||||
r2 = x2 * cos + x1 * sin
|
||||
|
||||
# MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache.
|
||||
if mla_cache_entry_stride == 0:
|
||||
return
|
||||
|
||||
mla_block_size = mla_cache_block_stride // mla_cache_entry_stride
|
||||
mla_block_idx = slot_idx // mla_block_size
|
||||
mla_block_off = slot_idx % mla_block_size
|
||||
@@ -363,7 +435,7 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
|
||||
|
||||
def fused_norm_rope(
|
||||
def _fused_norm_rope_impl(
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_rms_norm_w: torch.Tensor,
|
||||
@@ -465,12 +537,17 @@ def fused_norm_rope(
|
||||
# Dummy values — pid 2 will skip the MLA cache write because
|
||||
# slot_mapping is all -1.
|
||||
mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device)
|
||||
mla_block_stride = 0
|
||||
mla_entry_stride = 0
|
||||
# Torch's Triton HOP mutation analysis compiles every program-id
|
||||
# branch even though the dummy slot mapping returns before cache
|
||||
# access. Keep the unused strides nonzero so that analysis can lower
|
||||
# the block-size calculation safely.
|
||||
mla_block_stride = 1
|
||||
mla_entry_stride = 1
|
||||
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
|
||||
if q_c_out is None:
|
||||
q_c_out = torch.empty_like(q_c)
|
||||
use_pdl = _is_arch_support_pdl()
|
||||
_fused_norm_rope_kernel[(4, num_tokens)](
|
||||
positions,
|
||||
# Q RMS norm
|
||||
@@ -529,6 +606,157 @@ def fused_norm_rope(
|
||||
TOPK_BLOCK_SIZE=1024,
|
||||
HAS_INDEXER=has_indexer,
|
||||
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
||||
USE_PDL=use_pdl,
|
||||
launch_pdl=use_pdl,
|
||||
)
|
||||
return q_c_out
|
||||
|
||||
|
||||
def _fused_norm_rope_op(
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_rms_norm_w: torch.Tensor,
|
||||
q_rms_eps: float,
|
||||
kv_c: torch.Tensor,
|
||||
kv_rms_norm_w: torch.Tensor,
|
||||
kv_rms_eps: float,
|
||||
k_pe: torch.Tensor,
|
||||
k_rope_cos_sin_cache: torch.Tensor,
|
||||
index_k: torch.Tensor | None,
|
||||
index_k_layer_norm_w: torch.Tensor | None,
|
||||
index_k_layer_norm_bias: torch.Tensor | None,
|
||||
index_k_layer_norm_eps: float,
|
||||
index_k_rope_cos_sin_cache: torch.Tensor | None,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None,
|
||||
indexer_k_cache: torch.Tensor | None,
|
||||
mla_kv_cache: torch.Tensor | None,
|
||||
mla_kv_cache_dtype: str,
|
||||
mla_k_scale: torch.Tensor | None,
|
||||
has_indexer: bool,
|
||||
index_rope_interleave: bool,
|
||||
q_c_out: torch.Tensor,
|
||||
) -> None:
|
||||
_fused_norm_rope_impl(
|
||||
positions,
|
||||
q_c,
|
||||
q_rms_norm_w,
|
||||
q_rms_eps,
|
||||
kv_c,
|
||||
kv_rms_norm_w,
|
||||
kv_rms_eps,
|
||||
k_pe,
|
||||
k_rope_cos_sin_cache,
|
||||
index_k,
|
||||
index_k_layer_norm_w,
|
||||
index_k_layer_norm_bias,
|
||||
index_k_layer_norm_eps,
|
||||
index_k_rope_cos_sin_cache,
|
||||
topk_indices_buffer,
|
||||
slot_mapping=slot_mapping,
|
||||
indexer_k_cache=indexer_k_cache,
|
||||
mla_kv_cache=mla_kv_cache,
|
||||
mla_kv_cache_dtype=mla_kv_cache_dtype,
|
||||
mla_k_scale=mla_k_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=index_rope_interleave,
|
||||
q_c_out=q_c_out,
|
||||
)
|
||||
|
||||
|
||||
def _fused_norm_rope_fake(
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_rms_norm_w: torch.Tensor,
|
||||
q_rms_eps: float,
|
||||
kv_c: torch.Tensor,
|
||||
kv_rms_norm_w: torch.Tensor,
|
||||
kv_rms_eps: float,
|
||||
k_pe: torch.Tensor,
|
||||
k_rope_cos_sin_cache: torch.Tensor,
|
||||
index_k: torch.Tensor | None,
|
||||
index_k_layer_norm_w: torch.Tensor | None,
|
||||
index_k_layer_norm_bias: torch.Tensor | None,
|
||||
index_k_layer_norm_eps: float,
|
||||
index_k_rope_cos_sin_cache: torch.Tensor | None,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None,
|
||||
indexer_k_cache: torch.Tensor | None,
|
||||
mla_kv_cache: torch.Tensor | None,
|
||||
mla_kv_cache_dtype: str,
|
||||
mla_k_scale: torch.Tensor | None,
|
||||
has_indexer: bool,
|
||||
index_rope_interleave: bool,
|
||||
q_c_out: torch.Tensor,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_norm_rope_deepseek_v32",
|
||||
op_func=_fused_norm_rope_op,
|
||||
mutates_args=[
|
||||
"topk_indices_buffer",
|
||||
"indexer_k_cache",
|
||||
"mla_kv_cache",
|
||||
"q_c_out",
|
||||
],
|
||||
fake_impl=_fused_norm_rope_fake,
|
||||
dispatch_key="CUDA",
|
||||
)
|
||||
|
||||
|
||||
def fused_norm_rope(
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_rms_norm_w: torch.Tensor,
|
||||
q_rms_eps: float,
|
||||
kv_c: torch.Tensor,
|
||||
kv_rms_norm_w: torch.Tensor,
|
||||
kv_rms_eps: float,
|
||||
k_pe: torch.Tensor,
|
||||
k_rope_cos_sin_cache: torch.Tensor,
|
||||
index_k: torch.Tensor | None,
|
||||
index_k_layer_norm_w: torch.Tensor | None,
|
||||
index_k_layer_norm_bias: torch.Tensor | None,
|
||||
index_k_layer_norm_eps: float,
|
||||
index_k_rope_cos_sin_cache: torch.Tensor | None,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
indexer_k_cache: torch.Tensor | None = None,
|
||||
mla_kv_cache: torch.Tensor | None = None,
|
||||
mla_kv_cache_dtype: str = "auto",
|
||||
mla_k_scale: torch.Tensor | None = None,
|
||||
has_indexer: bool = True,
|
||||
index_rope_interleave: bool = False,
|
||||
q_c_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if q_c_out is None:
|
||||
q_c_out = torch.empty_like(q_c)
|
||||
torch.ops.vllm.fused_norm_rope_deepseek_v32(
|
||||
positions,
|
||||
q_c,
|
||||
q_rms_norm_w,
|
||||
q_rms_eps,
|
||||
kv_c,
|
||||
kv_rms_norm_w,
|
||||
kv_rms_eps,
|
||||
k_pe,
|
||||
k_rope_cos_sin_cache,
|
||||
index_k,
|
||||
index_k_layer_norm_w,
|
||||
index_k_layer_norm_bias,
|
||||
index_k_layer_norm_eps,
|
||||
index_k_rope_cos_sin_cache,
|
||||
topk_indices_buffer,
|
||||
slot_mapping,
|
||||
indexer_k_cache,
|
||||
mla_kv_cache,
|
||||
mla_kv_cache_dtype,
|
||||
mla_k_scale,
|
||||
has_indexer,
|
||||
index_rope_interleave,
|
||||
q_c_out,
|
||||
)
|
||||
return q_c_out
|
||||
|
||||
@@ -582,10 +810,14 @@ def _fused_q_kernel(
|
||||
HAS_INDEXER: tl.constexpr,
|
||||
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
||||
QUANTIZE_MQA: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
tok_idx = tl.program_id(1)
|
||||
head_idx = tl.program_id(2)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
if pid == 2:
|
||||
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
|
||||
@@ -773,10 +1005,12 @@ def fused_q(
|
||||
``(ql_nope, q_pe)`` tuple the backend expects.
|
||||
"""
|
||||
assert positions.ndim == 1
|
||||
assert positions.dtype == torch.int64
|
||||
assert q_pe.ndim == 3
|
||||
assert q_pe_cos_sin_cache.ndim == 2
|
||||
assert ql_nope.ndim == 3
|
||||
assert ql_nope.shape[:2] == q_pe.shape[:2]
|
||||
assert q_scale.dtype == torch.float32 and q_scale.numel() == 1
|
||||
|
||||
num_tokens = positions.shape[0]
|
||||
num_q_heads = q_pe.shape[1]
|
||||
@@ -794,7 +1028,18 @@ def fused_q(
|
||||
assert index_weights is not None
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
index_q_head_dim = index_q.shape[2]
|
||||
grid_heads = max(mqa_grid_heads, num_index_q_heads)
|
||||
use_cutedsl = False
|
||||
if _can_use_fused_q_cutedsl():
|
||||
from .ops.fused_q_cutedsl import is_fused_q_cutedsl_supported
|
||||
|
||||
use_cutedsl = is_fused_q_cutedsl_supported(
|
||||
q_pe,
|
||||
index_q,
|
||||
ql_nope,
|
||||
has_indexer=has_indexer,
|
||||
quantize_mqa=quantize_mqa,
|
||||
)
|
||||
grid_heads = max(mqa_grid_heads, num_index_q_heads if has_indexer else 1)
|
||||
if quantize_mqa:
|
||||
# fp8 path: pack [ql_nope; q_pe] into a single fp8 tensor.
|
||||
mqa_q_fp8 = torch.empty(
|
||||
@@ -815,6 +1060,27 @@ def fused_q(
|
||||
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn)
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
if use_cutedsl:
|
||||
torch.ops.vllm.fused_q_cutedsl(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_cos_sin_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_q,
|
||||
index_q,
|
||||
index_q_cos_sin_cache,
|
||||
index_weights,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=index_rope_interleave,
|
||||
)
|
||||
return index_q_fp8, index_weights_out, mqa_q
|
||||
|
||||
use_pdl = _is_arch_support_pdl()
|
||||
_fused_q_kernel[(3, num_tokens, grid_heads)](
|
||||
positions,
|
||||
q_pe,
|
||||
@@ -856,6 +1122,8 @@ def fused_q(
|
||||
HAS_INDEXER=has_indexer,
|
||||
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
||||
QUANTIZE_MQA=quantize_mqa,
|
||||
USE_PDL=use_pdl,
|
||||
launch_pdl=use_pdl,
|
||||
# num_warps=1 is optimal here: each program is a single 128-element
|
||||
# rope+quant, so the kernel is program-count/occupancy bound, not
|
||||
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).
|
||||
|
||||
@@ -6,6 +6,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
@@ -39,6 +40,7 @@ from vllm.model_executor.models.utils import (
|
||||
sequence_parallel_chunk,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import register_phys_shadow
|
||||
|
||||
from .attention import DeepseekV32Attention
|
||||
from .fused_ops import fused_allreduce_rms_norm
|
||||
@@ -177,6 +179,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class DeepseekV32Model(torch.nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
@@ -200,6 +203,7 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
register_phys_shadow(topk_indices_buffer)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
|
||||
@@ -6,9 +6,9 @@ from collections.abc import Callable, Iterable
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -40,7 +40,12 @@ from vllm.model_executor.models.utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
phys_shadow,
|
||||
register_phys_shadow,
|
||||
)
|
||||
|
||||
from .fused_ops import fused_allreduce_rms_norm
|
||||
from .kernels import fused_eh_norm
|
||||
from .model import DeepseekV32DecoderLayer
|
||||
|
||||
@@ -56,6 +61,19 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
# bf16 skinny GEMM for the eh_proj, B300+B200 measured: GLM-5.2
|
||||
# (6144, 12288) wins at M <= 3, DSv3.2 (7168, 14336) at M <= 2;
|
||||
# cuBLAS holds above (the 151/205MB weights stream at ~5.7TB/s).
|
||||
eh_dispatch = {(6144, 12288): 3, (7168, 14336): 2}
|
||||
eh_weight = getattr(self.eh_proj, "weight", None)
|
||||
self._eh_skinny_max = (
|
||||
eh_dispatch.get(tuple(eh_weight.shape), 0)
|
||||
if current_platform.is_device_capability_family(100)
|
||||
and hasattr(torch.ops._C, "bf16_skinny_gemm")
|
||||
and eh_weight is not None
|
||||
and eh_weight.dtype == torch.bfloat16
|
||||
else 0
|
||||
)
|
||||
|
||||
topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
@@ -63,6 +81,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
register_phys_shadow(topk_indices_buffer)
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
@@ -91,7 +110,13 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
if (
|
||||
eh_input.shape[0] <= self._eh_skinny_max
|
||||
and eh_input.dtype == torch.bfloat16
|
||||
):
|
||||
hidden_states = ops.bf16_skinny_gemm(eh_input, self.eh_proj.weight)
|
||||
else:
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
)
|
||||
@@ -101,9 +126,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
positions.shape[0],
|
||||
is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe,
|
||||
)
|
||||
if not self.mtp_block.use_sequence_parallel_moe:
|
||||
# Without sequence parallelism, the MoE output is left un-reduced.
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
# Recycle the POST-final-norm hidden into the next draft step. The
|
||||
# residual-add is fused into the final RMSNorm so it is computed
|
||||
# exactly once, and the result is returned for both tuple positions:
|
||||
@@ -114,7 +136,14 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
# is understood by both the V2 speculator (isinstance-tuple check) and
|
||||
# the legacy proposer (model_returns_tuple is True for the
|
||||
# DeepSeekMTPModel architecture).
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
if self.mtp_block.use_sequence_parallel_moe:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
else:
|
||||
# The MoE output is left un-reduced; fuse its all-reduce into the
|
||||
# final norm, as the main model does at layer boundaries.
|
||||
hidden_states, _ = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.shared_head.norm
|
||||
)
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
@@ -157,6 +186,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
|
||||
topk_indices_buffer = self_attn.topk_indices_buffer
|
||||
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
|
||||
# The physical-index shadow mirrors this buffer row-wise
|
||||
# (shadow[j] == convert(logical[j])); re-arrange it the same
|
||||
# way or skip_topk draft steps read the step-0 multi-token
|
||||
# layout's rows for the wrong tokens/requests.
|
||||
shadow = phys_shadow(topk_indices_buffer)
|
||||
if shadow is not None:
|
||||
phys, seq = shadow
|
||||
phys[:num_slots] = phys[slot_ids]
|
||||
seq[:num_slots] = seq[slot_ids]
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
@@ -192,6 +230,19 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
# second RMSNorm.
|
||||
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# Greedy draft token ids via vocab-parallel local argmax: skips the
|
||||
# full-vocab logits all-gather (use_local_argmax_reduction).
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
return self.logits_processor.get_top_tokens(
|
||||
mtp_layer.shared_head.head, hidden_states
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
@@ -240,6 +291,25 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
# The V2 speculator calls this without spec_step_idx; that is only
|
||||
# correct while a single MTP layer is cycled for every draft step
|
||||
# (num_nextn_predict_layers == 1, as in GLM-5.2/DSV3.2). Guard the
|
||||
# assumption so a future multi-layer MTP fails loudly instead of
|
||||
# silently using step 0's head for every step.
|
||||
if spec_step_idx is None:
|
||||
assert self.model.num_mtp_layers == 1, (
|
||||
"get_top_tokens called without spec_step_idx on a "
|
||||
"multi-layer MTP; thread the draft step index through "
|
||||
"the speculator."
|
||||
)
|
||||
spec_step_idx = 0
|
||||
return self.model.get_top_tokens(hidden_states, spec_step_idx)
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,532 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float8E4M3FN, Float32, Int64, Uint8, Uint16, Uint32
|
||||
|
||||
from vllm.cute_utils import TORCH_TO_CUTE_DTYPE, cvt
|
||||
|
||||
|
||||
def _make_fake_tensor(dtype, shape, divisibility):
|
||||
stride = tuple(
|
||||
1 if i == len(shape) - 1 else cute.sym_int64(divisibility=divisibility)
|
||||
for i in range(len(shape))
|
||||
)
|
||||
return cute.runtime.make_fake_tensor(
|
||||
dtype,
|
||||
shape,
|
||||
stride,
|
||||
assumed_align=divisibility * dtype.width // 8,
|
||||
)
|
||||
|
||||
|
||||
def is_fused_q_cutedsl_supported(
|
||||
q_pe: torch.Tensor,
|
||||
index_q: torch.Tensor | None,
|
||||
ql_nope: torch.Tensor,
|
||||
*,
|
||||
has_indexer: bool,
|
||||
quantize_mqa: bool,
|
||||
) -> bool:
|
||||
if not (
|
||||
quantize_mqa
|
||||
and q_pe.dtype == ql_nope.dtype == torch.bfloat16
|
||||
and q_pe.shape[-1] == 64
|
||||
and ql_nope.shape[-1] == 512
|
||||
):
|
||||
return False
|
||||
return not has_indexer or (
|
||||
index_q is not None
|
||||
and index_q.dtype == torch.bfloat16
|
||||
and index_q.shape[1] % 16 == 0
|
||||
and index_q.shape[-1] == 128
|
||||
)
|
||||
|
||||
|
||||
def fused_q_cutedsl(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
rope_cache: torch.Tensor,
|
||||
ql_nope: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
mqa_output: torch.Tensor,
|
||||
idx_q: torch.Tensor,
|
||||
idx_rope_cache: torch.Tensor,
|
||||
idx_weights: torch.Tensor,
|
||||
idx_weights_softmax_scale: float,
|
||||
idx_weights_head_scale: float,
|
||||
idx_q_fp8: torch.Tensor,
|
||||
idx_weights_out: torch.Tensor,
|
||||
has_indexer: bool = True,
|
||||
index_rope_interleave: bool = True,
|
||||
) -> None:
|
||||
_, num_heads, rope_dim = q_pe.shape
|
||||
_, _, nope_dim = ql_nope.shape
|
||||
_, num_idx_heads, idx_dim = idx_q.shape
|
||||
|
||||
if has_indexer:
|
||||
idx_rope_type = TORCH_TO_CUTE_DTYPE[idx_rope_cache.dtype]
|
||||
idx_weights_type = TORCH_TO_CUTE_DTYPE[idx_weights.dtype]
|
||||
else:
|
||||
idx_dim = num_idx_heads = 0
|
||||
idx_q = idx_rope_cache = idx_q_fp8 = None
|
||||
idx_weights = idx_weights_out = None
|
||||
idx_rope_type = idx_weights_type = None
|
||||
|
||||
rope_type = TORCH_TO_CUTE_DTYPE[rope_cache.dtype]
|
||||
compiled = FusedQKernel.compile(
|
||||
rope_dim,
|
||||
nope_dim,
|
||||
num_heads,
|
||||
rope_type,
|
||||
idx_dim,
|
||||
num_idx_heads,
|
||||
idx_rope_type,
|
||||
idx_weights_type,
|
||||
index_rope_interleave,
|
||||
)
|
||||
compiled(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale.view(1),
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
float(idx_weights_softmax_scale * idx_weights_head_scale),
|
||||
)
|
||||
|
||||
|
||||
class FusedQKernel:
|
||||
def __init__(
|
||||
self,
|
||||
rope_dim: int,
|
||||
nope_dim: int,
|
||||
num_heads: int,
|
||||
idx_dim: int,
|
||||
num_idx_heads: int,
|
||||
index_rope_interleave: bool,
|
||||
) -> None:
|
||||
assert rope_dim == 64
|
||||
assert nope_dim == 512
|
||||
assert idx_dim in (128, 0)
|
||||
|
||||
self.rope_dim = rope_dim
|
||||
self.nope_dim = nope_dim
|
||||
self.num_heads = num_heads
|
||||
self.idx_dim = idx_dim
|
||||
self.num_idx_heads = num_idx_heads
|
||||
self.index_rope_interleave = index_rope_interleave
|
||||
|
||||
# mqa: rope_dim=64, nope_dim=512, num_heads=64/TP
|
||||
# indexer: rope_dim=64, nope_dim=64, num_heads=32
|
||||
|
||||
self.num_warps = 4
|
||||
assert num_heads % self.num_warps == 0
|
||||
assert num_idx_heads % (4 * self.num_warps) == 0
|
||||
self.num_ctas_per_tok = num_heads // self.num_warps
|
||||
self.num_ctas_per_idx_tok = num_idx_heads // (4 * self.num_warps)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
stream: CUstream,
|
||||
):
|
||||
num_tokens = positions.shape[0]
|
||||
if cutlass.const_expr(self.idx_dim == 0):
|
||||
grid = (num_tokens, self.num_ctas_per_tok, 1)
|
||||
else:
|
||||
num_mqa_ctas = num_tokens * self.num_ctas_per_tok
|
||||
num_idx_ctas = num_tokens * self.num_ctas_per_idx_tok
|
||||
grid = (num_mqa_ctas + num_idx_ctas, 1, 1)
|
||||
|
||||
self.kernel(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
weight_scale,
|
||||
).launch(
|
||||
grid=grid,
|
||||
block=(self.num_warps * 32, 1, 1),
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
q_pe_rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_q_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
):
|
||||
if cutlass.const_expr(self.idx_dim == 0):
|
||||
token_id, group_id, _ = cute.arch.block_idx()
|
||||
self.mqa(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
token_id,
|
||||
group_id,
|
||||
)
|
||||
else:
|
||||
# CTA-specialization
|
||||
bid, _, _ = cute.arch.block_idx()
|
||||
num_mqa_ctas = positions.shape[0] * self.num_ctas_per_tok
|
||||
if bid < num_mqa_ctas:
|
||||
self.mqa(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
bid // self.num_ctas_per_tok,
|
||||
bid % self.num_ctas_per_tok,
|
||||
)
|
||||
else:
|
||||
bid -= num_mqa_ctas
|
||||
self.indexer(
|
||||
positions,
|
||||
idx_q,
|
||||
idx_q_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
weight_scale,
|
||||
bid // self.num_ctas_per_idx_tok,
|
||||
bid % self.num_ctas_per_idx_tok,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def mqa(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
q_pe_rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
token_id,
|
||||
group_id,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
head_id = group_id * self.num_warps + warp_id
|
||||
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
pos = positions[token_id]
|
||||
inv_scale = 1.0 / q_scale[0]
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_32B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256)
|
||||
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
|
||||
cp_2B = cute.make_copy_atom(cp_op, Uint8, num_bits_per_copy=16)
|
||||
|
||||
##### issue all loads asap #####
|
||||
rQ_nope_bf16 = cute.make_rmem_tensor(16, BFloat16)
|
||||
rQ_rope_bf16 = cute.make_rmem_tensor(2, BFloat16)
|
||||
|
||||
src_ql_nope = cute.local_tile(
|
||||
ql_nope[token_id, head_id, None], (16,), (lane_id,)
|
||||
)
|
||||
src_q_rope = cute.local_tile(q_pe[token_id, head_id, None], (2,), (lane_id,))
|
||||
cute.copy(cp_32B, src_ql_nope, rQ_nope_bf16)
|
||||
cute.copy(cp_4B, src_q_rope, rQ_rope_bf16)
|
||||
|
||||
rCos_raw = q_pe_rope_cache[pos, 0 + lane_id]
|
||||
rSin_raw = q_pe_rope_cache[pos, 32 + lane_id]
|
||||
|
||||
##### process NoPE #####
|
||||
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16).load() * inv_scale
|
||||
rQ_nope_f8 = cute.make_rmem_tensor(16, Float8E4M3FN)
|
||||
rQ_nope_f8.store(rQ_nope_f32.to(Float8E4M3FN))
|
||||
dst_Q_nope = cute.local_tile(
|
||||
mqa_output[token_id, head_id, None], (16,), (lane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, rQ_nope_f8, dst_Q_nope)
|
||||
|
||||
##### process RoPE ######
|
||||
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
|
||||
rCos = rCos_raw.to(Float32)
|
||||
rSin = rSin_raw.to(Float32)
|
||||
r0 = (rQ_rope_f32[0] * rCos - rQ_rope_f32[1] * rSin) * inv_scale
|
||||
r1 = (rQ_rope_f32[1] * rCos + rQ_rope_f32[0] * rSin) * inv_scale
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
# TensorSSA fp32->fp8 cvt has a bug. rely on direct PTX
|
||||
rQ_rope_f8 = cute.make_rmem_tensor(2, Float8E4M3FN)
|
||||
cute.recast_tensor(rQ_rope_f8, Uint16)[0] = cvt.fp32x2_to_fp8x2(r0, r1)
|
||||
dst_Q_rope = cute.local_tile(
|
||||
mqa_output[token_id, head_id, None], (2,), (256 + lane_id,)
|
||||
)
|
||||
cute.copy(cp_2B, rQ_rope_f8, dst_Q_rope)
|
||||
|
||||
@cute.jit
|
||||
def indexer(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_q_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
token_id,
|
||||
group_id,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
subwarp_id = tid // 8
|
||||
sublane_id = tid % 8
|
||||
head_id = group_id * (4 * self.num_warps) + subwarp_id
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
pos = positions[token_id]
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
cp_8B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
|
||||
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
|
||||
|
||||
##### issue all loads first #####
|
||||
rQ_rope_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
src_idx_q_rope = cute.local_tile(
|
||||
idx_q[token_id, head_id, None], (8,), (sublane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, src_idx_q_rope, rQ_rope_bf16)
|
||||
else:
|
||||
src_idx_q_rope = cute.zipped_divide(
|
||||
idx_q[token_id, head_id, None], (4,)
|
||||
) # (4,32)
|
||||
cute.copy(
|
||||
cp_8B,
|
||||
src_idx_q_rope[None, 0 + sublane_id],
|
||||
cute.local_tile(rQ_rope_bf16, (4,), (0,)),
|
||||
)
|
||||
cute.copy(
|
||||
cp_8B,
|
||||
src_idx_q_rope[None, 8 + sublane_id],
|
||||
cute.local_tile(rQ_rope_bf16, (4,), (1,)),
|
||||
)
|
||||
|
||||
rQ_nope_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
src_idx_q_nope = cute.local_tile(
|
||||
idx_q[token_id, head_id, None], (8,), (8 + sublane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, src_idx_q_nope, rQ_nope_bf16)
|
||||
|
||||
rCos_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
|
||||
rSin_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
|
||||
rope_cache_view = cute.zipped_divide(idx_q_rope_cache[pos, None], (4,))
|
||||
|
||||
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
|
||||
cute.copy(cp_16B, rope_cache_view[None, sublane_id], rCos_raw)
|
||||
cute.copy(cp_16B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
|
||||
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
|
||||
cute.copy(cp_8B, rope_cache_view[None, sublane_id], rCos_raw)
|
||||
cute.copy(cp_8B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
|
||||
|
||||
# unpack to FP32
|
||||
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
|
||||
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16)
|
||||
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
|
||||
rCos = rCos_raw
|
||||
rSin = rSin_raw
|
||||
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
|
||||
rCos = cvt.bf16x2_to_fp32x2(rCos_raw)
|
||||
rSin = cvt.bf16x2_to_fp32x2(rSin_raw)
|
||||
|
||||
# apply rope
|
||||
for i in cutlass.range_constexpr(4):
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
r0 = rQ_rope_f32[i * 2 + 0] * rCos[i] - rQ_rope_f32[i * 2 + 1] * rSin[i]
|
||||
r1 = rQ_rope_f32[i * 2 + 1] * rCos[i] + rQ_rope_f32[i * 2 + 0] * rSin[i]
|
||||
rQ_rope_f32[i * 2 + 0] = r0
|
||||
rQ_rope_f32[i * 2 + 1] = r1
|
||||
else:
|
||||
r0 = rQ_rope_f32[0 + i] * rCos[i] - rQ_rope_f32[4 + i] * rSin[i]
|
||||
r1 = rQ_rope_f32[4 + i] * rCos[i] + rQ_rope_f32[0 + i] * rSin[i]
|
||||
rQ_rope_f32[0 + i] = r0
|
||||
rQ_rope_f32[4 + i] = r1
|
||||
|
||||
# amax
|
||||
amax = Float32(1e-4)
|
||||
for i in cutlass.range_constexpr(8):
|
||||
amax = cute.arch.fmax(amax, cute.math.absf(rQ_rope_f32[i]))
|
||||
amax = cute.arch.fmax(amax, cute.math.absf(rQ_nope_f32[i]))
|
||||
|
||||
# warp reduction among 8 lanes
|
||||
for i in cutlass.range_constexpr(3):
|
||||
other = cute.arch.shuffle_sync_bfly(amax, 1 << i)
|
||||
amax = cute.arch.fmax(amax, other)
|
||||
|
||||
# compute scale from amax
|
||||
# exp2(ceil(log2(scale))) via bit manipulation
|
||||
scale = amax * (1.0 / 448.0)
|
||||
bits = scale.bitcast(Uint32)
|
||||
exp_bits = (bits + Uint32(0x007FFFFF)) & Uint32(0x7F800000)
|
||||
scale = exp_bits.bitcast(Float32)
|
||||
inv_scale = (Uint32(0x7F000000) - exp_bits).bitcast(Float32)
|
||||
|
||||
for i in cutlass.range_constexpr(8):
|
||||
rQ_nope_f32[i] *= inv_scale
|
||||
rQ_rope_f32[i] *= inv_scale
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
# quantize and store
|
||||
rQ_nope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
|
||||
rQ_nope_f8.store(rQ_nope_f32.load().to(Float8E4M3FN))
|
||||
dst_idx_q_nope = cute.local_tile(
|
||||
idx_q_fp8[token_id, head_id, None], (8,), (8 + sublane_id,)
|
||||
)
|
||||
cute.copy(cp_8B, rQ_nope_f8, dst_idx_q_nope)
|
||||
|
||||
rQ_rope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
|
||||
rQ_rope_f8.store(rQ_rope_f32.load().to(Float8E4M3FN))
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
dst_idx_q_rope = cute.local_tile(
|
||||
idx_q_fp8[token_id, head_id, None], (8,), (sublane_id,)
|
||||
)
|
||||
cute.copy(cp_8B, rQ_rope_f8, dst_idx_q_rope)
|
||||
else:
|
||||
dst_idx_q_rope = cute.zipped_divide(
|
||||
idx_q_fp8[token_id, head_id, None], (4,)
|
||||
) # (4,32)
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
cute.local_tile(rQ_rope_f8, (4,), (0,)),
|
||||
dst_idx_q_rope[None, 0 + sublane_id],
|
||||
)
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
cute.local_tile(rQ_rope_f8, (4,), (1,)),
|
||||
dst_idx_q_rope[None, 8 + sublane_id],
|
||||
)
|
||||
|
||||
# scale indexer weights
|
||||
if sublane_id == 0:
|
||||
w = idx_weights[token_id, head_id].to(Float32)
|
||||
idx_weights_out[token_id, head_id] = w * scale * weight_scale
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(
|
||||
rope_dim: int,
|
||||
nope_dim: int,
|
||||
num_heads: int,
|
||||
rope_type: type[cutlass.Numeric],
|
||||
idx_dim: int,
|
||||
num_idx_heads: int,
|
||||
idx_rope_type: type[cutlass.Numeric] | None,
|
||||
idx_weights_type: type[cutlass.Numeric] | None,
|
||||
index_rope_interleave: bool,
|
||||
):
|
||||
num_tokens = cute.sym_int()
|
||||
max_pos = cute.sym_int()
|
||||
|
||||
positions = _make_fake_tensor(Int64, (num_tokens,), divisibility=1)
|
||||
q_pe = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16
|
||||
)
|
||||
rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8)
|
||||
ql_nope = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
|
||||
)
|
||||
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4)
|
||||
mqa_output = _make_fake_tensor(
|
||||
Float8E4M3FN,
|
||||
(num_tokens, num_heads, nope_dim + rope_dim),
|
||||
divisibility=16,
|
||||
)
|
||||
|
||||
if idx_rope_type is not None:
|
||||
index_q = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_idx_heads, idx_dim), divisibility=16
|
||||
)
|
||||
index_rope_cache = _make_fake_tensor(
|
||||
idx_rope_type, (max_pos, rope_dim), divisibility=8
|
||||
)
|
||||
index_weights = _make_fake_tensor(
|
||||
idx_weights_type, (num_tokens, num_idx_heads), divisibility=8
|
||||
)
|
||||
index_q_fp8 = _make_fake_tensor(
|
||||
Float8E4M3FN, (num_tokens, num_idx_heads, idx_dim), divisibility=16
|
||||
)
|
||||
index_weights_out = _make_fake_tensor(
|
||||
Float32, (num_tokens, num_idx_heads), divisibility=4
|
||||
)
|
||||
else:
|
||||
index_q = index_rope_cache = index_q_fp8 = None
|
||||
index_weights = index_weights_out = None
|
||||
|
||||
kernel = FusedQKernel(
|
||||
rope_dim,
|
||||
nope_dim,
|
||||
num_heads,
|
||||
idx_dim,
|
||||
num_idx_heads,
|
||||
index_rope_interleave,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
index_q,
|
||||
index_rope_cache,
|
||||
index_weights,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
Float32(0.0),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
Reference in New Issue
Block a user