[ROCm][Model][Bugfix] Enable GLM-5.2-MXFP4 on the deepseek_v32 path and fix sparse attention correctness (#51915)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Signed-off-by: Jack Hu <jack.hu@amd.com> Signed-off-by: Douglas Lehr <Doug.Lehr@amd.com> Co-authored-by: James E T Smith <jamesETsmith@users.noreply.github.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Douglas Lehr <Doug.Lehr@amd.com>
This commit is contained in:
co-authored by
James E T Smith
Douglas Lehr
Douglas Lehr
parent
9646f53064
commit
1c0eee919d
@@ -7,9 +7,9 @@ MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
|
||||
attend. The same model code serves any DSA checkpoint, including GLM-5.2
|
||||
(``glm_moe_dsa``), which reuses this architecture.
|
||||
|
||||
The CUDA implementation selects capability-specific kernels internally and
|
||||
falls back when an optimization is unavailable. Other platforms use the
|
||||
generic implementation by default.
|
||||
The CUDA and ROCm implementations select capability-specific kernels
|
||||
internally and fall back when an optimization is unavailable. Other platforms
|
||||
use the generic implementation by default.
|
||||
"""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
@@ -20,8 +20,12 @@ if current_platform.is_cuda():
|
||||
from .nvidia.model import DeepseekV32ForCausalLM
|
||||
from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
|
||||
from .nvidia.mtp import DeepseekV32MTP
|
||||
elif current_platform.is_rocm():
|
||||
from .amd.model import DeepseekV32ForCausalLM
|
||||
from .amd.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
|
||||
from .amd.mtp import DeepseekV32MTP
|
||||
else:
|
||||
# ROCm, XPU, and CPU keep the generic implementation.
|
||||
# XPU and CPU keep the generic implementation.
|
||||
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
|
||||
|
||||
@@ -3,12 +3,14 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
@@ -16,21 +18,15 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32MLASparseBackend(ROCMAiterMLASparseBackend):
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list:
|
||||
return [16, 32]
|
||||
|
||||
|
||||
class DeepseekV32ROCmIndexerBackend(DeepseekV32IndexerBackend):
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list:
|
||||
return [16, 32]
|
||||
|
||||
|
||||
class DeepseekV32ROCmIndexerCache(DeepseekV32IndexerCache):
|
||||
def get_attn_backend(self):
|
||||
return DeepseekV32ROCmIndexerBackend
|
||||
return DeepseekV32IndexerBackend
|
||||
|
||||
@property
|
||||
def uses_shuffled_layout(self) -> bool:
|
||||
# aiter's gather/insert pair shuffles the cache above block size 1:
|
||||
# [n_blocks, blk/16, head_dim/16, 16, 16] instead of [n_blocks, blk, head_dim].
|
||||
return self.kv_cache.ndim == 3 and self.kv_cache.shape[1] != 1
|
||||
|
||||
|
||||
class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
|
||||
@@ -46,7 +42,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
config,
|
||||
prefix,
|
||||
topk_indices_buffer,
|
||||
attn_backend=DeepseekV32MLASparseBackend,
|
||||
attn_backend=ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
self.indexer_op: SparseAttnIndexer | None = None
|
||||
@@ -100,14 +96,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
q_nope = q_nope.transpose(0, 1) # (N, tokens, P)
|
||||
|
||||
if self.is_aiter_triton_fp4_bmm_enabled:
|
||||
from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4
|
||||
|
||||
ql_nope = batched_gemm_a16wfp4(
|
||||
ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4(
|
||||
q_nope, self.W_K, self.W_K_scale, transpose_bm=True, prequant=True
|
||||
)
|
||||
elif self.is_aiter_triton_fp8_bmm_enabled:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
ql_nope = rocm_aiter_ops.triton_fp8_bmm(
|
||||
q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True
|
||||
)
|
||||
@@ -152,14 +144,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
)
|
||||
|
||||
if self.is_aiter_triton_fp4_bmm_enabled:
|
||||
from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4
|
||||
|
||||
batched_gemm_a16wfp4(
|
||||
rocm_aiter_ops.batched_gemm_a16wfp4(
|
||||
x, self.W_V, self.W_V_scale, out_view, transpose_bm=True, prequant=True
|
||||
)
|
||||
elif self.is_aiter_triton_fp8_bmm_enabled:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.triton_fp8_bmm(
|
||||
x,
|
||||
self.W_V,
|
||||
@@ -202,6 +190,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
indexer_k_norm_eps = self.indexer.k_norm.eps
|
||||
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
|
||||
indexer_k_cache = self.indexer.k_cache.kv_cache
|
||||
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
|
||||
indexer_softmax_scale = self.indexer.softmax_scale
|
||||
indexer_n_head_scale = self.indexer.n_head**-0.5
|
||||
else:
|
||||
@@ -211,6 +200,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
indexer_k_norm_eps = 1e-6
|
||||
indexer_k_rope_cos_sin_cache = None
|
||||
indexer_k_cache = None
|
||||
indexer_cache_shuffled = False
|
||||
indexer_softmax_scale = 0.0
|
||||
indexer_n_head_scale = 0.0
|
||||
|
||||
@@ -241,6 +231,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
self.topk_indices_buffer,
|
||||
slot_mapping=mla_slot,
|
||||
indexer_k_cache=indexer_k_cache,
|
||||
indexer_cache_shuffled=indexer_cache_shuffled,
|
||||
mla_kv_cache=mla_kv_cache,
|
||||
mla_kv_cache_dtype=self.kv_cache_dtype,
|
||||
mla_k_scale=mla_k_scale,
|
||||
@@ -282,7 +273,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
|
||||
kv_cache = self.kv_cache
|
||||
if self._fp8_kv_needs_view:
|
||||
kv_cache = kv_cache.view(torch.float8_e4m3fn)
|
||||
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
||||
|
||||
q_for_attn = self._build_q_for_attn(ql_nope, mqa_q, num_actual)
|
||||
attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
|
||||
@@ -327,6 +327,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
indexer_k_norm_bias = self.indexer.k_norm.bias
|
||||
indexer_k_norm_eps = self.indexer.k_norm.eps
|
||||
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
|
||||
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
|
||||
indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache
|
||||
index_k_out = torch.empty_like(index_k) if self.use_pcp else None
|
||||
indexer_softmax_scale = self.indexer.softmax_scale
|
||||
@@ -338,6 +339,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
indexer_k_norm_eps = 1e-6
|
||||
indexer_k_rope_cos_sin_cache = None
|
||||
indexer_k_cache = None
|
||||
indexer_cache_shuffled = False
|
||||
index_k_out = None
|
||||
indexer_softmax_scale = 0.0
|
||||
indexer_n_head_scale = 0.0
|
||||
@@ -377,6 +379,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
slot_mapping=mla_slot,
|
||||
indexer_slot_mapping=indexer_slot,
|
||||
indexer_k_cache=indexer_k_cache,
|
||||
indexer_cache_shuffled=indexer_cache_shuffled,
|
||||
mla_kv_cache=mla_kv_cache,
|
||||
mla_kv_cache_dtype=self.kv_cache_dtype,
|
||||
mla_k_scale=mla_k_scale,
|
||||
@@ -544,7 +547,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
return
|
||||
|
||||
if self._fp8_kv_needs_view:
|
||||
kv_cache = kv_cache.view(torch.float8_e4m3fn)
|
||||
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
||||
if self._fp8_query:
|
||||
# FlashInfer sparse: single packed fp8 query.
|
||||
mqa_q_arg: torch.Tensor | tuple[torch.Tensor, torch.Tensor] = mqa_q[
|
||||
|
||||
@@ -12,6 +12,18 @@ from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
# has_indexer=False path so the indexer args don't allocate every call.
|
||||
_DUMMY_CACHE: dict[tuple, torch.Tensor] = {}
|
||||
|
||||
# Tile shape of the indexer-K cache's shuffled layout, used when the cache reports
|
||||
# uses_shuffled_layout. The block tile is a token count; the head tile is a byte
|
||||
# count, which the kernel converts to cache elements.
|
||||
_INDEXER_CACHE_BLOCK_TILE = 16
|
||||
_INDEXER_CACHE_HEAD_TILE_BYTES = 16
|
||||
|
||||
# FNUZ (e4m3fnuz) on gfx942, OCP (e4m3fn) elsewhere. 224.0 keeps the ue8m0
|
||||
# power-of-two scale inside the FNUZ range after its ceil.
|
||||
_FP8_DTYPE = current_platform.fp8_dtype()
|
||||
_USE_FNUZ = torch.float8_e4m3fnuz == _FP8_DTYPE
|
||||
_FP8_MAX = 224.0 if _USE_FNUZ else 448.0
|
||||
|
||||
|
||||
def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
|
||||
key = (shape, dtype, device)
|
||||
@@ -47,16 +59,17 @@ def _get_cos_sin(
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp8_ue8m0_quantize(vals):
|
||||
def _fp8_ue8m0_quantize(vals, FP8_MAX: tl.constexpr, USE_FNUZ: tl.constexpr):
|
||||
"""Quantize float32 values to FP8 E4M3 with a ue8m0 (power-of-2) scale.
|
||||
|
||||
Returns (fp8_vals, scale) so the caller can store them or reuse the scale.
|
||||
"""
|
||||
vals = vals.to(tl.float32)
|
||||
amax = tl.max(tl.abs(vals))
|
||||
scale = tl.div_rn(tl.maximum(amax, 1e-4), 448.0)
|
||||
scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX)
|
||||
scale = tl.math.exp2(tl.math.ceil(tl.math.log2(scale)))
|
||||
fp8_vals = tl.div_rn(vals, scale).to(tl.float8e4nv)
|
||||
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
|
||||
fp8_vals = tl.div_rn(vals, scale).to(fp8_dtype)
|
||||
return fp8_vals, scale
|
||||
|
||||
|
||||
@@ -71,18 +84,32 @@ def _fp8_quant_and_cache_write(
|
||||
cache_block_stride,
|
||||
offsets,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
SHUFFLE: tl.constexpr,
|
||||
BLOCK_TILE: tl.constexpr,
|
||||
HEAD_TILE: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
USE_FNUZ: tl.constexpr,
|
||||
):
|
||||
k_fp8, scale = _fp8_ue8m0_quantize(vals)
|
||||
k_fp8, scale = _fp8_ue8m0_quantize(vals, FP8_MAX, USE_FNUZ)
|
||||
|
||||
block_idx = slot_idx // cache_block_size
|
||||
block_offset = slot_idx % cache_block_size
|
||||
block_start = block_idx * cache_block_stride
|
||||
|
||||
tl.store(
|
||||
kv_cache_ptr + block_start + block_offset * HEAD_DIM + offsets,
|
||||
k_fp8,
|
||||
mask=mask,
|
||||
)
|
||||
# Shuffled layout: [blk/BLOCK_TILE, HEAD_DIM/HEAD_TILE, BLOCK_TILE, HEAD_TILE],
|
||||
# so one contiguous run holds BLOCK_TILE tokens x HEAD_TILE bytes and the
|
||||
# reader's coalesced loads land on the bytes we wrote.
|
||||
if SHUFFLE:
|
||||
value_off = (
|
||||
block_offset // BLOCK_TILE * BLOCK_TILE * HEAD_DIM
|
||||
+ block_offset % BLOCK_TILE * HEAD_TILE
|
||||
+ offsets // HEAD_TILE * BLOCK_TILE * HEAD_TILE
|
||||
+ offsets % HEAD_TILE
|
||||
)
|
||||
else:
|
||||
value_off = block_offset * HEAD_DIM + offsets
|
||||
|
||||
tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask)
|
||||
scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4
|
||||
tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale)
|
||||
|
||||
@@ -137,6 +164,9 @@ def _fused_norm_rope_kernel(
|
||||
indexer_cache_scale_ptr,
|
||||
indexer_cache_block_size,
|
||||
indexer_cache_stride,
|
||||
INDEXER_CACHE_SHUFFLE: tl.constexpr,
|
||||
INDEXER_CACHE_BLOCK_TILE: tl.constexpr,
|
||||
INDEXER_CACHE_HEAD_TILE: tl.constexpr,
|
||||
# MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr)
|
||||
mla_cache_ptr,
|
||||
mla_cache_block_stride,
|
||||
@@ -161,12 +191,15 @@ def _fused_norm_rope_kernel(
|
||||
HAS_INDEXER: tl.constexpr,
|
||||
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
USE_FNUZ: tl.constexpr,
|
||||
):
|
||||
tok_idx = tl.program_id(0).to(tl.int64)
|
||||
pid = tl.program_id(1)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
|
||||
if pid == 3:
|
||||
if not HAS_INDEXER:
|
||||
# Shared layer: reuse the previous indexer layer's top-k; do not
|
||||
@@ -327,9 +360,11 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM))
|
||||
tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True)
|
||||
tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1e-4)
|
||||
# scale = amax / FP8_MAX (fp8 e4m3 max), matching the reference
|
||||
# concat_and_cache_ds_mla kernel; floored to FLT_MIN.
|
||||
tile_scale = tl.maximum(tile_amax * (1.0 / FP8_MAX), 1e-4)
|
||||
tile_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(tile_scale)))
|
||||
kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,))
|
||||
kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(fp8_dtype), (KV_DIM,))
|
||||
tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8)
|
||||
tile_off = tl.arange(0, MLA_NUM_TILES)
|
||||
tl.store(
|
||||
@@ -349,16 +384,16 @@ def _fused_norm_rope_kernel(
|
||||
# kv_c_normed (KV_DIM elements)
|
||||
if MLA_CACHE_FP8:
|
||||
scale = tl.load(mla_cache_scale_ptr)
|
||||
kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv)
|
||||
kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(fp8_dtype)
|
||||
tl.store(dst + kv_block, kv_c_fp8)
|
||||
else:
|
||||
tl.store(dst + kv_block, kv_c)
|
||||
# k_pe_roped (from registers, interleaved layout)
|
||||
if MLA_CACHE_FP8:
|
||||
tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv))
|
||||
tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(fp8_dtype))
|
||||
tl.store(
|
||||
dst + KV_DIM + dim_off * 2 + 1,
|
||||
(r2 / scale).to(tl.float8e4nv),
|
||||
(r2 / scale).to(fp8_dtype),
|
||||
)
|
||||
else:
|
||||
tl.store(dst + KV_DIM + dim_off * 2, r1)
|
||||
@@ -463,6 +498,11 @@ def _fused_norm_rope_kernel(
|
||||
indexer_cache_stride,
|
||||
index_k_block,
|
||||
INDEX_K_DIM,
|
||||
INDEXER_CACHE_SHUFFLE,
|
||||
INDEXER_CACHE_BLOCK_TILE,
|
||||
INDEXER_CACHE_HEAD_TILE,
|
||||
FP8_MAX,
|
||||
USE_FNUZ,
|
||||
)
|
||||
|
||||
|
||||
@@ -486,6 +526,7 @@ def fused_norm_rope(
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
indexer_slot_mapping: torch.Tensor | None = None,
|
||||
indexer_k_cache: torch.Tensor | None = None,
|
||||
indexer_cache_shuffled: bool = False,
|
||||
mla_kv_cache: torch.Tensor | None = None,
|
||||
mla_kv_cache_dtype: str = "auto",
|
||||
mla_k_scale: torch.Tensor | None = None,
|
||||
@@ -532,13 +573,28 @@ def fused_norm_rope(
|
||||
indexer_slot_mapping = slot_mapping
|
||||
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
|
||||
idx_cache_block_size = indexer_k_cache.shape[1]
|
||||
idx_cache_stride = indexer_k_cache.shape[2]
|
||||
# The caller's cache reports whether its reader expects the shuffled
|
||||
# layout; see DeepseekV32IndexerCache.uses_shuffled_layout.
|
||||
idx_cache_shuffle = indexer_cache_shuffled
|
||||
if idx_cache_shuffle:
|
||||
assert idx_cache_block_size % _INDEXER_CACHE_BLOCK_TILE == 0, (
|
||||
f"indexer K cache block size {idx_cache_block_size} must be a "
|
||||
f"multiple of {_INDEXER_CACHE_BLOCK_TILE} for the shuffled layout"
|
||||
)
|
||||
idx_cache_stride = indexer_k_cache.stride(0)
|
||||
if indexer_k_cache.dtype == torch.uint8:
|
||||
indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn)
|
||||
indexer_k_cache = indexer_k_cache.view(_FP8_DTYPE)
|
||||
# The head tile is a byte count; the kernel indexes the cache in elements.
|
||||
idx_cache_head_tile = (
|
||||
_INDEXER_CACHE_HEAD_TILE_BYTES // indexer_k_cache.element_size()
|
||||
)
|
||||
else:
|
||||
idx_cache_scale_view = None
|
||||
idx_cache_block_size = 1
|
||||
idx_cache_stride = 0
|
||||
idx_cache_shuffle = False
|
||||
idx_cache_head_tile = _INDEXER_CACHE_HEAD_TILE_BYTES
|
||||
|
||||
# --- MLA KV cache setup ---
|
||||
mla_cache_nvfp4 = mla_kv_cache_dtype == "nvfp4_ds_mla"
|
||||
@@ -561,12 +617,12 @@ def fused_norm_rope(
|
||||
mla_entry_stride = u8_cache.stride(1)
|
||||
mla_ds_scale_view = u8_cache.view(torch.float32)
|
||||
mla_ds_rope_view = u8_cache.view(torch.bfloat16)
|
||||
mla_kv_cache = u8_cache.view(torch.float8_e4m3fn)
|
||||
mla_kv_cache = u8_cache.view(_FP8_DTYPE)
|
||||
else:
|
||||
mla_block_stride = mla_kv_cache.stride(0)
|
||||
mla_entry_stride = mla_kv_cache.stride(1)
|
||||
if mla_cache_fp8 and mla_kv_cache.dtype == torch.uint8:
|
||||
mla_kv_cache = mla_kv_cache.view(torch.float8_e4m3fn)
|
||||
mla_kv_cache = mla_kv_cache.view(_FP8_DTYPE)
|
||||
if mla_k_scale is None:
|
||||
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
else:
|
||||
@@ -592,6 +648,9 @@ def fused_norm_rope(
|
||||
assert index_k_out.shape == index_k.shape
|
||||
index_k_out_stride = index_k_out.stride(0)
|
||||
use_pdl = current_platform.is_arch_support_pdl()
|
||||
pdl_kwargs = (
|
||||
{"USE_PDL": use_pdl, "launch_pdl": True} if use_pdl else {"USE_PDL": use_pdl}
|
||||
)
|
||||
_fused_norm_rope_kernel[(num_tokens, 4)](
|
||||
positions,
|
||||
# Q RMS norm
|
||||
@@ -639,6 +698,9 @@ def fused_norm_rope(
|
||||
idx_cache_scale_view,
|
||||
idx_cache_block_size,
|
||||
idx_cache_stride,
|
||||
idx_cache_shuffle,
|
||||
_INDEXER_CACHE_BLOCK_TILE,
|
||||
idx_cache_head_tile,
|
||||
# MLA KV cache (uses same slot_mapping)
|
||||
mla_kv_cache,
|
||||
mla_block_stride,
|
||||
@@ -659,8 +721,9 @@ def fused_norm_rope(
|
||||
TOPK_BLOCK_SIZE=1024,
|
||||
HAS_INDEXER=has_indexer,
|
||||
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
||||
USE_PDL=use_pdl,
|
||||
launch_pdl=use_pdl,
|
||||
FP8_MAX=_FP8_MAX,
|
||||
USE_FNUZ=_USE_FNUZ,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
return q_c_out
|
||||
|
||||
@@ -715,6 +778,8 @@ def _fused_q_kernel(
|
||||
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
||||
QUANTIZE_MQA: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
USE_FNUZ: tl.constexpr,
|
||||
):
|
||||
tok_idx = tl.program_id(0).to(tl.int64)
|
||||
pid = tl.program_id(1)
|
||||
@@ -722,6 +787,7 @@ def _fused_q_kernel(
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
|
||||
|
||||
if pid == 2:
|
||||
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
|
||||
@@ -744,7 +810,7 @@ def _fused_q_kernel(
|
||||
+ ql_nope_off,
|
||||
mask=ql_nope_mask,
|
||||
).to(tl.float32)
|
||||
ql_nope_fp8 = (ql_nope / scale).to(tl.float8e4nv)
|
||||
ql_nope_fp8 = (ql_nope / scale).to(fp8_dtype)
|
||||
tl.store(
|
||||
mqa_q_fp8_ptr
|
||||
+ tok_idx * mqa_q_fp8_stride0
|
||||
@@ -794,7 +860,7 @@ def _fused_q_kernel(
|
||||
+ q_head_idx * mqa_q_fp8_stride1
|
||||
+ QL_NOPE_DIM
|
||||
+ rot_off * 2,
|
||||
(r1 / scale).to(tl.float8e4nv),
|
||||
(r1 / scale).to(fp8_dtype),
|
||||
)
|
||||
tl.store(
|
||||
mqa_q_fp8_ptr
|
||||
@@ -803,7 +869,7 @@ def _fused_q_kernel(
|
||||
+ QL_NOPE_DIM
|
||||
+ rot_off * 2
|
||||
+ 1,
|
||||
(r2 / scale).to(tl.float8e4nv),
|
||||
(r2 / scale).to(fp8_dtype),
|
||||
)
|
||||
else:
|
||||
# bf16 query: write the RoPE'd q_pe unquantized.
|
||||
@@ -861,7 +927,7 @@ def _fused_q_kernel(
|
||||
index_q = tl.where(in_rope, roped, index_q)
|
||||
|
||||
# Index Q Quantize (from registers)
|
||||
index_q_fp8, index_q_scale = _fp8_ue8m0_quantize(index_q)
|
||||
index_q_fp8, index_q_scale = _fp8_ue8m0_quantize(index_q, FP8_MAX, USE_FNUZ)
|
||||
tl.store(
|
||||
index_q_fp8_ptr
|
||||
+ tok_idx * index_q_fp8_stride0
|
||||
@@ -955,7 +1021,7 @@ def fused_q(
|
||||
q_pe.shape[0],
|
||||
q_pe.shape[1],
|
||||
ql_nope.shape[2] + q_pe.shape[2],
|
||||
dtype=torch.float8_e4m3fn,
|
||||
dtype=_FP8_DTYPE,
|
||||
device=q_pe.device,
|
||||
)
|
||||
# Placeholder; pid 0 packs q_pe into mqa_q_fp8 instead.
|
||||
@@ -967,7 +1033,7 @@ def fused_q(
|
||||
mqa_q_fp8 = q_pe_out # unused placeholder for the fp8 pack pointer
|
||||
mqa_q = q_pe_out
|
||||
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn)
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=_FP8_DTYPE)
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
if cutedsl_kernel is not None:
|
||||
cutedsl_kernel(
|
||||
@@ -990,6 +1056,9 @@ def fused_q(
|
||||
return index_q_fp8, index_weights_out, mqa_q
|
||||
|
||||
use_pdl = current_platform.is_arch_support_pdl()
|
||||
pdl_kwargs = (
|
||||
{"USE_PDL": use_pdl, "launch_pdl": True} if use_pdl else {"USE_PDL": use_pdl}
|
||||
)
|
||||
_fused_q_kernel[(num_tokens, 3, grid_heads)](
|
||||
positions,
|
||||
q_pe,
|
||||
@@ -1031,8 +1100,9 @@ def fused_q(
|
||||
HAS_INDEXER=has_indexer,
|
||||
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
||||
QUANTIZE_MQA=quantize_mqa,
|
||||
USE_PDL=use_pdl,
|
||||
launch_pdl=use_pdl,
|
||||
FP8_MAX=_FP8_MAX,
|
||||
USE_FNUZ=_USE_FNUZ,
|
||||
**pdl_kwargs,
|
||||
# num_warps=1 is optimal here: each program is a single 128-element
|
||||
# rope+quant, so the kernel is program-count/occupancy bound, not
|
||||
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).
|
||||
|
||||
Reference in New Issue
Block a user