[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-12 23:24:43 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 86b1c2ef15
commit a6009ea85b
23 changed files with 516 additions and 208 deletions
+13 -1
View File
@@ -6,7 +6,9 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.ops.common import pack_ue8m0_x4
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
# has_indexer=False path so the indexer args don't allocate every call.
@@ -87,6 +89,7 @@ def _fp8_quant_and_cache_write(
SHUFFLE: tl.constexpr,
BLOCK_TILE: tl.constexpr,
HEAD_TILE: tl.constexpr,
MX_SCALE: tl.constexpr,
FP8_MAX: tl.constexpr,
USE_FNUZ: tl.constexpr,
):
@@ -111,6 +114,8 @@ def _fp8_quant_and_cache_write(
tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask)
scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4
if MX_SCALE:
scale = pack_ue8m0_x4(scale)
tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale)
@@ -167,6 +172,7 @@ def _fused_norm_rope_kernel(
INDEXER_CACHE_SHUFFLE: tl.constexpr,
INDEXER_CACHE_BLOCK_TILE: tl.constexpr,
INDEXER_CACHE_HEAD_TILE: tl.constexpr,
INDEXER_CACHE_MX_SCALE: tl.constexpr,
# MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr)
mla_cache_ptr,
mla_cache_block_stride,
@@ -501,6 +507,7 @@ def _fused_norm_rope_kernel(
INDEXER_CACHE_SHUFFLE,
INDEXER_CACHE_BLOCK_TILE,
INDEXER_CACHE_HEAD_TILE,
INDEXER_CACHE_MX_SCALE,
FP8_MAX,
USE_FNUZ,
)
@@ -571,7 +578,10 @@ def fused_norm_rope(
assert slot_mapping is not None
if indexer_slot_mapping is None:
indexer_slot_mapping = slot_mapping
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
idx_cache_mx_scale = mqa_logits_use_mx_scales()
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(
torch.int32 if idx_cache_mx_scale else torch.float32
)
idx_cache_block_size = indexer_k_cache.shape[1]
idx_cache_stride = indexer_k_cache.shape[2]
# The caller's cache reports whether its reader expects the shuffled
@@ -591,6 +601,7 @@ def fused_norm_rope(
)
else:
idx_cache_scale_view = None
idx_cache_mx_scale = False
idx_cache_block_size = 1
idx_cache_stride = 0
idx_cache_shuffle = False
@@ -701,6 +712,7 @@ def fused_norm_rope(
idx_cache_shuffle,
_INDEXER_CACHE_BLOCK_TILE,
idx_cache_head_tile,
idx_cache_mx_scale,
# MLA KV cache (uses same slot_mapping)
mla_kv_cache,
mla_block_stride,
@@ -33,7 +33,9 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.utils.math_utils import round_up
from vllm.v1.attention.ops.common import pack_ue8m0_x4
if current_platform.is_rocm():
from vllm.platforms.rocm import _ON_GFX950
@@ -742,6 +744,7 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
TOKEN_STRIDE: tl.constexpr, # 128 for indexer
SCALE_DIM: tl.constexpr, # 4 for indexer (1 float32)
KV_BLOCK_STRIDE: tl.constexpr,
MX_SCALE: tl.constexpr, # store the scale as 4 packed UE8M0 bytes
):
"""Fused compress → RMSNorm → RoPE → FP8 quant → store.
@@ -878,7 +881,10 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
# Single float32 scale
scale_val = tl.exp2(exponent)
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), scale_val)
if MX_SCALE:
tl.store(scale_ptr.to(tl.pointer_type(tl.int32)), pack_ue8m0_x4(scale_val))
else:
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), scale_val)
# =============================================================================
@@ -927,6 +933,7 @@ def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn(
TOKEN_STRIDE: tl.constexpr, # HEAD_SIZE // 2 = 64 packed bytes/token
SCALE_DIM: tl.constexpr, # HEAD_SIZE // QUANT_BLOCK = 4 ue8m0 bytes/token
KV_BLOCK_STRIDE: tl.constexpr,
MX_SCALE: tl.constexpr, # unused for MXFP4 (kept for signature parity)
):
"""Fused compress → RMSNorm → RoPE → MXFP4 quant → store.
@@ -1102,6 +1109,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
block_size: int
kv_cache_block_size: int
kv_block_stride: int
mx_scale: bool
# Canonical kernel for the launcher's arg_names mapping; the launched
# variant (fp8/mxfp4, same arg_names) is picked per call in __call__.
@@ -1181,6 +1189,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
block_size=state_block_size,
kv_cache_block_size=kv_cache_block_size,
kv_block_stride=kv_block_stride,
mx_scale=not use_fp4_cache and mqa_logits_use_mx_scales(),
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
@@ -1301,6 +1310,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
TOKEN_STRIDE=token_stride,
SCALE_DIM=scale_dim,
KV_BLOCK_STRIDE=kv_cache.stride(0),
MX_SCALE=not use_fp4_cache and mqa_logits_use_mx_scales(),
num_warps=1,
**pdl_kwargs,
)
@@ -17,6 +17,8 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.v1.attention.ops.common import pack_ue8m0_x4
from . import MXFP4_BLOCK_SIZE, _fp32x2_to_fp4x2
@@ -51,7 +53,8 @@ def indexer_k_norm_rope_store(
kv_slot_mapping: [num_tokens] slots in the indexer cache (-1 = skip).
compress_ratio: group size; keys are emitted at group boundaries.
use_fp4_cache: MXFP4 (2 nibbles/byte + ue8m0 per 32) when True, else
per-token FP8 with a single fp32 scale.
per-token FP8 with a single fp32 scale (four packed ue8m0 bytes
under `mqa_logits_use_mx_scales`).
"""
num_tokens = kv_slot_mapping.numel()
@@ -111,6 +114,7 @@ def indexer_k_norm_rope_store(
KV_BLOCK_STRIDE=k_cache.stride(0),
FP8_MAX=448.0,
USE_FP4=use_fp4_cache,
MX_SCALE=not use_fp4_cache and mqa_logits_use_mx_scales(),
SHUFFLE=shuffle,
BLOCK_TILE_SIZE=_BLOCK_TILE_SIZE,
HEAD_TILE_SIZE=_HEAD_TILE_SIZE,
@@ -139,6 +143,7 @@ def _indexer_k_norm_rope_quant_store_kernel(
KV_BLOCK_STRIDE: tl.constexpr,
FP8_MAX: tl.constexpr,
USE_FP4: tl.constexpr,
MX_SCALE: tl.constexpr,
SHUFFLE: tl.constexpr,
BLOCK_TILE_SIZE: tl.constexpr,
HEAD_TILE_SIZE: tl.constexpr,
@@ -252,4 +257,10 @@ def _indexer_k_norm_rope_quant_store_kernel(
tl.store(val_ptr + tiled, x_uint8)
else:
tl.store(val_ptr + block, x_uint8)
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), tl.exp2(exponent))
if MX_SCALE:
tl.store(
scale_ptr.to(tl.pointer_type(tl.int32)),
pack_ue8m0_x4(tl.exp2(exponent)),
)
else:
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), tl.exp2(exponent))
@@ -15,6 +15,8 @@ from __future__ import annotations
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.v1.attention.ops.common import pack_ue8m0_x4
# The GLM-5.3-Flash indexer head dimension is fixed at 128.
INDEX_HEAD_DIM = 128
@@ -157,6 +159,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
MX_SCALE: tl.constexpr,
HAS_WRITE_MASK: tl.constexpr,
RETURN_COMPRESSED: tl.constexpr,
WRITE_CACHE: tl.constexpr,
@@ -245,7 +248,14 @@ def _kpool_softmax_rotate_write_cache_kernel(
+ loc_token_offset_in_page
)
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask)
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
if MX_SCALE:
tl.store(
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
pack_ue8m0_x4(scale),
mask=do_write,
)
else:
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
if RETURN_COMPRESSED:
tl.store(
@@ -324,6 +334,7 @@ def kpool_compress_and_write_cache(
buf_fp8 = buf.view(torch.float8_e4m3fn)
buf_fp32 = buf.view(torch.float32)
mx_scale = write_cache and mqa_logits_use_mx_scales()
# bytes per page (last dim of kv_cache) viewed as uint8
buf_numel_per_page = buf.stride(0)
s_offset_nbytes_in_page = page_size * head_dim
@@ -361,7 +372,9 @@ def kpool_compress_and_write_cache(
POOL_SIZE=slot_k.shape[1],
HEAD_DIM=head_dim,
S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page,
ROUND_SCALE=round_scale,
# MX scales need power-of-two scales.
ROUND_SCALE=round_scale or mx_scale,
MX_SCALE=mx_scale,
HAS_WRITE_MASK=has_write_mask,
RETURN_COMPRESSED=return_compressed,
WRITE_CACHE=write_cache,
@@ -479,6 +492,7 @@ def _kpool_decode_update_batched_kernel(
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
MX_SCALE: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""One program per request; iterates its NEXT_N verify tokens in order.
@@ -604,7 +618,13 @@ def _kpool_decode_update_batched_kernel(
+ loc_token_offset_in_page
)
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask)
tl.store(buf_fp32_ptr + out_s_offset, scale)
if MX_SCALE:
tl.store(
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
pack_ue8m0_x4(scale),
)
else:
tl.store(buf_fp32_ptr + out_s_offset, scale)
# Stash the current token AFTER any completion read so the completion
# uses prior stashes (and the current token's own key/score via
@@ -718,7 +738,9 @@ def kpool_decode_update_and_maybe_write_cache_batched(
KPOOL_HEAD=tail_kv_cache.stride(1),
HEAD_DIM=head_dim,
S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim,
ROUND_SCALE=round_scale,
# MX scales need power-of-two scales.
ROUND_SCALE=round_scale or mqa_logits_use_mx_scales(),
MX_SCALE=mqa_logits_use_mx_scales(),
BLOCK_D=triton.next_power_of_2(head_dim),
)
+46 -26
View File
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.sparse_attn_indexer import (
PrefillRowShard,
_merge_dcp_topk_global,
_reserve_row_shard_exchange_transient,
as_fp32_logits,
)
from vllm.models.glm5next.common.sparse_indexer import (
RADIX_TOPK_WORKSPACE_SIZE,
@@ -29,7 +30,13 @@ from vllm.models.glm5next.common.sparse_indexer import (
)
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import has_deep_gemm
from vllm.utils.deep_gemm import (
fp8_fp4_mqa_logits,
fp8_fp4_paged_mqa_logits,
has_deep_gemm,
mqa_logits_mx_inputs,
mqa_logits_use_mx_scales,
)
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.torch_utils import (
LayerNameType,
@@ -253,7 +260,7 @@ def sparse_attn_indexer_kpool(
kv_cache,
slot_mapping,
quant_block_size,
scale_fmt,
"mxfp8" if mqa_logits_use_mx_scales() else scale_fmt,
)
topk_indices_buffer[: hidden_states.shape[0]] = -1
@@ -343,20 +350,29 @@ def sparse_attn_indexer_kpool(
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
k_scale_cast = k_scale.view(
torch.int32 if mqa_logits_use_mx_scales() else torch.float32
).squeeze(-1)
num_rows = q_slice.shape[0]
if chunk.local_total_seq_lens == 0:
logits = q_slice.new_empty((num_rows, 0), dtype=torch.float32)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[row_start:row_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
q_scale_slice, chunk_weights = mqa_logits_mx_inputs(
q_slice_cast, q_scale_slice, weights[row_start:row_end]
)
# Compressed: row i starts at K position cu_seqlen_ks[i].
row_ends = cu_seqlen_ke - cu_seqlen_ks
logits = as_fp32_logits(
fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
chunk_weights,
cu_seqlen_ks,
cu_seqlen_ke,
chunk.max_seqlen_k,
),
row_ends,
)
# kpool: logits are pool-granular (compress_ratio == index_kpool),
@@ -376,8 +392,8 @@ def sparse_attn_indexer_kpool(
else:
torch.ops._C.top_k_per_row_prefill(
logits,
cu_seqlen_ks,
cu_seqlen_ke,
torch.zeros_like(row_ends),
row_ends,
topk_dst,
num_rows,
logits.stride(0),
@@ -392,7 +408,7 @@ def sparse_attn_indexer_kpool(
dcp_rank,
dcp_world_size,
pool_cp_interleave,
row_starts=cu_seqlen_ks,
row_starts=None,
)
if index_kpool > 1:
@@ -594,18 +610,22 @@ def sparse_attn_indexer_kpool(
if use_fp4_cache
else padded_q_quant_decode_tokens
)
from vllm.utils.deep_gemm import fp8_fp4_paged_mqa_logits
logits = fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
padded_weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_pool_len,
clean_logits=False,
indices=decode_metadata.indices,
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
padded_q_quant_cast, padded_q_scale, padded_weights[:num_padded_tokens]
)
# SM100 decode is varlen: one row and context length per token.
logits = as_fp32_logits(
fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
decode_weights,
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_pool_len,
indices=decode_metadata.indices,
),
seq_lens.view(-1),
)
num_rows = logits.shape[0]
# kpool: logits are pool-granular -> select topk_tokens//kpool pools,