[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-12 23:24:43 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 86b1c2ef15
commit a6009ea85b
23 changed files with 516 additions and 208 deletions
@@ -15,6 +15,8 @@ from __future__ import annotations
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.v1.attention.ops.common import pack_ue8m0_x4
# The GLM-5.3-Flash indexer head dimension is fixed at 128.
INDEX_HEAD_DIM = 128
@@ -157,6 +159,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
MX_SCALE: tl.constexpr,
HAS_WRITE_MASK: tl.constexpr,
RETURN_COMPRESSED: tl.constexpr,
WRITE_CACHE: tl.constexpr,
@@ -245,7 +248,14 @@ def _kpool_softmax_rotate_write_cache_kernel(
+ loc_token_offset_in_page
)
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask)
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
if MX_SCALE:
tl.store(
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
pack_ue8m0_x4(scale),
mask=do_write,
)
else:
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
if RETURN_COMPRESSED:
tl.store(
@@ -324,6 +334,7 @@ def kpool_compress_and_write_cache(
buf_fp8 = buf.view(torch.float8_e4m3fn)
buf_fp32 = buf.view(torch.float32)
mx_scale = write_cache and mqa_logits_use_mx_scales()
# bytes per page (last dim of kv_cache) viewed as uint8
buf_numel_per_page = buf.stride(0)
s_offset_nbytes_in_page = page_size * head_dim
@@ -361,7 +372,9 @@ def kpool_compress_and_write_cache(
POOL_SIZE=slot_k.shape[1],
HEAD_DIM=head_dim,
S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page,
ROUND_SCALE=round_scale,
# MX scales need power-of-two scales.
ROUND_SCALE=round_scale or mx_scale,
MX_SCALE=mx_scale,
HAS_WRITE_MASK=has_write_mask,
RETURN_COMPRESSED=return_compressed,
WRITE_CACHE=write_cache,
@@ -479,6 +492,7 @@ def _kpool_decode_update_batched_kernel(
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
MX_SCALE: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""One program per request; iterates its NEXT_N verify tokens in order.
@@ -604,7 +618,13 @@ def _kpool_decode_update_batched_kernel(
+ loc_token_offset_in_page
)
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask)
tl.store(buf_fp32_ptr + out_s_offset, scale)
if MX_SCALE:
tl.store(
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
pack_ue8m0_x4(scale),
)
else:
tl.store(buf_fp32_ptr + out_s_offset, scale)
# Stash the current token AFTER any completion read so the completion
# uses prior stashes (and the current token's own key/score via
@@ -718,7 +738,9 @@ def kpool_decode_update_and_maybe_write_cache_batched(
KPOOL_HEAD=tail_kv_cache.stride(1),
HEAD_DIM=head_dim,
S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim,
ROUND_SCALE=round_scale,
# MX scales need power-of-two scales.
ROUND_SCALE=round_scale or mqa_logits_use_mx_scales(),
MX_SCALE=mqa_logits_use_mx_scales(),
BLOCK_D=triton.next_power_of_2(head_dim),
)