[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
86b1c2ef15
commit
a6009ea85b
@@ -15,6 +15,8 @@ from __future__ import annotations
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
# The GLM-5.3-Flash indexer head dimension is fixed at 128.
|
||||
INDEX_HEAD_DIM = 128
|
||||
@@ -157,6 +159,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
HAS_WRITE_MASK: tl.constexpr,
|
||||
RETURN_COMPRESSED: tl.constexpr,
|
||||
WRITE_CACHE: tl.constexpr,
|
||||
@@ -245,7 +248,14 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
|
||||
if MX_SCALE:
|
||||
tl.store(
|
||||
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
|
||||
pack_ue8m0_x4(scale),
|
||||
mask=do_write,
|
||||
)
|
||||
else:
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
|
||||
|
||||
if RETURN_COMPRESSED:
|
||||
tl.store(
|
||||
@@ -324,6 +334,7 @@ def kpool_compress_and_write_cache(
|
||||
|
||||
buf_fp8 = buf.view(torch.float8_e4m3fn)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
mx_scale = write_cache and mqa_logits_use_mx_scales()
|
||||
# bytes per page (last dim of kv_cache) viewed as uint8
|
||||
buf_numel_per_page = buf.stride(0)
|
||||
s_offset_nbytes_in_page = page_size * head_dim
|
||||
@@ -361,7 +372,9 @@ def kpool_compress_and_write_cache(
|
||||
POOL_SIZE=slot_k.shape[1],
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page,
|
||||
ROUND_SCALE=round_scale,
|
||||
# MX scales need power-of-two scales.
|
||||
ROUND_SCALE=round_scale or mx_scale,
|
||||
MX_SCALE=mx_scale,
|
||||
HAS_WRITE_MASK=has_write_mask,
|
||||
RETURN_COMPRESSED=return_compressed,
|
||||
WRITE_CACHE=write_cache,
|
||||
@@ -479,6 +492,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per request; iterates its NEXT_N verify tokens in order.
|
||||
@@ -604,7 +618,13 @@ def _kpool_decode_update_batched_kernel(
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale)
|
||||
if MX_SCALE:
|
||||
tl.store(
|
||||
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
|
||||
pack_ue8m0_x4(scale),
|
||||
)
|
||||
else:
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale)
|
||||
|
||||
# Stash the current token AFTER any completion read so the completion
|
||||
# uses prior stashes (and the current token's own key/score via
|
||||
@@ -718,7 +738,9 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
# MX scales need power-of-two scales.
|
||||
ROUND_SCALE=round_scale or mqa_logits_use_mx_scales(),
|
||||
MX_SCALE=mqa_logits_use_mx_scales(),
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user