[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-12 23:24:43 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 86b1c2ef15
commit a6009ea85b
23 changed files with 516 additions and 208 deletions
@@ -42,6 +42,8 @@ from vllm.utils.deep_gemm import (
fp8_fp4_mqa_logits,
fp8_fp4_paged_mqa_logits,
has_deep_gemm,
mqa_logits_mx_inputs,
mqa_logits_use_mx_scales,
)
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.torch_utils import (
@@ -286,6 +288,42 @@ def fused_indexer_q_rope_quant(
return q_fp8, weights_out
@triton.jit
def _upcast_logits_kernel(src_ptr, dst_ptr, row_ends_ptr, stride, BLOCK: tl.constexpr):
row = tl.program_id(0)
end = tl.load(row_ends_ptr + row)
src_ptr += row.to(tl.int64) * stride
dst_ptr += row.to(tl.int64) * stride
for start in range(tl.program_id(1) * BLOCK, end, tl.num_programs(1) * BLOCK):
offs = start + tl.arange(0, BLOCK)
mask = offs < end
vals = tl.load(src_ptr + offs, mask=mask)
tl.store(dst_ptr + offs, vals.to(tl.float32), mask=mask)
def as_fp32_logits(logits: torch.Tensor, row_ends: torch.Tensor) -> torch.Tensor:
"""Upcast SM100's BF16 MQA logits for the FP32 top-k kernels.
Only columns below ``row_ends`` (int32, one per row) are converted, the
span the consumers read; DeepGEMM's aligned row stride is kept.
"""
if logits.dtype == torch.float32:
return logits
out = torch.empty_strided(
logits.shape, logits.stride(), dtype=torch.float32, device=logits.device
)
num_rows, width = logits.shape
if num_rows == 0 or width == 0:
return out
assert row_ends.numel() == num_rows and row_ends.dtype == torch.int32
block = 1024
splits = min(triton.cdiv(width, block), max(1, 1024 // num_rows))
_upcast_logits_kernel[(num_rows, splits)](
logits, out, row_ends.contiguous(), logits.stride(0), BLOCK=block
)
return out
def _gather_workspace_shapes(
total_seq_lens: int,
head_dim: int,
@@ -537,7 +575,7 @@ def sparse_attn_indexer(
kv_cache,
slot_mapping_for_cache,
quant_block_size,
scale_fmt,
"mxfp8" if mqa_logits_use_mx_scales() else scale_fmt,
)
# The indexer and main MLA may classify the same short extend differently
# because they use independent decode thresholds. Only the main MLA route
@@ -649,6 +687,9 @@ def sparse_attn_indexer(
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
topk_indices = topk_indices_buffer[row_start:row_end, :topk_tokens]
# DeepGEMM returns compressed logits: row i starts at K position
# cu_seqlen_ks[i]. None means every row starts at column zero.
row_starts: torch.Tensor | None = None
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
topk_indices.fill_(-1)
@@ -662,7 +703,9 @@ def sparse_attn_indexer(
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
k_scale_cast = k_scale.view(
torch.int32 if mqa_logits_use_mx_scales() else torch.float32
).squeeze(-1)
if current_platform.is_xpu():
if q_scale_slice is not None:
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
@@ -674,14 +717,23 @@ def sparse_attn_indexer(
cu_seqlen_ks,
cu_seqlen_ke,
)
row_starts = cu_seqlen_ks
row_ends = cu_seqlen_ke
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[row_start:row_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
q_scale_slice, chunk_weights = mqa_logits_mx_inputs(
q_slice_cast, q_scale_slice, weights[row_start:row_end]
)
row_ends = cu_seqlen_ke - cu_seqlen_ks
logits = as_fp32_logits(
fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
chunk_weights,
cu_seqlen_ks,
cu_seqlen_ke,
chunk.max_seqlen_k,
),
row_ends,
)
num_rows = logits.shape[0]
if candidate_blocks is not None:
@@ -694,8 +746,8 @@ def sparse_attn_indexer(
if candidate_write:
_select_candidate_blocks(
logits,
cu_seqlen_ks,
cu_seqlen_ke,
row_starts,
row_ends,
chunk_candidates.shape[1],
candidate_block_size,
chunk_candidates,
@@ -703,15 +755,15 @@ def sparse_attn_indexer(
else:
_apply_candidate_mask(
logits,
cu_seqlen_ks,
cu_seqlen_ke,
row_starts,
row_ends,
chunk_candidates,
candidate_block_size,
)
ops.top_k_per_row_prefill(
logits,
cu_seqlen_ks,
cu_seqlen_ke,
torch.zeros_like(row_ends) if row_starts is None else row_starts,
row_ends,
topk_indices,
num_rows,
logits.stride(0),
@@ -729,7 +781,7 @@ def sparse_attn_indexer(
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=cu_seqlen_ks,
row_starts=row_starts,
)
if row_shard is not None:
@@ -822,16 +874,22 @@ def sparse_attn_indexer(
max_model_len,
)
else:
logits = fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
clean_logits=False,
indices=decode_metadata.indices,
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
padded_q_quant_cast, padded_q_scale, weights[:num_padded_tokens]
)
# SM100 decode is varlen: one row and context length per token.
logits = as_fp32_logits(
fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
decode_weights,
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
indices=decode_metadata.indices,
),
seq_lens.view(-1),
)
num_rows = logits.shape[0]
if candidate_blocks is not None: