[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
86b1c2ef15
commit
a6009ea85b
@@ -42,6 +42,8 @@ from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
has_deep_gemm,
|
||||
mqa_logits_mx_inputs,
|
||||
mqa_logits_use_mx_scales,
|
||||
)
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -286,6 +288,42 @@ def fused_indexer_q_rope_quant(
|
||||
return q_fp8, weights_out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _upcast_logits_kernel(src_ptr, dst_ptr, row_ends_ptr, stride, BLOCK: tl.constexpr):
|
||||
row = tl.program_id(0)
|
||||
end = tl.load(row_ends_ptr + row)
|
||||
src_ptr += row.to(tl.int64) * stride
|
||||
dst_ptr += row.to(tl.int64) * stride
|
||||
for start in range(tl.program_id(1) * BLOCK, end, tl.num_programs(1) * BLOCK):
|
||||
offs = start + tl.arange(0, BLOCK)
|
||||
mask = offs < end
|
||||
vals = tl.load(src_ptr + offs, mask=mask)
|
||||
tl.store(dst_ptr + offs, vals.to(tl.float32), mask=mask)
|
||||
|
||||
|
||||
def as_fp32_logits(logits: torch.Tensor, row_ends: torch.Tensor) -> torch.Tensor:
|
||||
"""Upcast SM100's BF16 MQA logits for the FP32 top-k kernels.
|
||||
|
||||
Only columns below ``row_ends`` (int32, one per row) are converted, the
|
||||
span the consumers read; DeepGEMM's aligned row stride is kept.
|
||||
"""
|
||||
if logits.dtype == torch.float32:
|
||||
return logits
|
||||
out = torch.empty_strided(
|
||||
logits.shape, logits.stride(), dtype=torch.float32, device=logits.device
|
||||
)
|
||||
num_rows, width = logits.shape
|
||||
if num_rows == 0 or width == 0:
|
||||
return out
|
||||
assert row_ends.numel() == num_rows and row_ends.dtype == torch.int32
|
||||
block = 1024
|
||||
splits = min(triton.cdiv(width, block), max(1, 1024 // num_rows))
|
||||
_upcast_logits_kernel[(num_rows, splits)](
|
||||
logits, out, row_ends.contiguous(), logits.stride(0), BLOCK=block
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _gather_workspace_shapes(
|
||||
total_seq_lens: int,
|
||||
head_dim: int,
|
||||
@@ -537,7 +575,7 @@ def sparse_attn_indexer(
|
||||
kv_cache,
|
||||
slot_mapping_for_cache,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
"mxfp8" if mqa_logits_use_mx_scales() else scale_fmt,
|
||||
)
|
||||
# The indexer and main MLA may classify the same short extend differently
|
||||
# because they use independent decode thresholds. Only the main MLA route
|
||||
@@ -649,6 +687,9 @@ def sparse_attn_indexer(
|
||||
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
|
||||
topk_indices = topk_indices_buffer[row_start:row_end, :topk_tokens]
|
||||
|
||||
# DeepGEMM returns compressed logits: row i starts at K position
|
||||
# cu_seqlen_ks[i]. None means every row starts at column zero.
|
||||
row_starts: torch.Tensor | None = None
|
||||
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
|
||||
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
|
||||
topk_indices.fill_(-1)
|
||||
@@ -662,7 +703,9 @@ def sparse_attn_indexer(
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
k_scale_cast = k_scale.view(
|
||||
torch.int32 if mqa_logits_use_mx_scales() else torch.float32
|
||||
).squeeze(-1)
|
||||
if current_platform.is_xpu():
|
||||
if q_scale_slice is not None:
|
||||
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
|
||||
@@ -674,14 +717,23 @@ def sparse_attn_indexer(
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
)
|
||||
row_starts = cu_seqlen_ks
|
||||
row_ends = cu_seqlen_ke
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[row_start:row_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
q_scale_slice, chunk_weights = mqa_logits_mx_inputs(
|
||||
q_slice_cast, q_scale_slice, weights[row_start:row_end]
|
||||
)
|
||||
row_ends = cu_seqlen_ke - cu_seqlen_ks
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
chunk_weights,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
chunk.max_seqlen_k,
|
||||
),
|
||||
row_ends,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
if candidate_blocks is not None:
|
||||
@@ -694,8 +746,8 @@ def sparse_attn_indexer(
|
||||
if candidate_write:
|
||||
_select_candidate_blocks(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
row_starts,
|
||||
row_ends,
|
||||
chunk_candidates.shape[1],
|
||||
candidate_block_size,
|
||||
chunk_candidates,
|
||||
@@ -703,15 +755,15 @@ def sparse_attn_indexer(
|
||||
else:
|
||||
_apply_candidate_mask(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
row_starts,
|
||||
row_ends,
|
||||
chunk_candidates,
|
||||
candidate_block_size,
|
||||
)
|
||||
ops.top_k_per_row_prefill(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
torch.zeros_like(row_ends) if row_starts is None else row_starts,
|
||||
row_ends,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
@@ -729,7 +781,7 @@ def sparse_attn_indexer(
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
row_starts=cu_seqlen_ks,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
|
||||
if row_shard is not None:
|
||||
@@ -822,16 +874,22 @@ def sparse_attn_indexer(
|
||||
max_model_len,
|
||||
)
|
||||
else:
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
clean_logits=False,
|
||||
indices=decode_metadata.indices,
|
||||
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
|
||||
padded_q_quant_cast, padded_q_scale, weights[:num_padded_tokens]
|
||||
)
|
||||
# SM100 decode is varlen: one row and context length per token.
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
decode_weights,
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
indices=decode_metadata.indices,
|
||||
),
|
||||
seq_lens.view(-1),
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
if candidate_blocks is not None:
|
||||
|
||||
Reference in New Issue
Block a user