[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
86b1c2ef15
commit
a6009ea85b
@@ -42,6 +42,8 @@ from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
has_deep_gemm,
|
||||
mqa_logits_mx_inputs,
|
||||
mqa_logits_use_mx_scales,
|
||||
)
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -286,6 +288,42 @@ def fused_indexer_q_rope_quant(
|
||||
return q_fp8, weights_out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _upcast_logits_kernel(src_ptr, dst_ptr, row_ends_ptr, stride, BLOCK: tl.constexpr):
|
||||
row = tl.program_id(0)
|
||||
end = tl.load(row_ends_ptr + row)
|
||||
src_ptr += row.to(tl.int64) * stride
|
||||
dst_ptr += row.to(tl.int64) * stride
|
||||
for start in range(tl.program_id(1) * BLOCK, end, tl.num_programs(1) * BLOCK):
|
||||
offs = start + tl.arange(0, BLOCK)
|
||||
mask = offs < end
|
||||
vals = tl.load(src_ptr + offs, mask=mask)
|
||||
tl.store(dst_ptr + offs, vals.to(tl.float32), mask=mask)
|
||||
|
||||
|
||||
def as_fp32_logits(logits: torch.Tensor, row_ends: torch.Tensor) -> torch.Tensor:
|
||||
"""Upcast SM100's BF16 MQA logits for the FP32 top-k kernels.
|
||||
|
||||
Only columns below ``row_ends`` (int32, one per row) are converted, the
|
||||
span the consumers read; DeepGEMM's aligned row stride is kept.
|
||||
"""
|
||||
if logits.dtype == torch.float32:
|
||||
return logits
|
||||
out = torch.empty_strided(
|
||||
logits.shape, logits.stride(), dtype=torch.float32, device=logits.device
|
||||
)
|
||||
num_rows, width = logits.shape
|
||||
if num_rows == 0 or width == 0:
|
||||
return out
|
||||
assert row_ends.numel() == num_rows and row_ends.dtype == torch.int32
|
||||
block = 1024
|
||||
splits = min(triton.cdiv(width, block), max(1, 1024 // num_rows))
|
||||
_upcast_logits_kernel[(num_rows, splits)](
|
||||
logits, out, row_ends.contiguous(), logits.stride(0), BLOCK=block
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _gather_workspace_shapes(
|
||||
total_seq_lens: int,
|
||||
head_dim: int,
|
||||
@@ -537,7 +575,7 @@ def sparse_attn_indexer(
|
||||
kv_cache,
|
||||
slot_mapping_for_cache,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
"mxfp8" if mqa_logits_use_mx_scales() else scale_fmt,
|
||||
)
|
||||
# The indexer and main MLA may classify the same short extend differently
|
||||
# because they use independent decode thresholds. Only the main MLA route
|
||||
@@ -649,6 +687,9 @@ def sparse_attn_indexer(
|
||||
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
|
||||
topk_indices = topk_indices_buffer[row_start:row_end, :topk_tokens]
|
||||
|
||||
# DeepGEMM returns compressed logits: row i starts at K position
|
||||
# cu_seqlen_ks[i]. None means every row starts at column zero.
|
||||
row_starts: torch.Tensor | None = None
|
||||
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
|
||||
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
|
||||
topk_indices.fill_(-1)
|
||||
@@ -662,7 +703,9 @@ def sparse_attn_indexer(
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
k_scale_cast = k_scale.view(
|
||||
torch.int32 if mqa_logits_use_mx_scales() else torch.float32
|
||||
).squeeze(-1)
|
||||
if current_platform.is_xpu():
|
||||
if q_scale_slice is not None:
|
||||
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
|
||||
@@ -674,14 +717,23 @@ def sparse_attn_indexer(
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
)
|
||||
row_starts = cu_seqlen_ks
|
||||
row_ends = cu_seqlen_ke
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[row_start:row_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
q_scale_slice, chunk_weights = mqa_logits_mx_inputs(
|
||||
q_slice_cast, q_scale_slice, weights[row_start:row_end]
|
||||
)
|
||||
row_ends = cu_seqlen_ke - cu_seqlen_ks
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
chunk_weights,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
chunk.max_seqlen_k,
|
||||
),
|
||||
row_ends,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
if candidate_blocks is not None:
|
||||
@@ -694,8 +746,8 @@ def sparse_attn_indexer(
|
||||
if candidate_write:
|
||||
_select_candidate_blocks(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
row_starts,
|
||||
row_ends,
|
||||
chunk_candidates.shape[1],
|
||||
candidate_block_size,
|
||||
chunk_candidates,
|
||||
@@ -703,15 +755,15 @@ def sparse_attn_indexer(
|
||||
else:
|
||||
_apply_candidate_mask(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
row_starts,
|
||||
row_ends,
|
||||
chunk_candidates,
|
||||
candidate_block_size,
|
||||
)
|
||||
ops.top_k_per_row_prefill(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
torch.zeros_like(row_ends) if row_starts is None else row_starts,
|
||||
row_ends,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
@@ -729,7 +781,7 @@ def sparse_attn_indexer(
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
row_starts=cu_seqlen_ks,
|
||||
row_starts=row_starts,
|
||||
)
|
||||
|
||||
if row_shard is not None:
|
||||
@@ -822,16 +874,22 @@ def sparse_attn_indexer(
|
||||
max_model_len,
|
||||
)
|
||||
else:
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
clean_logits=False,
|
||||
indices=decode_metadata.indices,
|
||||
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
|
||||
padded_q_quant_cast, padded_q_scale, weights[:num_padded_tokens]
|
||||
)
|
||||
# SM100 decode is varlen: one row and context length per token.
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
decode_weights,
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
indices=decode_metadata.indices,
|
||||
),
|
||||
seq_lens.view(-1),
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
if candidate_blocks is not None:
|
||||
|
||||
@@ -6,7 +6,9 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
|
||||
# has_indexer=False path so the indexer args don't allocate every call.
|
||||
@@ -87,6 +89,7 @@ def _fp8_quant_and_cache_write(
|
||||
SHUFFLE: tl.constexpr,
|
||||
BLOCK_TILE: tl.constexpr,
|
||||
HEAD_TILE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
USE_FNUZ: tl.constexpr,
|
||||
):
|
||||
@@ -111,6 +114,8 @@ def _fp8_quant_and_cache_write(
|
||||
|
||||
tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask)
|
||||
scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4
|
||||
if MX_SCALE:
|
||||
scale = pack_ue8m0_x4(scale)
|
||||
tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale)
|
||||
|
||||
|
||||
@@ -167,6 +172,7 @@ def _fused_norm_rope_kernel(
|
||||
INDEXER_CACHE_SHUFFLE: tl.constexpr,
|
||||
INDEXER_CACHE_BLOCK_TILE: tl.constexpr,
|
||||
INDEXER_CACHE_HEAD_TILE: tl.constexpr,
|
||||
INDEXER_CACHE_MX_SCALE: tl.constexpr,
|
||||
# MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr)
|
||||
mla_cache_ptr,
|
||||
mla_cache_block_stride,
|
||||
@@ -501,6 +507,7 @@ def _fused_norm_rope_kernel(
|
||||
INDEXER_CACHE_SHUFFLE,
|
||||
INDEXER_CACHE_BLOCK_TILE,
|
||||
INDEXER_CACHE_HEAD_TILE,
|
||||
INDEXER_CACHE_MX_SCALE,
|
||||
FP8_MAX,
|
||||
USE_FNUZ,
|
||||
)
|
||||
@@ -571,7 +578,10 @@ def fused_norm_rope(
|
||||
assert slot_mapping is not None
|
||||
if indexer_slot_mapping is None:
|
||||
indexer_slot_mapping = slot_mapping
|
||||
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
|
||||
idx_cache_mx_scale = mqa_logits_use_mx_scales()
|
||||
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(
|
||||
torch.int32 if idx_cache_mx_scale else torch.float32
|
||||
)
|
||||
idx_cache_block_size = indexer_k_cache.shape[1]
|
||||
idx_cache_stride = indexer_k_cache.shape[2]
|
||||
# The caller's cache reports whether its reader expects the shuffled
|
||||
@@ -591,6 +601,7 @@ def fused_norm_rope(
|
||||
)
|
||||
else:
|
||||
idx_cache_scale_view = None
|
||||
idx_cache_mx_scale = False
|
||||
idx_cache_block_size = 1
|
||||
idx_cache_stride = 0
|
||||
idx_cache_shuffle = False
|
||||
@@ -701,6 +712,7 @@ def fused_norm_rope(
|
||||
idx_cache_shuffle,
|
||||
_INDEXER_CACHE_BLOCK_TILE,
|
||||
idx_cache_head_tile,
|
||||
idx_cache_mx_scale,
|
||||
# MLA KV cache (uses same slot_mapping)
|
||||
mla_kv_cache,
|
||||
mla_block_stride,
|
||||
|
||||
@@ -33,7 +33,9 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import _ON_GFX950
|
||||
@@ -742,6 +744,7 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
|
||||
TOKEN_STRIDE: tl.constexpr, # 128 for indexer
|
||||
SCALE_DIM: tl.constexpr, # 4 for indexer (1 float32)
|
||||
KV_BLOCK_STRIDE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr, # store the scale as 4 packed UE8M0 bytes
|
||||
):
|
||||
"""Fused compress → RMSNorm → RoPE → FP8 quant → store.
|
||||
|
||||
@@ -878,7 +881,10 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
|
||||
|
||||
# Single float32 scale
|
||||
scale_val = tl.exp2(exponent)
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), scale_val)
|
||||
if MX_SCALE:
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.int32)), pack_ue8m0_x4(scale_val))
|
||||
else:
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), scale_val)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -927,6 +933,7 @@ def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn(
|
||||
TOKEN_STRIDE: tl.constexpr, # HEAD_SIZE // 2 = 64 packed bytes/token
|
||||
SCALE_DIM: tl.constexpr, # HEAD_SIZE // QUANT_BLOCK = 4 ue8m0 bytes/token
|
||||
KV_BLOCK_STRIDE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr, # unused for MXFP4 (kept for signature parity)
|
||||
):
|
||||
"""Fused compress → RMSNorm → RoPE → MXFP4 quant → store.
|
||||
|
||||
@@ -1102,6 +1109,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
|
||||
block_size: int
|
||||
kv_cache_block_size: int
|
||||
kv_block_stride: int
|
||||
mx_scale: bool
|
||||
|
||||
# Canonical kernel for the launcher's arg_names mapping; the launched
|
||||
# variant (fp8/mxfp4, same arg_names) is picked per call in __call__.
|
||||
@@ -1181,6 +1189,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
|
||||
block_size=state_block_size,
|
||||
kv_cache_block_size=kv_cache_block_size,
|
||||
kv_block_stride=kv_block_stride,
|
||||
mx_scale=not use_fp4_cache and mqa_logits_use_mx_scales(),
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
@@ -1301,6 +1310,7 @@ class FusedKVCompressNormRopeInsertIndexerTritonKernel(
|
||||
TOKEN_STRIDE=token_stride,
|
||||
SCALE_DIM=scale_dim,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
MX_SCALE=not use_fp4_cache and mqa_logits_use_mx_scales(),
|
||||
num_warps=1,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
|
||||
@@ -17,6 +17,8 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
from . import MXFP4_BLOCK_SIZE, _fp32x2_to_fp4x2
|
||||
|
||||
@@ -51,7 +53,8 @@ def indexer_k_norm_rope_store(
|
||||
kv_slot_mapping: [num_tokens] slots in the indexer cache (-1 = skip).
|
||||
compress_ratio: group size; keys are emitted at group boundaries.
|
||||
use_fp4_cache: MXFP4 (2 nibbles/byte + ue8m0 per 32) when True, else
|
||||
per-token FP8 with a single fp32 scale.
|
||||
per-token FP8 with a single fp32 scale (four packed ue8m0 bytes
|
||||
under `mqa_logits_use_mx_scales`).
|
||||
|
||||
"""
|
||||
num_tokens = kv_slot_mapping.numel()
|
||||
@@ -111,6 +114,7 @@ def indexer_k_norm_rope_store(
|
||||
KV_BLOCK_STRIDE=k_cache.stride(0),
|
||||
FP8_MAX=448.0,
|
||||
USE_FP4=use_fp4_cache,
|
||||
MX_SCALE=not use_fp4_cache and mqa_logits_use_mx_scales(),
|
||||
SHUFFLE=shuffle,
|
||||
BLOCK_TILE_SIZE=_BLOCK_TILE_SIZE,
|
||||
HEAD_TILE_SIZE=_HEAD_TILE_SIZE,
|
||||
@@ -139,6 +143,7 @@ def _indexer_k_norm_rope_quant_store_kernel(
|
||||
KV_BLOCK_STRIDE: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
USE_FP4: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
SHUFFLE: tl.constexpr,
|
||||
BLOCK_TILE_SIZE: tl.constexpr,
|
||||
HEAD_TILE_SIZE: tl.constexpr,
|
||||
@@ -252,4 +257,10 @@ def _indexer_k_norm_rope_quant_store_kernel(
|
||||
tl.store(val_ptr + tiled, x_uint8)
|
||||
else:
|
||||
tl.store(val_ptr + block, x_uint8)
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), tl.exp2(exponent))
|
||||
if MX_SCALE:
|
||||
tl.store(
|
||||
scale_ptr.to(tl.pointer_type(tl.int32)),
|
||||
pack_ue8m0_x4(tl.exp2(exponent)),
|
||||
)
|
||||
else:
|
||||
tl.store(scale_ptr.to(tl.pointer_type(tl.float32)), tl.exp2(exponent))
|
||||
|
||||
@@ -15,6 +15,8 @@ from __future__ import annotations
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.v1.attention.ops.common import pack_ue8m0_x4
|
||||
|
||||
# The GLM-5.3-Flash indexer head dimension is fixed at 128.
|
||||
INDEX_HEAD_DIM = 128
|
||||
@@ -157,6 +159,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
HAS_WRITE_MASK: tl.constexpr,
|
||||
RETURN_COMPRESSED: tl.constexpr,
|
||||
WRITE_CACHE: tl.constexpr,
|
||||
@@ -245,7 +248,14 @@ def _kpool_softmax_rotate_write_cache_kernel(
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
|
||||
if MX_SCALE:
|
||||
tl.store(
|
||||
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
|
||||
pack_ue8m0_x4(scale),
|
||||
mask=do_write,
|
||||
)
|
||||
else:
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale, mask=do_write)
|
||||
|
||||
if RETURN_COMPRESSED:
|
||||
tl.store(
|
||||
@@ -324,6 +334,7 @@ def kpool_compress_and_write_cache(
|
||||
|
||||
buf_fp8 = buf.view(torch.float8_e4m3fn)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
mx_scale = write_cache and mqa_logits_use_mx_scales()
|
||||
# bytes per page (last dim of kv_cache) viewed as uint8
|
||||
buf_numel_per_page = buf.stride(0)
|
||||
s_offset_nbytes_in_page = page_size * head_dim
|
||||
@@ -361,7 +372,9 @@ def kpool_compress_and_write_cache(
|
||||
POOL_SIZE=slot_k.shape[1],
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=s_offset_nbytes_in_page,
|
||||
ROUND_SCALE=round_scale,
|
||||
# MX scales need power-of-two scales.
|
||||
ROUND_SCALE=round_scale or mx_scale,
|
||||
MX_SCALE=mx_scale,
|
||||
HAS_WRITE_MASK=has_write_mask,
|
||||
RETURN_COMPRESSED=return_compressed,
|
||||
WRITE_CACHE=write_cache,
|
||||
@@ -479,6 +492,7 @@ def _kpool_decode_update_batched_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
MX_SCALE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per request; iterates its NEXT_N verify tokens in order.
|
||||
@@ -604,7 +618,13 @@ def _kpool_decode_update_batched_kernel(
|
||||
+ loc_token_offset_in_page
|
||||
)
|
||||
tl.store(buf_fp8_ptr + out_k_offsets, quantized, mask=dim_mask)
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale)
|
||||
if MX_SCALE:
|
||||
tl.store(
|
||||
buf_fp32_ptr.to(tl.pointer_type(tl.int32)) + out_s_offset,
|
||||
pack_ue8m0_x4(scale),
|
||||
)
|
||||
else:
|
||||
tl.store(buf_fp32_ptr + out_s_offset, scale)
|
||||
|
||||
# Stash the current token AFTER any completion read so the completion
|
||||
# uses prior stashes (and the current token's own key/score via
|
||||
@@ -718,7 +738,9 @@ def kpool_decode_update_and_maybe_write_cache_batched(
|
||||
KPOOL_HEAD=tail_kv_cache.stride(1),
|
||||
HEAD_DIM=head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=page_size * head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
# MX scales need power-of-two scales.
|
||||
ROUND_SCALE=round_scale or mqa_logits_use_mx_scales(),
|
||||
MX_SCALE=mqa_logits_use_mx_scales(),
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
)
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
PrefillRowShard,
|
||||
_merge_dcp_topk_global,
|
||||
_reserve_row_shard_exchange_transient,
|
||||
as_fp32_logits,
|
||||
)
|
||||
from vllm.models.glm5next.common.sparse_indexer import (
|
||||
RADIX_TOPK_WORKSPACE_SIZE,
|
||||
@@ -29,7 +30,13 @@ from vllm.models.glm5next.common.sparse_indexer import (
|
||||
)
|
||||
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import has_deep_gemm
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
has_deep_gemm,
|
||||
mqa_logits_mx_inputs,
|
||||
mqa_logits_use_mx_scales,
|
||||
)
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
@@ -253,7 +260,7 @@ def sparse_attn_indexer_kpool(
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
"mxfp8" if mqa_logits_use_mx_scales() else scale_fmt,
|
||||
)
|
||||
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
@@ -343,20 +350,29 @@ def sparse_attn_indexer_kpool(
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
|
||||
k_scale_cast = k_scale.view(
|
||||
torch.int32 if mqa_logits_use_mx_scales() else torch.float32
|
||||
).squeeze(-1)
|
||||
|
||||
num_rows = q_slice.shape[0]
|
||||
if chunk.local_total_seq_lens == 0:
|
||||
logits = q_slice.new_empty((num_rows, 0), dtype=torch.float32)
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[row_start:row_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
q_scale_slice, chunk_weights = mqa_logits_mx_inputs(
|
||||
q_slice_cast, q_scale_slice, weights[row_start:row_end]
|
||||
)
|
||||
# Compressed: row i starts at K position cu_seqlen_ks[i].
|
||||
row_ends = cu_seqlen_ke - cu_seqlen_ks
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
chunk_weights,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
chunk.max_seqlen_k,
|
||||
),
|
||||
row_ends,
|
||||
)
|
||||
|
||||
# kpool: logits are pool-granular (compress_ratio == index_kpool),
|
||||
@@ -376,8 +392,8 @@ def sparse_attn_indexer_kpool(
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
torch.zeros_like(row_ends),
|
||||
row_ends,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
@@ -392,7 +408,7 @@ def sparse_attn_indexer_kpool(
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
pool_cp_interleave,
|
||||
row_starts=cu_seqlen_ks,
|
||||
row_starts=None,
|
||||
)
|
||||
|
||||
if index_kpool > 1:
|
||||
@@ -594,18 +610,22 @@ def sparse_attn_indexer_kpool(
|
||||
if use_fp4_cache
|
||||
else padded_q_quant_decode_tokens
|
||||
)
|
||||
from vllm.utils.deep_gemm import fp8_fp4_paged_mqa_logits
|
||||
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
padded_weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_pool_len,
|
||||
clean_logits=False,
|
||||
indices=decode_metadata.indices,
|
||||
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
|
||||
padded_q_quant_cast, padded_q_scale, padded_weights[:num_padded_tokens]
|
||||
)
|
||||
# SM100 decode is varlen: one row and context length per token.
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
decode_weights,
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_pool_len,
|
||||
indices=decode_metadata.indices,
|
||||
),
|
||||
seq_lens.view(-1),
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
# kpool: logits are pool-granular -> select topk_tokens//kpool pools,
|
||||
|
||||
+72
-29
@@ -591,37 +591,79 @@ def transform_sf_into_required_layout(*args, **kwargs):
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def mqa_logits_use_mx_scales() -> bool:
|
||||
"""Whether the MQA logits kernels take only MX-scaled inputs.
|
||||
|
||||
On SM100 they need MXFP8/MXFP4 Q and K with packed UE8M0 scales and BF16
|
||||
weights, and return BF16 logits. FP8 indexer K caches then store each
|
||||
token's power-of-two scale as four equal UE8M0 bytes instead of an fp32.
|
||||
"""
|
||||
return current_platform.is_cuda() and current_platform.is_device_capability_family(
|
||||
100
|
||||
)
|
||||
|
||||
|
||||
# Four packed UE8M0 bytes encoding 1.0.
|
||||
_UNIT_UE8M0_X4 = 0x7F7F7F7F
|
||||
# Grown buffers are kept alive: CUDA graphs captured earlier still read them.
|
||||
_unit_mx_scales: dict[torch.device, list[torch.Tensor]] = {}
|
||||
|
||||
|
||||
def mqa_logits_mx_inputs(
|
||||
q: torch.Tensor, q_scale: torch.Tensor | None, weights: torch.Tensor
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""Adapt indexer Q scales and weights to the SM100 MQA logits kernels.
|
||||
|
||||
FP8 Q folds its per-(token, head) power-of-two scale into ``weights``, so
|
||||
it is passed with unit MX scales, which is exact since relu commutes with
|
||||
a positive scale. Returns ``(q_scale, weights)`` unchanged off SM100.
|
||||
"""
|
||||
if not mqa_logits_use_mx_scales():
|
||||
return q_scale, weights
|
||||
if q_scale is None:
|
||||
numel = q.numel() // q.shape[-1]
|
||||
bufs = _unit_mx_scales.setdefault(q.device, [])
|
||||
if not bufs or bufs[-1].numel() < numel:
|
||||
size = max(numel, 2 * bufs[-1].numel() if bufs else 0)
|
||||
bufs.append(
|
||||
torch.full((size,), _UNIT_UE8M0_X4, dtype=torch.int32, device=q.device)
|
||||
)
|
||||
q_scale = bufs[-1][:numel].view(q.shape[:-1])
|
||||
return q_scale, weights.to(torch.bfloat16)
|
||||
|
||||
|
||||
def fp8_fp4_mqa_logits(
|
||||
q: tuple[torch.Tensor, torch.Tensor | None],
|
||||
kv: tuple[torch.Tensor, torch.Tensor],
|
||||
weights: torch.Tensor,
|
||||
cu_seqlen_ks: torch.Tensor,
|
||||
cu_seqlen_ke: torch.Tensor,
|
||||
clean_logits: bool,
|
||||
max_seqlen_k: int,
|
||||
) -> torch.Tensor:
|
||||
"""Compute MQA logits for a single sequence without KV paging.
|
||||
|
||||
Unified FP8/FP4 dispatch — the underlying DeepGEMM kernel takes
|
||||
``q = (values, scales_or_None)`` where ``scales`` is None for FP8 Q
|
||||
(per-token scale is folded into ``weights``) and a packed block-scale
|
||||
tensor for MXFP4 Q.
|
||||
``q = (values, scales_or_None)``. SM100 takes MXFP8 or MXFP4 with packed
|
||||
UE8M0 block scales; SM90/SM120 also take FP8 Q with ``q_scale=None``
|
||||
(per-token scale folded into ``weights``).
|
||||
|
||||
Args:
|
||||
q: Tuple ``(q_values, q_scale)``. FP8 path: q_values is [M, H, D]
|
||||
float8_e4m3fn and q_scale is None (per-token scale is folded
|
||||
into ``weights``). FP4 path: q_values is packed uint8 and
|
||||
q_scale is the companion block-scale tensor.
|
||||
kv: Tuple `(k_packed, k_scales)` — FP8 layout is [N, D]
|
||||
float8_e4m3fn plus fp32 scales [N]; FP4 layout is packed uint8.
|
||||
weights: weights of shape [M, H], dtype `torch.float32`.
|
||||
q: Tuple ``(q_values, q_scale)``, q_values of shape [M, H, D] (packed
|
||||
to D/2 bytes for MXFP4) and q_scale int32 [M, H] or None.
|
||||
kv: Tuple ``(k_values, k_scales)`` of shape [N, D] and [N]; k_scales
|
||||
is int32 packed UE8M0 with MX scales, fp32 otherwise.
|
||||
weights: Tensor of shape [M, H], bf16 on SM100 and fp32 elsewhere.
|
||||
cu_seqlen_ks: Start indices (inclusive) for valid K per query
|
||||
position, shape [M], dtype int32.
|
||||
cu_seqlen_ke: End indices (exclusive) for valid K per query
|
||||
position, shape [M], dtype int32.
|
||||
clean_logits: Whether to clean the unfilled logits into `-inf`.
|
||||
max_seqlen_k: Upper bound on ``cu_seqlen_ke - cu_seqlen_ks``.
|
||||
|
||||
Returns:
|
||||
Logits tensor of shape [M, N], dtype `torch.float32`.
|
||||
Compressed logits of shape [M, max_seqlen_k]: row ``i`` holds K
|
||||
positions ``[cu_seqlen_ks[i], cu_seqlen_ke[i])`` starting at column
|
||||
zero, and the tail is not cleaned. bf16 on SM100, fp32 elsewhere.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
@@ -633,7 +675,7 @@ def fp8_fp4_mqa_logits(
|
||||
weights,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=clean_logits,
|
||||
max_seqlen_k,
|
||||
)
|
||||
|
||||
|
||||
@@ -699,24 +741,25 @@ def fp8_fp4_paged_mqa_logits(
|
||||
block_tables: torch.Tensor,
|
||||
schedule_metadata: torch.Tensor,
|
||||
max_model_len: int,
|
||||
clean_logits: bool,
|
||||
indices: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute MQA logits using a paged KV-cache.
|
||||
|
||||
Unified FP8/FP4 dispatch — the underlying DeepGEMM kernel takes
|
||||
``q = (values, scales_or_None)``; pass ``(q_tensor, None)`` for the FP8
|
||||
path and ``(q_values, q_scale)`` for MXFP4.
|
||||
``q = (values, scales_or_None)``; see `fp8_fp4_mqa_logits` for the
|
||||
per-architecture formats. SM100 requires the varlen form (``indices``
|
||||
with next_n == 1).
|
||||
|
||||
Args:
|
||||
q: Tuple ``(q_values, q_scale)``. FP8 path: q_values is
|
||||
[B, next_n, H, D] float8_e4m3fn and q_scale is None. FP4 path:
|
||||
q_values is packed uint8 and q_scale is the companion
|
||||
block-scale tensor.
|
||||
kv_cache: Paged KV-cache. FP8 layout is [num_blocks, block_size, 1,
|
||||
D+4], dtype `torch.uint8`, with the last 4 bytes per (block, pos)
|
||||
storing the float dequant scale.
|
||||
weights: Tensor of shape [B * next_n, H], dtype `torch.float32`.
|
||||
q: Tuple ``(q_values, q_scale)``, q_values of shape
|
||||
[B, next_n, H, D] (packed to D/2 bytes for MXFP4) and q_scale
|
||||
int32 [B, next_n, H] or None.
|
||||
kv_cache: Paged KV-cache of shape [num_blocks, block_size, 1, D+4]
|
||||
(D/2+4 for MXFP4), dtype `torch.uint8`; the last 4 bytes per
|
||||
(block, pos) hold the int32 packed UE8M0 scales with MX scales,
|
||||
or the fp32 dequant scale otherwise.
|
||||
weights: Tensor of shape [B * next_n, H], bf16 on SM100 and fp32
|
||||
elsewhere.
|
||||
context_lens: Tensor of shape [B], dtype int32; effective context length
|
||||
for each batch element.
|
||||
block_tables: Tensor of shape [B, max_blocks], dtype int32; maps logical
|
||||
@@ -724,12 +767,11 @@ def fp8_fp4_paged_mqa_logits(
|
||||
schedule_metadata: Returned by `get_paged_mqa_logits_metadata`;
|
||||
used to distribute work across SMs.
|
||||
max_model_len: Maximum sequence length used to size the logits output.
|
||||
clean_logits: Whether to clean the unfilled logits into `-inf`.
|
||||
indices: Optional request index for each varlen row.
|
||||
|
||||
Returns:
|
||||
Logits tensor of shape [B * next_n, max_model_len], dtype
|
||||
`torch.float32`.
|
||||
Logits tensor of shape [B * next_n, max_model_len]; columns past each
|
||||
row's context length are not cleaned. bf16 on SM100, fp32 elsewhere.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
@@ -751,7 +793,6 @@ def fp8_fp4_paged_mqa_logits(
|
||||
block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
clean_logits=clean_logits,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -1052,6 +1093,8 @@ __all__ = [
|
||||
"fp8_fp4_mqa_logits",
|
||||
"fp8_fp4_paged_mqa_logits",
|
||||
"get_paged_mqa_logits_metadata",
|
||||
"mqa_logits_mx_inputs",
|
||||
"mqa_logits_use_mx_scales",
|
||||
"native_next_n_supported",
|
||||
"per_block_cast_to_fp8",
|
||||
"is_deep_gemm_e8m0_used",
|
||||
|
||||
@@ -467,6 +467,9 @@ class DeepseekV32IndexerPrefillChunkMetadata:
|
||||
cu_seq_lens: torch.Tensor
|
||||
token_to_seq: torch.Tensor
|
||||
total_seq_lens: int
|
||||
# Host upper bound on cu_seqlen_ke - cu_seqlen_ks, sizing the compressed
|
||||
# DeepGEMM prefill logits.
|
||||
max_seqlen_k: int
|
||||
token_start: int
|
||||
token_end: int
|
||||
num_reqs: int
|
||||
@@ -1957,6 +1960,7 @@ def build_prefill_chunk_metadata(
|
||||
cu_seq_lens=cu_seq_lens,
|
||||
token_to_seq=token_to_seq,
|
||||
total_seq_lens=total_seq_lens,
|
||||
max_seqlen_k=int(compressed_seq_lens_cpu[start_idx:end_idx].max()),
|
||||
block_table=block_table[start_idx:end_idx],
|
||||
token_start=token_start,
|
||||
token_end=token_end,
|
||||
|
||||
@@ -154,6 +154,13 @@ class PackSeqTritonKernel(VllmTritonJitKernel["PackSeqTritonKernel.CompileKey"])
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def pack_ue8m0_x4(scale):
|
||||
"""Encode a power-of-two fp32 scale as four equal packed UE8M0 bytes, the
|
||||
MX scales of a 128-wide row (see `mqa_logits_use_mx_scales`)."""
|
||||
return ((scale.to(tl.int32, bitcast=True) >> 23) & 0xFF) * 0x01010101
|
||||
|
||||
|
||||
def pack_seq_triton(
|
||||
x: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user