[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-12 23:24:43 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 86b1c2ef15
commit a6009ea85b
23 changed files with 516 additions and 208 deletions
+72 -29
View File
@@ -591,37 +591,79 @@ def transform_sf_into_required_layout(*args, **kwargs):
)
@functools.cache
def mqa_logits_use_mx_scales() -> bool:
"""Whether the MQA logits kernels take only MX-scaled inputs.
On SM100 they need MXFP8/MXFP4 Q and K with packed UE8M0 scales and BF16
weights, and return BF16 logits. FP8 indexer K caches then store each
token's power-of-two scale as four equal UE8M0 bytes instead of an fp32.
"""
return current_platform.is_cuda() and current_platform.is_device_capability_family(
100
)
# Four packed UE8M0 bytes encoding 1.0.
_UNIT_UE8M0_X4 = 0x7F7F7F7F
# Grown buffers are kept alive: CUDA graphs captured earlier still read them.
_unit_mx_scales: dict[torch.device, list[torch.Tensor]] = {}
def mqa_logits_mx_inputs(
q: torch.Tensor, q_scale: torch.Tensor | None, weights: torch.Tensor
) -> tuple[torch.Tensor | None, torch.Tensor]:
"""Adapt indexer Q scales and weights to the SM100 MQA logits kernels.
FP8 Q folds its per-(token, head) power-of-two scale into ``weights``, so
it is passed with unit MX scales, which is exact since relu commutes with
a positive scale. Returns ``(q_scale, weights)`` unchanged off SM100.
"""
if not mqa_logits_use_mx_scales():
return q_scale, weights
if q_scale is None:
numel = q.numel() // q.shape[-1]
bufs = _unit_mx_scales.setdefault(q.device, [])
if not bufs or bufs[-1].numel() < numel:
size = max(numel, 2 * bufs[-1].numel() if bufs else 0)
bufs.append(
torch.full((size,), _UNIT_UE8M0_X4, dtype=torch.int32, device=q.device)
)
q_scale = bufs[-1][:numel].view(q.shape[:-1])
return q_scale, weights.to(torch.bfloat16)
def fp8_fp4_mqa_logits(
q: tuple[torch.Tensor, torch.Tensor | None],
kv: tuple[torch.Tensor, torch.Tensor],
weights: torch.Tensor,
cu_seqlen_ks: torch.Tensor,
cu_seqlen_ke: torch.Tensor,
clean_logits: bool,
max_seqlen_k: int,
) -> torch.Tensor:
"""Compute MQA logits for a single sequence without KV paging.
Unified FP8/FP4 dispatch — the underlying DeepGEMM kernel takes
``q = (values, scales_or_None)`` where ``scales`` is None for FP8 Q
(per-token scale is folded into ``weights``) and a packed block-scale
tensor for MXFP4 Q.
``q = (values, scales_or_None)``. SM100 takes MXFP8 or MXFP4 with packed
UE8M0 block scales; SM90/SM120 also take FP8 Q with ``q_scale=None``
(per-token scale folded into ``weights``).
Args:
q: Tuple ``(q_values, q_scale)``. FP8 path: q_values is [M, H, D]
float8_e4m3fn and q_scale is None (per-token scale is folded
into ``weights``). FP4 path: q_values is packed uint8 and
q_scale is the companion block-scale tensor.
kv: Tuple `(k_packed, k_scales)` — FP8 layout is [N, D]
float8_e4m3fn plus fp32 scales [N]; FP4 layout is packed uint8.
weights: weights of shape [M, H], dtype `torch.float32`.
q: Tuple ``(q_values, q_scale)``, q_values of shape [M, H, D] (packed
to D/2 bytes for MXFP4) and q_scale int32 [M, H] or None.
kv: Tuple ``(k_values, k_scales)`` of shape [N, D] and [N]; k_scales
is int32 packed UE8M0 with MX scales, fp32 otherwise.
weights: Tensor of shape [M, H], bf16 on SM100 and fp32 elsewhere.
cu_seqlen_ks: Start indices (inclusive) for valid K per query
position, shape [M], dtype int32.
cu_seqlen_ke: End indices (exclusive) for valid K per query
position, shape [M], dtype int32.
clean_logits: Whether to clean the unfilled logits into `-inf`.
max_seqlen_k: Upper bound on ``cu_seqlen_ke - cu_seqlen_ks``.
Returns:
Logits tensor of shape [M, N], dtype `torch.float32`.
Compressed logits of shape [M, max_seqlen_k]: row ``i`` holds K
positions ``[cu_seqlen_ks[i], cu_seqlen_ke[i])`` starting at column
zero, and the tail is not cleaned. bf16 on SM100, fp32 elsewhere.
"""
_lazy_init()
@@ -633,7 +675,7 @@ def fp8_fp4_mqa_logits(
weights,
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=clean_logits,
max_seqlen_k,
)
@@ -699,24 +741,25 @@ def fp8_fp4_paged_mqa_logits(
block_tables: torch.Tensor,
schedule_metadata: torch.Tensor,
max_model_len: int,
clean_logits: bool,
indices: torch.Tensor | None = None,
) -> torch.Tensor:
"""Compute MQA logits using a paged KV-cache.
Unified FP8/FP4 dispatch — the underlying DeepGEMM kernel takes
``q = (values, scales_or_None)``; pass ``(q_tensor, None)`` for the FP8
path and ``(q_values, q_scale)`` for MXFP4.
``q = (values, scales_or_None)``; see `fp8_fp4_mqa_logits` for the
per-architecture formats. SM100 requires the varlen form (``indices``
with next_n == 1).
Args:
q: Tuple ``(q_values, q_scale)``. FP8 path: q_values is
[B, next_n, H, D] float8_e4m3fn and q_scale is None. FP4 path:
q_values is packed uint8 and q_scale is the companion
block-scale tensor.
kv_cache: Paged KV-cache. FP8 layout is [num_blocks, block_size, 1,
D+4], dtype `torch.uint8`, with the last 4 bytes per (block, pos)
storing the float dequant scale.
weights: Tensor of shape [B * next_n, H], dtype `torch.float32`.
q: Tuple ``(q_values, q_scale)``, q_values of shape
[B, next_n, H, D] (packed to D/2 bytes for MXFP4) and q_scale
int32 [B, next_n, H] or None.
kv_cache: Paged KV-cache of shape [num_blocks, block_size, 1, D+4]
(D/2+4 for MXFP4), dtype `torch.uint8`; the last 4 bytes per
(block, pos) hold the int32 packed UE8M0 scales with MX scales,
or the fp32 dequant scale otherwise.
weights: Tensor of shape [B * next_n, H], bf16 on SM100 and fp32
elsewhere.
context_lens: Tensor of shape [B], dtype int32; effective context length
for each batch element.
block_tables: Tensor of shape [B, max_blocks], dtype int32; maps logical
@@ -724,12 +767,11 @@ def fp8_fp4_paged_mqa_logits(
schedule_metadata: Returned by `get_paged_mqa_logits_metadata`;
used to distribute work across SMs.
max_model_len: Maximum sequence length used to size the logits output.
clean_logits: Whether to clean the unfilled logits into `-inf`.
indices: Optional request index for each varlen row.
Returns:
Logits tensor of shape [B * next_n, max_model_len], dtype
`torch.float32`.
Logits tensor of shape [B * next_n, max_model_len]; columns past each
row's context length are not cleaned. bf16 on SM100, fp32 elsewhere.
"""
_lazy_init()
@@ -751,7 +793,6 @@ def fp8_fp4_paged_mqa_logits(
block_tables,
schedule_metadata,
max_model_len,
clean_logits=clean_logits,
**kwargs,
)
@@ -1052,6 +1093,8 @@ __all__ = [
"fp8_fp4_mqa_logits",
"fp8_fp4_paged_mqa_logits",
"get_paged_mqa_logits_metadata",
"mqa_logits_mx_inputs",
"mqa_logits_use_mx_scales",
"native_next_n_supported",
"per_block_cast_to_fp8",
"is_deep_gemm_e8m0_used",