[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
86b1c2ef15
commit
a6009ea85b
+72
-29
@@ -591,37 +591,79 @@ def transform_sf_into_required_layout(*args, **kwargs):
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def mqa_logits_use_mx_scales() -> bool:
|
||||
"""Whether the MQA logits kernels take only MX-scaled inputs.
|
||||
|
||||
On SM100 they need MXFP8/MXFP4 Q and K with packed UE8M0 scales and BF16
|
||||
weights, and return BF16 logits. FP8 indexer K caches then store each
|
||||
token's power-of-two scale as four equal UE8M0 bytes instead of an fp32.
|
||||
"""
|
||||
return current_platform.is_cuda() and current_platform.is_device_capability_family(
|
||||
100
|
||||
)
|
||||
|
||||
|
||||
# Four packed UE8M0 bytes encoding 1.0.
|
||||
_UNIT_UE8M0_X4 = 0x7F7F7F7F
|
||||
# Grown buffers are kept alive: CUDA graphs captured earlier still read them.
|
||||
_unit_mx_scales: dict[torch.device, list[torch.Tensor]] = {}
|
||||
|
||||
|
||||
def mqa_logits_mx_inputs(
|
||||
q: torch.Tensor, q_scale: torch.Tensor | None, weights: torch.Tensor
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""Adapt indexer Q scales and weights to the SM100 MQA logits kernels.
|
||||
|
||||
FP8 Q folds its per-(token, head) power-of-two scale into ``weights``, so
|
||||
it is passed with unit MX scales, which is exact since relu commutes with
|
||||
a positive scale. Returns ``(q_scale, weights)`` unchanged off SM100.
|
||||
"""
|
||||
if not mqa_logits_use_mx_scales():
|
||||
return q_scale, weights
|
||||
if q_scale is None:
|
||||
numel = q.numel() // q.shape[-1]
|
||||
bufs = _unit_mx_scales.setdefault(q.device, [])
|
||||
if not bufs or bufs[-1].numel() < numel:
|
||||
size = max(numel, 2 * bufs[-1].numel() if bufs else 0)
|
||||
bufs.append(
|
||||
torch.full((size,), _UNIT_UE8M0_X4, dtype=torch.int32, device=q.device)
|
||||
)
|
||||
q_scale = bufs[-1][:numel].view(q.shape[:-1])
|
||||
return q_scale, weights.to(torch.bfloat16)
|
||||
|
||||
|
||||
def fp8_fp4_mqa_logits(
|
||||
q: tuple[torch.Tensor, torch.Tensor | None],
|
||||
kv: tuple[torch.Tensor, torch.Tensor],
|
||||
weights: torch.Tensor,
|
||||
cu_seqlen_ks: torch.Tensor,
|
||||
cu_seqlen_ke: torch.Tensor,
|
||||
clean_logits: bool,
|
||||
max_seqlen_k: int,
|
||||
) -> torch.Tensor:
|
||||
"""Compute MQA logits for a single sequence without KV paging.
|
||||
|
||||
Unified FP8/FP4 dispatch — the underlying DeepGEMM kernel takes
|
||||
``q = (values, scales_or_None)`` where ``scales`` is None for FP8 Q
|
||||
(per-token scale is folded into ``weights``) and a packed block-scale
|
||||
tensor for MXFP4 Q.
|
||||
``q = (values, scales_or_None)``. SM100 takes MXFP8 or MXFP4 with packed
|
||||
UE8M0 block scales; SM90/SM120 also take FP8 Q with ``q_scale=None``
|
||||
(per-token scale folded into ``weights``).
|
||||
|
||||
Args:
|
||||
q: Tuple ``(q_values, q_scale)``. FP8 path: q_values is [M, H, D]
|
||||
float8_e4m3fn and q_scale is None (per-token scale is folded
|
||||
into ``weights``). FP4 path: q_values is packed uint8 and
|
||||
q_scale is the companion block-scale tensor.
|
||||
kv: Tuple `(k_packed, k_scales)` — FP8 layout is [N, D]
|
||||
float8_e4m3fn plus fp32 scales [N]; FP4 layout is packed uint8.
|
||||
weights: weights of shape [M, H], dtype `torch.float32`.
|
||||
q: Tuple ``(q_values, q_scale)``, q_values of shape [M, H, D] (packed
|
||||
to D/2 bytes for MXFP4) and q_scale int32 [M, H] or None.
|
||||
kv: Tuple ``(k_values, k_scales)`` of shape [N, D] and [N]; k_scales
|
||||
is int32 packed UE8M0 with MX scales, fp32 otherwise.
|
||||
weights: Tensor of shape [M, H], bf16 on SM100 and fp32 elsewhere.
|
||||
cu_seqlen_ks: Start indices (inclusive) for valid K per query
|
||||
position, shape [M], dtype int32.
|
||||
cu_seqlen_ke: End indices (exclusive) for valid K per query
|
||||
position, shape [M], dtype int32.
|
||||
clean_logits: Whether to clean the unfilled logits into `-inf`.
|
||||
max_seqlen_k: Upper bound on ``cu_seqlen_ke - cu_seqlen_ks``.
|
||||
|
||||
Returns:
|
||||
Logits tensor of shape [M, N], dtype `torch.float32`.
|
||||
Compressed logits of shape [M, max_seqlen_k]: row ``i`` holds K
|
||||
positions ``[cu_seqlen_ks[i], cu_seqlen_ke[i])`` starting at column
|
||||
zero, and the tail is not cleaned. bf16 on SM100, fp32 elsewhere.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
@@ -633,7 +675,7 @@ def fp8_fp4_mqa_logits(
|
||||
weights,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=clean_logits,
|
||||
max_seqlen_k,
|
||||
)
|
||||
|
||||
|
||||
@@ -699,24 +741,25 @@ def fp8_fp4_paged_mqa_logits(
|
||||
block_tables: torch.Tensor,
|
||||
schedule_metadata: torch.Tensor,
|
||||
max_model_len: int,
|
||||
clean_logits: bool,
|
||||
indices: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute MQA logits using a paged KV-cache.
|
||||
|
||||
Unified FP8/FP4 dispatch — the underlying DeepGEMM kernel takes
|
||||
``q = (values, scales_or_None)``; pass ``(q_tensor, None)`` for the FP8
|
||||
path and ``(q_values, q_scale)`` for MXFP4.
|
||||
``q = (values, scales_or_None)``; see `fp8_fp4_mqa_logits` for the
|
||||
per-architecture formats. SM100 requires the varlen form (``indices``
|
||||
with next_n == 1).
|
||||
|
||||
Args:
|
||||
q: Tuple ``(q_values, q_scale)``. FP8 path: q_values is
|
||||
[B, next_n, H, D] float8_e4m3fn and q_scale is None. FP4 path:
|
||||
q_values is packed uint8 and q_scale is the companion
|
||||
block-scale tensor.
|
||||
kv_cache: Paged KV-cache. FP8 layout is [num_blocks, block_size, 1,
|
||||
D+4], dtype `torch.uint8`, with the last 4 bytes per (block, pos)
|
||||
storing the float dequant scale.
|
||||
weights: Tensor of shape [B * next_n, H], dtype `torch.float32`.
|
||||
q: Tuple ``(q_values, q_scale)``, q_values of shape
|
||||
[B, next_n, H, D] (packed to D/2 bytes for MXFP4) and q_scale
|
||||
int32 [B, next_n, H] or None.
|
||||
kv_cache: Paged KV-cache of shape [num_blocks, block_size, 1, D+4]
|
||||
(D/2+4 for MXFP4), dtype `torch.uint8`; the last 4 bytes per
|
||||
(block, pos) hold the int32 packed UE8M0 scales with MX scales,
|
||||
or the fp32 dequant scale otherwise.
|
||||
weights: Tensor of shape [B * next_n, H], bf16 on SM100 and fp32
|
||||
elsewhere.
|
||||
context_lens: Tensor of shape [B], dtype int32; effective context length
|
||||
for each batch element.
|
||||
block_tables: Tensor of shape [B, max_blocks], dtype int32; maps logical
|
||||
@@ -724,12 +767,11 @@ def fp8_fp4_paged_mqa_logits(
|
||||
schedule_metadata: Returned by `get_paged_mqa_logits_metadata`;
|
||||
used to distribute work across SMs.
|
||||
max_model_len: Maximum sequence length used to size the logits output.
|
||||
clean_logits: Whether to clean the unfilled logits into `-inf`.
|
||||
indices: Optional request index for each varlen row.
|
||||
|
||||
Returns:
|
||||
Logits tensor of shape [B * next_n, max_model_len], dtype
|
||||
`torch.float32`.
|
||||
Logits tensor of shape [B * next_n, max_model_len]; columns past each
|
||||
row's context length are not cleaned. bf16 on SM100, fp32 elsewhere.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
@@ -751,7 +793,6 @@ def fp8_fp4_paged_mqa_logits(
|
||||
block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
clean_logits=clean_logits,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -1052,6 +1093,8 @@ __all__ = [
|
||||
"fp8_fp4_mqa_logits",
|
||||
"fp8_fp4_paged_mqa_logits",
|
||||
"get_paged_mqa_logits_metadata",
|
||||
"mqa_logits_mx_inputs",
|
||||
"mqa_logits_use_mx_scales",
|
||||
"native_next_n_supported",
|
||||
"per_block_cast_to_fp8",
|
||||
"is_deep_gemm_e8m0_used",
|
||||
|
||||
Reference in New Issue
Block a user