[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-12 23:24:43 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 86b1c2ef15
commit a6009ea85b
23 changed files with 516 additions and 208 deletions
@@ -5,6 +5,7 @@ import random
import pytest
import torch
from vllm.model_executor.layers.sparse_attn_indexer import as_fp32_logits
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
_ceil_to_ue8m0,
@@ -13,18 +14,29 @@ from vllm.utils.deep_gemm import (
fp8_fp4_paged_mqa_logits,
get_num_sms,
get_paged_mqa_logits_metadata,
mqa_logits_mx_inputs,
mqa_logits_use_mx_scales,
native_next_n_supported,
)
from vllm.utils.import_utils import has_deep_gemm
from vllm.utils.math_utils import cdiv
def encode_indexer_scales(sf: torch.Tensor) -> torch.Tensor:
"""Per-token power-of-two fp32 scales as the indexer K cache stores them:
four equal packed UE8M0 bytes under MX scales, else the fp32 itself."""
if not mqa_logits_use_mx_scales():
return sf
ue8m0 = (sf.view(torch.int32) >> 23).to(torch.uint8)
return ue8m0[..., None].repeat_interleave(4, dim=-1).view(torch.int32)[..., 0]
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
# x: (num_blocks, block_size, 1, head_dim)
num_blocks, block_size, num_heads, head_dim = x.shape
assert num_heads == 1
x_amax = x.abs().float().amax(dim=3, keepdim=True).clamp(1e-4)
sf = x_amax / 448.0
sf = _ceil_to_ue8m0(x_amax / 448.0)
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
x_fp8 = torch.empty(
(num_blocks, block_size * (head_dim + 4)),
@@ -34,9 +46,9 @@ def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
x_fp8[:, : block_size * head_dim] = x_scaled.view(
num_blocks, block_size * head_dim
).view(dtype=torch.uint8)
x_fp8[:, block_size * head_dim :] = sf.view(num_blocks, block_size).view(
dtype=torch.uint8
)
x_fp8[:, block_size * head_dim :] = encode_indexer_scales(
sf.view(num_blocks, block_size)
).view(dtype=torch.uint8)
return x_fp8.view(num_blocks, block_size, num_heads, head_dim + 4)
@@ -91,64 +103,72 @@ def _ref_fp8_mqa_logits(
return logits
def _packed_requests_test_data(seq_len: int, seq_len_kv: int):
# Two causal prefill requests packed into one KV buffer, so the second
# request's rows start at a nonzero K position.
half, half_kv = seq_len // 2, seq_len_kv // 2
rows = torch.arange(seq_len, dtype=torch.int, device="cuda")
ks = torch.where(rows < half, 0, half_kv).to(torch.int)
ke = ks + half_kv - half + 1 + rows % half
return ks, ke
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
@pytest.mark.parametrize("clean_logits", [True, False])
def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
@pytest.mark.parametrize("layout", ["cp", "no_cp", "packed"])
def test_deepgemm_fp8_mqa_logits(layout: str):
"""FP8 indexer prefill logits, compressed to each row's valid K span."""
torch.manual_seed(0)
random.seed(0)
num_heads, head_dim = 32, 128
for seq_len in (512,):
for seq_len_kv in (1024,):
for disable_cp in (False, True):
q = torch.randn(
seq_len,
num_heads,
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
kv = torch.randn(
seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16
)
weights = torch.randn(
seq_len, num_heads, device="cuda", dtype=torch.float32
)
seq_len, seq_len_kv = 512, 1024
q = torch.randn(seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
kv = torch.randn(seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16)
weights = torch.randn(seq_len, num_heads, device="cuda", dtype=torch.float32)
if disable_cp:
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
seq_len_kv - seq_len
)
else:
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
if layout == "cp":
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
elif layout == "no_cp":
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
seq_len_kv - seq_len
)
else:
ks, ke = _packed_requests_test_data(seq_len, seq_len_kv)
q_fp8 = q.to(torch.float8_e4m3fn)
kv_fp8 = per_custom_dims_cast_to_fp8(kv, (0,), False)
logits = fp8_fp4_mqa_logits(
(q_fp8, None), kv_fp8, weights, ks, ke, clean_logits=clean_logits
)
q_fp8 = q.to(torch.float8_e4m3fn)
kv_fp8, kv_sf = per_custom_dims_cast_to_fp8(kv, (0,), True)
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
max_seqlen_k = int((ke - ks).max())
logits = fp8_fp4_mqa_logits(
(q_fp8, q_scale),
(kv_fp8, encode_indexer_scales(kv_sf)),
kernel_weights,
ks,
ke,
max_seqlen_k,
)
assert logits.shape == (seq_len, max_seqlen_k)
logits = as_fp32_logits(logits, ke - ks)
ref_logits = _ref_fp8_mqa_logits(
q=q,
kv=kv,
weights=weights,
cu_seqlen_ks=ks,
cu_seqlen_ke=ke,
)
ref_neginf_mask = ref_logits == float("-inf")
if clean_logits:
neginf_mask = logits == float("-inf")
assert torch.equal(neginf_mask, ref_neginf_mask)
ref_logits = ref_logits.masked_fill(ref_neginf_mask, 0)
logits = logits.masked_fill(ref_neginf_mask, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
ref_logits = _ref_fp8_mqa_logits(
q=q,
kv=kv,
weights=kernel_weights.float(),
cu_seqlen_ks=ks,
cu_seqlen_ke=ke,
)
# Compressed row i holds K positions [ks[i], ke[i]) from column zero.
cols = ks[:, None] + torch.arange(max_seqlen_k, device="cuda")[None, :]
valid = cols < ke[:, None]
ref_logits = ref_logits.gather(1, cols.clamp(max=seq_len_kv - 1).long())
ref_logits = ref_logits.masked_fill(~valid, 0)
logits = logits.masked_fill(~valid, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
def _ref_fp8_fp4_paged_mqa_logits(
@@ -212,9 +232,6 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
if not native_next_n_supported(next_n):
pytest.skip(f"next_n={next_n} has no native kernel on this architecture")
# NOTE: clean_logits=True is incompatible with the 2D context_lens
# required by csrc/apis/attention.hpp; only the False path is exercised.
clean_logits = False
torch.manual_seed(0)
random.seed(0)
@@ -273,26 +290,42 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
context_lens_2d = (
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
).contiguous()
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
kernel_q = (q_fp8, q_scale)
kernel_context_lens, kernel_block_tables = context_lens_2d, block_tables
indices = None
if mqa_logits_use_mx_scales():
# SM100 only takes varlen rows: one Q token each, tied to its
# request by `indices`.
assert q_scale is not None
kernel_q = (
q_fp8.flatten(0, 1)[:, None],
q_scale.flatten(0, 1)[:, None],
)
kernel_context_lens = context_lens_2d.view(-1, 1)
kernel_block_tables = block_tables.repeat_interleave(next_n, dim=0)
indices = torch.arange(batch_size, device="cuda", dtype=torch.int32)
indices = indices.repeat_interleave(next_n)
schedule_metadata = get_paged_mqa_logits_metadata(
context_lens_2d,
blocksize,
get_num_sms(),
kernel_context_lens, blocksize, get_num_sms(), indices=indices
)
logits = fp8_fp4_paged_mqa_logits(
(q_fp8, None),
kernel_q,
kv_cache_fp8,
weights,
context_lens_2d,
block_tables,
kernel_weights,
kernel_context_lens,
kernel_block_tables,
schedule_metadata,
max_model_len,
clean_logits=clean_logits,
indices=indices,
)
if indices is not None:
logits = as_fp32_logits(logits, kernel_context_lens.view(-1))
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
q,
kv_cache,
weights,
kernel_weights.float(),
context_lens,
block_tables,
max_model_len,
@@ -309,7 +342,7 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
context_lens[row_indices] - next_n + next_n_offset
).unsqueeze(1)
logits = logits.masked_fill(~mask, 0)
logits = logits.float().masked_fill(~mask, 0)
ref_logits = ref_logits.masked_fill(~mask, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
@@ -349,12 +382,28 @@ def test_deepgemm_paged_mqa_packed_manager_block_stride():
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
torch.float8_e4m3fn
)
weights = torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
q_scale, weights = mqa_logits_mx_inputs(
q, None, torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
)
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
schedule = get_paged_mqa_logits_metadata(context_lens, page_size, get_num_sms())
indices = (
torch.zeros(1, device="cuda", dtype=torch.int32)
if mqa_logits_use_mx_scales()
else None
)
schedule = get_paged_mqa_logits_metadata(
context_lens, page_size, get_num_sms(), indices=indices
)
logits = [
fp8_fp4_paged_mqa_logits(
(q, None), cache, weights, context_lens, table, schedule, block_size, False
(q, q_scale),
cache,
weights,
context_lens,
table,
schedule,
block_size,
indices=indices,
)
for cache, table in zip((compact_pages, packed_pages), tables)
]
@@ -253,6 +253,7 @@ def _prefill_metadata(
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
total_seq_lens=1,
max_seqlen_k=int(ends.max()),
token_start=t0,
token_end=t1,
num_reqs=req_hi - req_lo,
+32 -10
View File
@@ -1422,7 +1422,10 @@ def test_dequantize_and_gather_k_cache(
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
@pytest.mark.parametrize("block_size", [16, 64])
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
@pytest.mark.parametrize("scale_fmt", ["ue8m0", "mxfp8"])
def test_indexer_quant_cache_roundtrip(
num_tokens: int, block_size: int, scale_fmt: str
):
"""K → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
→ manual dequant → compare against original."""
HEAD_DIM = 128
@@ -1442,7 +1445,9 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, "ue8m0")
ops.indexer_k_quant_and_cache(
k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, scale_fmt
)
# ── Gather ──────────────────────────────────────────────────────────
max_blocks_per_seq = num_blocks
@@ -1465,7 +1470,12 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
# ── Manual dequant ──────────────────────────────────────────────────
k_fp8 = dst_k.view(torch.float8_e4m3fn).float() # [num_tokens, 128]
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
if scale_fmt == "mxfp8":
# One UE8M0 byte per 32 elements, all equal to the token's scale.
assert (dst_scale == dst_scale[:, :1]).all()
scale = (dst_scale[:, :1].to(torch.int32) << 23).view(torch.float32)
else:
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
k_recovered = k_fp8 * scale # [num_tokens, 128]
# ── Compare ─────────────────────────────────────────────────────────
@@ -1749,8 +1759,13 @@ def _reference_kv_compress_norm_rope(
),
],
)
def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: bool):
@pytest.mark.parametrize("mx_scale", [False, True])
def test_fused_kv_insert_indexer(
num_tokens: int, kv_block_size: int, use_fp4: bool, mx_scale: bool
):
"""Fused K compress+norm+rope+quant+insert for the indexer KV cache."""
if use_fp4 and mx_scale:
pytest.skip("MX scales only change the FP8 cache")
HEAD_DIM = 128
ROPE_DIM = 64
BLOCK_SIZE = 16
@@ -1833,6 +1848,7 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
TOKEN_STRIDE=TOKEN_STRIDE,
SCALE_DIM=SCALE_DIM,
KV_BLOCK_STRIDE=kv_cache.stride(0),
MX_SCALE=mx_scale,
num_warps=1,
)
@@ -1876,12 +1892,18 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
), f"token {i}: FP8 bytes differ"
scale_off = kv_block_size * TOKEN_STRIDE + pos * SCALE_DIM
actual_scale = kv_cache[blk, scale_off : scale_off + SCALE_DIM].view(
torch.float32
)
assert torch.equal(actual_scale, scale[i : i + 1]), (
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
)
actual_scale = kv_cache[blk, scale_off : scale_off + SCALE_DIM]
if mx_scale:
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = (scale[i : i + 1].view(torch.int32) >> 23).to(torch.uint8)
assert torch.equal(actual_scale, ue8m0.repeat(SCALE_DIM)), (
f"token {i}: mx scale {actual_scale.tolist()} != {ue8m0.item()}"
)
else:
actual_scale = actual_scale.view(torch.float32)
assert torch.equal(actual_scale, scale[i : i + 1]), (
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
)
@pytest.mark.parametrize("compress_ratio", [4, 128])
@@ -32,6 +32,7 @@ import torch
from vllm.models.deepseek_v32.common import kernels as K
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
FP8 = torch.float8_e4m3fn
FP8_MAX = 448.0
@@ -116,6 +117,16 @@ def rope(
return out
def indexer_scales(scale_bytes: torch.Tensor) -> torch.Tensor:
"""Per-token fp32 scales from the indexer cache's 4 scale bytes per token."""
if not mqa_logits_use_mx_scales():
return scale_bytes.view(torch.float32)
# MX scales: the power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = scale_bytes.view(-1, 4)
assert (ue8m0 == ue8m0[:, :1]).all(), "MX scale bytes differ within a token"
return (ue8m0[:, 0].to(torch.int32) << 23).view(torch.float32)
def ue8m0_quant(vals: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-row (last dim) UE8M0 fp8 quant matching kernels._fp8_ue8m0_quantize."""
amax = vals.float().abs().amax(dim=-1, keepdim=True)
@@ -243,7 +254,7 @@ def test_fused_norm_rope(
q_ref, s_ref = ue8m0_quant(ik_ref)
flat = idx_cache[0].reshape(-1)
vals = flat[: bs * cfg.index_head_dim].view(FP8).reshape(bs, cfg.index_head_dim)
scales = flat[bs * cfg.index_head_dim :].view(torch.float32)
scales = indexer_scales(flat[bs * cfg.index_head_dim :])
assert_fp8(vals[:num_tokens], q_ref, "indexer-K fp8")
torch.testing.assert_close(scales[:num_tokens], s_ref, rtol=0, atol=0)
@@ -318,7 +329,7 @@ def test_fused_norm_rope_packed_indexer_block_stride(cfg: ModelConfig):
.view(FP8)
.view(block_size, cfg.index_head_dim)
)
scales = packed[block_size * cfg.index_head_dim :].view(torch.float32)
scales = indexer_scales(packed[block_size * cfg.index_head_dim :])
assert_fp8(values, q_ref, "packed indexer-K fp8")
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
)
from vllm.models.deepseek_v4.common.ops import fused_indexer_q_rope_quant
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.utils.import_utils import has_cutedsl
HEAD_DIM = 128
@@ -347,6 +348,10 @@ def _indexer_k_reference(
exponent = k.float().abs().amax(dim=-1, keepdim=True).clamp(min=1e-4) / 448
scale = exponent.log2().ceil().exp2()
values = (k.float() / scale).clamp(-448, 448).to(torch.float8_e4m3fn)
if mqa_logits_use_mx_scales():
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = (scale.view(torch.int32) >> 23).to(torch.uint8)
return values.view(torch.uint8), ue8m0.repeat(1, 4)
return values.view(torch.uint8), scale.view(torch.uint8)
@@ -29,6 +29,7 @@ import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
if current_platform.is_rocm():
from vllm.models.glm5next.amd.ops.kpool_compress import (
@@ -206,7 +207,12 @@ def _torch_reference(
k_off = page_base + loc_tok * HEAD_DIM + torch.arange(HEAD_DIM)
s_off = page_base + k_region + loc_tok * 4
kv_flat[k_off] = quantized.view(torch.uint8)
kv_flat[s_off : s_off + 4] = scale.detach().reshape(1).view(torch.uint8)
scale_bytes = scale.detach().reshape(1).view(torch.uint8)
if mqa_logits_use_mx_scales():
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = scale.reshape(1).view(torch.int32) >> 23
scale_bytes = ue8m0.to(torch.uint8).repeat(4)
kv_flat[s_off : s_off + 4] = scale_bytes
# stash -- gated on the TOKEN-granular tail slot, not on pos_valid.
# pos_valid keys off the pool-granular cache_loc, which is -1 for
@@ -771,6 +771,8 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
# The logits are compressed: row r starts at K position ks[r].
max_len = int(lens.max())
dense = deep_gemm.fp8_fp4_mqa_logits(
(
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
@@ -780,19 +782,13 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
weights,
ks,
ke,
clean_logits=False,
logits_dtype=torch.bfloat16,
max_len,
)
cols = torch.arange(total, device="cuda")
in_range = (cols - ks.unsqueeze(1) >= 0) & (
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
)
masked = dense.masked_fill(
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
float("-inf"),
).float()
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
in_range = torch.arange(max_len, device="cuda") < lens.unsqueeze(1)
# A row's last candidate block may extend past max_len.
in_cand = _candidate_token_mask(candidates, cbk, max_len + cbk)[:, :max_len]
masked = dense.masked_fill(~(in_range & in_cand), float("-inf")).float()
ref = _reference_topk(masked, topk)
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
scratch = _sparse_scratch(rows, num_candidates, topk)
@@ -827,11 +823,11 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
graph.replay()
else:
kernel_metadata = run_sparse()
_assert_same_selection(dense, out, ref, row_starts=ks)
_assert_same_selection(dense, out, ref)
out.fill_(-1)
run_sparse(kernel_metadata)
_assert_same_selection(dense, out, ref, row_starts=ks)
_assert_same_selection(dense, out, ref)
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
@@ -902,21 +898,22 @@ def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
def reference_topk():
context_lens_2d = row_lens.view(batch, next_n)
# SM100 paged MQA logits take varlen rows: one Q token per row.
context_lens_2d = row_lens.view(rows, 1)
indices = torch.arange(batch, dtype=torch.int32, device="cuda")
indices = indices.repeat_interleave(next_n)
sched = deep_gemm.get_paged_mqa_logits_metadata(
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
context_lens_2d, page_kv, deep_gemm.get_num_sms(), indices
)
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
(q, q_scale),
(q.view(rows, 1, num_heads, -1), q_scale.view(rows, 1, num_heads)),
kv_cache,
weights,
context_lens_2d,
block_table,
block_table.repeat_interleave(next_n, dim=0),
sched,
int(req_lens.max()),
False,
torch.bfloat16,
None,
indices,
)
lens_flat = row_lens.flatten()
cols = torch.arange(logits.shape[1], device="cuda")
@@ -14,6 +14,7 @@ from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
)
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla import indexer as indexer_module
from vllm.v1.attention.backends.mla.compressor_utils import (
@@ -113,10 +114,17 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
num_reqs, 4
)
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
# SM100's paged MQA logits only take varlen rows.
indices = (
torch.arange(num_reqs, dtype=torch.int32, device=device)
if mqa_logits_use_mx_scales()
else None
)
schedule_metadata = indexer_module.get_paged_mqa_logits_metadata(
torch.div(seq_lens, 16, rounding_mode="floor").unsqueeze(1),
64,
num_sms,
indices=indices,
).clone()
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
@@ -139,6 +147,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
decode_lens=torch.zeros(num_reqs, dtype=torch.int32, device=device),
requires_padding=False,
schedule_metadata=schedule_metadata,
indices=indices,
),
block_table=block_table,
)
@@ -157,7 +166,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
torch.testing.assert_close(
schedule_metadata,
indexer_module.get_paged_mqa_logits_metadata(
expected_decode_seq_lens.unsqueeze(1), 64, num_sms
expected_decode_seq_lens.unsqueeze(1), 64, num_sms, indices=indices
),
)
assert metadata.decode is not None
@@ -36,6 +36,7 @@ def _chunk(block_table, token_start, token_end, row_ends):
cu_seq_lens=torch.zeros(1, dtype=torch.int32),
token_to_seq=torch.zeros(1, dtype=torch.int32),
total_seq_lens=1,
max_seqlen_k=int(ends.max()),
token_start=token_start,
token_end=token_end,
num_reqs=block_table.shape[0],