[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
86b1c2ef15
commit
a6009ea85b
@@ -5,6 +5,7 @@ import random
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import as_fp32_logits
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
_ceil_to_ue8m0,
|
||||
@@ -13,18 +14,29 @@ from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
get_num_sms,
|
||||
get_paged_mqa_logits_metadata,
|
||||
mqa_logits_mx_inputs,
|
||||
mqa_logits_use_mx_scales,
|
||||
native_next_n_supported,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
|
||||
def encode_indexer_scales(sf: torch.Tensor) -> torch.Tensor:
|
||||
"""Per-token power-of-two fp32 scales as the indexer K cache stores them:
|
||||
four equal packed UE8M0 bytes under MX scales, else the fp32 itself."""
|
||||
if not mqa_logits_use_mx_scales():
|
||||
return sf
|
||||
ue8m0 = (sf.view(torch.int32) >> 23).to(torch.uint8)
|
||||
return ue8m0[..., None].repeat_interleave(4, dim=-1).view(torch.int32)[..., 0]
|
||||
|
||||
|
||||
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
|
||||
# x: (num_blocks, block_size, 1, head_dim)
|
||||
num_blocks, block_size, num_heads, head_dim = x.shape
|
||||
assert num_heads == 1
|
||||
x_amax = x.abs().float().amax(dim=3, keepdim=True).clamp(1e-4)
|
||||
sf = x_amax / 448.0
|
||||
sf = _ceil_to_ue8m0(x_amax / 448.0)
|
||||
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
|
||||
x_fp8 = torch.empty(
|
||||
(num_blocks, block_size * (head_dim + 4)),
|
||||
@@ -34,9 +46,9 @@ def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
|
||||
x_fp8[:, : block_size * head_dim] = x_scaled.view(
|
||||
num_blocks, block_size * head_dim
|
||||
).view(dtype=torch.uint8)
|
||||
x_fp8[:, block_size * head_dim :] = sf.view(num_blocks, block_size).view(
|
||||
dtype=torch.uint8
|
||||
)
|
||||
x_fp8[:, block_size * head_dim :] = encode_indexer_scales(
|
||||
sf.view(num_blocks, block_size)
|
||||
).view(dtype=torch.uint8)
|
||||
return x_fp8.view(num_blocks, block_size, num_heads, head_dim + 4)
|
||||
|
||||
|
||||
@@ -91,64 +103,72 @@ def _ref_fp8_mqa_logits(
|
||||
return logits
|
||||
|
||||
|
||||
def _packed_requests_test_data(seq_len: int, seq_len_kv: int):
|
||||
# Two causal prefill requests packed into one KV buffer, so the second
|
||||
# request's rows start at a nonzero K position.
|
||||
half, half_kv = seq_len // 2, seq_len_kv // 2
|
||||
rows = torch.arange(seq_len, dtype=torch.int, device="cuda")
|
||||
ks = torch.where(rows < half, 0, half_kv).to(torch.int)
|
||||
ke = ks + half_kv - half + 1 + rows % half
|
||||
return ks, ke
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
@pytest.mark.parametrize("clean_logits", [True, False])
|
||||
def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
|
||||
@pytest.mark.parametrize("layout", ["cp", "no_cp", "packed"])
|
||||
def test_deepgemm_fp8_mqa_logits(layout: str):
|
||||
"""FP8 indexer prefill logits, compressed to each row's valid K span."""
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
num_heads, head_dim = 32, 128
|
||||
for seq_len in (512,):
|
||||
for seq_len_kv in (1024,):
|
||||
for disable_cp in (False, True):
|
||||
q = torch.randn(
|
||||
seq_len,
|
||||
num_heads,
|
||||
head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv = torch.randn(
|
||||
seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
weights = torch.randn(
|
||||
seq_len, num_heads, device="cuda", dtype=torch.float32
|
||||
)
|
||||
seq_len, seq_len_kv = 512, 1024
|
||||
q = torch.randn(seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
kv = torch.randn(seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
weights = torch.randn(seq_len, num_heads, device="cuda", dtype=torch.float32)
|
||||
|
||||
if disable_cp:
|
||||
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
|
||||
seq_len_kv - seq_len
|
||||
)
|
||||
else:
|
||||
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
|
||||
if layout == "cp":
|
||||
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
|
||||
elif layout == "no_cp":
|
||||
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
|
||||
seq_len_kv - seq_len
|
||||
)
|
||||
else:
|
||||
ks, ke = _packed_requests_test_data(seq_len, seq_len_kv)
|
||||
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_fp8 = per_custom_dims_cast_to_fp8(kv, (0,), False)
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_fp8, None), kv_fp8, weights, ks, ke, clean_logits=clean_logits
|
||||
)
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_fp8, kv_sf = per_custom_dims_cast_to_fp8(kv, (0,), True)
|
||||
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
|
||||
max_seqlen_k = int((ke - ks).max())
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_fp8, q_scale),
|
||||
(kv_fp8, encode_indexer_scales(kv_sf)),
|
||||
kernel_weights,
|
||||
ks,
|
||||
ke,
|
||||
max_seqlen_k,
|
||||
)
|
||||
assert logits.shape == (seq_len, max_seqlen_k)
|
||||
logits = as_fp32_logits(logits, ke - ks)
|
||||
|
||||
ref_logits = _ref_fp8_mqa_logits(
|
||||
q=q,
|
||||
kv=kv,
|
||||
weights=weights,
|
||||
cu_seqlen_ks=ks,
|
||||
cu_seqlen_ke=ke,
|
||||
)
|
||||
ref_neginf_mask = ref_logits == float("-inf")
|
||||
|
||||
if clean_logits:
|
||||
neginf_mask = logits == float("-inf")
|
||||
assert torch.equal(neginf_mask, ref_neginf_mask)
|
||||
|
||||
ref_logits = ref_logits.masked_fill(ref_neginf_mask, 0)
|
||||
logits = logits.masked_fill(ref_neginf_mask, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
ref_logits = _ref_fp8_mqa_logits(
|
||||
q=q,
|
||||
kv=kv,
|
||||
weights=kernel_weights.float(),
|
||||
cu_seqlen_ks=ks,
|
||||
cu_seqlen_ke=ke,
|
||||
)
|
||||
# Compressed row i holds K positions [ks[i], ke[i]) from column zero.
|
||||
cols = ks[:, None] + torch.arange(max_seqlen_k, device="cuda")[None, :]
|
||||
valid = cols < ke[:, None]
|
||||
ref_logits = ref_logits.gather(1, cols.clamp(max=seq_len_kv - 1).long())
|
||||
ref_logits = ref_logits.masked_fill(~valid, 0)
|
||||
logits = logits.masked_fill(~valid, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
def _ref_fp8_fp4_paged_mqa_logits(
|
||||
@@ -212,9 +232,6 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
if not native_next_n_supported(next_n):
|
||||
pytest.skip(f"next_n={next_n} has no native kernel on this architecture")
|
||||
|
||||
# NOTE: clean_logits=True is incompatible with the 2D context_lens
|
||||
# required by csrc/apis/attention.hpp; only the False path is exercised.
|
||||
clean_logits = False
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
|
||||
@@ -273,26 +290,42 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
context_lens_2d = (
|
||||
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
|
||||
).contiguous()
|
||||
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
|
||||
kernel_q = (q_fp8, q_scale)
|
||||
kernel_context_lens, kernel_block_tables = context_lens_2d, block_tables
|
||||
indices = None
|
||||
if mqa_logits_use_mx_scales():
|
||||
# SM100 only takes varlen rows: one Q token each, tied to its
|
||||
# request by `indices`.
|
||||
assert q_scale is not None
|
||||
kernel_q = (
|
||||
q_fp8.flatten(0, 1)[:, None],
|
||||
q_scale.flatten(0, 1)[:, None],
|
||||
)
|
||||
kernel_context_lens = context_lens_2d.view(-1, 1)
|
||||
kernel_block_tables = block_tables.repeat_interleave(next_n, dim=0)
|
||||
indices = torch.arange(batch_size, device="cuda", dtype=torch.int32)
|
||||
indices = indices.repeat_interleave(next_n)
|
||||
schedule_metadata = get_paged_mqa_logits_metadata(
|
||||
context_lens_2d,
|
||||
blocksize,
|
||||
get_num_sms(),
|
||||
kernel_context_lens, blocksize, get_num_sms(), indices=indices
|
||||
)
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(q_fp8, None),
|
||||
kernel_q,
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
context_lens_2d,
|
||||
block_tables,
|
||||
kernel_weights,
|
||||
kernel_context_lens,
|
||||
kernel_block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
clean_logits=clean_logits,
|
||||
indices=indices,
|
||||
)
|
||||
if indices is not None:
|
||||
logits = as_fp32_logits(logits, kernel_context_lens.view(-1))
|
||||
|
||||
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
|
||||
q,
|
||||
kv_cache,
|
||||
weights,
|
||||
kernel_weights.float(),
|
||||
context_lens,
|
||||
block_tables,
|
||||
max_model_len,
|
||||
@@ -309,7 +342,7 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
context_lens[row_indices] - next_n + next_n_offset
|
||||
).unsqueeze(1)
|
||||
|
||||
logits = logits.masked_fill(~mask, 0)
|
||||
logits = logits.float().masked_fill(~mask, 0)
|
||||
ref_logits = ref_logits.masked_fill(~mask, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
@@ -349,12 +382,28 @@ def test_deepgemm_paged_mqa_packed_manager_block_stride():
|
||||
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
weights = torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
|
||||
q_scale, weights = mqa_logits_mx_inputs(
|
||||
q, None, torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
|
||||
)
|
||||
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
|
||||
schedule = get_paged_mqa_logits_metadata(context_lens, page_size, get_num_sms())
|
||||
indices = (
|
||||
torch.zeros(1, device="cuda", dtype=torch.int32)
|
||||
if mqa_logits_use_mx_scales()
|
||||
else None
|
||||
)
|
||||
schedule = get_paged_mqa_logits_metadata(
|
||||
context_lens, page_size, get_num_sms(), indices=indices
|
||||
)
|
||||
logits = [
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(q, None), cache, weights, context_lens, table, schedule, block_size, False
|
||||
(q, q_scale),
|
||||
cache,
|
||||
weights,
|
||||
context_lens,
|
||||
table,
|
||||
schedule,
|
||||
block_size,
|
||||
indices=indices,
|
||||
)
|
||||
for cache, table in zip((compact_pages, packed_pages), tables)
|
||||
]
|
||||
|
||||
@@ -253,6 +253,7 @@ def _prefill_metadata(
|
||||
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
total_seq_lens=1,
|
||||
max_seqlen_k=int(ends.max()),
|
||||
token_start=t0,
|
||||
token_end=t1,
|
||||
num_reqs=req_hi - req_lo,
|
||||
|
||||
@@ -1422,7 +1422,10 @@ def test_dequantize_and_gather_k_cache(
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
@pytest.mark.parametrize("scale_fmt", ["ue8m0", "mxfp8"])
|
||||
def test_indexer_quant_cache_roundtrip(
|
||||
num_tokens: int, block_size: int, scale_fmt: str
|
||||
):
|
||||
"""K → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
|
||||
→ manual dequant → compare against original."""
|
||||
HEAD_DIM = 128
|
||||
@@ -1442,7 +1445,9 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, "ue8m0")
|
||||
ops.indexer_k_quant_and_cache(
|
||||
k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, scale_fmt
|
||||
)
|
||||
|
||||
# ── Gather ──────────────────────────────────────────────────────────
|
||||
max_blocks_per_seq = num_blocks
|
||||
@@ -1465,7 +1470,12 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
|
||||
# ── Manual dequant ──────────────────────────────────────────────────
|
||||
k_fp8 = dst_k.view(torch.float8_e4m3fn).float() # [num_tokens, 128]
|
||||
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
|
||||
if scale_fmt == "mxfp8":
|
||||
# One UE8M0 byte per 32 elements, all equal to the token's scale.
|
||||
assert (dst_scale == dst_scale[:, :1]).all()
|
||||
scale = (dst_scale[:, :1].to(torch.int32) << 23).view(torch.float32)
|
||||
else:
|
||||
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
|
||||
k_recovered = k_fp8 * scale # [num_tokens, 128]
|
||||
|
||||
# ── Compare ─────────────────────────────────────────────────────────
|
||||
@@ -1749,8 +1759,13 @@ def _reference_kv_compress_norm_rope(
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: bool):
|
||||
@pytest.mark.parametrize("mx_scale", [False, True])
|
||||
def test_fused_kv_insert_indexer(
|
||||
num_tokens: int, kv_block_size: int, use_fp4: bool, mx_scale: bool
|
||||
):
|
||||
"""Fused K compress+norm+rope+quant+insert for the indexer KV cache."""
|
||||
if use_fp4 and mx_scale:
|
||||
pytest.skip("MX scales only change the FP8 cache")
|
||||
HEAD_DIM = 128
|
||||
ROPE_DIM = 64
|
||||
BLOCK_SIZE = 16
|
||||
@@ -1833,6 +1848,7 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
|
||||
TOKEN_STRIDE=TOKEN_STRIDE,
|
||||
SCALE_DIM=SCALE_DIM,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
MX_SCALE=mx_scale,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
@@ -1876,12 +1892,18 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
|
||||
), f"token {i}: FP8 bytes differ"
|
||||
|
||||
scale_off = kv_block_size * TOKEN_STRIDE + pos * SCALE_DIM
|
||||
actual_scale = kv_cache[blk, scale_off : scale_off + SCALE_DIM].view(
|
||||
torch.float32
|
||||
)
|
||||
assert torch.equal(actual_scale, scale[i : i + 1]), (
|
||||
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
|
||||
)
|
||||
actual_scale = kv_cache[blk, scale_off : scale_off + SCALE_DIM]
|
||||
if mx_scale:
|
||||
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = (scale[i : i + 1].view(torch.int32) >> 23).to(torch.uint8)
|
||||
assert torch.equal(actual_scale, ue8m0.repeat(SCALE_DIM)), (
|
||||
f"token {i}: mx scale {actual_scale.tolist()} != {ue8m0.item()}"
|
||||
)
|
||||
else:
|
||||
actual_scale = actual_scale.view(torch.float32)
|
||||
assert torch.equal(actual_scale, scale[i : i + 1]), (
|
||||
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("compress_ratio", [4, 128])
|
||||
|
||||
@@ -32,6 +32,7 @@ import torch
|
||||
|
||||
from vllm.models.deepseek_v32.common import kernels as K
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
|
||||
FP8 = torch.float8_e4m3fn
|
||||
FP8_MAX = 448.0
|
||||
@@ -116,6 +117,16 @@ def rope(
|
||||
return out
|
||||
|
||||
|
||||
def indexer_scales(scale_bytes: torch.Tensor) -> torch.Tensor:
|
||||
"""Per-token fp32 scales from the indexer cache's 4 scale bytes per token."""
|
||||
if not mqa_logits_use_mx_scales():
|
||||
return scale_bytes.view(torch.float32)
|
||||
# MX scales: the power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = scale_bytes.view(-1, 4)
|
||||
assert (ue8m0 == ue8m0[:, :1]).all(), "MX scale bytes differ within a token"
|
||||
return (ue8m0[:, 0].to(torch.int32) << 23).view(torch.float32)
|
||||
|
||||
|
||||
def ue8m0_quant(vals: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-row (last dim) UE8M0 fp8 quant matching kernels._fp8_ue8m0_quantize."""
|
||||
amax = vals.float().abs().amax(dim=-1, keepdim=True)
|
||||
@@ -243,7 +254,7 @@ def test_fused_norm_rope(
|
||||
q_ref, s_ref = ue8m0_quant(ik_ref)
|
||||
flat = idx_cache[0].reshape(-1)
|
||||
vals = flat[: bs * cfg.index_head_dim].view(FP8).reshape(bs, cfg.index_head_dim)
|
||||
scales = flat[bs * cfg.index_head_dim :].view(torch.float32)
|
||||
scales = indexer_scales(flat[bs * cfg.index_head_dim :])
|
||||
assert_fp8(vals[:num_tokens], q_ref, "indexer-K fp8")
|
||||
torch.testing.assert_close(scales[:num_tokens], s_ref, rtol=0, atol=0)
|
||||
|
||||
@@ -318,7 +329,7 @@ def test_fused_norm_rope_packed_indexer_block_stride(cfg: ModelConfig):
|
||||
.view(FP8)
|
||||
.view(block_size, cfg.index_head_dim)
|
||||
)
|
||||
scales = packed[block_size * cfg.index_head_dim :].view(torch.float32)
|
||||
scales = indexer_scales(packed[block_size * cfg.index_head_dim :])
|
||||
assert_fp8(values, q_ref, "packed indexer-K fp8")
|
||||
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
|
||||
assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import fused_indexer_q_rope_quant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
|
||||
HEAD_DIM = 128
|
||||
@@ -347,6 +348,10 @@ def _indexer_k_reference(
|
||||
exponent = k.float().abs().amax(dim=-1, keepdim=True).clamp(min=1e-4) / 448
|
||||
scale = exponent.log2().ceil().exp2()
|
||||
values = (k.float() / scale).clamp(-448, 448).to(torch.float8_e4m3fn)
|
||||
if mqa_logits_use_mx_scales():
|
||||
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = (scale.view(torch.int32) >> 23).to(torch.uint8)
|
||||
return values.view(torch.uint8), ue8m0.repeat(1, 4)
|
||||
return values.view(torch.uint8), scale.view(torch.uint8)
|
||||
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.glm5next.amd.ops.kpool_compress import (
|
||||
@@ -206,7 +207,12 @@ def _torch_reference(
|
||||
k_off = page_base + loc_tok * HEAD_DIM + torch.arange(HEAD_DIM)
|
||||
s_off = page_base + k_region + loc_tok * 4
|
||||
kv_flat[k_off] = quantized.view(torch.uint8)
|
||||
kv_flat[s_off : s_off + 4] = scale.detach().reshape(1).view(torch.uint8)
|
||||
scale_bytes = scale.detach().reshape(1).view(torch.uint8)
|
||||
if mqa_logits_use_mx_scales():
|
||||
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = scale.reshape(1).view(torch.int32) >> 23
|
||||
scale_bytes = ue8m0.to(torch.uint8).repeat(4)
|
||||
kv_flat[s_off : s_off + 4] = scale_bytes
|
||||
|
||||
# stash -- gated on the TOKEN-granular tail slot, not on pos_valid.
|
||||
# pos_valid keys off the pool-granular cache_loc, which is -1 for
|
||||
|
||||
@@ -771,6 +771,8 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
|
||||
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
|
||||
# The logits are compressed: row r starts at K position ks[r].
|
||||
max_len = int(lens.max())
|
||||
dense = deep_gemm.fp8_fp4_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
@@ -780,19 +782,13 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
clean_logits=False,
|
||||
logits_dtype=torch.bfloat16,
|
||||
max_len,
|
||||
)
|
||||
cols = torch.arange(total, device="cuda")
|
||||
in_range = (cols - ks.unsqueeze(1) >= 0) & (
|
||||
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
|
||||
)
|
||||
masked = dense.masked_fill(
|
||||
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
|
||||
float("-inf"),
|
||||
).float()
|
||||
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
|
||||
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
|
||||
in_range = torch.arange(max_len, device="cuda") < lens.unsqueeze(1)
|
||||
# A row's last candidate block may extend past max_len.
|
||||
in_cand = _candidate_token_mask(candidates, cbk, max_len + cbk)[:, :max_len]
|
||||
masked = dense.masked_fill(~(in_range & in_cand), float("-inf")).float()
|
||||
ref = _reference_topk(masked, topk)
|
||||
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
@@ -827,11 +823,11 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse()
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
_assert_same_selection(dense, out, ref)
|
||||
out.fill_(-1)
|
||||
run_sparse(kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
_assert_same_selection(dense, out, ref)
|
||||
|
||||
|
||||
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
|
||||
@@ -902,21 +898,22 @@ def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool
|
||||
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
|
||||
|
||||
def reference_topk():
|
||||
context_lens_2d = row_lens.view(batch, next_n)
|
||||
# SM100 paged MQA logits take varlen rows: one Q token per row.
|
||||
context_lens_2d = row_lens.view(rows, 1)
|
||||
indices = torch.arange(batch, dtype=torch.int32, device="cuda")
|
||||
indices = indices.repeat_interleave(next_n)
|
||||
sched = deep_gemm.get_paged_mqa_logits_metadata(
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), indices
|
||||
)
|
||||
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
(q.view(rows, 1, num_heads, -1), q_scale.view(rows, 1, num_heads)),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens_2d,
|
||||
block_table,
|
||||
block_table.repeat_interleave(next_n, dim=0),
|
||||
sched,
|
||||
int(req_lens.max()),
|
||||
False,
|
||||
torch.bfloat16,
|
||||
None,
|
||||
indices,
|
||||
)
|
||||
lens_flat = row_lens.flatten()
|
||||
cols = torch.arange(logits.shape[1], device="cuda")
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
|
||||
)
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla import indexer as indexer_module
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
@@ -113,10 +114,17 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
num_reqs, 4
|
||||
)
|
||||
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||
# SM100's paged MQA logits only take varlen rows.
|
||||
indices = (
|
||||
torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
if mqa_logits_use_mx_scales()
|
||||
else None
|
||||
)
|
||||
schedule_metadata = indexer_module.get_paged_mqa_logits_metadata(
|
||||
torch.div(seq_lens, 16, rounding_mode="floor").unsqueeze(1),
|
||||
64,
|
||||
num_sms,
|
||||
indices=indices,
|
||||
).clone()
|
||||
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
@@ -139,6 +147,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
decode_lens=torch.zeros(num_reqs, dtype=torch.int32, device=device),
|
||||
requires_padding=False,
|
||||
schedule_metadata=schedule_metadata,
|
||||
indices=indices,
|
||||
),
|
||||
block_table=block_table,
|
||||
)
|
||||
@@ -157,7 +166,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
torch.testing.assert_close(
|
||||
schedule_metadata,
|
||||
indexer_module.get_paged_mqa_logits_metadata(
|
||||
expected_decode_seq_lens.unsqueeze(1), 64, num_sms
|
||||
expected_decode_seq_lens.unsqueeze(1), 64, num_sms, indices=indices
|
||||
),
|
||||
)
|
||||
assert metadata.decode is not None
|
||||
|
||||
@@ -36,6 +36,7 @@ def _chunk(block_table, token_start, token_end, row_ends):
|
||||
cu_seq_lens=torch.zeros(1, dtype=torch.int32),
|
||||
token_to_seq=torch.zeros(1, dtype=torch.int32),
|
||||
total_seq_lens=1,
|
||||
max_seqlen_k=int(ends.max()),
|
||||
token_start=token_start,
|
||||
token_end=token_end,
|
||||
num_reqs=block_table.shape[0],
|
||||
|
||||
Reference in New Issue
Block a user