[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,471 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the AMX-only CPU MLA backend: the vendored
|
||||
decode/extend/bmm kernels, the KV cache write, and the ``AMXMLAImpl``
|
||||
backend built on top of them.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
if not torch.cpu._is_amx_tile_supported():
|
||||
pytest.skip("AMX MLA requires an AMX-capable host", allow_module_level=True)
|
||||
|
||||
torch.cpu._init_amx()
|
||||
|
||||
from vllm import _custom_ops as ops # noqa: E402
|
||||
from vllm.utils.torch_utils import set_random_seed # noqa: E402
|
||||
from vllm.v1.attention.backends.mla.amx_mla import ( # noqa: E402
|
||||
AMXMLAImpl,
|
||||
_compute_num_kv_splits,
|
||||
_expand_block_table,
|
||||
)
|
||||
|
||||
KV_LORA_RANK = 64
|
||||
QK_ROPE_HEAD_DIM = 32
|
||||
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM # 96, kv-cache row width
|
||||
# The extend/decode kernels transpose by 512-bit lanes: both the cache row
|
||||
# width and the (kv_lora_rank-wide) value width must be 32-element aligned.
|
||||
DTYPE = torch.bfloat16
|
||||
ATOL = 2e-2
|
||||
RTOL = 2e-2
|
||||
# Backend-level tests chain absorb -> attend -> de-absorb through bf16,
|
||||
# accumulating more rounding error than a single kernel op.
|
||||
_IMPL_ATOL = 1.5e-1
|
||||
_IMPL_RTOL = 1.5e-1
|
||||
|
||||
|
||||
def _flatten_cache(kv_cache: torch.Tensor) -> torch.Tensor:
|
||||
"""(num_blocks, block_size, head_size) -> (num_slots, head_size)."""
|
||||
return kv_cache.view(-1, kv_cache.size(-1))
|
||||
|
||||
|
||||
def _random_paged_cache(num_slots: int, head_size: int) -> torch.Tensor:
|
||||
return torch.randn(num_slots, head_size, dtype=DTYPE)
|
||||
|
||||
|
||||
def _ref_latent_attn(
|
||||
q: torch.Tensor,
|
||||
keys: torch.Tensor,
|
||||
scale: float,
|
||||
kv_lora_rank: int,
|
||||
) -> torch.Tensor:
|
||||
"""Plain-PyTorch causal-free latent-space MQA attention for one request.
|
||||
|
||||
q: (num_heads, head_size); keys: (ctx_len, head_size). Values are the
|
||||
first ``kv_lora_rank`` columns of the same latent rows (MLA's K/V
|
||||
aliasing), matching what decode_attention_cpu/extend_attention_cpu
|
||||
compute internally for the MLA-shaped case.
|
||||
"""
|
||||
values = keys[:, :kv_lora_rank]
|
||||
scores = (q.float() @ keys.float().T) * scale
|
||||
probs = torch.softmax(scores, dim=-1)
|
||||
return (probs @ values.float()).to(q.dtype)
|
||||
|
||||
|
||||
def test_bmm_cpu_matches_torch_bmm():
|
||||
set_random_seed(0)
|
||||
# bmm_cpu requires the output's last dim (mat2's out-features) to be a
|
||||
# multiple of 32 (tinygemm's tile width); the contraction dim is free.
|
||||
n, b, p, l = 8, 32, 32, 32 # noqa: E741
|
||||
mat1 = torch.randn(n, b, p, dtype=DTYPE)
|
||||
mat2 = torch.randn(n, l, p, dtype=DTYPE) # (N, OUT, IN) Linear convention
|
||||
|
||||
ref = torch.bmm(mat1.float(), mat2.float().transpose(1, 2)).to(DTYPE)
|
||||
|
||||
out = torch.empty(n, b, l, dtype=DTYPE)
|
||||
ops.bmm_cpu(out, mat1, mat2, False, None)
|
||||
torch.testing.assert_close(out, ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
packed = torch.ops._C.convert_weight_packed(mat2.contiguous())
|
||||
out_vnni = torch.empty(n, b, l, dtype=DTYPE)
|
||||
ops.bmm_cpu(out_vnni, mat1, packed, True, None)
|
||||
torch.testing.assert_close(out_vnni, ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
def test_concat_and_cache_mla_round_trip():
|
||||
set_random_seed(0)
|
||||
num_tokens = 37
|
||||
num_blocks, block_size = 8, 16
|
||||
kv_cache = torch.zeros(num_blocks, block_size, HEAD_SIZE, dtype=DTYPE)
|
||||
|
||||
kv_c_normed = torch.randn(num_tokens, KV_LORA_RANK, dtype=DTYPE)
|
||||
k_pe = torch.randn(num_tokens, QK_ROPE_HEAD_DIM, dtype=DTYPE)
|
||||
slot_mapping = torch.randperm(num_blocks * block_size)[:num_tokens].to(torch.int64)
|
||||
|
||||
ops.amx_mla_concat_and_cache(kv_c_normed, k_pe, kv_cache, slot_mapping)
|
||||
|
||||
flat = _flatten_cache(kv_cache)
|
||||
read_back = flat[slot_mapping]
|
||||
expected = torch.cat([kv_c_normed, k_pe], dim=-1)
|
||||
torch.testing.assert_close(read_back, expected)
|
||||
|
||||
untouched = torch.ones(num_blocks * block_size, dtype=torch.bool)
|
||||
untouched[slot_mapping] = False
|
||||
assert (flat[untouched] == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens", [[5], [1, 300, 33], [513, 1, 129, 7]])
|
||||
def test_decode_attention_cpu_matches_reference(seq_lens):
|
||||
set_random_seed(0)
|
||||
num_seqs = len(seq_lens)
|
||||
num_heads = 4
|
||||
block_size = 32
|
||||
max_len = max(seq_lens)
|
||||
max_blocks = (max_len + block_size - 1) // block_size
|
||||
num_blocks = num_seqs * max_blocks
|
||||
|
||||
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
|
||||
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
|
||||
req_to_token = _expand_block_table(block_table, block_size)
|
||||
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
|
||||
scale = HEAD_SIZE**-0.5
|
||||
|
||||
q = torch.randn(num_seqs, num_heads, HEAD_SIZE, dtype=DTYPE)
|
||||
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
|
||||
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
|
||||
|
||||
o = torch.zeros(num_seqs, num_heads, KV_LORA_RANK, dtype=DTYPE)
|
||||
attn_logits = torch.zeros(
|
||||
num_seqs, num_heads, 4, KV_LORA_RANK + 1, dtype=torch.float32
|
||||
)
|
||||
ops.cpu_mla_decode(
|
||||
q,
|
||||
kv_cache_flat,
|
||||
v_buffer,
|
||||
o,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
attn_logits,
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens_t,
|
||||
scale,
|
||||
0.0,
|
||||
False,
|
||||
0,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
flat_cache = _flatten_cache(kv_cache)
|
||||
for i, seq_len in enumerate(seq_lens):
|
||||
token_ids = req_to_token[i, :seq_len]
|
||||
keys = flat_cache[token_ids]
|
||||
ref = _ref_latent_attn(q[i], keys, scale, KV_LORA_RANK)
|
||||
torch.testing.assert_close(o[i], ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prefix_lens,extend_lens",
|
||||
[
|
||||
([0], [9]), # fresh prefill, no cached prefix
|
||||
([64], [17]), # continuation of a cached prefix
|
||||
([0, 40, 128], [23, 5, 31]), # mixed batch
|
||||
],
|
||||
)
|
||||
def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
|
||||
set_random_seed(0)
|
||||
num_seqs = len(prefix_lens)
|
||||
num_heads = 4
|
||||
block_size = 32
|
||||
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
|
||||
max_len = max(seq_lens)
|
||||
max_blocks = (max_len + block_size - 1) // block_size
|
||||
num_blocks = num_seqs * max_blocks
|
||||
|
||||
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
|
||||
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
|
||||
req_to_token = _expand_block_table(block_table, block_size)
|
||||
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
|
||||
extend_seq_lens_t = torch.tensor(extend_lens, dtype=torch.int64)
|
||||
extend_start_loc = torch.cumsum(
|
||||
torch.tensor([0, *extend_lens[:-1]], dtype=torch.int64), dim=0
|
||||
)
|
||||
scale = HEAD_SIZE**-0.5
|
||||
total_new_tokens = sum(extend_lens)
|
||||
|
||||
# The new tokens' latent K/V, already written into the cache (mirrors
|
||||
# do_kv_cache_update running before forward_mha in real usage).
|
||||
k_extend_flat = torch.randn(total_new_tokens, HEAD_SIZE, dtype=DTYPE)
|
||||
flat_cache = _flatten_cache(kv_cache)
|
||||
for i in range(num_seqs):
|
||||
prefix_len = prefix_lens[i]
|
||||
ext_len = extend_lens[i]
|
||||
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
|
||||
start = extend_start_loc[i].item()
|
||||
flat_cache[new_token_ids] = k_extend_flat[start : start + ext_len]
|
||||
|
||||
q_extend = torch.randn(total_new_tokens, num_heads, HEAD_SIZE, dtype=DTYPE)
|
||||
k_extend = k_extend_flat.unsqueeze(1)
|
||||
v_extend = k_extend[..., :KV_LORA_RANK]
|
||||
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
|
||||
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
|
||||
|
||||
o_extend = torch.empty(total_new_tokens, num_heads, KV_LORA_RANK, dtype=DTYPE)
|
||||
ops.cpu_mla_extend(
|
||||
q_extend,
|
||||
k_extend,
|
||||
v_extend,
|
||||
o_extend,
|
||||
kv_cache_flat,
|
||||
v_buffer,
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens_t,
|
||||
extend_seq_lens_t,
|
||||
extend_start_loc,
|
||||
max(extend_lens),
|
||||
scale,
|
||||
0.0,
|
||||
False,
|
||||
0,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
for i in range(num_seqs):
|
||||
prefix_len = prefix_lens[i]
|
||||
ext_len = extend_lens[i]
|
||||
start = extend_start_loc[i].item()
|
||||
for t in range(ext_len):
|
||||
visible = prefix_len + t + 1
|
||||
token_ids = req_to_token[i, :visible]
|
||||
keys = flat_cache[token_ids]
|
||||
ref = _ref_latent_attn(q_extend[start + t], keys, scale, KV_LORA_RANK)
|
||||
torch.testing.assert_close(o_extend[start + t], ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
class _FakeLinear:
|
||||
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
|
||||
.quant_method are read by get_and_maybe_dequant_weights()."""
|
||||
|
||||
def __init__(self, weight: torch.Tensor):
|
||||
self.weight = weight
|
||||
self.quant_method = None
|
||||
|
||||
|
||||
def _make_amx_mla_impl(num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank):
|
||||
weight = torch.randn(
|
||||
num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank, dtype=DTYPE
|
||||
)
|
||||
kv_b_proj = _FakeLinear(weight)
|
||||
impl = AMXMLAImpl(
|
||||
num_heads=num_heads,
|
||||
head_size=kv_lora_rank + QK_ROPE_HEAD_DIM,
|
||||
scale=(qk_nope_head_dim + QK_ROPE_HEAD_DIM) ** -0.5,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="auto",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
qk_head_dim=qk_nope_head_dim + QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=v_head_dim,
|
||||
kv_b_proj=kv_b_proj,
|
||||
)
|
||||
impl.process_weights_after_loading(DTYPE)
|
||||
return impl, weight
|
||||
|
||||
|
||||
class _FakePrefillMetadata:
|
||||
def __init__(
|
||||
self, block_table, cpu_seq_lens, query_start_loc, req_to_token, req_pool_indices
|
||||
):
|
||||
self.block_table = block_table
|
||||
self.cpu_seq_lens = cpu_seq_lens
|
||||
query_start_loc_i64 = query_start_loc.to(torch.int64)
|
||||
extend_seq_lens = query_start_loc_i64[1:] - query_start_loc_i64[:-1]
|
||||
self.extend_seq_lens = extend_seq_lens
|
||||
self.extend_start_loc = query_start_loc_i64[:-1]
|
||||
self.max_len_extend = int(extend_seq_lens.max().item())
|
||||
self.req_to_token = req_to_token
|
||||
self.req_pool_indices = req_pool_indices
|
||||
|
||||
|
||||
class _FakeDecodeMetadata:
|
||||
def __init__(self, block_table, seq_lens, req_to_token, req_pool_indices):
|
||||
self.block_table = block_table
|
||||
self.seq_lens_i64 = seq_lens.to(torch.int64)
|
||||
self.req_to_token = req_to_token
|
||||
self.req_pool_indices = req_pool_indices
|
||||
self.num_kv_splits = _compute_num_kv_splits(
|
||||
int(seq_lens.max().item()), current_platform.num_compute_units()
|
||||
)
|
||||
|
||||
|
||||
class _FakeAttnMetadata:
|
||||
def __init__(self, decode=None, prefill=None, max_seq_len=0):
|
||||
self.decode = decode
|
||||
self.prefill = prefill
|
||||
self.max_seq_len = max_seq_len
|
||||
|
||||
|
||||
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
||||
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
|
||||
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
|
||||
decode attention through W_UK/W_UV."""
|
||||
set_random_seed(1)
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||
impl, kv_b_weight = _make_amx_mla_impl(
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
|
||||
)
|
||||
w_uk, w_uv = kv_b_weight.T.view(
|
||||
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
|
||||
).split([qk_nope_head_dim, v_head_dim], dim=-1)
|
||||
|
||||
seq_lens = [1, 40, 5]
|
||||
num_seqs = len(seq_lens)
|
||||
block_size = 32
|
||||
max_blocks = (max(seq_lens) + block_size - 1) // block_size
|
||||
num_blocks = num_seqs * max_blocks
|
||||
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
|
||||
|
||||
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
|
||||
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
|
||||
req_to_token = _expand_block_table(block_table, block_size)
|
||||
|
||||
q_nope = torch.randn(num_seqs, num_heads, qk_nope_head_dim, dtype=DTYPE)
|
||||
q_pe = torch.randn(num_seqs, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
|
||||
# Pre-absorb, mirroring MLAAttention.forward_impl's own bmm against W_UK_T.
|
||||
ql_nope = torch.einsum("bnp,lnp->bnl", q_nope.float(), w_uk.float()).to(DTYPE)
|
||||
|
||||
attn_metadata = _FakeAttnMetadata(
|
||||
decode=_FakeDecodeMetadata(
|
||||
block_table=block_table,
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
|
||||
req_to_token=req_to_token,
|
||||
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
|
||||
),
|
||||
max_seq_len=max(seq_lens),
|
||||
)
|
||||
o, lse = impl.forward_mqa((ql_nope, q_pe), kv_cache, attn_metadata, layer=None)
|
||||
assert lse is None
|
||||
# De-absorb, mirroring MLAAttention.forward_impl's own bmm against
|
||||
# layer._v_up_proj, to get back into real per-head V space for
|
||||
# comparison against a from-scratch reference.
|
||||
out_real = torch.einsum("bnl,lnv->bnv", o.float(), w_uv.float())
|
||||
|
||||
flat_cache = _flatten_cache(kv_cache)
|
||||
scale = impl.scale
|
||||
for i, seq_len in enumerate(seq_lens):
|
||||
token_ids = req_to_token[i, :seq_len]
|
||||
keys_latent = flat_cache[token_ids] # (L, head_size)
|
||||
k_nope_latent, k_pe = keys_latent.split(
|
||||
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
|
||||
)
|
||||
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
|
||||
k_pe_b = k_pe.float().unsqueeze(0).expand(num_heads, -1, -1)
|
||||
k_full = torch.cat([k_real, k_pe_b], dim=-1) # (num_heads, L, qk_head_dim)
|
||||
q_full = torch.cat(
|
||||
[q_nope[i], q_pe[i]], dim=-1
|
||||
).float() # (num_heads, qk_head_dim)
|
||||
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
|
||||
probs = torch.softmax(scores, dim=-1)
|
||||
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
|
||||
ref = torch.einsum("nl,nlv->nv", probs, v_real)
|
||||
torch.testing.assert_close(out_real[i], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL)
|
||||
|
||||
|
||||
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
|
||||
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
|
||||
prefill batch with a mix of fresh and continued (cached-prefix)
|
||||
sequences."""
|
||||
set_random_seed(2)
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||
impl, kv_b_weight = _make_amx_mla_impl(
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
|
||||
)
|
||||
w_uk, w_uv = kv_b_weight.T.view(
|
||||
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
|
||||
).split([qk_nope_head_dim, v_head_dim], dim=-1)
|
||||
|
||||
prefix_lens = [0, 20]
|
||||
extend_lens = [13, 7]
|
||||
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
|
||||
num_seqs = len(prefix_lens)
|
||||
block_size = 32
|
||||
max_blocks = (max(seq_lens) + block_size - 1) // block_size
|
||||
num_blocks = num_seqs * max_blocks
|
||||
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
|
||||
|
||||
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
|
||||
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
|
||||
req_to_token = _expand_block_table(block_table, block_size)
|
||||
flat_cache = _flatten_cache(kv_cache)
|
||||
|
||||
total_new_tokens = sum(extend_lens)
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.cumsum(torch.tensor(extend_lens), dim=0).tolist()],
|
||||
dtype=torch.int64,
|
||||
)
|
||||
|
||||
q_nope = torch.randn(total_new_tokens, num_heads, qk_nope_head_dim, dtype=DTYPE)
|
||||
q_pe = torch.randn(total_new_tokens, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
|
||||
q = torch.cat([q_nope, q_pe], dim=-1)
|
||||
|
||||
kv_c_normed = torch.randn(total_new_tokens, kv_lora_rank, dtype=DTYPE)
|
||||
k_pe = torch.randn(total_new_tokens, 1, QK_ROPE_HEAD_DIM, dtype=DTYPE)
|
||||
|
||||
# Write the new tokens' latent K/V into the cache ahead of time, mirroring
|
||||
# do_kv_cache_update running before forward_mha in real usage.
|
||||
for i in range(num_seqs):
|
||||
prefix_len = prefix_lens[i]
|
||||
ext_len = extend_lens[i]
|
||||
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
|
||||
start = query_start_loc[i].item()
|
||||
flat_cache[new_token_ids, :kv_lora_rank] = kv_c_normed[start : start + ext_len]
|
||||
flat_cache[new_token_ids, kv_lora_rank:] = k_pe[start : start + ext_len, 0]
|
||||
|
||||
attn_metadata = _FakeAttnMetadata(
|
||||
prefill=_FakePrefillMetadata(
|
||||
block_table=block_table,
|
||||
cpu_seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
|
||||
query_start_loc=query_start_loc,
|
||||
req_to_token=req_to_token,
|
||||
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
|
||||
)
|
||||
)
|
||||
output = torch.empty(total_new_tokens, num_heads * v_head_dim, dtype=DTYPE)
|
||||
impl.forward_mha(
|
||||
q, kv_c_normed, k_pe, kv_cache, attn_metadata, k_scale=None, output=output
|
||||
)
|
||||
out_view = output.view(total_new_tokens, num_heads, v_head_dim).float()
|
||||
|
||||
scale = impl.scale
|
||||
for i in range(num_seqs):
|
||||
prefix_len = prefix_lens[i]
|
||||
ext_len = extend_lens[i]
|
||||
start = query_start_loc[i].item()
|
||||
for t in range(ext_len):
|
||||
visible = prefix_len + t + 1
|
||||
token_ids = req_to_token[i, :visible]
|
||||
keys_latent = flat_cache[token_ids]
|
||||
k_nope_latent, k_pe_ref = keys_latent.split(
|
||||
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
|
||||
)
|
||||
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
|
||||
k_pe_b = k_pe_ref.float().unsqueeze(0).expand(num_heads, -1, -1)
|
||||
k_full = torch.cat([k_real, k_pe_b], dim=-1)
|
||||
q_full = torch.cat([q_nope[start + t], q_pe[start + t]], dim=-1).float()
|
||||
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
|
||||
probs = torch.softmax(scores, dim=-1)
|
||||
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
|
||||
ref = torch.einsum("nl,nlv->nv", probs, v_real)
|
||||
torch.testing.assert_close(
|
||||
out_view[start + t], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL
|
||||
)
|
||||
@@ -1,919 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Literal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
CacheConfig,
|
||||
ParallelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
if current_platform.is_cuda():
|
||||
from vllm.platforms.cuda import CudaPlatform
|
||||
else:
|
||||
CudaPlatform = None
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
else:
|
||||
RocmPlatform = None # type: ignore[misc] # Unavailable platform import.
|
||||
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import (
|
||||
AttentionSelectorConfig,
|
||||
_cached_get_attn_backend,
|
||||
get_attn_backend,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_cache():
|
||||
"""Clear lru cache to ensure each test case runs without caching."""
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
|
||||
# Define MLA and non-MLA backends separately
|
||||
DEVICE_MLA_BACKENDS = {
|
||||
"cuda": [
|
||||
"TRITON_MLA",
|
||||
"FLASHMLA",
|
||||
"FLASHINFER_MLA",
|
||||
"FLASH_ATTN_MLA",
|
||||
"CUTLASS_MLA",
|
||||
],
|
||||
"hip": ["TRITON_MLA", "ROCM_AITER_MLA"],
|
||||
"cpu": [],
|
||||
}
|
||||
|
||||
DEVICE_REGULAR_ATTN_BACKENDS = {
|
||||
"cuda": ["FLASHINFER", "FLASH_ATTN"],
|
||||
"hip": ["ROCM_ATTN"],
|
||||
"cpu": ["CPU_ATTN"],
|
||||
}
|
||||
|
||||
DEVICE_MLA_BLOCK_SIZES = {
|
||||
"cuda": [16, 64], # CUDA supports both standard and extended block sizes
|
||||
"hip": [16, 1], # HIP requires special handling for block_size=1
|
||||
# "cpu": [16] # CPU uses fixed block size from test cases
|
||||
"cpu": [], # FIXME(woosuk): Temporarily disable CPU tests
|
||||
}
|
||||
|
||||
|
||||
def generate_params():
|
||||
is_rocm = current_platform.is_rocm()
|
||||
params = []
|
||||
device_list = ["cuda", "cpu"] if not is_rocm else ["hip", "cpu"]
|
||||
for use_mla in [True, False]:
|
||||
for device in device_list:
|
||||
backends = (
|
||||
DEVICE_MLA_BACKENDS[device]
|
||||
if use_mla
|
||||
else DEVICE_REGULAR_ATTN_BACKENDS[device]
|
||||
)
|
||||
for name in backends:
|
||||
block_sizes = (
|
||||
[128]
|
||||
if name == "CUTLASS_MLA"
|
||||
else DEVICE_MLA_BLOCK_SIZES[device]
|
||||
if use_mla
|
||||
else [16]
|
||||
)
|
||||
for block_size in block_sizes:
|
||||
for use_dcp in [False, True] if device != "cpu" else [False]:
|
||||
params.append(
|
||||
pytest.param(
|
||||
device,
|
||||
name,
|
||||
use_mla,
|
||||
block_size,
|
||||
use_dcp,
|
||||
id=(
|
||||
f"{device}_{name}_mla_{str(use_mla)[0]}"
|
||||
f"_blks{block_size}_dcp{use_dcp}"
|
||||
),
|
||||
)
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device, name, use_mla, block_size, use_dcp", generate_params()
|
||||
)
|
||||
def test_backend_selection(
|
||||
device: str,
|
||||
name: str,
|
||||
use_mla: bool,
|
||||
block_size: int,
|
||||
use_dcp: bool,
|
||||
):
|
||||
"""Supported GPU backends remain selectable when DCP is enabled."""
|
||||
# Create AttentionConfig with the specified backend
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum[name])
|
||||
cache_config = CacheConfig(block_size=block_size)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config,
|
||||
cache_config=cache_config,
|
||||
parallel_config=ParallelConfig(
|
||||
# Skip GPU-count autodetection: selection does not launch workers.
|
||||
distributed_executor_backend="mp" if use_dcp else None,
|
||||
tensor_parallel_size=2 if use_dcp else 1,
|
||||
decode_context_parallel_size=2 if use_dcp else 1,
|
||||
),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if device == "cpu":
|
||||
with patch("vllm.platforms.current_platform", CpuPlatform()):
|
||||
backend = get_attn_backend(16, torch.float16, None)
|
||||
assert backend.get_name() == "CPU_ATTN"
|
||||
|
||||
elif device == "hip":
|
||||
if RocmPlatform is None:
|
||||
pytest.skip("RocmPlatform not available")
|
||||
with patch("vllm.platforms.current_platform", RocmPlatform()):
|
||||
if use_mla:
|
||||
# ROCm MLA backend logic:
|
||||
# - TRITON_MLA: supported when block_size != 1
|
||||
# - ROCM_AITER_MLA: supported when block_size == 1
|
||||
# If backend is forced but doesn't match block_size,
|
||||
# should raise ValueError
|
||||
|
||||
if name == "TRITON_MLA" and block_size == 1:
|
||||
# TRITON_MLA doesn't support block_size == 1
|
||||
with pytest.raises(ValueError):
|
||||
get_attn_backend(576, torch.float16, None, use_mla=use_mla)
|
||||
else:
|
||||
# Valid backend-block_size combination
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = name
|
||||
assert backend.get_name() == expected
|
||||
else:
|
||||
if use_dcp:
|
||||
with pytest.raises(ValueError, match="DCP not supported"):
|
||||
get_attn_backend(32, torch.float16, None, use_mla=use_mla)
|
||||
else:
|
||||
backend = get_attn_backend(
|
||||
32, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "ROCM_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
elif device == "cuda":
|
||||
if CudaPlatform is None:
|
||||
pytest.skip("CudaPlatform not available")
|
||||
with patch("vllm.platforms.current_platform", CudaPlatform()):
|
||||
capability = torch.cuda.get_device_capability()
|
||||
if use_mla:
|
||||
# CUDA MLA backend logic:
|
||||
# - CUTLASS_MLA: only supported with block_size == 128
|
||||
# and Blackwell GPUs (SM 10.x), V1 only
|
||||
# - FLASHINFER_MLA: only supported on Blackwell GPUs
|
||||
# (SM 10.x), V1 only
|
||||
# - FLASHMLA: only supported with block_size == 64
|
||||
# - FLASH_ATTN_MLA: V1 only
|
||||
# - TRITON_MLA: fallback for other cases
|
||||
|
||||
if name == "CUTLASS_MLA":
|
||||
if block_size != 128:
|
||||
# CUTLASS_MLA only supports block_size == 128
|
||||
pytest.skip("CUTLASS_MLA only supports block_size 128")
|
||||
if capability[0] != 10:
|
||||
pytest.skip("CUTLASS MLA is not supported on this platform")
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "CUTLASS_MLA"
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASHINFER_MLA":
|
||||
if capability[0] != 10:
|
||||
pytest.skip(
|
||||
"FlashInfer MLA is not supported on this platform"
|
||||
)
|
||||
if block_size not in [32, 64]:
|
||||
# FlashInfer MLA only supports block_size 32 or 64
|
||||
pytest.skip(
|
||||
"FlashInfer MLA only supports block_size 32 or 64"
|
||||
)
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "FLASHINFER_MLA"
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASHMLA":
|
||||
if block_size != 64:
|
||||
# FlashMLA only supports block_size == 64
|
||||
pytest.skip("FlashMLA only supports block_size 64")
|
||||
from vllm.v1.attention.backends.mla.flashmla import (
|
||||
is_flashmla_dense_supported,
|
||||
)
|
||||
|
||||
is_supported, _ = is_flashmla_dense_supported()
|
||||
if not is_supported:
|
||||
pytest.skip("FlashMLA not supported on this platform")
|
||||
backend = get_attn_backend(
|
||||
576,
|
||||
torch.float16,
|
||||
None,
|
||||
use_mla=use_mla,
|
||||
)
|
||||
expected = name
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASH_ATTN_MLA":
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
flash_attn_supports_mla,
|
||||
)
|
||||
|
||||
if not flash_attn_supports_mla():
|
||||
pytest.skip(
|
||||
"FlashAttention MLA not supported on this platform"
|
||||
)
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "FLASH_ATTN_MLA"
|
||||
assert backend.get_name() == expected
|
||||
else:
|
||||
# TRITON_MLA or other fallback
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "TRITON_MLA"
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASHINFER":
|
||||
backend = get_attn_backend(64, torch.float16, None, use_mla=use_mla)
|
||||
expected = "FLASHINFER"
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASH_ATTN":
|
||||
backend = get_attn_backend(32, torch.float16, None, use_mla=use_mla)
|
||||
expected = "FLASH_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="GPU attention backends"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name",
|
||||
[
|
||||
"FLASH_ATTN",
|
||||
"FLASH_ATTN_DIFFKV",
|
||||
"FLASHINFER",
|
||||
"TRITON_MLA",
|
||||
"ROCM_AITER_MLA",
|
||||
"ROCM_AITER_TRITON_MLA",
|
||||
"CUTLASS_MLA",
|
||||
"FLASHMLA",
|
||||
"FLASHINFER_MLA",
|
||||
"TOKENSPEED_MLA",
|
||||
"FLASH_ATTN_MLA",
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
],
|
||||
)
|
||||
def test_supported_backend_preserves_dcp_eligibility(backend_name, default_vllm_config):
|
||||
"""DCP opt-ins preserve eligibility and unrelated configuration restrictions."""
|
||||
if backend_name.startswith("ROCM_") and not current_platform.is_rocm():
|
||||
pytest.skip("ROCm-specific backend")
|
||||
try:
|
||||
backend_cls = AttentionBackendEnum[backend_name].get_class()
|
||||
except ImportError as error:
|
||||
pytest.skip(f"Optional backend dependency unavailable: {error}")
|
||||
|
||||
config = AttentionSelectorConfig(
|
||||
head_size=576 if backend_cls.is_mla() else 128,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=backend_cls.get_preferred_block_size(64),
|
||||
use_mla=backend_cls.is_mla(),
|
||||
use_sparse=backend_cls.is_sparse(),
|
||||
)
|
||||
kwargs = dict(
|
||||
device_capability=current_platform.get_device_capability(), **config._asdict()
|
||||
)
|
||||
invalid_without_dcp = backend_cls.validate_configuration(**kwargs)
|
||||
kwargs["use_dcp"] = True
|
||||
assert backend_cls.validate_configuration(**kwargs) == invalid_without_dcp
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda", "hip"])
|
||||
def test_fp32_fallback(device: str):
|
||||
"""Test attention backend selection with fp32."""
|
||||
# Use default config (no backend specified)
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if device == "cpu":
|
||||
with patch("vllm.platforms.current_platform", CpuPlatform()):
|
||||
backend = get_attn_backend(16, torch.float32, None)
|
||||
assert backend.get_name() == "CPU_ATTN"
|
||||
|
||||
elif device == "cuda":
|
||||
if CudaPlatform is None:
|
||||
pytest.skip("CudaPlatform not available")
|
||||
with patch("vllm.platforms.current_platform", CudaPlatform()):
|
||||
backend = get_attn_backend(16, torch.float32, None)
|
||||
assert backend.get_name() == "FLEX_ATTENTION"
|
||||
|
||||
elif device == "hip":
|
||||
if RocmPlatform is None:
|
||||
pytest.skip("RocmPlatform not available")
|
||||
# ROCm backends do not support head_size=16 (minimum is 32).
|
||||
# No known HuggingFace transformer model uses head_size=16.
|
||||
# Revisit if a real model with this head size is identified
|
||||
# and accuracy-tested.
|
||||
with (
|
||||
patch("vllm.platforms.current_platform", RocmPlatform()),
|
||||
pytest.raises(ValueError, match="No valid attention backend"),
|
||||
):
|
||||
get_attn_backend(16, torch.float32, None)
|
||||
|
||||
|
||||
def test_invalid_backend():
|
||||
"""Test that invalid attention backend names raise ValueError."""
|
||||
with (
|
||||
pytest.raises(ValueError),
|
||||
):
|
||||
# Invalid backend name should raise ValueError when creating enum
|
||||
AttentionConfig(backend=AttentionBackendEnum["INVALID"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("auto_value", ["auto", "AUTO", "Auto"])
|
||||
def test_auto_backend_string(auto_value: str):
|
||||
"""Test that 'auto' string value triggers automatic backend selection."""
|
||||
# Using "auto" should result in backend=None (automatic selection)
|
||||
attention_config = AttentionConfig(backend=auto_value)
|
||||
assert attention_config.backend is None
|
||||
|
||||
|
||||
def test_auto_backend_selection_behavior():
|
||||
"""Test that 'auto' backend behaves same as None (automatic selection)."""
|
||||
# Create config with explicit "auto"
|
||||
auto_config = AttentionConfig(backend="auto")
|
||||
|
||||
# Create config with None (default)
|
||||
none_config = AttentionConfig(backend=None)
|
||||
|
||||
# Both should have backend=None
|
||||
assert auto_config.backend is None
|
||||
assert none_config.backend is None
|
||||
|
||||
# Both configs should result in the same automatic backend selection
|
||||
vllm_config_auto = VllmConfig(attention_config=auto_config)
|
||||
vllm_config_none = VllmConfig(attention_config=none_config)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config_auto),
|
||||
patch("vllm.platforms.current_platform", CpuPlatform()),
|
||||
):
|
||||
backend_auto = get_attn_backend(16, torch.float16, None)
|
||||
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config_none),
|
||||
patch("vllm.platforms.current_platform", CpuPlatform()),
|
||||
):
|
||||
backend_none = get_attn_backend(16, torch.float16, None)
|
||||
|
||||
# Both should select the same backend
|
||||
assert backend_auto.get_name() == backend_none.get_name()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,flash_attn_version,should_succeed",
|
||||
[
|
||||
("FLASH_ATTN", 3, True), # FA3 supports per-head quant scales
|
||||
("FLASH_ATTN", 2, False), # FA2 does not support per-head quant scales
|
||||
("FLASHINFER", None, False), # FlashInfer does not support
|
||||
("FLEX_ATTENTION", None, False), # Flex does not support
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",
|
||||
)
|
||||
def test_per_head_quant_scales_backend_selection(
|
||||
backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool
|
||||
):
|
||||
"""Test backend selection when use_per_head_quant_scales=True."""
|
||||
# Clear cache to ensure fresh backend selection
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
attention_config = AttentionConfig(
|
||||
backend=AttentionBackendEnum[backend_name],
|
||||
flash_attn_version=flash_attn_version,
|
||||
)
|
||||
cache_config = CacheConfig(block_size=64)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config, cache_config=cache_config
|
||||
)
|
||||
|
||||
if CudaPlatform is None:
|
||||
pytest.skip("CudaPlatform not available")
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", CudaPlatform()),
|
||||
):
|
||||
if backend_name == "FLASH_ATTN" and flash_attn_version == 3:
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("FA3 requires CUDA")
|
||||
capability = torch.cuda.get_device_capability()
|
||||
if capability[0] != 9:
|
||||
pytest.skip("FA3 is only supported on Hopper (SM 9.x) GPUs")
|
||||
|
||||
if should_succeed:
|
||||
backend = get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="fp8",
|
||||
use_per_head_quant_scales=True,
|
||||
)
|
||||
assert backend.get_name() == backend_name
|
||||
else:
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="fp8",
|
||||
use_per_head_quant_scales=True,
|
||||
)
|
||||
assert backend_name in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
CudaPlatform is None, reason="CUDA platform is required for this test"
|
||||
)
|
||||
@pytest.mark.parametrize("head_size", [64, 80, 96, 128, 192, 256])
|
||||
def test_int4_per_token_head_head_size_selection(head_size: int):
|
||||
"""INT4 per-token-head KV cache is selectable at any even head size.
|
||||
|
||||
The Hadamard rotation the INT4 write path applies is now defined for
|
||||
non-power-of-two rows as well (it runs block-diagonally, see
|
||||
``fast_hadamard_transform``), so a model such as Phi-3 (head_size=96) must
|
||||
keep selecting TRITON_ATTN instead of being rejected.
|
||||
"""
|
||||
vllm_config = VllmConfig(cache_config=CacheConfig(block_size=64))
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", CudaPlatform()),
|
||||
):
|
||||
backend = get_attn_backend(
|
||||
head_size=head_size,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="int4_per_token_head",
|
||||
)
|
||||
assert backend.get_name() == "TRITON_ATTN"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,use_non_causal,should_succeed",
|
||||
[
|
||||
("FLASH_ATTN", True, True), # FlashAttn supports non-causal
|
||||
("FLASH_ATTN", False, True), # FlashAttn also works with causal
|
||||
]
|
||||
+ (
|
||||
[
|
||||
("FLASHINFER", True, True), # FlashInfer supports non-causal
|
||||
("FLASHINFER", False, True), # FlashInfer works with causal
|
||||
]
|
||||
if CudaPlatform is not None
|
||||
else []
|
||||
),
|
||||
)
|
||||
def test_non_causal_backend_selection(
|
||||
backend_name: str, use_non_causal: bool, should_succeed: bool
|
||||
):
|
||||
"""Test that use_non_causal on AttentionConfig controls backend filtering.
|
||||
|
||||
DFlashProposer sets use_non_causal=True on the draft model's
|
||||
AttentionConfig so only non-causal-capable backends are selected.
|
||||
The target model keeps use_non_causal=False (default) and can use
|
||||
any backend.
|
||||
"""
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
attention_config = AttentionConfig(
|
||||
backend=AttentionBackendEnum[backend_name],
|
||||
use_non_causal=use_non_causal,
|
||||
)
|
||||
cache_config = CacheConfig(block_size=16)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config, cache_config=cache_config
|
||||
)
|
||||
|
||||
platform = CudaPlatform or RocmPlatform
|
||||
if platform is None:
|
||||
pytest.skip("CudaPlatform and RocmPlatform are not available")
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", platform()),
|
||||
):
|
||||
if should_succeed:
|
||||
backend = get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype=None,
|
||||
)
|
||||
assert backend.get_name() == backend_name
|
||||
else:
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype=None,
|
||||
)
|
||||
assert "non-causal" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
def test_non_causal_autoselect_backend():
|
||||
"""Test that when backend=None with use_non_causal=True, auto-selection
|
||||
picks a compatible backend.
|
||||
|
||||
This simulates the DFlash scenario where the user doesn't specify
|
||||
--attention-backend or --speculative-config.attention_backend.
|
||||
The drafter inherits backend=None and auto-selects a backend that
|
||||
supports non-causal attention.
|
||||
"""
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
attention_config = AttentionConfig(
|
||||
backend=None,
|
||||
use_non_causal=True,
|
||||
)
|
||||
cache_config = CacheConfig(block_size=16)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config, cache_config=cache_config
|
||||
)
|
||||
|
||||
if CudaPlatform is None:
|
||||
pytest.skip("CudaPlatform not available")
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", CudaPlatform()),
|
||||
):
|
||||
backend = get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype=None,
|
||||
)
|
||||
assert backend.supports_non_causal()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
[
|
||||
"fp8_e5m2",
|
||||
"fp8_ds_mla",
|
||||
"fp8_inc",
|
||||
"nvfp4",
|
||||
"nvfp4_4over6",
|
||||
"fp8_per_token_head",
|
||||
"int8_per_token_head",
|
||||
],
|
||||
)
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType):
|
||||
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
||||
that it cannot handle."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
|
||||
def test_flash_attn_accepts_handled_fp8_variants(
|
||||
kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
|
||||
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
# The fp8 decision is made in fa_utils, using its own current_platform
|
||||
# binding, so patch is_xpu there (not on flash_attn's) to stay robust to
|
||||
# import order across earlier tests that patch vllm.platforms.current_platform.
|
||||
monkeypatch.setattr(fa_utils_mod.current_platform, "is_xpu", lambda: True)
|
||||
assert FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
|
||||
|
||||
blackwell_only = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="FA4 is CUDA-only"
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _blackwell(vllm_config=None):
|
||||
platform = MagicMock()
|
||||
platform.is_xpu.return_value = False
|
||||
platform.is_rocm.return_value = False
|
||||
platform.get_device_capability.return_value = DeviceCapability(10, 0)
|
||||
with (
|
||||
patch("vllm.v1.attention.backends.fa_utils.current_platform", platform),
|
||||
patch(
|
||||
"vllm.vllm_flash_attn.flash_attn_interface.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
patch("vllm.config.get_current_vllm_config_or_none", return_value=vllm_config),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.flash_attn.get_current_vllm_config_or_none",
|
||||
return_value=vllm_config,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def _hd256_config(
|
||||
*,
|
||||
is_mm_prefix_lm=False,
|
||||
rswa_window=None,
|
||||
dcp_size=1,
|
||||
softcap=None,
|
||||
head_size=256,
|
||||
cache_dtype="auto",
|
||||
):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.attention_config.flash_attn_version = None
|
||||
vllm_config.model_config.is_mm_prefix_lm = is_mm_prefix_lm
|
||||
vllm_config.model_config.rswa_window = rswa_window
|
||||
vllm_config.model_config.hf_text_config.attn_logit_softcapping = softcap
|
||||
vllm_config.model_config.get_head_size.return_value = head_size
|
||||
vllm_config.cache_config.cache_dtype = cache_dtype
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
return vllm_config
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs,config_kwargs,expected",
|
||||
[
|
||||
({}, {}, 4),
|
||||
({"supports_fa4_hd256": False}, {}, 2),
|
||||
({"has_sinks": True}, {}, 2),
|
||||
({"requires_softcap": True}, {}, 2),
|
||||
# Larger block sizes normalize to 128.
|
||||
({"kv_cache_block_size": 16}, {}, 2),
|
||||
({"kv_cache_block_size": 64}, {}, 2),
|
||||
({"kv_cache_block_size": 256}, {}, 4),
|
||||
({}, {"is_mm_prefix_lm": True}, 2),
|
||||
({}, {"rswa_window": 512}, 2),
|
||||
({}, {"dcp_size": 2}, 2),
|
||||
({}, {"softcap": 50.0}, 2),
|
||||
({}, {"cache_dtype": "fp8"}, 2),
|
||||
({"head_size": 128, "kv_cache_block_size": 16}, {}, 4),
|
||||
({"head_size": 192, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 4),
|
||||
({"head_size": 256, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 2),
|
||||
({"head_size": 256, "head_size_v": 64}, {}, 2),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_fallback_matrix(kwargs, config_kwargs, expected):
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
|
||||
kwargs = {
|
||||
"head_size": 256,
|
||||
"kv_cache_block_size": 128,
|
||||
"supports_fa4_hd256": True,
|
||||
**kwargs,
|
||||
}
|
||||
with _blackwell(_hd256_config(head_size=kwargs["head_size"], **config_kwargs)):
|
||||
assert get_flash_attn_version(**kwargs) == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,config_kwargs,expected",
|
||||
[
|
||||
("FLASH_ATTN", {}, 128),
|
||||
("FLASH_ATTN", {"softcap": 50.0}, 16),
|
||||
("FLASH_ATTN", {"head_size": 128}, 16),
|
||||
("FLASH_ATTN", {"cache_dtype": "fp8"}, 16),
|
||||
("FLASH_ATTN_DIFFKV", {}, 16),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_block_size_advertisement(
|
||||
backend_name: str, config_kwargs: dict, expected: int
|
||||
):
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm.v1.attention.backends.flash_attn_diffkv import (
|
||||
FlashAttentionDiffKVBackend,
|
||||
)
|
||||
|
||||
backend = {
|
||||
"FLASH_ATTN": FlashAttentionBackend,
|
||||
"FLASH_ATTN_DIFFKV": FlashAttentionDiffKVBackend,
|
||||
}[backend_name]
|
||||
vllm_config = _hd256_config(**config_kwargs)
|
||||
with _blackwell(vllm_config):
|
||||
(size,) = backend.get_supported_kernel_block_sizes()
|
||||
preferred = backend.get_preferred_block_size(16)
|
||||
assert preferred == expected
|
||||
if expected == 128:
|
||||
assert size == 128
|
||||
else:
|
||||
assert isinstance(size, MultipleOf) and size.base == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
def test_fa4_hd256_mm_prefix_deselects_flash_attn():
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
with _blackwell(_hd256_config(is_mm_prefix_lm=True)):
|
||||
assert (
|
||||
FlashAttentionBackend.supports_combination(
|
||||
head_size=256,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=None,
|
||||
block_size=128,
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=True,
|
||||
device_capability=DeviceCapability(10, 0),
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda()
|
||||
or not current_platform.is_device_capability_family(100),
|
||||
reason="requires a Blackwell GPU",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_type,sliding_window,expected",
|
||||
[
|
||||
(AttentionType.DECODER, None, 4),
|
||||
(AttentionType.DECODER, 512, 4),
|
||||
# Local encoder attention lacks the required seqused tensors.
|
||||
(AttentionType.ENCODER_ONLY, None, 4),
|
||||
(AttentionType.ENCODER_ONLY, 512, 2),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionImpl
|
||||
|
||||
# Implementations are constructed before the platform updates the block size.
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
impl = FlashAttentionImpl(
|
||||
num_heads=8,
|
||||
head_size=256,
|
||||
scale=0.0625,
|
||||
num_kv_heads=8,
|
||||
alibi_slopes=None,
|
||||
sliding_window=sliding_window,
|
||||
kv_cache_dtype="auto",
|
||||
attn_type=attn_type,
|
||||
)
|
||||
assert impl.vllm_flash_attn_version == expected
|
||||
assert impl.fa4_hd256 == (expected == 4)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def blackwell_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(10, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hopper_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(9, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
|
||||
def test_hopper_mm_prefix_selects_triton_flash_attn(
|
||||
use_mm_prefix, flash_attn_version, hopper_selection
|
||||
):
|
||||
"""Hopper selects the composite when its causal route resolves FA3 or FA4."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
expected = "TRITON_FLASH_ATTN" if use_mm_prefix else "FLASH_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
|
||||
def test_mm_prefix_selects_composite_without_changing_causal_default(
|
||||
use_mm_prefix, kv_cache_dtype, blackwell_selection
|
||||
):
|
||||
"""The image-mask requirement must reach CUDA's automatic backend priority."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"feature_kwargs,reason",
|
||||
[
|
||||
({"use_rswa": True}, "R-SWA not supported"),
|
||||
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
|
||||
({"block_size": 16}, "block_size not supported"),
|
||||
({"use_dcp": True}, "DCP not supported"),
|
||||
({"use_pcp": True}, "PCP not supported"),
|
||||
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
|
||||
],
|
||||
)
|
||||
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
|
||||
"""Automatic selection must not admit features only one child can execute."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
args = dict(
|
||||
head_size=256,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=128,
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=True,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(10, 0),
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
args.update(feature_kwargs)
|
||||
with set_current_vllm_config(config):
|
||||
reasons = TritonFlashInferBackend.validate_configuration(**args)
|
||||
assert any(reason in message for message in reasons), reasons
|
||||
|
||||
|
||||
@blackwell_only
|
||||
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
|
||||
"""An R-SWA requirement must invalidate a previously cached causal selection."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config=AttentionConfig(
|
||||
backend=AttentionBackendEnum.TRITON_FLASHINFER
|
||||
),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
assert (
|
||||
get_attn_backend(256, torch.bfloat16, None).get_name()
|
||||
== "TRITON_FLASHINFER"
|
||||
)
|
||||
config.model_config.model_arch_config.rswa_window = 128
|
||||
with pytest.raises(ValueError, match="R-SWA"):
|
||||
get_attn_backend(256, torch.bfloat16, None)
|
||||
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,410 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import as_fp32_logits
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
_ceil_to_ue8m0,
|
||||
calc_diff,
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
get_num_sms,
|
||||
get_paged_mqa_logits_metadata,
|
||||
mqa_logits_mx_inputs,
|
||||
mqa_logits_use_mx_scales,
|
||||
native_next_n_supported,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
|
||||
def encode_indexer_scales(sf: torch.Tensor) -> torch.Tensor:
|
||||
"""Per-token power-of-two fp32 scales as the indexer K cache stores them:
|
||||
four equal packed UE8M0 bytes under MX scales, else the fp32 itself."""
|
||||
if not mqa_logits_use_mx_scales():
|
||||
return sf
|
||||
ue8m0 = (sf.view(torch.int32) >> 23).to(torch.uint8)
|
||||
return ue8m0[..., None].repeat_interleave(4, dim=-1).view(torch.int32)[..., 0]
|
||||
|
||||
|
||||
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
|
||||
# x: (num_blocks, block_size, 1, head_dim)
|
||||
num_blocks, block_size, num_heads, head_dim = x.shape
|
||||
assert num_heads == 1
|
||||
x_amax = x.abs().float().amax(dim=3, keepdim=True).clamp(1e-4)
|
||||
sf = _ceil_to_ue8m0(x_amax / 448.0)
|
||||
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
|
||||
x_fp8 = torch.empty(
|
||||
(num_blocks, block_size * (head_dim + 4)),
|
||||
device=x.device,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
x_fp8[:, : block_size * head_dim] = x_scaled.view(
|
||||
num_blocks, block_size * head_dim
|
||||
).view(dtype=torch.uint8)
|
||||
x_fp8[:, block_size * head_dim :] = encode_indexer_scales(
|
||||
sf.view(num_blocks, block_size)
|
||||
).view(dtype=torch.uint8)
|
||||
return x_fp8.view(num_blocks, block_size, num_heads, head_dim + 4)
|
||||
|
||||
|
||||
def per_custom_dims_cast_to_fp8(
|
||||
x: torch.Tensor, dims: tuple, use_ue8m0: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
excluded_dims = tuple([i for i in range(x.dim()) if i not in set(dims)])
|
||||
x_amax = x.abs().float().amax(dim=excluded_dims, keepdim=True).clamp(1e-4)
|
||||
sf = x_amax / 448.0
|
||||
sf = _ceil_to_ue8m0(sf) if use_ue8m0 else sf
|
||||
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
|
||||
return x_scaled, sf.squeeze()
|
||||
|
||||
|
||||
def _generate_cp_test_data(seq_len: int, seq_len_kv: int):
|
||||
assert seq_len_kv % seq_len == 0 and seq_len % 2 == 0
|
||||
chunk_size = seq_len // 2
|
||||
cp_size = seq_len_kv // seq_len
|
||||
cp_id = cp_size // 3
|
||||
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
ke = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
for i in range(chunk_size):
|
||||
ke[i] = cp_id * chunk_size + i
|
||||
ke[i + chunk_size] = (cp_size * 2 - 1 - cp_id) * chunk_size + i
|
||||
return ks, ke
|
||||
|
||||
|
||||
def _ref_fp8_mqa_logits(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
cu_seqlen_ks: torch.Tensor,
|
||||
cu_seqlen_ke: torch.Tensor,
|
||||
):
|
||||
seq_len_kv = kv.shape[0]
|
||||
|
||||
k = kv
|
||||
q = q.float()
|
||||
k = k.float()
|
||||
|
||||
mask_lo = (
|
||||
torch.arange(0, seq_len_kv, device="cuda")[None, :] >= cu_seqlen_ks[:, None]
|
||||
)
|
||||
mask_hi = (
|
||||
torch.arange(0, seq_len_kv, device="cuda")[None, :] < cu_seqlen_ke[:, None]
|
||||
)
|
||||
mask = mask_lo & mask_hi
|
||||
score = torch.einsum("mhd,nd->hmn", q, k)
|
||||
logits = (score.relu() * weights.unsqueeze(-1).transpose(0, 1)).sum(dim=0)
|
||||
logits = logits.masked_fill(~mask, float("-inf"))
|
||||
|
||||
return logits
|
||||
|
||||
|
||||
def _packed_requests_test_data(seq_len: int, seq_len_kv: int):
|
||||
# Two causal prefill requests packed into one KV buffer, so the second
|
||||
# request's rows start at a nonzero K position.
|
||||
half, half_kv = seq_len // 2, seq_len_kv // 2
|
||||
rows = torch.arange(seq_len, dtype=torch.int, device="cuda")
|
||||
ks = torch.where(rows < half, 0, half_kv).to(torch.int)
|
||||
ke = ks + half_kv - half + 1 + rows % half
|
||||
return ks, ke
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["cp", "no_cp", "packed"])
|
||||
def test_deepgemm_fp8_mqa_logits(layout: str):
|
||||
"""FP8 indexer prefill logits, compressed to each row's valid K span."""
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
num_heads, head_dim = 32, 128
|
||||
seq_len, seq_len_kv = 512, 1024
|
||||
q = torch.randn(seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
kv = torch.randn(seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
weights = torch.randn(seq_len, num_heads, device="cuda", dtype=torch.float32)
|
||||
|
||||
if layout == "cp":
|
||||
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
|
||||
elif layout == "no_cp":
|
||||
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
|
||||
seq_len_kv - seq_len
|
||||
)
|
||||
else:
|
||||
ks, ke = _packed_requests_test_data(seq_len, seq_len_kv)
|
||||
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_fp8, kv_sf = per_custom_dims_cast_to_fp8(kv, (0,), True)
|
||||
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
|
||||
max_seqlen_k = int((ke - ks).max())
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_fp8, q_scale),
|
||||
(kv_fp8, encode_indexer_scales(kv_sf)),
|
||||
kernel_weights,
|
||||
ks,
|
||||
ke,
|
||||
max_seqlen_k,
|
||||
)
|
||||
assert logits.shape == (seq_len, max_seqlen_k)
|
||||
logits = as_fp32_logits(logits, ke - ks)
|
||||
|
||||
ref_logits = _ref_fp8_mqa_logits(
|
||||
q=q,
|
||||
kv=kv,
|
||||
weights=kernel_weights.float(),
|
||||
cu_seqlen_ks=ks,
|
||||
cu_seqlen_ke=ke,
|
||||
)
|
||||
# Compressed row i holds K positions [ks[i], ke[i]) from column zero.
|
||||
cols = ks[:, None] + torch.arange(max_seqlen_k, device="cuda")[None, :]
|
||||
valid = cols < ke[:, None]
|
||||
ref_logits = ref_logits.gather(1, cols.clamp(max=seq_len_kv - 1).long())
|
||||
ref_logits = ref_logits.masked_fill(~valid, 0)
|
||||
logits = logits.masked_fill(~valid, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
def _ref_fp8_fp4_paged_mqa_logits(
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
context_lens: torch.Tensor,
|
||||
block_tables: torch.Tensor,
|
||||
max_model_len: int,
|
||||
):
|
||||
batch_size, next_n, _, _ = q.size()
|
||||
_, block_size, _, _ = kv_cache.size()
|
||||
logits = torch.full(
|
||||
[batch_size * next_n, max_model_len],
|
||||
float("-inf"),
|
||||
device=q.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
context_lens_list = context_lens.tolist()
|
||||
for i in range(batch_size):
|
||||
context_len = context_lens_list[i]
|
||||
q_offsets = torch.arange(context_len - next_n, context_len, device="cuda")
|
||||
weight_slice = (
|
||||
weights[i * next_n : (i + 1) * next_n, :].transpose(0, 1).contiguous()
|
||||
)
|
||||
for block_rk in range(cdiv(context_len, block_size)):
|
||||
block_idx = block_tables[i][block_rk]
|
||||
qx, kx = q[i], kv_cache[block_idx]
|
||||
k_offsets = torch.arange(
|
||||
block_rk * block_size,
|
||||
(block_rk + 1) * block_size,
|
||||
device="cuda",
|
||||
)
|
||||
mask = (k_offsets[None, :] < context_len) & (
|
||||
k_offsets[None, :] <= q_offsets[:, None]
|
||||
)
|
||||
s = torch.where(
|
||||
mask[None, :, :],
|
||||
(qx.transpose(0, 1) @ kx.transpose(0, 1).transpose(1, 2)).to(
|
||||
logits.dtype
|
||||
),
|
||||
float("-inf"),
|
||||
)
|
||||
s = torch.relu(s) * weight_slice[..., None]
|
||||
s = s.sum(dim=0)
|
||||
logits[
|
||||
i * next_n : (i + 1) * next_n,
|
||||
block_rk * block_size : (block_rk + 1) * block_size,
|
||||
] = torch.where(k_offsets[None, :] <= q_offsets[:, None], s, float("-inf"))
|
||||
return logits
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
# next_n = 1 + num_speculative_tokens, so next_n=4 is MTP=3 (issue #35878).
|
||||
@pytest.mark.parametrize("batch_size,next_n", [(4, 1), (2, 2), (2, 4)])
|
||||
def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
if not native_next_n_supported(next_n):
|
||||
pytest.skip(f"next_n={next_n} has no native kernel on this architecture")
|
||||
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
|
||||
max_model_len = 4096
|
||||
for heads, index_dim in [(32, 128)]:
|
||||
for avg_kv in (2048,):
|
||||
num_blocks, blocksize = max_model_len * 2, 64
|
||||
|
||||
q = torch.randn(
|
||||
(batch_size, next_n, heads, index_dim),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
(num_blocks, blocksize, 1, index_dim),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
weights = torch.randn(
|
||||
(batch_size * next_n, heads),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
context_lens = (
|
||||
torch.randint(int(0.8 * avg_kv), int(1.2 * avg_kv), (batch_size,))
|
||||
.cuda()
|
||||
.to(torch.int32)
|
||||
)
|
||||
max_block_len = (
|
||||
(context_lens.max().item() + blocksize - 1) // blocksize * blocksize
|
||||
)
|
||||
block_tables = torch.zeros(
|
||||
(batch_size, max_block_len),
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
counter = 0
|
||||
block_idx_pool = list(range(num_blocks))
|
||||
random.shuffle(block_idx_pool)
|
||||
for i in range(batch_size):
|
||||
ctx_len = int(context_lens[i].item())
|
||||
for j in range((ctx_len + blocksize - 1) // blocksize):
|
||||
block_tables[i][j] = block_idx_pool[counter]
|
||||
counter += 1
|
||||
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache)
|
||||
|
||||
# deep_gemm paged MQA logits requires 2D context_lens of
|
||||
# shape (B, next_n) (csrc/apis/attention.hpp:332-335);
|
||||
# see indexer.py:607-608. For each batch/next_n token, the
|
||||
# effective context length is context_lens[b] - next_n + j + 1.
|
||||
next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32)
|
||||
context_lens_2d = (
|
||||
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
|
||||
).contiguous()
|
||||
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
|
||||
kernel_q = (q_fp8, q_scale)
|
||||
kernel_context_lens, kernel_block_tables = context_lens_2d, block_tables
|
||||
indices = None
|
||||
if mqa_logits_use_mx_scales():
|
||||
# SM100 only takes varlen rows: one Q token each, tied to its
|
||||
# request by `indices`.
|
||||
assert q_scale is not None
|
||||
kernel_q = (
|
||||
q_fp8.flatten(0, 1)[:, None],
|
||||
q_scale.flatten(0, 1)[:, None],
|
||||
)
|
||||
kernel_context_lens = context_lens_2d.view(-1, 1)
|
||||
kernel_block_tables = block_tables.repeat_interleave(next_n, dim=0)
|
||||
indices = torch.arange(batch_size, device="cuda", dtype=torch.int32)
|
||||
indices = indices.repeat_interleave(next_n)
|
||||
schedule_metadata = get_paged_mqa_logits_metadata(
|
||||
kernel_context_lens, blocksize, get_num_sms(), indices=indices
|
||||
)
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
kernel_q,
|
||||
kv_cache_fp8,
|
||||
kernel_weights,
|
||||
kernel_context_lens,
|
||||
kernel_block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
indices=indices,
|
||||
)
|
||||
if indices is not None:
|
||||
logits = as_fp32_logits(logits, kernel_context_lens.view(-1))
|
||||
|
||||
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
|
||||
q,
|
||||
kv_cache,
|
||||
kernel_weights.float(),
|
||||
context_lens,
|
||||
block_tables,
|
||||
max_model_len,
|
||||
)
|
||||
|
||||
positions = (
|
||||
torch.arange(max_model_len, device="cuda")
|
||||
.unsqueeze(0)
|
||||
.expand(batch_size * next_n, -1)
|
||||
)
|
||||
row_indices = torch.arange(batch_size * next_n, device="cuda") // next_n
|
||||
next_n_offset = torch.arange(batch_size * next_n, device="cuda") % next_n
|
||||
mask = positions <= (
|
||||
context_lens[row_indices] - next_n + next_n_offset
|
||||
).unsqueeze(1)
|
||||
|
||||
logits = logits.float().masked_fill(~mask, 0)
|
||||
ref_logits = ref_logits.masked_fill(~mask, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
def test_deepgemm_paged_mqa_packed_manager_block_stride():
|
||||
torch.manual_seed(0)
|
||||
num_blocks, block_size, packed_rows = 4, 256, 320
|
||||
num_heads, head_dim, row_bytes = 64, 128, 132
|
||||
page_size, pages_per_block = 64, block_size // 64
|
||||
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
|
||||
compact_pages = kv_cache_cast_to_fp8(
|
||||
torch.randn(
|
||||
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
|
||||
)
|
||||
packed_backing = torch.zeros(
|
||||
num_blocks,
|
||||
packed_rows,
|
||||
row_bytes,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
packed_manager = packed_backing[:, :block_size]
|
||||
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
|
||||
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
|
||||
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
|
||||
tables = [
|
||||
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
|
||||
]
|
||||
|
||||
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
q_scale, weights = mqa_logits_mx_inputs(
|
||||
q, None, torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
|
||||
)
|
||||
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
|
||||
indices = (
|
||||
torch.zeros(1, device="cuda", dtype=torch.int32)
|
||||
if mqa_logits_use_mx_scales()
|
||||
else None
|
||||
)
|
||||
schedule = get_paged_mqa_logits_metadata(
|
||||
context_lens, page_size, get_num_sms(), indices=indices
|
||||
)
|
||||
logits = [
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
cache,
|
||||
weights,
|
||||
context_lens,
|
||||
table,
|
||||
schedule,
|
||||
block_size,
|
||||
indices=indices,
|
||||
)
|
||||
for cache, table in zip((compact_pages, packed_pages), tables)
|
||||
]
|
||||
torch.testing.assert_close(*logits, rtol=0, atol=0)
|
||||
@@ -1,737 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
try:
|
||||
import flashinfer
|
||||
except ImportError:
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"flashinfer is not supported for vLLM on ROCm.", allow_module_level=True
|
||||
)
|
||||
|
||||
import torch
|
||||
|
||||
NUM_HEADS = [(32, 8), (6, 1)]
|
||||
HEAD_SIZES = [128, 256]
|
||||
BLOCK_SIZES = [16, 32]
|
||||
DTYPES = [torch.bfloat16]
|
||||
NUM_BLOCKS = 32768 # Large enough to test overflow in index calculation.
|
||||
SOFT_CAPS = [None, 30.0]
|
||||
SLIDING_WINDOWS = [None, 64]
|
||||
|
||||
|
||||
def ref_paged_attn(
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
query_lens: list[int],
|
||||
kv_lens: list[int],
|
||||
block_tables: torch.Tensor,
|
||||
scale: float,
|
||||
sliding_window: int | None = None,
|
||||
soft_cap: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
num_seqs = len(query_lens)
|
||||
block_tables = block_tables.cpu().numpy()
|
||||
_, block_size, num_kv_heads, head_size = key_cache.shape
|
||||
|
||||
outputs: list[torch.Tensor] = []
|
||||
start_idx = 0
|
||||
for i in range(num_seqs):
|
||||
query_len = query_lens[i]
|
||||
kv_len = kv_lens[i]
|
||||
q = query[start_idx : start_idx + query_len]
|
||||
q *= scale
|
||||
|
||||
num_kv_blocks = (kv_len + block_size - 1) // block_size
|
||||
block_indices = block_tables[i, :num_kv_blocks]
|
||||
|
||||
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)
|
||||
k = k[:kv_len]
|
||||
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)
|
||||
v = v[:kv_len]
|
||||
|
||||
if q.shape[1] != k.shape[1]:
|
||||
k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
|
||||
v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
|
||||
attn = torch.einsum("qhd,khd->hqk", q, k).float()
|
||||
empty_mask = torch.ones(query_len, kv_len)
|
||||
mask = torch.triu(empty_mask, diagonal=kv_len - query_len + 1).bool()
|
||||
if sliding_window is not None:
|
||||
sliding_window_mask = (
|
||||
torch.triu(
|
||||
empty_mask, diagonal=kv_len - (query_len + sliding_window) + 1
|
||||
)
|
||||
.bool()
|
||||
.logical_not()
|
||||
)
|
||||
mask |= sliding_window_mask
|
||||
if soft_cap is not None:
|
||||
attn = soft_cap * torch.tanh(attn / soft_cap)
|
||||
attn.masked_fill_(mask, float("-inf"))
|
||||
attn = torch.softmax(attn, dim=-1).to(v.dtype)
|
||||
out = torch.einsum("hqk,khd->qhd", attn, v)
|
||||
|
||||
outputs.append(out)
|
||||
start_idx += query_len
|
||||
|
||||
return torch.cat(outputs, dim=0)
|
||||
|
||||
|
||||
def _make_paged_kv_metadata(
|
||||
kv_lens: list[int],
|
||||
block_size: int,
|
||||
num_blocks: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build paged-KV metadata tensors for fast_plan_decode tests.
|
||||
|
||||
Returns:
|
||||
kv_indptr – CPU int32, shape [num_seqs + 1]
|
||||
kv_indices – CUDA int32, shape [total_blocks]
|
||||
kv_last_page_lens – CPU int32, shape [num_seqs]
|
||||
block_tables – CUDA int32, shape [num_seqs, max_blocks_per_seq]
|
||||
|
||||
"""
|
||||
num_seqs = len(kv_lens)
|
||||
max_blocks = (max(kv_lens) + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, num_blocks, (num_seqs, max_blocks), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
indptr_list = [0]
|
||||
indices_list: list[int] = []
|
||||
last_lens_list: list[int] = []
|
||||
for i, seq_len in enumerate(kv_lens):
|
||||
n = (seq_len + block_size - 1) // block_size
|
||||
indices_list.extend(block_tables[i, :n].cpu().tolist())
|
||||
indptr_list.append(indptr_list[-1] + n)
|
||||
last_lens_list.append(seq_len % block_size or block_size)
|
||||
|
||||
return (
|
||||
torch.tensor(indptr_list, dtype=torch.int32, device="cpu"),
|
||||
torch.tensor(indices_list, dtype=torch.int32, device="cuda"),
|
||||
torch.tensor(last_lens_list, dtype=torch.int32, device="cpu"),
|
||||
block_tables,
|
||||
)
|
||||
|
||||
|
||||
def _make_cg_decode_wrapper(
|
||||
num_seqs: int,
|
||||
kv_indices_buffer: torch.Tensor,
|
||||
workspace_buffer: torch.Tensor,
|
||||
use_tensor_cores: bool = True,
|
||||
) -> "flashinfer.BatchDecodeWithPagedKVCacheWrapper":
|
||||
"""Create a cudagraph-enabled BatchDecodeWithPagedKVCacheWrapper.
|
||||
|
||||
*kv_indices_buffer* is shared with the caller so that fast_plan_decode
|
||||
can avoid the device-to-device index copy on subsequent (cudagraph) calls.
|
||||
"""
|
||||
return flashinfer.BatchDecodeWithPagedKVCacheWrapper(
|
||||
workspace_buffer,
|
||||
"NHD",
|
||||
use_cuda_graph=True,
|
||||
paged_kv_indptr_buffer=torch.zeros(
|
||||
num_seqs + 1, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
paged_kv_indices_buffer=kv_indices_buffer,
|
||||
paged_kv_last_page_len_buffer=torch.zeros(
|
||||
num_seqs, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
use_tensor_cores=use_tensor_cores,
|
||||
)
|
||||
|
||||
|
||||
def test_fast_decode_plan_importable() -> None:
|
||||
"""fast_decode_plan must be importable from flashinfer.decode.
|
||||
|
||||
This is a forward-compatibility smoke test: if FlashInfer reorganises its
|
||||
public API the import will fail before any other test does.
|
||||
"""
|
||||
from flashinfer.decode import fast_decode_plan # noqa: F401
|
||||
|
||||
assert callable(fast_decode_plan)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode
|
||||
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
|
||||
"""On the first call fast_plan_decode must route through self.plan() and
|
||||
flip vllm_first_call to False on the wrapper object."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
|
||||
kv_lens = [128, 64]
|
||||
block_size = 16
|
||||
num_seqs = len(kv_lens)
|
||||
num_query_heads, num_kv_heads = 8, 2
|
||||
head_size = 128
|
||||
|
||||
kv_indptr, kv_indices, kv_last_page_lens, _ = _make_paged_kv_metadata(
|
||||
kv_lens, block_size, NUM_BLOCKS
|
||||
)
|
||||
|
||||
workspace = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = _make_cg_decode_wrapper(num_seqs, kv_indices.clone(), workspace)
|
||||
|
||||
assert getattr(wrapper, "vllm_first_call", True) is True
|
||||
|
||||
with patch.object(wrapper, "plan", wraps=wrapper.plan) as mock_plan:
|
||||
fast_plan_decode(
|
||||
wrapper,
|
||||
indptr_cpu=kv_indptr,
|
||||
indices=kv_indices,
|
||||
last_page_len_cpu=kv_last_page_lens,
|
||||
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
|
||||
num_qo_heads=num_query_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_size,
|
||||
page_size=block_size,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
)
|
||||
mock_plan.assert_called_once()
|
||||
|
||||
assert wrapper.vllm_first_call is False, (
|
||||
"vllm_first_call should be False after the first fast_plan_decode call"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode
|
||||
def test_fast_plan_decode_matches_full_plan(
|
||||
kv_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""fast_plan_decode's cudagraph path (delegating to FlashInfer's
|
||||
fast_decode_plan) must produce attention output numerically identical to
|
||||
a standard plan() call.
|
||||
|
||||
Both the warmup call (self.plan) and the subsequent fast call
|
||||
(fast_decode_plan) are verified against the same reference.
|
||||
"""
|
||||
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(kv_lens)
|
||||
num_query_heads, num_kv_heads = num_heads
|
||||
|
||||
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
|
||||
kv_indptr, kv_indices, kv_last_page_lens, _ = _make_paged_kv_metadata(
|
||||
kv_lens, block_size, NUM_BLOCKS
|
||||
)
|
||||
|
||||
# Reference output via the standard plan()
|
||||
workspace_ref = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
ref_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
|
||||
workspace_ref, "NHD", use_tensor_cores=True
|
||||
)
|
||||
ref_wrapper.plan(
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
"NONE",
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
)
|
||||
ref_output = ref_wrapper.run(query, key_value_cache)
|
||||
|
||||
# CUDAGraph wrapper exercised through fast_plan_decode
|
||||
kv_indices_buf = kv_indices.clone()
|
||||
workspace_cg = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
cg_wrapper = _make_cg_decode_wrapper(num_seqs, kv_indices_buf, workspace_cg)
|
||||
|
||||
plan_kwargs: dict = dict(
|
||||
indptr_cpu=kv_indptr,
|
||||
indices=kv_indices_buf,
|
||||
last_page_len_cpu=kv_last_page_lens,
|
||||
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
|
||||
num_qo_heads=num_query_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_size,
|
||||
page_size=block_size,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
)
|
||||
|
||||
# First call – warmup path (routes through self.plan)
|
||||
fast_plan_decode(cg_wrapper, **plan_kwargs)
|
||||
warmup_output = cg_wrapper.run(query, key_value_cache)
|
||||
torch.testing.assert_close(warmup_output, ref_output, atol=1e-2, rtol=1e-2)
|
||||
|
||||
# Second call – fast path (routes through fast_decode_plan from FlashInfer)
|
||||
fast_plan_decode(cg_wrapper, **plan_kwargs)
|
||||
fast_output = cg_wrapper.run(query, key_value_cache)
|
||||
torch.testing.assert_close(fast_output, ref_output, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
|
||||
@torch.inference_mode
|
||||
def test_flashinfer_decode_with_paged_kv(
|
||||
kv_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
sliding_window: int | None,
|
||||
) -> None:
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(kv_lens)
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
|
||||
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
|
||||
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
|
||||
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(num_seqs):
|
||||
seq_len = kv_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
|
||||
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
|
||||
workspace_buffer, "NHD", use_tensor_cores=True
|
||||
)
|
||||
wrapper.plan(
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
"NONE",
|
||||
window_left=sliding_window - 1 if sliding_window is not None else -1,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
logits_soft_cap=soft_cap,
|
||||
)
|
||||
|
||||
output = wrapper.run(query, key_value_cache)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
query_lens=[1] * num_seqs,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
soft_cap=soft_cap,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens", [[(1, 1328), (5, 18), (129, 463)]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
|
||||
@torch.inference_mode
|
||||
def test_flashinfer_prefill_with_paged_kv(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
sliding_window: int | None,
|
||||
) -> None:
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [x[0] for x in seq_lens]
|
||||
kv_lens = [x[1] for x in seq_lens]
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
|
||||
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
|
||||
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
|
||||
|
||||
# Normalize the scale of the key and value caches to mitigate
|
||||
# numerical instability.
|
||||
key_cache /= head_size**0.5
|
||||
value_cache /= head_size**0.5
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
qo_indptr = [0]
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(num_seqs):
|
||||
seq_len = kv_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
qo_indptr.append(qo_indptr[-1] + query_lens[i])
|
||||
|
||||
qo_indptr = torch.tensor(qo_indptr, dtype=torch.int32)
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
|
||||
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(workspace_buffer, "NHD")
|
||||
wrapper.plan(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
window_left=sliding_window - 1 if sliding_window is not None else -1,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
logits_soft_cap=soft_cap,
|
||||
)
|
||||
|
||||
output = wrapper.run(
|
||||
query,
|
||||
key_value_cache,
|
||||
)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
query_lens=query_lens,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
soft_cap=soft_cap,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens", [[(1, 132), (5, 18)]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
|
||||
def test_flashinfer_prefill_with_paged_fp8_kv(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
) -> None:
|
||||
pytest.skip("TODO: fix the accuracy issue")
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [x[0] for x in seq_lens]
|
||||
kv_lens = [x[1] for x in seq_lens]
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
|
||||
kv_cache_dtype = torch.float8_e4m3fn
|
||||
|
||||
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
|
||||
NUM_BLOCKS_FP8 = 2048
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS_FP8, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
key_cache, value_cache = torch.chunk(key_value_cache, 2, dim=1)
|
||||
key_cache /= head_size**0.5
|
||||
value_cache /= head_size**0.5
|
||||
|
||||
k_scale = key_cache.amax().item() / 448.0
|
||||
v_scale = value_cache.amax().item() / 448.0
|
||||
|
||||
kv_cache_fp8 = torch.cat([key_cache / k_scale, value_cache / v_scale], dim=1).to(
|
||||
kv_cache_dtype
|
||||
)
|
||||
|
||||
assert kv_cache_fp8.shape == key_value_cache.shape
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS_FP8, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
qo_indptr = [0]
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(num_seqs):
|
||||
seq_len = kv_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
qo_indptr.append(qo_indptr[-1] + query_lens[i])
|
||||
|
||||
qo_indptr = torch.tensor(qo_indptr, dtype=torch.int32)
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
|
||||
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(workspace_buffer, "NHD")
|
||||
wrapper.plan(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=kv_cache_dtype,
|
||||
logits_soft_cap=soft_cap,
|
||||
)
|
||||
|
||||
output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache.squeeze(1),
|
||||
value_cache=value_cache.squeeze(1),
|
||||
query_lens=query_lens,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
soft_cap=soft_cap,
|
||||
)
|
||||
del query
|
||||
del block_tables
|
||||
# verify prefill fp8
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
|
||||
@pytest.mark.skip(reason="TODO: fix the accuracy issue")
|
||||
@torch.inference_mode
|
||||
def test_flashinfer_decode_with_paged_fp8_kv(
|
||||
kv_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
) -> None:
|
||||
# test doesn't work for num_heads = (16,16)
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(kv_lens)
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
use_tensor_cores = True
|
||||
kv_cache_dtype = torch.float8_e4m3fn
|
||||
|
||||
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
|
||||
NUM_BLOCKS_FP8 = 2048
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS_FP8, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
key_cache, value_cache = torch.chunk(key_value_cache, 2, dim=1)
|
||||
key_cache /= head_size**0.5
|
||||
value_cache /= head_size**0.5
|
||||
|
||||
k_scale = key_cache.amax().item() / 448.0
|
||||
v_scale = value_cache.amax().item() / 448.0
|
||||
|
||||
key_cache_fp8 = (key_cache / k_scale).to(kv_cache_dtype)
|
||||
value_cache_fp8 = (value_cache / v_scale).to(kv_cache_dtype)
|
||||
assert key_cache_fp8.shape[1] == 1 and value_cache_fp8.shape[1] == 1
|
||||
kv_cache_fp8 = torch.cat([key_cache_fp8, value_cache_fp8], dim=1)
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS_FP8, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(num_seqs):
|
||||
seq_len = kv_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
|
||||
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
|
||||
workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores
|
||||
)
|
||||
wrapper.plan(
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
"NONE",
|
||||
q_data_type=dtype,
|
||||
kv_data_type=kv_cache_dtype,
|
||||
logits_soft_cap=soft_cap,
|
||||
)
|
||||
output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
|
||||
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
|
||||
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
query_lens=[1] * num_seqs,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
soft_cap=soft_cap,
|
||||
)
|
||||
# Temporary fix: Increasing the tolerance. Seems like a flashinfer issue
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not any(current_platform.is_device_capability_family(f) for f in (80, 90, 120)),
|
||||
reason="NVFP4 fa2 path",
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
|
||||
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
order = KVCacheLayout[layout].stride_order
|
||||
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
|
||||
shape = (2, 6, 4, 16, 64 + 8)
|
||||
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
|
||||
layers = layers.permute(*(order.index(i) for i in range(5)))
|
||||
one = torch.ones(())
|
||||
layer = torch.nn.Module()
|
||||
layer._k_scale = layer._v_scale = one
|
||||
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
|
||||
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
|
||||
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
|
||||
|
||||
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
|
||||
pages = torch.arange(6, dtype=torch.int32)[None]
|
||||
seq_lens = torch.tensor([96], dtype=torch.int32)
|
||||
deq = torch.empty_like(kv)[:, None]
|
||||
flashinfer.nvfp4_kv_dequantize_paged(
|
||||
data, scales, pages, seq_lens, one, one, *deq, kv_layout
|
||||
)
|
||||
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
|
||||
assert not layers[0].any()
|
||||
@@ -1,389 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
|
||||
|
||||
if not current_platform.is_cuda() or not current_platform.has_device_capability(90):
|
||||
pytest.skip(
|
||||
reason="FlashInfer MLA requires CUDA compute capability 9.0 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
else:
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
from flashinfer.mla import BatchMLAPagedAttentionWrapper
|
||||
|
||||
requires_sm90 = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(90),
|
||||
reason="This test requires an SM90 GPU.",
|
||||
)
|
||||
requires_sm10x = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="This test requires an SM10x GPU.",
|
||||
)
|
||||
|
||||
# Deepseek R1 MLA config.
|
||||
NUM_HEADS = 128
|
||||
KV_LORA_RANK = 512
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
|
||||
|
||||
|
||||
def _make_decode_inputs(bs: int, block_size: int, dtype: torch.dtype):
|
||||
"""Build valid trtllm MLA decode inputs on the current CUDA device."""
|
||||
max_seq_len_cap = 1024
|
||||
seq_lens = [torch.randint(2, max_seq_len_cap, (1,)).item() for _ in range(bs)]
|
||||
seq_lens[-1] = max_seq_len_cap
|
||||
max_seq_len = max(seq_lens)
|
||||
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
|
||||
|
||||
# Generate block tables with random but unique block IDs
|
||||
# From https://github.com/flashinfer-ai/flashinfer/pull/1222
|
||||
blocks_per_seq = (seq_lens_tensor + block_size - 1) // block_size
|
||||
max_num_blocks_per_seq = max(blocks_per_seq.max().item(), 4)
|
||||
total_blocks_needed = int(sum(blocks_per_seq))
|
||||
all_block_ids = torch.randperm(total_blocks_needed)
|
||||
|
||||
block_tables = torch.zeros((bs, max_num_blocks_per_seq), dtype=torch.int32)
|
||||
block_id = 0
|
||||
for i in range(bs):
|
||||
num_blocks_needed = blocks_per_seq[i]
|
||||
block_tables[i, :num_blocks_needed] = all_block_ids[
|
||||
block_id : block_id + num_blocks_needed
|
||||
]
|
||||
block_id += num_blocks_needed
|
||||
|
||||
kv_cache = torch.randn(block_tables.numel(), block_size, QK_HEAD_DIM).to(dtype)
|
||||
q = torch.randn(bs, NUM_HEADS, QK_HEAD_DIM).to(dtype)
|
||||
return q, kv_cache, block_tables, seq_lens_tensor, max_seq_len
|
||||
|
||||
|
||||
def ref_mla(
|
||||
out: Tensor, # (bs, num_heads, v_head_dim)
|
||||
query: Tensor, # (bs, num_heads, head_dim)
|
||||
kv_cache: Tensor, # (num_blocks, block_size, head_dim)
|
||||
scale: float,
|
||||
block_tables: Tensor, # (bs, max_num_blocks)
|
||||
seq_lens: Tensor, # (bs,)
|
||||
):
|
||||
bs, num_heads, v_head_dim = out.shape
|
||||
head_dim = query.shape[2]
|
||||
|
||||
for i in range(bs):
|
||||
# gather and flatten KV-cache
|
||||
kv = kv_cache[block_tables[i]] # (max_num_blocks, block_size, head_dim)
|
||||
kv = kv.view(1, -1, head_dim)[:, : seq_lens[i]] # (1, seq_len, head_dim)
|
||||
v = kv[:, :, :v_head_dim]
|
||||
|
||||
q = query[i].view(num_heads, 1, head_dim)
|
||||
o = F.scaled_dot_product_attention(q, kv, v, scale=scale, enable_gqa=True)
|
||||
out[i] = o.view(num_heads, v_head_dim)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("bs", [1, 2, 4, 16])
|
||||
@pytest.mark.parametrize("block_size", [32, 64])
|
||||
@requires_sm10x
|
||||
def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(42)
|
||||
|
||||
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
|
||||
bs, block_size, dtype
|
||||
)
|
||||
|
||||
out_ref = q.new_zeros(bs, NUM_HEADS, KV_LORA_RANK)
|
||||
ref_mla(out_ref, q, kv_cache, SCALE, block_tables, seq_lens_tensor)
|
||||
|
||||
workspace_buffer = torch.zeros(
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
device=q.device,
|
||||
)
|
||||
# Flashinfer MLA expects the query to be of shape
|
||||
# (bs, q_len_per_request, num_heads, qk_head_dim),
|
||||
# where q_len_per_request is the MTP query length (=1 without MTP)
|
||||
q = q.unsqueeze(1)
|
||||
|
||||
out_ans = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q,
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace_buffer,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_tensor,
|
||||
max_seq_len=max_seq_len,
|
||||
bmm1_scale=SCALE,
|
||||
)
|
||||
out_ans = out_ans.squeeze(1)
|
||||
torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("mode", ["tp", "dcp", "hisparse"])
|
||||
@requires_sm10x
|
||||
def test_flashinfer_trtllm_sparse_mla_decode_without_rope(dtype, mode, monkeypatch):
|
||||
"""TP skips preparation; empty DCP contributions and HiSparse outputs vanish."""
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as backend
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(42)
|
||||
batch_size, num_heads, block_size, num_blocks, sparse_topk = 3, 8, 64, 4, 2176
|
||||
valid_lens = [1 if mode == "tp" else 0, 17, 73]
|
||||
counts = torch.tensor(valid_lens, dtype=torch.int32)
|
||||
query = torch.randn(batch_size, 1, num_heads, KV_LORA_RANK).to(dtype)
|
||||
kv_cache = torch.randn(num_blocks, block_size, KV_LORA_RANK).to(dtype)
|
||||
num_slots = num_blocks * block_size
|
||||
indices = torch.full((batch_size, sparse_topk), -1, dtype=torch.int32)
|
||||
for token, length in enumerate(valid_lens):
|
||||
indices[token, :length] = torch.randperm(num_slots)[:length].int()
|
||||
empty = counts == 0
|
||||
reference_indices = indices.clone()
|
||||
reference_indices[:, 0].masked_fill_(empty, 0)
|
||||
workspace = torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE, dtype=torch.int8)
|
||||
|
||||
impl = object.__new__(backend.FlashInferMLASparseImpl)
|
||||
impl.qk_nope_head_dim = QK_NOPE_HEAD_DIM
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = 0
|
||||
impl.index_group = (
|
||||
object.__new__(HiSparseMLAIndexGroup) if mode == "hisparse" else None
|
||||
)
|
||||
impl._workspace_buffer = workspace
|
||||
impl.bmm1_scale = QK_NOPE_HEAD_DIM**-0.5
|
||||
impl.bmm2_scale = 1.0
|
||||
impl.is_nope_mla = True
|
||||
impl.need_to_return_lse_for_decode = mode == "dcp"
|
||||
if mode == "tp":
|
||||
|
||||
def unexpected_helper(*args, **kwargs):
|
||||
pytest.fail("TP must not launch empty-query preparation")
|
||||
|
||||
monkeypatch.setattr(
|
||||
backend, "prepare_sparse_mla_safe_lengths", unexpected_helper
|
||||
)
|
||||
|
||||
out, lse = impl._forward_mqa_kernel(
|
||||
query.squeeze(1),
|
||||
kv_cache,
|
||||
indices,
|
||||
counts,
|
||||
layer=None,
|
||||
block_size=block_size,
|
||||
is_decode=True,
|
||||
)
|
||||
reference = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query,
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=0,
|
||||
block_tables=reference_indices.unsqueeze(1),
|
||||
seq_lens=counts,
|
||||
max_seq_len=sparse_topk,
|
||||
sparse_mla_top_k=sparse_topk,
|
||||
sparse_mla_top_k_lens=counts.clamp(min=1),
|
||||
bmm1_scale=QK_NOPE_HEAD_DIM**-0.5,
|
||||
bmm2_scale=1.0,
|
||||
return_lse=mode == "dcp",
|
||||
)
|
||||
if mode == "dcp":
|
||||
reference, reference_lse = reference
|
||||
reference_lse = reference_lse.reshape(batch_size, num_heads)
|
||||
reference_lse.masked_fill_(empty[:, None], float("-inf"))
|
||||
torch.testing.assert_close(lse, reference_lse, atol=0, rtol=0)
|
||||
|
||||
from vllm.v1.attention.ops.dcp import CPTritonContext, correct_attn_out
|
||||
|
||||
# The peer owns only our empty selections. Check our contribution after
|
||||
# the production merge, without zeroing the local attention output.
|
||||
peer_lse = torch.full_like(lse, float("-inf")).masked_fill(empty[:, None], 0)
|
||||
out, _ = correct_attn_out(
|
||||
out, torch.stack((lse, peer_lse)), cp_rank=0, ctx=CPTritonContext()
|
||||
)
|
||||
else:
|
||||
assert lse is None
|
||||
reference = reference.squeeze(1).masked_fill(empty[:, None, None], 0)
|
||||
torch.testing.assert_close(out, reference, atol=0, rtol=0)
|
||||
|
||||
if dtype == torch.bfloat16:
|
||||
flat_cache = kv_cache.view(num_slots, KV_LORA_RANK).float()
|
||||
refs = []
|
||||
for token, length in enumerate(valid_lens):
|
||||
selected_kv = flat_cache[reference_indices[token, :length].long()]
|
||||
scores = torch.einsum("hd,kd->hk", query[token, 0].float(), selected_kv)
|
||||
probs = torch.softmax(scores * QK_NOPE_HEAD_DIM**-0.5, dim=-1)
|
||||
refs.append(torch.einsum("hk,kd->hd", probs, selected_kv))
|
||||
ref = torch.stack(refs).to(torch.bfloat16)
|
||||
torch.testing.assert_close(out, ref, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_sm10x
|
||||
def test_sparse_mla_safe_lengths():
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
prepare_sparse_mla_safe_lengths,
|
||||
)
|
||||
|
||||
device = "cuda"
|
||||
num_tokens = 259
|
||||
# Include a partial tile and expose accidental writes into adjacent storage.
|
||||
counts_storage = torch.full((num_tokens, 2), 17, dtype=torch.int32, device=device)
|
||||
counts = counts_storage[:, 0]
|
||||
counts[::3] = 0
|
||||
original_counts = counts_storage.clone()
|
||||
empty = counts == 0
|
||||
indices_storage = torch.full(
|
||||
(num_tokens, 256), 42, dtype=torch.int32, device=device
|
||||
)
|
||||
indices = indices_storage[:, ::2]
|
||||
expected_indices = indices_storage.clone()
|
||||
expected_indices[:, 0].masked_fill_(empty, 0)
|
||||
safe = prepare_sparse_mla_safe_lengths(indices, counts)
|
||||
torch.testing.assert_close(safe, counts.clamp(min=1), rtol=0, atol=0)
|
||||
torch.testing.assert_close(indices_storage, expected_indices, rtol=0, atol=0)
|
||||
torch.testing.assert_close(counts_storage, original_counts, rtol=0, atol=0)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
def test_flashinfer_sm90_fp8_mla_decode_without_rope():
|
||||
"""Hopper FA3 supports BF16 queries over an FP8 cache without KPE."""
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
batch_size = 2
|
||||
num_heads = 16
|
||||
page_size = 16
|
||||
num_pages = 6
|
||||
|
||||
q_nope = torch.randn(
|
||||
batch_size,
|
||||
num_heads,
|
||||
KV_LORA_RANK,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
q_pe = torch.empty(
|
||||
batch_size,
|
||||
num_heads,
|
||||
0,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
ckv = torch.randn(
|
||||
num_pages,
|
||||
page_size,
|
||||
KV_LORA_RANK,
|
||||
device=device,
|
||||
)
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
ckv_scale = ckv.abs().max().item() / fp8_max
|
||||
ckv_fp8 = (ckv / ckv_scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
|
||||
scale_bf16 = torch.tensor(ckv_scale, dtype=torch.bfloat16, device=device)
|
||||
ckv_ref = ckv_fp8.to(torch.bfloat16) * scale_bf16
|
||||
kpe_fp8 = torch.empty(
|
||||
num_pages,
|
||||
page_size,
|
||||
0,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=device,
|
||||
)
|
||||
kpe_ref = torch.empty(
|
||||
num_pages,
|
||||
page_size,
|
||||
0,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
qo_indptr = torch.tensor([0, 1, 2], dtype=torch.int32, device=device)
|
||||
kv_indptr = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
|
||||
kv_indices = torch.tensor([4, 1, 3, 0, 5], dtype=torch.int32, device=device)
|
||||
kv_lens = torch.tensor([45, 29], dtype=torch.int32, device=device)
|
||||
sm_scale = QK_NOPE_HEAD_DIM**-0.5
|
||||
|
||||
def run(
|
||||
ckv_cache: torch.Tensor,
|
||||
kpe_cache: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
workspace = torch.empty(
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
wrapper = BatchMLAPagedAttentionWrapper(workspace, backend="fa3")
|
||||
wrapper.plan(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_lens,
|
||||
num_heads,
|
||||
KV_LORA_RANK,
|
||||
0,
|
||||
page_size,
|
||||
False,
|
||||
sm_scale,
|
||||
q_data_type=torch.bfloat16,
|
||||
kv_data_type=ckv_cache.dtype,
|
||||
)
|
||||
return wrapper.run(q_nope, q_pe, ckv_cache, kpe_cache, **kwargs)
|
||||
|
||||
out_ref = run(ckv_ref, kpe_ref)
|
||||
out = run(ckv_fp8, kpe_fp8, ckv_scale=ckv_scale, kpe_scale=1.0)
|
||||
torch.testing.assert_close(out, out_ref, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_sm10x
|
||||
def test_flashinfer_mla_decode_workspace_supports_autotune():
|
||||
"""VLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
|
||||
|
||||
Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes
|
||||
the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts
|
||||
``workspace_buffer.dtype == torch.int8``; the trtllm-gen path (used for
|
||||
normal, non-autotuned inference) instead views the buffer as uint8, so a
|
||||
uint8 workspace only fails once the autotuner tries CuteDSL. That regressed
|
||||
every DeepSeek MLA test on Blackwell under V2 with
|
||||
``workspace_buffer must be torch.int8`` (vllm-project/vllm#46646).
|
||||
"""
|
||||
from flashinfer.autotuner import autotune
|
||||
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla import _get_workspace_buffer
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
workspace_buffer = _get_workspace_buffer(return_lse=False)
|
||||
assert workspace_buffer.dtype == torch.int8
|
||||
|
||||
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
|
||||
bs=1, block_size=64, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Under the autotuner the CuteDSL tactic is instantiated with our workspace;
|
||||
# a uint8 buffer raises AssertionError here, an int8 buffer succeeds.
|
||||
with torch.inference_mode(), autotune(True):
|
||||
trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q.unsqueeze(1),
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace_buffer,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_tensor,
|
||||
max_seq_len=max_seq_len,
|
||||
bmm1_scale=SCALE,
|
||||
)
|
||||
@@ -1,842 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sm120", [False, True])
|
||||
def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sm120: bool,
|
||||
):
|
||||
from vllm.models.deepseek_v4 import sparse_mla
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mla.current_platform,
|
||||
"get_device_capability",
|
||||
lambda: DeviceCapability(12, 0) if sm120 else DeviceCapability(10, 0),
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
capacity_width = 512
|
||||
global_decode_buffer = torch.empty(
|
||||
(2, capacity_width), dtype=torch.int32, device=device
|
||||
)
|
||||
prefill_buffer = torch.empty_like(global_decode_buffer)
|
||||
kwargs = dict(
|
||||
positions=torch.tensor([255, 511, 383, 639], device=device),
|
||||
compress_ratio=128,
|
||||
num_decode_tokens=2,
|
||||
token_to_req_indices=torch.tensor(
|
||||
[0, 1, 0, 1], dtype=torch.int32, device=device
|
||||
),
|
||||
block_table=torch.tensor([[3], [5]], dtype=torch.int32, device=device),
|
||||
block_size=capacity_width,
|
||||
slot_mapping=torch.arange(4, dtype=torch.int64, device=device),
|
||||
global_decode_buffer=global_decode_buffer,
|
||||
decode_lens_buffer=torch.empty(2, dtype=torch.int32, device=device),
|
||||
prefill_buffer=prefill_buffer,
|
||||
)
|
||||
captured_decode, _, captured_prefill = sparse_mla.build_c128a_topk_metadata(
|
||||
max_compressed_tokens=256,
|
||||
**kwargs,
|
||||
)
|
||||
# SM120 keeps the decode view contiguous across the full buffer width;
|
||||
# other backends get the active-width slice. The prefill view is always
|
||||
# narrowed.
|
||||
expected_decode_width = capacity_width if sm120 else 256
|
||||
assert captured_decode.shape == (2, expected_decode_width)
|
||||
assert captured_prefill.shape == (2, 256)
|
||||
assert captured_decode.stride(0) == captured_prefill.stride(0) == capacity_width
|
||||
assert captured_decode.is_contiguous() == sm120
|
||||
|
||||
captured_rows = torch.empty((4, 4), dtype=torch.int32, device=device)
|
||||
captured_rows[:2].copy_(captured_decode[:, :4])
|
||||
captured_rows[2:].copy_(captured_prefill[:, :4])
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured_rows[:2].copy_(captured_decode[:, :4])
|
||||
captured_rows[2:].copy_(captured_prefill[:, :4])
|
||||
|
||||
global_decode_buffer.fill_(-99)
|
||||
prefill_buffer.fill_(-99)
|
||||
sparse_mla.build_c128a_topk_metadata(
|
||||
max_compressed_tokens=128,
|
||||
**kwargs,
|
||||
)
|
||||
graph.replay()
|
||||
|
||||
assert captured_rows.cpu().tolist() == [
|
||||
[1536, 1537, -1, -1],
|
||||
[2560, 2561, 2562, 2563],
|
||||
[0, 1, 2, -1],
|
||||
[0, 1, 2, 3],
|
||||
]
|
||||
assert torch.all(global_decode_buffer[:, 128:] == -99)
|
||||
assert torch.all(prefill_buffer[:, 128:] == -99)
|
||||
|
||||
|
||||
def test_sparse_flashmla_metadata_smoke():
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_sparse_supported()
|
||||
if not ok:
|
||||
pytest.skip(reason)
|
||||
|
||||
device = torch.device("cuda")
|
||||
batch_size = 1
|
||||
seqlen_q = 1
|
||||
num_heads_q = 128
|
||||
num_heads_k = 1
|
||||
q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k
|
||||
topk = 128
|
||||
|
||||
cache_seqlens = torch.zeros(batch_size, dtype=torch.int32, device=device)
|
||||
|
||||
tile_md, num_splits = fm.get_mla_metadata(
|
||||
cache_seqlens,
|
||||
q_seq_per_hk,
|
||||
num_heads_k,
|
||||
num_heads_q=num_heads_q,
|
||||
topk=topk,
|
||||
is_fp8_kvcache=True,
|
||||
)
|
||||
assert isinstance(tile_md, fm.FlashMLASchedMeta)
|
||||
assert tile_md.tile_scheduler_metadata is None
|
||||
assert tile_md.num_splits is None
|
||||
assert num_splits is None
|
||||
|
||||
|
||||
def test_sparse_flashmla_decode_matches_cache_writer_scales():
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
ok, reason = fm.is_flashmla_sparse_supported()
|
||||
if not ok:
|
||||
pytest.skip(reason)
|
||||
|
||||
device = torch.device("cuda")
|
||||
batch_size = 1
|
||||
seqlen_q = 1
|
||||
num_heads_q = 64
|
||||
head_dim_k = 576
|
||||
head_dim_v = 512
|
||||
num_heads_k = 1
|
||||
page_block_size = 64
|
||||
bytes_per_token = 656
|
||||
topk = 128
|
||||
|
||||
# Metadata
|
||||
q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k
|
||||
# q_heads_per_hk = num_heads_q // num_heads_k
|
||||
cache_seqlens = torch.ones(batch_size, dtype=torch.int32, device=device)
|
||||
tile_md, num_splits = fm.get_mla_metadata(
|
||||
cache_seqlens,
|
||||
q_seq_per_hk,
|
||||
num_heads_k,
|
||||
num_heads_q=num_heads_q,
|
||||
topk=topk,
|
||||
is_fp8_kvcache=True,
|
||||
)
|
||||
|
||||
# Inputs
|
||||
q = torch.zeros(
|
||||
(batch_size, seqlen_q, num_heads_q, head_dim_k),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
cache_rows = torch.zeros(
|
||||
(1, page_block_size, bytes_per_token), dtype=torch.uint8, device=device
|
||||
)
|
||||
# 336 / 448 = 0.75 distinguishes arbitrary FP32 from power-of-two scales.
|
||||
kv_c = torch.full((1, 512), 336.0, dtype=torch.bfloat16, device=device)
|
||||
kv_c[:, -128:] = 0
|
||||
ops.concat_and_cache_mla(
|
||||
kv_c,
|
||||
torch.zeros((1, 64), dtype=torch.bfloat16, device=device),
|
||||
cache_rows,
|
||||
torch.zeros(1, dtype=torch.int64, device=device),
|
||||
"fp8_ds_mla",
|
||||
torch.ones(1, dtype=torch.float32, device=device),
|
||||
)
|
||||
scales = cache_rows[0, 0].view(torch.float32)[128:132]
|
||||
expected_scales = torch.tensor([1.0, 1.0, 1.0, 2**-13], device=device)
|
||||
torch.testing.assert_close(scales, expected_scales, rtol=0, atol=0)
|
||||
cached_nope = cache_rows[0, 0, :512].view(torch.float8_e4m3fn).float()
|
||||
expected = (cached_nope * scales.repeat_interleave(128)).to(torch.bfloat16)
|
||||
|
||||
k_cache = cache_rows.unsqueeze(2)
|
||||
indices = torch.full(
|
||||
(batch_size, seqlen_q, topk), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
indices[..., 0] = 0
|
||||
# SM90 sparse decode only supports a fixed top-k width. Padding with -1
|
||||
# exercises the same cache data on SM90 and SM100 without topk_length.
|
||||
|
||||
block_table = torch.zeros((batch_size, 128), dtype=torch.int32, device=device)
|
||||
out, lse = fm.flash_mla_with_kvcache(
|
||||
q,
|
||||
k_cache,
|
||||
block_table,
|
||||
cache_seqlens,
|
||||
head_dim_v,
|
||||
tile_md,
|
||||
num_splits,
|
||||
indices=indices,
|
||||
is_fp8_kvcache=True,
|
||||
)
|
||||
assert out.shape[0] == batch_size
|
||||
assert out.shape[-1] == head_dim_v
|
||||
assert lse.shape[0] == batch_size
|
||||
torch.testing.assert_close(
|
||||
out, expected.view(1, 1, 1, 512).expand_as(out), rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("h_q", [64, 128])
|
||||
def test_sparse_flashmla_prefill_smoke(h_q: int):
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_sparse_supported()
|
||||
if not ok:
|
||||
pytest.skip(reason)
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
s_q = 1
|
||||
s_kv = 8
|
||||
h_kv = 1
|
||||
d_qk = 576
|
||||
d_v = 512
|
||||
topk = 128
|
||||
q = torch.randn((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device)
|
||||
kv = torch.randn((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device)
|
||||
indices = torch.randint(s_kv, (s_q, h_kv, topk), dtype=torch.int32, device=device)
|
||||
reference_indices = indices.clone()
|
||||
reference_indices[..., 1:] = -1
|
||||
kwargs = {"topk_length": torch.ones(1, dtype=torch.int32, device=device)}
|
||||
reference = fm.flash_mla_sparse_fwd(q, kv, reference_indices, 1.0, d_v, **kwargs)
|
||||
actual = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v, **kwargs)
|
||||
|
||||
for actual_tensor, reference_tensor in zip(actual, reference):
|
||||
torch.testing.assert_close(actual_tensor, reference_tensor, rtol=0, atol=0)
|
||||
assert actual[0].shape == (s_q, h_q, d_v)
|
||||
|
||||
|
||||
def test_deepseek_v4_prefill_chunk_planning_expands_for_short_sequences():
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.empty(0, dtype=torch.int32),
|
||||
slot_mapping=torch.empty(0, dtype=torch.int32),
|
||||
block_size=64,
|
||||
num_prefills=5,
|
||||
prefill_seq_lens_cpu=torch.tensor([80, 96, 112, 128, 144], dtype=torch.int32),
|
||||
prefill_query_lens_cpu=torch.tensor([4, 4, 4, 4, 4], dtype=torch.int32),
|
||||
prefill_window_size=64,
|
||||
prefill_max_model_len=1024,
|
||||
prefill_max_num_batched_tokens=128,
|
||||
)
|
||||
|
||||
chunk_plan = metadata.get_prefill_chunk_plan(compress_ratio=4, prefill_chunk_size=4)
|
||||
|
||||
# the adaptive plan keeps all 5 in one chunk
|
||||
assert chunk_plan == [(0, 5, 36, 103)]
|
||||
|
||||
|
||||
def test_flashinfer_sparse_indices_cache(monkeypatch):
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
|
||||
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
builder_calls = 0
|
||||
|
||||
def fake_build(*args, **kwargs):
|
||||
nonlocal builder_calls
|
||||
builder_calls += 1
|
||||
return (
|
||||
torch.tensor([[builder_calls]], dtype=torch.int32),
|
||||
torch.tensor([builder_calls], dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
|
||||
)
|
||||
|
||||
def make_attn(compress_ratio: int, topk_width: int):
|
||||
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = compress_ratio
|
||||
attn.window_size = 4
|
||||
attn.topk_indices_buffer = torch.tensor(
|
||||
[[0, 1], [2, 3], [4, 5]], dtype=torch.int32
|
||||
)[:, :topk_width]
|
||||
return attn
|
||||
|
||||
def make_swa_metadata():
|
||||
return DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8, 10], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 1, 3], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
|
||||
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=4,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=2,
|
||||
)
|
||||
|
||||
def make_flashmla_metadata():
|
||||
return DeepseekV4FlashMLAMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=2,
|
||||
max_seq_len=10,
|
||||
num_actual_tokens=3,
|
||||
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1, 2], dtype=torch.int64),
|
||||
block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
|
||||
req_id_per_token=torch.tensor([0, 1, 1], dtype=torch.int32),
|
||||
block_size=256,
|
||||
topk_tokens=2,
|
||||
c128a_global_decode_topk_indices=torch.tensor(
|
||||
[[[9, 10]]], dtype=torch.int32
|
||||
),
|
||||
c128a_decode_topk_lens=torch.tensor([2], dtype=torch.int32),
|
||||
c128a_prefill_topk_indices=torch.tensor(
|
||||
[[0, 1], [1, 2]], dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
swa_attn = make_attn(1, 0)
|
||||
swa_metadata = make_swa_metadata()
|
||||
_, _, sparse_indices_first, sparse_lens_first = (
|
||||
swa_attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
)
|
||||
_, _, sparse_indices_second, sparse_lens_second = (
|
||||
swa_attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
)
|
||||
assert builder_calls == 1
|
||||
assert sparse_indices_first is sparse_indices_second
|
||||
assert sparse_lens_first is sparse_lens_second
|
||||
|
||||
c128a_attn = make_attn(128, 2)
|
||||
c128a_metadata = make_swa_metadata()
|
||||
c128a_flashmla_md = make_flashmla_metadata()
|
||||
_, _, sparse_indices_first, sparse_lens_first = (
|
||||
c128a_attn._build_sparse_index_metadata(
|
||||
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=c128a_metadata,
|
||||
attn_metadata=c128a_flashmla_md,
|
||||
swa_only=False,
|
||||
)
|
||||
)
|
||||
_, _, sparse_indices_second, sparse_lens_second = (
|
||||
c128a_attn._build_sparse_index_metadata(
|
||||
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=c128a_metadata,
|
||||
attn_metadata=c128a_flashmla_md,
|
||||
swa_only=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert builder_calls == 2
|
||||
assert sparse_indices_first is sparse_indices_second
|
||||
assert sparse_lens_first is sparse_lens_second
|
||||
|
||||
c4a_attn = make_attn(4, 2)
|
||||
c4a_metadata = make_swa_metadata()
|
||||
c4a_flashmla_md = make_flashmla_metadata()
|
||||
c4a_flashmla_md.c128a_global_decode_topk_indices = None
|
||||
c4a_flashmla_md.c128a_decode_topk_lens = None
|
||||
c4a_flashmla_md.c128a_prefill_topk_indices = None
|
||||
_, _, sparse_indices_third, sparse_lens_third = (
|
||||
c4a_attn._build_sparse_index_metadata(
|
||||
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=c4a_metadata,
|
||||
attn_metadata=c4a_flashmla_md,
|
||||
swa_only=False,
|
||||
)
|
||||
)
|
||||
_, _, sparse_indices_fourth, sparse_lens_fourth = (
|
||||
c4a_attn._build_sparse_index_metadata(
|
||||
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=c4a_metadata,
|
||||
attn_metadata=c4a_flashmla_md,
|
||||
swa_only=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert builder_calls == 4
|
||||
assert sparse_indices_third is not sparse_indices_fourth
|
||||
assert sparse_lens_third is not sparse_lens_fourth
|
||||
|
||||
|
||||
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
captured_shapes_and_windows: list[tuple[int, int]] = []
|
||||
|
||||
def fake_build(*args, **kwargs):
|
||||
# window_size is the 12th positional arg of
|
||||
# build_flashinfer_mixed_sparse_indices.
|
||||
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
|
||||
num_tokens = args[0].shape[0] + args[3].shape[0]
|
||||
return (
|
||||
torch.zeros((num_tokens, 1), dtype=torch.int32),
|
||||
torch.zeros((num_tokens,), dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
|
||||
)
|
||||
|
||||
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 4
|
||||
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
|
||||
|
||||
wide_width = 8
|
||||
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
|
||||
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
|
||||
wide_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
|
||||
decode_swa_indices=wide_indices,
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=wide_width,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=0,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=wide_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
|
||||
|
||||
empty_width = 8
|
||||
empty_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
|
||||
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
|
||||
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
|
||||
decode_swa_width=empty_width,
|
||||
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
|
||||
num_decodes=0,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=0,
|
||||
num_prefill_tokens=2,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=empty_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [
|
||||
(wide_width, attn.window_size),
|
||||
(empty_width, attn.window_size),
|
||||
]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
padded_width = 8
|
||||
logical_window = 4
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=torch.empty(
|
||||
(0, padded_width), dtype=torch.int32, device=device
|
||||
),
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=64,
|
||||
window_size=logical_window,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_indices.shape == (1, padded_width)
|
||||
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
|
||||
assert sparse_lens.cpu().tolist() == [padded_width]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_noncausal_rows_have_no_active_gaps():
|
||||
"""A DSpark non-causal row keeps -1 out of the kernel's active ranges.
|
||||
|
||||
Each draft token's causal window holds its first visible entries, the rest
|
||||
of the block starts at column `window`, and the compressed entries follow
|
||||
directly; `sparse_topk_lens` ends right after them.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
window, width = 4, 8
|
||||
# One request: 1 context token, a 2-token draft block, slots 0..2 visible.
|
||||
decode_swa = torch.full((2, width), -1, dtype=torch.int32, device=device)
|
||||
decode_swa[:, :3] = torch.arange(3, dtype=torch.int32, device=device)
|
||||
compressed = torch.tensor(
|
||||
[[100, 101, -1, -1]] * 2, dtype=torch.int32, device=device
|
||||
)
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=decode_swa,
|
||||
decode_compressed_indices=compressed,
|
||||
decode_compressed_topk_lens=torch.tensor(
|
||||
[2, 2], dtype=torch.int32, device=device
|
||||
),
|
||||
prefill_topk_indices=torch.empty((0, 4), dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 2], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([3], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
compressed_block_size=64,
|
||||
window_size=window,
|
||||
compress_ratio=128,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_indices.cpu().tolist() == [
|
||||
# pos 1: window [0, 1]; slot 2 spills to column 4; compressed after it.
|
||||
[0, 1, -1, -1, 2, 100, 101, -1, -1, -1, -1, -1],
|
||||
# pos 2: window [0, 1, 2]; nothing spills; compressed at column 4.
|
||||
[0, 1, 2, -1, 100, 101, -1, -1, -1, -1, -1, -1],
|
||||
]
|
||||
assert sparse_lens.cpu().tolist() == [7, 6]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_noncausal_pad_rows_keep_min_length():
|
||||
"""A CUDA-graph pad token of the DSpark graph (a zero-length request) gets
|
||||
a causal pad row's shape instead of negative spill arithmetic."""
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
window, width = 128, 192
|
||||
decode_swa = torch.full((3, width), -1, dtype=torch.int32, device=device)
|
||||
decode_swa[:2, :3] = torch.arange(3, dtype=torch.int32, device=device)
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=decode_swa,
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((0, 0), dtype=torch.int32, device=device),
|
||||
# Request 1 is the padding: no query tokens, seq_len 0.
|
||||
query_start_loc=torch.tensor([0, 2, 2], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([3, 0], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0, 0, 1], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.zeros((2, 1), dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=64,
|
||||
window_size=window,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_lens.cpu().tolist() == [129, 128, 128]
|
||||
assert (sparse_indices[2] == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"])
|
||||
@pytest.mark.parametrize("page_padding", [0, 64])
|
||||
def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
|
||||
"""Decode and prefill must read storage rows after vLLM remaps packed pages."""
|
||||
import importlib
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
|
||||
mod = importlib.import_module(f"vllm.models.{model}.nvidia.flashinfer_sparse")
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
storage = torch.full(
|
||||
(3, block_size + page_padding, 512), 99, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
cache = storage[:, :block_size]
|
||||
cache[1].fill_(2)
|
||||
cache[2].fill_(3)
|
||||
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 128
|
||||
attn.scale = 512**-0.5
|
||||
attn.kv_cache_torch_dtype = torch.bfloat16
|
||||
attn.attn_sink = torch.full((64,), -float("inf"), device=device)
|
||||
attn.topk_indices_buffer = torch.empty((3, 0), dtype=torch.int32, device=device)
|
||||
decode_indices = torch.full((1, 128), -1, dtype=torch.int32, device=device)
|
||||
decode_indices[0, :4] = torch.arange(64, 68, dtype=torch.int32, device=device)
|
||||
query_start = torch.tensor([0, 1, 3], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[1], [2]], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.tensor([67, 130, 131], device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=torch.tensor([4, 4], dtype=torch.int32, device=device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32, device=device),
|
||||
decode_swa_indices=decode_indices,
|
||||
decode_swa_lens=torch.tensor([4], dtype=torch.int32, device=device),
|
||||
decode_swa_width=128,
|
||||
is_valid_token=torch.ones(3, dtype=torch.bool, device=device),
|
||||
replay_start=torch.zeros(2, dtype=torch.int32, device=device),
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=2,
|
||||
max_decode_query_len=1,
|
||||
)
|
||||
query = torch.zeros((3, 64, 512), dtype=torch.bfloat16, device=device)
|
||||
output = torch.empty_like(query)
|
||||
attn._forward(query, None, cache, metadata, None, True, output)
|
||||
expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device)
|
||||
torch.testing.assert_close(output, expected[:, None, None].expand_as(output))
|
||||
|
||||
|
||||
def _make_rope_quant_attn(mod, num_rows: int, device: str):
|
||||
"""A bare DSv4 FlashInfer layer set up for RopeQuant: 128 heads, fp8 KV."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
heads, groups, rank = 128, 16, 256
|
||||
angles = torch.arange(2048, device=device, dtype=torch.float32)[:, None] * 0.05
|
||||
angles = angles + torch.arange(32, device=device)[None] * 0.1
|
||||
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 128
|
||||
attn.scale = 512**-0.5
|
||||
attn.kv_cache_torch_dtype = torch.float8_e4m3fn
|
||||
attn._flashinfer_fp8_bmm1_scale = attn.scale
|
||||
attn._flashinfer_fp8_bmm2_scale = 1.0
|
||||
attn.attn_sink = torch.linspace(-1, 1, heads, device=device)
|
||||
attn.padded_heads = heads
|
||||
attn.topk_indices_buffer = torch.empty(
|
||||
(num_rows, 0), dtype=torch.int32, device=device
|
||||
)
|
||||
attn.rotary_emb = SimpleNamespace(
|
||||
cos_sin_cache=torch.cat((angles.cos(), angles.sin()), -1).contiguous()
|
||||
)
|
||||
attn.n_local_heads, attn.n_local_groups, attn.o_lora_rank = heads, groups, rank
|
||||
attn.nope_head_dim, attn.rope_head_dim = 448, 64
|
||||
attn._einsum_recipe, attn._tma_aligned_scales = (1, 1, 128), True
|
||||
attn.wo_a = SimpleNamespace(
|
||||
weight=(torch.randn((groups, rank, 4096), device=device) * 0.05).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
weight_scale=torch.ones((groups, rank, 32), device=device),
|
||||
)
|
||||
attn.wo_b = lambda z: z
|
||||
return attn
|
||||
|
||||
|
||||
def _rope_quant_vs_unfused(attn, q, cache, metadata, positions, num_tokens):
|
||||
"""Run `_forward` unfused and fused; return (bf16 attention, z_unfused, z_fused)."""
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import rope_quant_attn_out
|
||||
|
||||
heads, groups = attn.n_local_heads, attn.n_local_groups
|
||||
unfused = torch.empty(q.shape, dtype=torch.bfloat16, device=q.device)
|
||||
attn._forward(q, None, cache, metadata, None, True, unfused)
|
||||
fused = rope_quant_attn_out(q.shape[0], q.device)
|
||||
metadata.flashinfer_sparse_index_cache.clear()
|
||||
attn._forward(q, None, cache, metadata, None, True, fused)
|
||||
z_unfused = attn._o_proj(unfused[:num_tokens, :heads], positions)
|
||||
z_fused = attn._o_proj(fused, positions)[:num_tokens]
|
||||
return unfused[:num_tokens], z_unfused, z_fused.view(num_tokens, groups, -1)
|
||||
|
||||
|
||||
def _assert_projects_alike(z_unfused, z_fused):
|
||||
z_unfused = z_unfused.float().view_as(z_fused)
|
||||
rel = (z_fused.float() - z_unfused).norm() / z_unfused.norm()
|
||||
assert rel < 2e-2, rel
|
||||
|
||||
|
||||
def _skip_unless_rope_quant():
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
|
||||
|
||||
|
||||
def test_flashinfer_rope_quant_matches_unfused_o_proj():
|
||||
"""RopeQuant's fused output must project like the bf16 + inv-RoPE/quant path.
|
||||
|
||||
One mixed step (two decodes, a three-token prefill) plus CUDA-graph pad rows,
|
||||
so both calls run over the full buffer with un-rebased cum_seq_lens_q.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_skip_unless_rope_quant()
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
q_lens, seq_lens = [1, 1, 3], [5, 9, 70]
|
||||
num_tokens, num_rows = sum(q_lens), sum(q_lens) + 3
|
||||
cache = torch.randn((3, block_size, 512), device=device).to(torch.float8_e4m3fn)
|
||||
attn = _make_rope_quant_attn(mod, num_rows, device)
|
||||
|
||||
decode_indices = torch.full((2, 128), -1, dtype=torch.int32, device=device)
|
||||
for i, seq_len in enumerate(seq_lens[:2]):
|
||||
decode_indices[i, :seq_len] = torch.arange(
|
||||
i * block_size, i * block_size + seq_len, device=device
|
||||
)
|
||||
query_start = torch.tensor([0, 1, 2, 5], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0], [1], [2]], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.tensor(
|
||||
[0, 1, 2, 2, 2], dtype=torch.int32, device=device
|
||||
),
|
||||
decode_swa_indices=decode_indices,
|
||||
decode_swa_lens=torch.tensor(seq_lens[:2], dtype=torch.int32, device=device),
|
||||
decode_swa_width=128,
|
||||
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
|
||||
num_decodes=2,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=2,
|
||||
num_prefill_tokens=3,
|
||||
max_decode_query_len=1,
|
||||
)
|
||||
positions = torch.tensor([4, 8, 67, 68, 69], device=device)
|
||||
q = torch.randn((num_rows, 128, 512), device=device).to(torch.float8_e4m3fn)
|
||||
_, z_unfused, z_fused = _rope_quant_vs_unfused(
|
||||
attn, q, cache, metadata, positions, num_tokens
|
||||
)
|
||||
_assert_projects_alike(z_unfused, z_fused)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("context_len", [0, 20, 127, 900])
|
||||
def test_flashinfer_dspark_noncausal_block_sees_future_tokens(context_len):
|
||||
"""Every DSpark draft token attends to the whole block, fused or not.
|
||||
|
||||
The kernel counts every entry of its active index ranges as a key, so the
|
||||
builder must place the block around each query's causal window without -1
|
||||
gaps, down to an empty context.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_skip_unless_rope_quant()
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
block_size, window, draft = 64, 128, 4
|
||||
width = get_dspark_swa_index_width(window, draft)
|
||||
contexts = [context_len, context_len + 37]
|
||||
num_reqs, num_tokens = len(contexts), len(contexts) * draft
|
||||
num_rows = num_tokens + 2
|
||||
pages = -(-(max(contexts) + draft) // block_size)
|
||||
cache = torch.randn((num_reqs * pages, block_size, 512), device=device)
|
||||
cache = cache.clamp_(-1, 1).to(torch.float8_e4m3fn)
|
||||
attn = _make_rope_quant_attn(mod, num_rows, device)
|
||||
attn.attn_sink = torch.full((128,), -float("inf"), device=device)
|
||||
|
||||
# As the DSpark SWA builder lays them out: the trailing window of context
|
||||
# and the whole block, contiguous from column 0.
|
||||
decode_indices = torch.full((num_tokens, width), -1, dtype=torch.int32)
|
||||
visible, positions = [], []
|
||||
for r, ctx in enumerate(contexts):
|
||||
slots = torch.arange(max(ctx - window, 0), ctx + draft) + r * pages * block_size
|
||||
for k in range(draft):
|
||||
decode_indices[r * draft + k, : slots.numel()] = slots.to(torch.int32)
|
||||
visible.append(slots)
|
||||
positions.append(ctx + k)
|
||||
query_start = torch.arange(0, num_tokens + 1, draft, dtype=torch.int32)
|
||||
seq_lens = torch.tensor([c + draft for c in contexts], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.arange(
|
||||
num_reqs * pages, dtype=torch.int32, device=device
|
||||
).view(num_reqs, pages),
|
||||
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=seq_lens.to(device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.arange(
|
||||
num_reqs, dtype=torch.int32, device=device
|
||||
).repeat_interleave(draft),
|
||||
decode_swa_indices=decode_indices.to(device),
|
||||
decode_swa_lens=torch.tensor(
|
||||
[v.numel() for v in visible], dtype=torch.int32, device=device
|
||||
),
|
||||
decode_swa_width=width,
|
||||
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
|
||||
num_decodes=num_reqs,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=num_tokens,
|
||||
num_prefill_tokens=0,
|
||||
max_decode_query_len=draft,
|
||||
)
|
||||
q = torch.randn((num_rows, 128, 512), device=device).clamp_(-1, 1)
|
||||
q = q.to(torch.float8_e4m3fn)
|
||||
attention, z_unfused, z_fused = _rope_quant_vs_unfused(
|
||||
attn, q, cache, metadata, torch.tensor(positions, device=device), num_tokens
|
||||
)
|
||||
|
||||
keys_all = cache.flatten(0, 1).float()
|
||||
for token, slots in enumerate(visible):
|
||||
keys = keys_all[slots.to(device)]
|
||||
weights = torch.softmax(q[token].float() @ keys.T * attn.scale, -1)
|
||||
torch.testing.assert_close(
|
||||
attention[token].float(), weights @ keys, atol=0.05, rtol=0.05
|
||||
)
|
||||
_assert_projects_alike(z_unfused, z_fused)
|
||||
@@ -1,310 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""RoPE equivalence tests for the fused Kimi-K3 MLA epilogues."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_kv_concat,
|
||||
fused_mla_kv_concat_quant_fp8,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
|
||||
)
|
||||
|
||||
_DTYPE = torch.bfloat16
|
||||
_NUM_TOKENS = 3
|
||||
_NUM_HEADS = 4
|
||||
_BLOCK_SIZE = 8
|
||||
_POSITIONS = (1, 7, 13)
|
||||
_SLOTS = (0, 3, 9)
|
||||
|
||||
|
||||
def _randn(*shape: int, dtype: torch.dtype = _DTYPE) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=dtype) * 0.2
|
||||
|
||||
|
||||
def _rope_cache(max_position: int = 32) -> torch.Tensor:
|
||||
inv_freq = 1.0 / (
|
||||
50000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device="cuda") / 64)
|
||||
)
|
||||
positions = torch.arange(max_position, dtype=torch.float32, device="cuda")
|
||||
freqs = torch.outer(positions, inv_freq)
|
||||
# The fused epilogue reads the cos/sin table in fp32 (RoPE math runs in fp32).
|
||||
return torch.cat((freqs.cos(), freqs.sin()), dim=-1)
|
||||
|
||||
|
||||
def _apply_gptj_rope(
|
||||
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
cos, sin = cos_sin_cache.index_select(0, positions).chunk(2, dim=-1)
|
||||
for _ in range(x.ndim - 2):
|
||||
cos = cos.unsqueeze(1)
|
||||
sin = sin.unsqueeze(1)
|
||||
x1 = x[..., ::2].float()
|
||||
x2 = x[..., 1::2].float()
|
||||
out1 = x1 * cos.float() - x2 * sin.float()
|
||||
out2 = x2 * cos.float() + x1 * sin.float()
|
||||
return torch.stack((out1, out2), dim=-1).flatten(-2).to(x.dtype)
|
||||
|
||||
|
||||
def _cache_rows(cache: torch.Tensor, slots: torch.Tensor) -> torch.Tensor:
|
||||
return cache.reshape(-1, cache.shape[-1]).index_select(0, slots)
|
||||
|
||||
|
||||
def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
torch.testing.assert_close(
|
||||
actual.float(),
|
||||
expected.to(torch.float8_e4m3fn).float(),
|
||||
atol=0.03125,
|
||||
rtol=0.15,
|
||||
)
|
||||
|
||||
|
||||
def _strided_context_inputs(
|
||||
num_tokens: int, num_heads: int, dtype: torch.dtype, k_pe_fp8: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
|
||||
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
|
||||
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
|
||||
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
|
||||
k_nope, v = kv_nope.split((128, 128), dim=-1)
|
||||
workspace = _randn(num_tokens, 576, dtype=dtype)
|
||||
if k_pe_fp8:
|
||||
workspace = workspace.to(torch.float8_e4m3fn)
|
||||
return k_nope, workspace[:, 512:].unsqueeze(1), v
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
|
||||
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
|
||||
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_context_kv_concat_accepts_strided_inputs(
|
||||
num_tokens: int, num_heads: int, input_dtype: torch.dtype
|
||||
) -> None:
|
||||
k_nope, k_pe, _ = _strided_context_inputs(
|
||||
num_tokens, num_heads, input_dtype, k_pe_fp8=False
|
||||
)
|
||||
|
||||
k = fused_mla_kv_concat(k_nope, k_pe)
|
||||
|
||||
assert k.is_contiguous()
|
||||
assert k.dtype == input_dtype
|
||||
torch.testing.assert_close(k[..., :128], k_nope, atol=0, rtol=0)
|
||||
torch.testing.assert_close(
|
||||
k[..., 128:], k_pe.expand(-1, num_heads, -1), atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k_pe_dtype", ["input", "fp8"])
|
||||
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
|
||||
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
|
||||
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_context_kv_pack_quantizes_strided_inputs(
|
||||
k_pe_dtype: str,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
input_dtype: torch.dtype,
|
||||
) -> None:
|
||||
k_nope, k_pe, v = _strided_context_inputs(
|
||||
num_tokens, num_heads, input_dtype, k_pe_fp8=k_pe_dtype == "fp8"
|
||||
)
|
||||
|
||||
k_actual, v_actual = fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
|
||||
|
||||
fp8 = torch.float8_e4m3fn
|
||||
k_expected = torch.empty_like(k_actual)
|
||||
k_expected[..., :128] = k_nope.to(fp8)
|
||||
k_expected[..., 128:] = k_pe.to(fp8)
|
||||
assert k_actual.is_contiguous()
|
||||
assert v_actual.is_contiguous()
|
||||
# The pack casts with the native pairwise converters, so it must be
|
||||
# bit-identical to torch's `.to(fp8)` rather than merely close.
|
||||
torch.testing.assert_close(k_actual.float(), k_expected.float(), atol=0, rtol=0)
|
||||
torch.testing.assert_close(v_actual.float(), v.to(fp8).float(), atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
torch.manual_seed(0)
|
||||
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
cos_sin_cache = _rope_cache()
|
||||
q = _randn(_NUM_TOKENS, _NUM_HEADS, 192)
|
||||
k_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
v = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
|
||||
|
||||
q_expected = q.clone()
|
||||
q_expected[..., 128:] = _apply_gptj_rope(
|
||||
q_expected[..., 128:], positions, cos_sin_cache
|
||||
)
|
||||
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
|
||||
k_expected = torch.cat(
|
||||
(k_nope, k_pe_expected[:, None, :].expand(-1, _NUM_HEADS, -1)), dim=-1
|
||||
)
|
||||
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
|
||||
|
||||
if cache_kind == "bf16":
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
q_actual = q.clone()
|
||||
k_actual = fused_mla_key_concat_kv_cache_insert(
|
||||
q_actual,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cache,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(k_actual, k_expected)
|
||||
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
|
||||
elif cache_kind == "fp8":
|
||||
cache = torch.zeros(
|
||||
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
|
||||
)
|
||||
one = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
q_actual, k_actual, v_actual = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
v,
|
||||
cache,
|
||||
slots,
|
||||
one,
|
||||
one,
|
||||
one,
|
||||
one,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
_assert_fp8_close(q_actual, q_expected)
|
||||
_assert_fp8_close(k_actual, k_expected)
|
||||
_assert_fp8_close(v_actual, v)
|
||||
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
|
||||
else:
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
|
||||
q_actual = q.clone()
|
||||
k_actual = fused_mla_key_concat_ds_mla_insert(
|
||||
q_actual,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cache,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(k_actual, k_expected)
|
||||
torch.testing.assert_close(rope_cache, k_pe_expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_decode_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
torch.manual_seed(1)
|
||||
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
cos_sin_cache = _rope_cache()
|
||||
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
|
||||
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
|
||||
q_pe_expected = _apply_gptj_rope(q_pe, positions, cos_sin_cache)
|
||||
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
|
||||
q_expected = torch.cat((ql_nope, q_pe_expected), dim=-1)
|
||||
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
|
||||
|
||||
kwargs = {"positions": positions, "cos_sin_cache": cos_sin_cache}
|
||||
if cache_kind == "bf16":
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots, **kwargs
|
||||
)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
|
||||
elif cache_kind == "fp8":
|
||||
cache = torch.zeros(
|
||||
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
|
||||
)
|
||||
one = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c,
|
||||
k_pe,
|
||||
cache,
|
||||
slots,
|
||||
q_scale_inv=one,
|
||||
cache_scale_inv=one,
|
||||
**kwargs,
|
||||
)
|
||||
_assert_fp8_close(q_actual, q_expected)
|
||||
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
|
||||
else:
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots, ds_mla=True, **kwargs
|
||||
)
|
||||
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(rope_cache, k_pe_expected)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_decode_epilogue_preserves_nope_path() -> None:
|
||||
torch.manual_seed(2)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
|
||||
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
|
||||
torch.testing.assert_close(
|
||||
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("num_tokens", [64, 65])
|
||||
def test_decode_epilogue_row_split_boundary(num_tokens: int) -> None:
|
||||
"""Both sides of the M-based warps-per-row dispatch produce the same rows."""
|
||||
torch.manual_seed(3)
|
||||
num_blocks = (num_tokens + _BLOCK_SIZE - 1) // _BLOCK_SIZE
|
||||
slots = torch.randperm(num_blocks * _BLOCK_SIZE, device="cuda")[:num_tokens].long()
|
||||
ql_nope = _randn(num_tokens, _NUM_HEADS, 512)
|
||||
q_pe = _randn(num_tokens, _NUM_HEADS, 64)
|
||||
kv_c = _randn(num_tokens, 512)
|
||||
k_pe = _randn(num_tokens, 64)
|
||||
cache = torch.zeros(num_blocks, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
|
||||
torch.testing.assert_close(
|
||||
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
|
||||
)
|
||||
@@ -1,257 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
NUM_TOKENS = 4
|
||||
NUM_HEADS = 2
|
||||
KV_LORA_RANK = 512
|
||||
NOPE_HEAD_DIM = 128
|
||||
ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
|
||||
V_HEAD_DIM = 128
|
||||
CACHE_ENTRY = KV_LORA_RANK + ROPE_HEAD_DIM
|
||||
DS_MLA_CACHE_ENTRY = 656
|
||||
BLOCK_SIZE = 8
|
||||
OWNED_TOKEN_INDICES = [1, 3]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA ops require CUDA"
|
||||
)
|
||||
|
||||
|
||||
def _inputs() -> dict[str, torch.Tensor]:
|
||||
torch.manual_seed(0)
|
||||
|
||||
def make(*shape: int) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
return {
|
||||
"q": make(NUM_TOKENS, NUM_HEADS, QK_HEAD_DIM),
|
||||
"k_nope": make(NUM_TOKENS, NUM_HEADS, NOPE_HEAD_DIM),
|
||||
"k_pe": make(NUM_TOKENS, ROPE_HEAD_DIM),
|
||||
"kv_c": make(NUM_TOKENS, KV_LORA_RANK),
|
||||
"v": make(NUM_TOKENS, NUM_HEADS, V_HEAD_DIM),
|
||||
"ql_nope": make(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK),
|
||||
"q_pe": make(NUM_TOKENS, NUM_HEADS, ROPE_HEAD_DIM),
|
||||
}
|
||||
|
||||
|
||||
def _slot_mapping() -> torch.Tensor:
|
||||
return torch.tensor([-1, 0, -1, 5], device="cuda", dtype=torch.int64)
|
||||
|
||||
|
||||
def _owned(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return tensor[OWNED_TOKEN_INDICES]
|
||||
|
||||
|
||||
def _full_key(k_nope: torch.Tensor, k_pe: torch.Tensor) -> torch.Tensor:
|
||||
shared_pe = k_pe[:, None, :].expand(-1, k_nope.shape[1], -1)
|
||||
return torch.cat((k_nope, shared_pe), dim=-1)
|
||||
|
||||
|
||||
def _latent_query(ql_nope: torch.Tensor, q_pe: torch.Tensor) -> torch.Tensor:
|
||||
return torch.cat((ql_nope, q_pe), dim=-1)
|
||||
|
||||
|
||||
def _make_cache(cache_format: str) -> torch.Tensor:
|
||||
if cache_format == "bf16":
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, CACHE_ENTRY),
|
||||
-7,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
if cache_format == "fp8":
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, CACHE_ENTRY),
|
||||
1.0,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
).to(torch.float8_e4m3fn)
|
||||
assert cache_format == "fp8_ds_mla"
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
|
||||
165,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
|
||||
|
||||
def _reference_cache(
|
||||
cache_format: str,
|
||||
inputs: dict[str, torch.Tensor],
|
||||
initial_cache: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
reference_cache = initial_cache.clone()
|
||||
valid_slots = _owned(slots)
|
||||
if cache_format == "fp8_ds_mla":
|
||||
ops.concat_and_cache_mla(
|
||||
_owned(inputs["kv_c"]),
|
||||
_owned(inputs["k_pe"]),
|
||||
reference_cache,
|
||||
valid_slots,
|
||||
cache_format,
|
||||
torch.ones(1, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
return reference_cache
|
||||
|
||||
latent = torch.cat(
|
||||
(_owned(inputs["kv_c"]), _owned(inputs["k_pe"])),
|
||||
dim=-1,
|
||||
)
|
||||
flat_reference_cache = reference_cache.view(-1, CACHE_ENTRY)
|
||||
for row, slot in zip(latent, valid_slots.tolist()):
|
||||
flat_reference_cache[slot].copy_(row.to(reference_cache.dtype))
|
||||
return reference_cache
|
||||
|
||||
|
||||
def _assert_cache_matches_reference(
|
||||
cache: torch.Tensor,
|
||||
reference_cache: torch.Tensor,
|
||||
initial_cache: torch.Tensor,
|
||||
) -> None:
|
||||
assert torch.equal(cache, reference_cache)
|
||||
for slot in (0, 5):
|
||||
assert not torch.equal(cache[0, slot], initial_cache[0, slot])
|
||||
for slot in (1, 2, 3, 4, 6, 7):
|
||||
assert torch.equal(cache[0, slot], initial_cache[0, slot])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_format", ["bf16", "fp8_ds_mla"])
|
||||
def test_prefill_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache(cache_format)
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
|
||||
op = (
|
||||
fused_mla_key_concat_kv_cache_insert
|
||||
if cache_format == "bf16"
|
||||
else fused_mla_key_concat_ds_mla_insert
|
||||
)
|
||||
output = op(
|
||||
inputs["q"],
|
||||
inputs["k_nope"],
|
||||
inputs["k_pe"],
|
||||
inputs["kv_c"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
output, _full_key(inputs["k_nope"], inputs["k_pe"]), rtol=0, atol=0
|
||||
)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
def test_fp8_prefill_outputs_ignore_negative_slots_for_cache() -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache("fp8")
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache("fp8", inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
|
||||
q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
inputs["q"],
|
||||
inputs["k_nope"],
|
||||
inputs["k_pe"],
|
||||
inputs["kv_c"],
|
||||
inputs["v"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
)
|
||||
|
||||
expected = (
|
||||
inputs["q"].to(torch.float8_e4m3fn),
|
||||
_full_key(inputs["k_nope"], inputs["k_pe"]).to(torch.float8_e4m3fn),
|
||||
inputs["v"].to(torch.float8_e4m3fn),
|
||||
)
|
||||
for actual, reference in zip((q, k, v), expected):
|
||||
torch.testing.assert_close(actual.float(), reference.float(), rtol=0, atol=0)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_format", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache(cache_format)
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
|
||||
output = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
inputs["ql_nope"],
|
||||
inputs["q_pe"],
|
||||
inputs["kv_c"],
|
||||
inputs["k_pe"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
ds_mla=cache_format == "fp8_ds_mla",
|
||||
q_scale_inv=scale_inv if cache_format == "fp8" else None,
|
||||
cache_scale_inv=scale_inv if cache_format == "fp8" else None,
|
||||
)
|
||||
|
||||
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
|
||||
if cache_format == "fp8":
|
||||
expected = expected.to(torch.float8_e4m3fn)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
else:
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode", [False, True])
|
||||
def test_ds_mla_cache_insert_bit_compatible_with_reference(decode: bool) -> None:
|
||||
"""Fused ds_mla insertion must match the reference bit-for-bit."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_blocks = 33, 16
|
||||
dt = torch.bfloat16
|
||||
kv_c = torch.randn(num_tokens, KV_LORA_RANK, device="cuda", dtype=dt)
|
||||
k_pe = torch.randn(num_tokens, 1, ROPE_HEAD_DIM, device="cuda", dtype=dt)
|
||||
k_nope = torch.randn(num_tokens, NUM_HEADS, NOPE_HEAD_DIM, device="cuda", dtype=dt)
|
||||
q = torch.randn(num_tokens, NUM_HEADS, QK_HEAD_DIM, device="cuda", dtype=dt)
|
||||
slots = torch.randperm(num_blocks * BLOCK_SIZE, device="cuda")[:num_tokens].long()
|
||||
slots[torch.rand(num_tokens, device="cuda") < 0.3] = -1
|
||||
ref = torch.full(
|
||||
(num_blocks, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
|
||||
165,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
got = ref.clone()
|
||||
scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale)
|
||||
if decode:
|
||||
ql_nope = torch.randn(
|
||||
num_tokens, NUM_HEADS, KV_LORA_RANK, device="cuda", dtype=dt
|
||||
)
|
||||
fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q[..., -ROPE_HEAD_DIM:], kv_c, k_pe, got, slots, ds_mla=True
|
||||
)
|
||||
else:
|
||||
fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots)
|
||||
assert torch.equal(ref, got)
|
||||
rows = got.view(-1, DS_MLA_CACHE_ENTRY)[slots[slots >= 0]]
|
||||
scales = rows[:, KV_LORA_RANK : KV_LORA_RANK + 16].view(torch.float32)
|
||||
torch.testing.assert_close(
|
||||
torch.log2(scales), torch.log2(scales).round(), rtol=0, atol=0
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,143 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""fp8 query and index-K cache on the Triton indexer must agree with bf16.
|
||||
|
||||
The MiniMax M3 lightning-indexer scores only feed a top-k block ranking (the
|
||||
attention itself reads the main KV cache), so an fp8 (e4m3) side cache trades
|
||||
~2% relative score error for half the per-step index-K read bandwidth — the
|
||||
dominant linear-in-context decode cost at long context. #45892 enabled fp8
|
||||
index caches on the SM100 MSA path; this covers the Triton impl used
|
||||
everywhere else (e.g. SM120), for both the decode and prefill scorers.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.minimax_m3.common.ops.index_topk import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_index_decode,
|
||||
minimax_m3_index_score,
|
||||
minimax_m3_index_topk,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
FP8_CUDA_SUPPORTED = current_platform.is_cuda() and current_platform.supports_fp8()
|
||||
PAGE = SPARSE_BLOCK_SIZE
|
||||
HEAD_DIM = 128
|
||||
HEADS = 1
|
||||
TOPK = 16
|
||||
NUM_PAGES = 100
|
||||
SEQ_LEN = 96 * PAGE + 37
|
||||
PLANTED = (7, 40, 88)
|
||||
|
||||
|
||||
def _make_inputs(device: str) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
torch.manual_seed(3)
|
||||
keys = torch.randn(NUM_PAGES * PAGE, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
# RMS-normalize like the real (post index_k_norm) keys.
|
||||
keys = keys * keys.float().pow(2).mean(-1, keepdim=True).add(1e-6).rsqrt().to(
|
||||
torch.bfloat16
|
||||
)
|
||||
q = torch.randn(1, HEADS, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
for b in PLANTED:
|
||||
keys[b * PAGE + 5] = (q[0, 0] * 3).to(torch.bfloat16)
|
||||
return q, keys
|
||||
|
||||
|
||||
def _check_selection(top_bf16: set[int], top_fp8: set[int]) -> None:
|
||||
# Planted (clearly relevant) blocks must never be lost.
|
||||
assert all(b in top_fp8 for b in PLANTED)
|
||||
# Selection may differ only at the noise floor (near-tied filler blocks).
|
||||
assert len(top_bf16 & top_fp8) >= TOPK - 2
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
|
||||
@torch.inference_mode()
|
||||
def test_fp8_index_query_and_cache_decode_topk_matches_bf16() -> None:
|
||||
dev = "cuda"
|
||||
q, keys = _make_inputs(dev)
|
||||
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
|
||||
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
|
||||
|
||||
def select(cache: torch.Tensor) -> set[int]:
|
||||
query = q.to(cache.dtype)
|
||||
idx = minimax_m3_index_decode(
|
||||
query,
|
||||
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
|
||||
block_table,
|
||||
seq_lens,
|
||||
SEQ_LEN,
|
||||
TOPK,
|
||||
0,
|
||||
0,
|
||||
HEADS,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
|
||||
@torch.inference_mode()
|
||||
def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
|
||||
dev = "cuda"
|
||||
q, keys = _make_inputs(dev)
|
||||
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
|
||||
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
|
||||
# single decode-like query token at the end of the sequence
|
||||
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=dev)
|
||||
prefix_lens = torch.tensor([SEQ_LEN - 1], dtype=torch.int32, device=dev)
|
||||
|
||||
def select(cache: torch.Tensor) -> set[int]:
|
||||
query = q.to(cache.dtype)
|
||||
score = minimax_m3_index_score(
|
||||
query,
|
||||
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
|
||||
block_table,
|
||||
cu_seqlens_q,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
1,
|
||||
SEQ_LEN,
|
||||
HEADS,
|
||||
)
|
||||
idx = minimax_m3_index_topk(score, cu_seqlens_q, prefix_lens, 1, TOPK, 0, 0)
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
|
||||
@torch.inference_mode()
|
||||
def test_bf16_query_against_fp8_cache_decode_topk_matches_bf16() -> None:
|
||||
"""A mixed pair must upcast rather than emit a mixed-dtype fp8 dot.
|
||||
|
||||
Production allocates index_q with the index-K cache dtype, so the scorers
|
||||
normally see a matched pair and keep the native FP8 MMA. This pins the
|
||||
other branch: a bf16 query against an e4m3 cache still compiles and still
|
||||
ranks the planted blocks.
|
||||
"""
|
||||
dev = "cuda"
|
||||
q, keys = _make_inputs(dev)
|
||||
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
|
||||
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
|
||||
|
||||
def select(cache: torch.Tensor) -> set[int]:
|
||||
idx = minimax_m3_index_decode(
|
||||
q, # bf16 query, deliberately not cast to the cache dtype
|
||||
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
|
||||
block_table,
|
||||
seq_lens,
|
||||
SEQ_LEN,
|
||||
TOPK,
|
||||
0,
|
||||
0,
|
||||
HEADS,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,226 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm correctness tests for the strided DSA candidate mask.
|
||||
|
||||
``_apply_candidate_mask_strided`` is a decode-path variant of the shared
|
||||
``apply_candidate_mask``. It differs in two ways that need pinning down:
|
||||
|
||||
* it launches a fixed number of programs that stride the column axis, rather
|
||||
than one program per 1024-column tile, so at ``max_model_len`` widths the
|
||||
grid no longer scales with a workspace that is mostly padding;
|
||||
* it stops at each row's ``end`` instead of sanitizing the whole width.
|
||||
|
||||
The second is only sound because every consumer bounds its scan by the same
|
||||
ends -- ``top_k_per_row_decode`` takes ``rowEnd`` from ``seq_lens``. So the
|
||||
contract is agreement with the shared kernel over ``[0, end)`` and nothing
|
||||
beyond it, which is what these tests assert rather than comparing whole rows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
BLOCK_SIZE = 8
|
||||
TOPK_BLOCKS = 64
|
||||
# Above 128 tiles the grid saturates and programs start striding; below it the
|
||||
# min() clamp applies instead. Both regimes need covering, so the widths here
|
||||
# deliberately straddle 128 * 1024.
|
||||
NARROW = 8192
|
||||
WIDE = 262144
|
||||
|
||||
|
||||
def _inputs(rows, width, ends, starts, *, block_size, col_stride, dtype):
|
||||
"""Logits, bounds and candidate blocks shared by both kernels."""
|
||||
torch.manual_seed(0)
|
||||
# A column stride > 1 keeps the kernels honest about stride_col; the decode
|
||||
# caller slices a workspace, so contiguous rows are not guaranteed.
|
||||
logits = torch.randn(rows, width * col_stride, device="cuda")[:, ::col_stride]
|
||||
logits[0, : min(16, width)] = 0.0
|
||||
if rows > 2:
|
||||
logits[2, 5] = float("nan")
|
||||
row_ke = torch.tensor(ends, device="cuda", dtype=dtype)
|
||||
row_ks = torch.tensor(starts, device="cuda", dtype=dtype) if starts else None
|
||||
nblocks = (width + block_size - 1) // block_size
|
||||
candidates = torch.randint(
|
||||
0, nblocks, (rows, TOPK_BLOCKS), device="cuda", dtype=torch.int32
|
||||
)
|
||||
# -1 is the "no block" sentinel the selector emits for short rows.
|
||||
candidates[:, ::7] = -1
|
||||
return logits, row_ks, row_ke, candidates
|
||||
|
||||
|
||||
def _run_both(logits, row_ks, row_ke, candidates, block_size, row_repeat):
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
apply_candidate_mask,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
ref = logits.clone()
|
||||
got = logits.clone()
|
||||
apply_candidate_mask(ref, row_ks, row_ke, candidates, block_size, row_repeat)
|
||||
_apply_candidate_mask_strided(
|
||||
got, row_ks, row_ke, candidates, block_size, row_repeat
|
||||
)
|
||||
return ref, got
|
||||
|
||||
|
||||
def _assert_agrees_within_ends(ref, got, row_ke, row_repeat):
|
||||
"""Compare only the region a consumer can read."""
|
||||
for row in range(ref.shape[0]):
|
||||
end = int(row_ke[row // row_repeat])
|
||||
if end <= 0:
|
||||
continue
|
||||
torch.testing.assert_close(
|
||||
got[row, :end],
|
||||
ref[row, :end],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
equal_nan=True,
|
||||
msg=lambda m, r=row, e=end: f"row {r} differs within [0, {e}): {m}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [NARROW, WIDE])
|
||||
@pytest.mark.parametrize("col_stride", [1, 2])
|
||||
def test_matches_shared_kernel_ragged_ends(width, col_stride):
|
||||
"""Ends on and around tile boundaries, plus the empty and full rows."""
|
||||
ends = [0, 1, 1023, 1024, 1025, width - 1, width, width // 2]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
width,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=col_stride,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [8, 16])
|
||||
def test_matches_shared_kernel_with_starts(block_size):
|
||||
"""Non-zero starts: the decode caller passes zeros, prefill does not."""
|
||||
ends = [0, 64, 4096, 8192]
|
||||
starts = [0, 7, 1024, 3]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
NARROW,
|
||||
ends,
|
||||
starts,
|
||||
block_size=block_size,
|
||||
col_stride=1,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, block_size, 1)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, 1)
|
||||
|
||||
|
||||
def test_matches_shared_kernel_row_repeat():
|
||||
"""Speculative decode: next_n logit rows share one bound."""
|
||||
row_repeat = 3
|
||||
ends = [0, 1025, WIDE]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends) * row_repeat,
|
||||
WIDE,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, row_repeat)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, row_repeat)
|
||||
|
||||
|
||||
def test_leaves_columns_past_end_untouched():
|
||||
"""The deliberate difference from the shared kernel, pinned explicitly.
|
||||
|
||||
Skipping the tail is where the speedup comes from, so a change that
|
||||
quietly restores full-width sanitizing should fail here rather than just
|
||||
get slower. Work stops at the tile boundary containing ``end``, not at
|
||||
``end`` itself, so that is the bound asserted.
|
||||
"""
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_MASK_TILE,
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
ends = [0, 1, 1025, WIDE // 2]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
WIDE,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
sentinel = 1.5
|
||||
logits.fill_(sentinel)
|
||||
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
for row, end in enumerate(ends):
|
||||
touched = min(-(-end // _MASK_TILE) * _MASK_TILE, WIDE)
|
||||
tail = logits[row, touched:]
|
||||
assert torch.equal(tail, torch.full_like(tail, sentinel)), (
|
||||
f"row {row} (end {end}) was written past column {touched}"
|
||||
)
|
||||
# And the bound is tight: a row that ends early must not have cost a
|
||||
# full-width pass, which is the whole point of the kernel.
|
||||
assert touched <= end + _MASK_TILE
|
||||
|
||||
|
||||
def test_cudagraph_replay_tracks_changing_ends():
|
||||
"""The grid is static so a FULL capture stays valid when ends change.
|
||||
|
||||
Capturing bakes the grid, so a data-dependent program count would freeze
|
||||
whatever lengths were resident at capture. Replaying against longer and
|
||||
shorter contexts than the captured ones is the property that matters.
|
||||
"""
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
apply_candidate_mask,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
rows = 4
|
||||
captured_ends = [1024, 4096, 8192, 2048]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
rows,
|
||||
WIDE,
|
||||
captured_ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
base = logits.clone()
|
||||
|
||||
# Warm up outside the capture: Triton compiles on first call, and autotune
|
||||
# launches would otherwise happen inside the graph.
|
||||
_apply_candidate_mask_strided(
|
||||
logits.clone(), row_ks, row_ke, candidates, BLOCK_SIZE, 1
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
|
||||
for replay_ends in ([1, 131072, WIDE, 0], [WIDE, 1025, 3, 65536]):
|
||||
row_ke.copy_(torch.tensor(replay_ends, device="cuda", dtype=row_ke.dtype))
|
||||
logits.copy_(base)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
ref = base.clone()
|
||||
apply_candidate_mask(ref, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
_assert_agrees_within_ends(ref, logits, row_ke, 1)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,262 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for causal masking in the AITER MLA verify MTP path.
|
||||
|
||||
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
|
||||
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify
|
||||
through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped
|
||||
``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the
|
||||
causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``.
|
||||
|
||||
The test drives the real builder and ``forward_mqa`` over a multi-token decode
|
||||
batch, intercepts the arguments actually handed to the Gluon kernel, and pins
|
||||
that the 4-D MTP contract is used with per-request ``block_table`` and
|
||||
``cache_seqlens`` rather than a flattened per-token paged-KV view.
|
||||
"""
|
||||
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_rocm_with_aiter() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found()
|
||||
|
||||
|
||||
# The production verify MTP path is gfx950-only. This test replaces the Gluon
|
||||
# kernel with a spy and forces the architecture feature probe so the metadata
|
||||
# regression remains testable on every ROCm AITER CI runner.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_rocm_with_aiter(),
|
||||
reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER",
|
||||
)
|
||||
|
||||
# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at
|
||||
# TP=16 and Kimi-K3 at TP=8 both land here.
|
||||
NUM_QUERY_HEADS = 12
|
||||
KV_LORA_RANK = 512
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
V_HEAD_DIM = 128
|
||||
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
|
||||
# 1 + num_speculative_tokens, i.e. the verify block length.
|
||||
QLEN = 8
|
||||
# Committed context per request, including a fresh request at zero context
|
||||
# where the whole KV range is the verify block itself.
|
||||
CONTEXT_LENS = [0, 1, 37, 512]
|
||||
# A cudagraph padding request: seq_len 0.
|
||||
PADDING_ROWS = 1
|
||||
|
||||
# The Gluon path flattens the KV cache to one page per token.
|
||||
PAGE_SIZE = 1
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_NUM_SEQS = 8
|
||||
|
||||
# The builder resolves the verify routing and records it on the metadata, so
|
||||
# the gfx950 feature probe has to be forced for the build as well as for the
|
||||
# forward; ROCm CI also runs gfx942, where the real probe is False.
|
||||
_GLUON_SUPPORTED_TARGET = (
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla._gluon_mla_decode_supported"
|
||||
)
|
||||
|
||||
|
||||
def _seq_lens() -> list[int]:
|
||||
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
|
||||
|
||||
|
||||
def _run_verify_block():
|
||||
"""Drive the real builder + forward_mqa over one multi-token verify block.
|
||||
|
||||
Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs.
|
||||
"""
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
seq_lens = _seq_lens()
|
||||
num_reqs = len(seq_lens)
|
||||
# One flat page per token, arange block indices, plus room to spare.
|
||||
num_gpu_blocks = num_reqs * max(seq_lens) + 256
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-R1",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
block_size=PAGE_SIZE,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
max_num_batched_tokens=8192,
|
||||
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
|
||||
)
|
||||
# What raises reorder_batch_threshold to QLEN, so a QLEN-token block is
|
||||
# classified as a decode instead of a prefill. ngram needs no draft model.
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=QLEN - 1
|
||||
)
|
||||
vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign]
|
||||
lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS,
|
||||
vllm_config.model_config,
|
||||
)
|
||||
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=PAGE_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
cache_dtype_str="auto",
|
||||
)
|
||||
|
||||
backend_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class()
|
||||
builder_cls = backend_cls.get_builder_cls()
|
||||
impl_cls = backend_cls.get_impl_cls()
|
||||
|
||||
# The builder reads layer.prefill_backend from static_forward_context; a
|
||||
# stub with the attribute is enough for metadata construction.
|
||||
layer_name = "placeholder"
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
types.SimpleNamespace(
|
||||
prefill_backend=torch.empty((1,)),
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=V_HEAD_DIM,
|
||||
)
|
||||
)
|
||||
|
||||
init_workspace_manager(device)
|
||||
|
||||
batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=[QLEN] * num_reqs)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def spy(**kwargs):
|
||||
captured.update(kwargs)
|
||||
captured["q_nope"] = kwargs["q_nope"].detach().clone()
|
||||
captured["page_table"] = kwargs["page_table"].detach().clone()
|
||||
captured["seq_info"] = kwargs["seq_info"].detach().clone()
|
||||
|
||||
with (
|
||||
patch(_GLUON_SUPPORTED_TARGET, lambda: True),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
builder = builder_cls(spec, [layer_name], vllm_config, device)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, PAGE_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
impl = impl_cls(
|
||||
num_heads=NUM_QUERY_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
scale=(QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="auto",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=V_HEAD_DIM,
|
||||
kv_b_proj=None,
|
||||
)
|
||||
|
||||
num_rows = num_reqs * QLEN
|
||||
dtype = torch.bfloat16
|
||||
q_nope = torch.zeros(
|
||||
num_rows, NUM_QUERY_HEADS, KV_LORA_RANK, dtype=dtype, device=device
|
||||
)
|
||||
q_pe = torch.zeros(
|
||||
num_rows, NUM_QUERY_HEADS, QK_ROPE_HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
kv_cache = torch.zeros(
|
||||
num_gpu_blocks, PAGE_SIZE, HEAD_SIZE, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
# The Gluon kernel only reads the metadata this test is about, so a spy in
|
||||
# its place keeps the assertions independent of the AITER build.
|
||||
with (
|
||||
patch(_GLUON_SUPPORTED_TARGET, lambda: True),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla._get_mla_gluon",
|
||||
lambda: spy,
|
||||
),
|
||||
):
|
||||
assert isinstance(impl, AiterMLAImpl)
|
||||
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
|
||||
|
||||
return metadata, captured
|
||||
|
||||
|
||||
def test_verify_mtp_uses_native_4d_gluon_entry():
|
||||
"""Verify uses 4-D q with per-request paged_kv metadata.
|
||||
|
||||
Regression guard: before the fix every flattened row of a request got that
|
||||
request's whole paged-KV range, so verify position t attended to the draft
|
||||
tokens after it. The native MTP path delegates causal masking to aiter.
|
||||
"""
|
||||
metadata, captured = _run_verify_block()
|
||||
|
||||
decode = metadata.decode
|
||||
assert decode is not None, "batch was not classified as a decode"
|
||||
assert decode.max_qo_len == QLEN, (
|
||||
f"expected a {QLEN}-token verify block, got max_qo_len="
|
||||
f"{decode.max_qo_len}; the MTP path under test was not reached"
|
||||
)
|
||||
assert decode.use_gluon_verify, (
|
||||
"the builder owns verify routing, so the metadata -- not the impl -- "
|
||||
"has to select the Gluon MTP entry for a small-head bf16 verify block"
|
||||
)
|
||||
assert captured, "forward_mqa did not reach the Gluon kernel"
|
||||
|
||||
seq_lens = _seq_lens()
|
||||
num_reqs = len(seq_lens)
|
||||
|
||||
q_nope = captured["q_nope"]
|
||||
assert q_nope.dim() == 4, (
|
||||
f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape "
|
||||
f"{tuple(q_nope.shape)}"
|
||||
)
|
||||
assert q_nope.shape[:2] == (num_reqs, QLEN), (
|
||||
f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], "
|
||||
f"got {tuple(q_nope.shape)}"
|
||||
)
|
||||
assert q_nope.shape[2] == NUM_QUERY_HEADS
|
||||
|
||||
assert captured["use_2d_view"] is False
|
||||
page_table = captured["page_table"]
|
||||
seq_info = captured["seq_info"]
|
||||
assert page_table.dim() == 1, (
|
||||
f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D"
|
||||
)
|
||||
assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, (
|
||||
f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape "
|
||||
f"{tuple(seq_info.shape)}"
|
||||
)
|
||||
assert decode is not None and decode.paged_kv_indptr is not None
|
||||
assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1])
|
||||
assert torch.equal(
|
||||
page_table[: int(seq_info[-1].item())],
|
||||
decode.paged_kv_indices[: int(seq_info[-1].item())],
|
||||
)
|
||||
@@ -1,225 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm kernel correctness tests for AITER MLA decode.
|
||||
|
||||
Compares ``rocm_aiter_ops.mla_decode_fwd`` against a pure PyTorch reference
|
||||
under the absorbed MLA formulation (DeepSeek-V3/V4) with varied batch sizes,
|
||||
head counts, and sequence lengths.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
_SKIP_NON_MI3XX = True
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
_SKIP_NON_MI3XX = not on_mi3xx()
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific tests"),
|
||||
pytest.mark.skipif(_SKIP_NON_MI3XX, reason="MI300/MI350 ROCm only"),
|
||||
]
|
||||
|
||||
# DeepSeek-V3/V4 MLA config.
|
||||
Q_HEAD_DIM = 576 # kv_lora_rank + qk_rope_head_dim
|
||||
V_HEAD_DIM = 512 # kv_lora_rank
|
||||
SM_SCALE = Q_HEAD_DIM**-0.5
|
||||
|
||||
NUM_HEADS = [16, 128]
|
||||
BATCH_SIZES = [1, 4, 16]
|
||||
KV_SEQ_LENS = [16, 256]
|
||||
|
||||
ATOL, RTOL = 1.5e-2, 1e-2
|
||||
|
||||
|
||||
def _require_aiter():
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip("aiter is required on supported ROCm hardware for this test")
|
||||
|
||||
|
||||
def _ref_mla_decode(
|
||||
q: torch.Tensor,
|
||||
kv_buffer: torch.Tensor,
|
||||
kv_indptr: torch.Tensor,
|
||||
kv_indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Pure PyTorch reference for absorbed MLA decode.
|
||||
|
||||
K uses full q_head_dim (576) for scoring, V uses first v_head_dim (512)
|
||||
for output. This mirrors the production kernel's absorbed formulation.
|
||||
"""
|
||||
batch_size, num_heads, _ = q.shape
|
||||
output = torch.zeros(
|
||||
batch_size, num_heads, V_HEAD_DIM, dtype=q.dtype, device=q.device
|
||||
)
|
||||
|
||||
for b in range(batch_size):
|
||||
start = kv_indptr[b].item()
|
||||
end = kv_indptr[b + 1].item()
|
||||
token_indices = kv_indices[start:end]
|
||||
|
||||
k = kv_buffer[token_indices].float()
|
||||
v = kv_buffer[token_indices, :V_HEAD_DIM].float()
|
||||
|
||||
for h in range(num_heads):
|
||||
q_h = q[b, h, :].float()
|
||||
scores = torch.mv(k, q_h) * SM_SCALE
|
||||
weights = torch.softmax(scores, dim=0)
|
||||
output[b, h, :] = torch.mv(v.t(), weights).to(q.dtype)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def _make_inputs(
|
||||
batch_size: int,
|
||||
nhead: int,
|
||||
kv_seq_len: int,
|
||||
*,
|
||||
contiguous_indices: bool = True,
|
||||
):
|
||||
"""Build valid MLA decode inputs on the current CUDA device.
|
||||
|
||||
When contiguous_indices=False, tokens are randomly scattered in a 2x pool
|
||||
to simulate real paged allocation.
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
|
||||
|
||||
actual_nhead = AiterMLAHelper.get_actual_mla_num_heads(nhead)
|
||||
total_kv_tokens = batch_size * kv_seq_len
|
||||
|
||||
pool_size = total_kv_tokens if contiguous_indices else total_kv_tokens * 2
|
||||
|
||||
q = torch.randn(
|
||||
batch_size, actual_nhead, Q_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
kv_buffer = torch.randn(
|
||||
pool_size, 1, Q_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
o = torch.zeros(
|
||||
batch_size, actual_nhead, V_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
|
||||
qo_indptr = torch.arange(0, batch_size + 1, dtype=torch.int32, device="cuda")
|
||||
kv_indptr = (
|
||||
torch.arange(0, batch_size + 1, dtype=torch.int32, device="cuda") * kv_seq_len
|
||||
)
|
||||
kv_last_page_lens = torch.ones(batch_size, dtype=torch.int32, device="cuda")
|
||||
|
||||
if contiguous_indices:
|
||||
kv_indices = torch.arange(0, total_kv_tokens, dtype=torch.int32, device="cuda")
|
||||
else:
|
||||
perm = torch.randperm(pool_size, device="cuda")[:total_kv_tokens]
|
||||
kv_indices = perm.to(torch.int32)
|
||||
|
||||
return {
|
||||
"q": q,
|
||||
"kv_buffer": kv_buffer,
|
||||
"o": o,
|
||||
"qo_indptr": qo_indptr,
|
||||
"kv_indptr": kv_indptr,
|
||||
"kv_indices": kv_indices,
|
||||
"kv_last_page_lens": kv_last_page_lens,
|
||||
"actual_nhead": actual_nhead,
|
||||
}
|
||||
|
||||
|
||||
def _run_kernel(inputs: dict) -> torch.Tensor:
|
||||
"""Run decode through the production rocm_aiter_ops path."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.mla_decode_fwd(
|
||||
inputs["q"],
|
||||
inputs["kv_buffer"],
|
||||
inputs["o"],
|
||||
SM_SCALE,
|
||||
inputs["qo_indptr"],
|
||||
1,
|
||||
inputs["kv_indptr"],
|
||||
inputs["kv_indices"],
|
||||
inputs["kv_last_page_lens"],
|
||||
)
|
||||
return inputs["o"]
|
||||
|
||||
|
||||
def _ref_output(inputs: dict) -> torch.Tensor:
|
||||
"""Compute reference output from the same inputs."""
|
||||
kv_flat = inputs["kv_buffer"].squeeze(1)
|
||||
return _ref_mla_decode(
|
||||
inputs["q"], kv_flat, inputs["kv_indptr"], inputs["kv_indices"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nhead", NUM_HEADS)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
|
||||
@pytest.mark.parametrize("kv_seq_len", KV_SEQ_LENS)
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_accuracy(
|
||||
nhead: int,
|
||||
batch_size: int,
|
||||
kv_seq_len: int,
|
||||
) -> None:
|
||||
"""BF16 decode accuracy vs PyTorch reference (contiguous indices)."""
|
||||
_require_aiter()
|
||||
set_random_seed(0)
|
||||
|
||||
inputs = _make_inputs(batch_size, nhead, kv_seq_len)
|
||||
output = _run_kernel(inputs)
|
||||
output_ref = _ref_output(inputs)
|
||||
|
||||
torch.testing.assert_close(output, output_ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nhead", NUM_HEADS)
|
||||
@pytest.mark.parametrize("batch_size", [1, 4, 8])
|
||||
@pytest.mark.parametrize("kv_seq_len", [16, 128])
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_noncontiguous_indices(
|
||||
nhead: int,
|
||||
batch_size: int,
|
||||
kv_seq_len: int,
|
||||
) -> None:
|
||||
"""BF16 decode accuracy with shuffled kv_indices (paged allocation)."""
|
||||
_require_aiter()
|
||||
set_random_seed(0)
|
||||
|
||||
inputs = _make_inputs(batch_size, nhead, kv_seq_len, contiguous_indices=False)
|
||||
output = _run_kernel(inputs)
|
||||
output_ref = _ref_output(inputs)
|
||||
|
||||
torch.testing.assert_close(output, output_ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_determinism() -> None:
|
||||
"""Repeated decode calls produce bitwise-identical results."""
|
||||
_require_aiter()
|
||||
set_random_seed(0)
|
||||
|
||||
inputs = _make_inputs(batch_size=4, nhead=128, kv_seq_len=16)
|
||||
out_first = _run_kernel(inputs).clone()
|
||||
|
||||
for _ in range(3):
|
||||
inputs["o"] = torch.zeros_like(inputs["o"])
|
||||
out = _run_kernel(inputs)
|
||||
torch.testing.assert_close(out, out_first, atol=0, rtol=0)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_smoke() -> None:
|
||||
"""Basic sanity: correct shape, dtype, finite, non-zero."""
|
||||
_require_aiter()
|
||||
set_random_seed(0)
|
||||
|
||||
inputs = _make_inputs(batch_size=4, nhead=128, kv_seq_len=64)
|
||||
output = _run_kernel(inputs)
|
||||
|
||||
assert output.shape == (4, 128, V_HEAD_DIM)
|
||||
assert output.dtype == torch.bfloat16
|
||||
assert torch.isfinite(output).all()
|
||||
assert not torch.all(output == 0)
|
||||
@@ -1,213 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for AITER MLA persistent decode metadata dtypes.
|
||||
|
||||
For the gfx950 fp8/fp8 nhead=32 qlen=1 fold path, the split/reduce metadata
|
||||
layout depends on the q/kv element size. The builder must forward dtype_q/dtype_kv
|
||||
to ``get_mla_metadata_v1``; omitting them lays out the work for the wrong dtype
|
||||
and corrupts decode output. The test pins the builder's metadata to a golden
|
||||
recomputed at runtime with the explicit correct dtypes.
|
||||
"""
|
||||
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not (current_platform.is_rocm() and is_aiter_found()):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_gfx950(),
|
||||
reason="AITER MLA fp8 persistent decode metadata is gfx950-only",
|
||||
)
|
||||
|
||||
# The fold path that the bug corrupted: fp8 query + fp8 KV-cache, 32 query
|
||||
# heads, single-token decode, batch 128, context 8192, page_size 1.
|
||||
NUM_QUERY_HEADS = 32
|
||||
DECODE_QLEN = 1
|
||||
BATCH_SIZE = 128
|
||||
CONTEXT_LEN = 8192
|
||||
PAGE_SIZE = 1
|
||||
|
||||
# On the fp8 KV-cache path the builder forwards the platform fp8 dtype (aiter
|
||||
# dtypes.fp8) for both q and kv. Mirror it via current_platform.fp8_dtype()
|
||||
# instead of hardcoding a literal (see #47276).
|
||||
EXPECTED_Q_DTYPE = current_platform.fp8_dtype()
|
||||
EXPECTED_KV_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
# The split/reduce content tensors filled by get_mla_metadata_v1. work_meta_data
|
||||
# is excluded: it holds raw device pointers, never equal across allocations.
|
||||
_CONTENT_METADATA_FIELDS = (
|
||||
"work_indptr",
|
||||
"work_info_set",
|
||||
"reduce_indptr",
|
||||
"reduce_final_map",
|
||||
"reduce_partial_map",
|
||||
)
|
||||
|
||||
# The builder's get_mla_metadata_v1 call passes 6 input args then 6 output
|
||||
# buffers (see AiterMLAMetadataBuilder._build_decode). Output order -> field.
|
||||
_NUM_INPUT_ARGS = 6
|
||||
_OUTPUT_ARG_FIELDS = (
|
||||
"work_meta_data",
|
||||
"work_info_set",
|
||||
"work_indptr",
|
||||
"reduce_indptr",
|
||||
"reduce_final_map",
|
||||
"reduce_partial_map",
|
||||
)
|
||||
|
||||
|
||||
def _build_decode_metadata():
|
||||
"""Build AITER MLA decode metadata for the fp8/fp8 nhead=32 fold path.
|
||||
|
||||
Returns ``(metadata, captured)`` where ``captured`` records the positional
|
||||
args/kwargs the builder passed to ``get_mla_metadata_v1``, so the golden can
|
||||
be recomputed from the identical inputs.
|
||||
"""
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-R1",
|
||||
max_model_len=CONTEXT_LEN,
|
||||
# One flat page per token (page_size=1); +buffer for the null block.
|
||||
num_gpu_blocks=BATCH_SIZE * CONTEXT_LEN + 200,
|
||||
block_size=PAGE_SIZE,
|
||||
max_num_seqs=BATCH_SIZE,
|
||||
max_num_batched_tokens=8192,
|
||||
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
|
||||
)
|
||||
vllm_config.cache_config.cache_dtype = "fp8"
|
||||
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=PAGE_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
cache_dtype_str="fp8",
|
||||
)
|
||||
|
||||
builder_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class().get_builder_cls()
|
||||
|
||||
# The builder reads prefill_backend and the MLA latent dims off the layer,
|
||||
# so the stub carries both; the dims come from the model config to stay in
|
||||
# step with the checkpoint.
|
||||
mla_dims = get_mla_dims(vllm_config.model_config)
|
||||
layer_name = "placeholder"
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
types.SimpleNamespace(
|
||||
prefill_backend=torch.empty((1,)),
|
||||
q_lora_rank=mla_dims.q_lora_rank,
|
||||
kv_lora_rank=mla_dims.kv_lora_rank,
|
||||
qk_nope_head_dim=mla_dims.qk_nope_head_dim,
|
||||
qk_rope_head_dim=mla_dims.qk_rope_head_dim,
|
||||
v_head_dim=mla_dims.v_head_dim,
|
||||
)
|
||||
)
|
||||
|
||||
init_workspace_manager(device)
|
||||
|
||||
batch_spec = BatchSpec(
|
||||
seq_lens=[CONTEXT_LEN] * BATCH_SIZE,
|
||||
query_lens=[DECODE_QLEN] * BATCH_SIZE,
|
||||
)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
builder = builder_cls(spec, [layer_name], vllm_config, device)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, PAGE_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
|
||||
import aiter
|
||||
|
||||
real_get_mla_metadata_v1 = aiter.get_mla_metadata_v1
|
||||
|
||||
def spy(*args, **kwargs):
|
||||
captured["args"] = args
|
||||
captured["kwargs"] = dict(kwargs)
|
||||
return real_get_mla_metadata_v1(*args, **kwargs)
|
||||
|
||||
with patch("aiter.get_mla_metadata_v1", spy):
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
)
|
||||
|
||||
return metadata, captured
|
||||
|
||||
|
||||
def _compute_golden_metadata(captured: dict) -> dict[str, torch.Tensor]:
|
||||
"""Recompute the persistent metadata with explicit fp8/bf16 dtypes.
|
||||
|
||||
Replays ``get_mla_metadata_v1`` on the builder's exact input tensors with
|
||||
fresh output buffers and the explicitly-correct dtypes. This reference must
|
||||
match the builder's output when the fix is in place.
|
||||
"""
|
||||
import aiter
|
||||
|
||||
args = captured["args"]
|
||||
inputs = args[:_NUM_INPUT_ARGS]
|
||||
# Fresh copies so the golden does not alias the builder's persistent buffers.
|
||||
fresh_outputs = [arg.clone() for arg in args[_NUM_INPUT_ARGS:]]
|
||||
|
||||
golden_kwargs = dict(captured["kwargs"])
|
||||
golden_kwargs["dtype_q"] = EXPECTED_Q_DTYPE
|
||||
golden_kwargs["dtype_kv"] = EXPECTED_KV_DTYPE
|
||||
|
||||
aiter.get_mla_metadata_v1(*inputs, *fresh_outputs, **golden_kwargs)
|
||||
|
||||
return dict(zip(_OUTPUT_ARG_FIELDS, fresh_outputs))
|
||||
|
||||
|
||||
def test_persistent_decode_metadata_matches_fp8_golden():
|
||||
"""The builder's metadata must match the dtype-correct golden.
|
||||
|
||||
Regression guard: the fixed builder forwards fp8/bf16 dtypes so its
|
||||
split/reduce metadata matches the golden recomputed with those explicit
|
||||
dtypes. Dropping the dtypes (the original bug) produces a different layout
|
||||
and fails this test.
|
||||
"""
|
||||
metadata, captured = _build_decode_metadata()
|
||||
|
||||
# qlen=1 must take the persistent-metadata path for this to be meaningful.
|
||||
assert metadata.decode is not None
|
||||
assert metadata.decode.has_persistent_metadata
|
||||
assert metadata.work_meta_data is not None
|
||||
|
||||
golden = _compute_golden_metadata(captured)
|
||||
|
||||
mismatched = [
|
||||
name
|
||||
for name in _CONTENT_METADATA_FIELDS
|
||||
if getattr(metadata, name).shape != golden[name].shape
|
||||
or not torch.equal(getattr(metadata, name), golden[name])
|
||||
]
|
||||
assert not mismatched, (
|
||||
"AITER MLA persistent decode metadata does not match the fp8/bf16 "
|
||||
f"golden for fields {mismatched}; the builder must forward "
|
||||
"dtype_q/dtype_kv to get_mla_metadata_v1."
|
||||
)
|
||||
@@ -1,246 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Accuracy test for the ROCm AITER FP8 MLA prefill path.
|
||||
|
||||
The FP8 MLA prefill path
|
||||
(``AiterMLAImpl._mla_fp8_prefill_attn`` -> ``mla_prefill_ps_asm_fwd`` +
|
||||
``mla_reduce_v1``) is auto-enabled on gfx950 (MI355) when AITER ships those
|
||||
kernels, and it has no other test coverage. This test drives the *real*
|
||||
metadata builder and impl methods (via ``object.__new__``, mirroring
|
||||
``test_rocm_aiter_mla_head_padding.py::test_h12_aiter_mla_decode_matches_reference``)
|
||||
so it exercises the actual persistent-scheduling metadata contract
|
||||
(``get_ps_metadata_v1``) plus both kernels -- rather than re-implementing them --
|
||||
and compares the output against a causal SDPA reference.
|
||||
|
||||
gfx950-only: ``mla_prefill_ps_asm_fwd`` has no gfx942 build, so this is skipped
|
||||
on every other platform.
|
||||
"""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _fp8_prefill_available() -> bool:
|
||||
if not (current_platform.is_rocm() and torch.cuda.is_available()):
|
||||
return False
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
_fp8_mla_prefill_supported,
|
||||
)
|
||||
|
||||
return _fp8_mla_prefill_supported()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _fp8_prefill_available(),
|
||||
reason="FP8 MLA prefill needs gfx950 (MI355) + an AITER build exporting "
|
||||
"mla_prefill_ps_asm_fwd / mla_reduce_v1",
|
||||
)
|
||||
|
||||
# DeepSeek-style MLA head dims after kv_b_proj decompression (q/k carry the
|
||||
# nope+rope dims, v carries v_head_dim).
|
||||
#
|
||||
# The kernel requires 16-aligned heads, so _mla_fp8_prefill_attn replicate-pads
|
||||
# q/k/v up to get_fp8_prefill_num_heads(num_heads) and slices the output back.
|
||||
# Cover all four regimes:
|
||||
# 8 -> padded to 16 (divisor, repeat_interleave), so the real heads sit at
|
||||
# stride 2 in the padded output. Live case: Kimi-K2.x at TP8.
|
||||
# 12 -> padded to 16 (non-divisor, tile+slice). Live case: a K3 rank at TP8.
|
||||
# 16 -> no padding, output aliases the caller's buffer.
|
||||
# 24 -> padded to 32. No current model and TP reaches this band (96 heads
|
||||
# would need TP4, which does not fit in 288 GiB; 128-head models give
|
||||
# 128/64/32/16/8), so this case exists to keep the general path honest
|
||||
# for future architectures: it is the only one where padding above
|
||||
# 16 actually happens, so it is what proves the replicate-pad/slice-back
|
||||
# argument holds for a target other than 16.
|
||||
HEAD_COUNTS = [8, 12, 16, 24]
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM # 192
|
||||
V_HEAD_DIM = 128
|
||||
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
|
||||
|
||||
# Dominated by fp8 (e4m3) rounding of q/k/v. Measured on MI355: worst-element
|
||||
# abs error ~0.054, so atol=0.1 is ~2x that noise floor; rtol=0.05 bounds larger
|
||||
# elements.
|
||||
ATOL, RTOL = 1e-1, 5e-2
|
||||
|
||||
# The assembly kernel writes bf16 through a raw output pointer. Keep its tensors
|
||||
# and workspace reservation on one dtype.
|
||||
ATTN_OUT_DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _workspace_manager():
|
||||
"""The FP8 prefill path draws per-call scratch from the global workspace
|
||||
manager, which ``GPUModelRunner.__init__`` normally initializes. Set it up
|
||||
for the test and tear it down so no global state leaks to other tests.
|
||||
"""
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
reset_workspace_manager,
|
||||
)
|
||||
|
||||
init_workspace_manager(torch.device("cuda"))
|
||||
yield
|
||||
reset_workspace_manager()
|
||||
|
||||
|
||||
def _make_impl(num_heads: int):
|
||||
"""Minimal AiterMLAImpl exposing only what _mla_fp8_prefill_attn reads."""
|
||||
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1
|
||||
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = num_heads
|
||||
impl.v_head_dim = V_HEAD_DIM
|
||||
impl.scale = SCALE
|
||||
impl._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
|
||||
impl._mla_reduce_v1 = mla_reduce_v1
|
||||
return impl
|
||||
|
||||
|
||||
def _build_prefill_metadata(
|
||||
seq_lens: list[int],
|
||||
device: torch.device,
|
||||
num_heads: int,
|
||||
attn_out_dtype: torch.dtype = ATTN_OUT_DTYPE,
|
||||
):
|
||||
"""Build metadata with the production builder and output dtype."""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
|
||||
|
||||
qo_indptr_cpu = torch.zeros(len(seq_lens) + 1, dtype=torch.int32)
|
||||
qo_indptr_cpu[1:] = torch.tensor(seq_lens, dtype=torch.int32).cumsum(0)
|
||||
qo_indptr = qo_indptr_cpu.to(device)
|
||||
total_q = int(qo_indptr_cpu[-1].item())
|
||||
max_q = max(seq_lens)
|
||||
|
||||
builder = object.__new__(AiterMLAMetadataBuilder)
|
||||
builder.num_heads = num_heads
|
||||
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM)
|
||||
builder._init_fp8_prefill_ps_buffers(
|
||||
max_num_reqs=len(seq_lens),
|
||||
max_prefill_qlen=max_q,
|
||||
max_num_batched_tokens=total_q,
|
||||
attn_out_dtype=attn_out_dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# SimpleNamespace stands in for the metadata dataclasses: the builder only
|
||||
# reads prefill.{query_start_loc,max_query_len}, num_decodes, and
|
||||
# common.query_start_loc_cpu, then populates metadata.fp8_prefill_*.
|
||||
prefill = SimpleNamespace(query_start_loc=qo_indptr, max_query_len=max_q)
|
||||
metadata = SimpleNamespace(prefill=prefill, num_decodes=0)
|
||||
common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu)
|
||||
builder._build_fp8_prefill_ps_metadata(metadata, common)
|
||||
return metadata, total_q, builder
|
||||
|
||||
|
||||
def _reference(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
|
||||
"""Causal SDPA reference on fp8-quantized inputs (matches kernel dtype)."""
|
||||
fp8 = current_platform.fp8_dtype()
|
||||
qf = q.to(fp8).float().transpose(0, 1) # [H, S, Dqk]
|
||||
kf = k.to(fp8).float().transpose(0, 1)
|
||||
vf = v.to(fp8).float().transpose(0, 1) # [H, S, Dv]
|
||||
out = F.scaled_dot_product_attention(qf, kf, vf, is_causal=True, scale=SCALE)
|
||||
return out.transpose(0, 1) # [S, H, Dv]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
|
||||
@pytest.mark.parametrize("seq_len", [128, 512])
|
||||
@torch.inference_mode()
|
||||
def test_fp8_prefill_matches_reference(seq_len: int, num_heads: int) -> None:
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
metadata, total_q, _ = _build_prefill_metadata([seq_len], device, num_heads)
|
||||
|
||||
# Lock the workspace, as GPUModelRunner does after warmup, so the forward
|
||||
# cannot quietly grow it. The builder reserves scratch at num_head_k and
|
||||
# the forward requests it at its own padded width; if a future change makes
|
||||
# the forward's width exceed the builder's, that is an under-reservation
|
||||
# which is invisible on an unlocked workspace (it just grows) and fatal in
|
||||
# a real serve. Locking turns it into a failure here.
|
||||
#
|
||||
# Note this does not catch the reverse -- a builder wider than the forward
|
||||
# merely over-reserves -- so it would not have flagged the pre-PR
|
||||
# max(16, num_heads) sizing, which over-reserves 2.8x at 24 heads.
|
||||
from vllm.v1.worker.workspace import lock_workspace
|
||||
|
||||
lock_workspace()
|
||||
|
||||
qkv_kwargs = dict(dtype=ATTN_OUT_DTYPE, device=device)
|
||||
q = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
|
||||
k = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
|
||||
v = torch.randn(total_q, num_heads, V_HEAD_DIM, **qkv_kwargs)
|
||||
out = torch.zeros(
|
||||
total_q, num_heads * V_HEAD_DIM, dtype=ATTN_OUT_DTYPE, device=device
|
||||
)
|
||||
|
||||
impl = _make_impl(num_heads)
|
||||
impl._mla_fp8_prefill_attn(q, k, v, metadata, out)
|
||||
|
||||
out_ref = _reference(q, k, v)
|
||||
|
||||
assert torch.isfinite(out).all()
|
||||
torch.testing.assert_close(
|
||||
out.view(total_q, num_heads, V_HEAD_DIM).float(),
|
||||
out_ref.float(),
|
||||
atol=ATOL,
|
||||
rtol=RTOL,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
|
||||
def test_fp8_prefill_metadata_width_matches_forward(num_heads: int) -> None:
|
||||
"""The PS metadata must be built for the head count the kernel is handed.
|
||||
|
||||
``_init_fp8_prefill_ps_buffers``/``build()`` size the work and reduce maps
|
||||
from ``num_head_k``, while ``_mla_fp8_prefill_attn`` pads q/k/v to its own
|
||||
target. Divergence is not a numerical bug in either direction -- maps built
|
||||
narrower than the kernel width still cover the real heads, since padding
|
||||
only ever appends duplicate heads above them. It is a sizing bug: narrower
|
||||
maps mean a lower head alignment, hence more partial tiles and a larger
|
||||
reservation, while wider ones under-reserve the forward's scratch. Pin the
|
||||
two to one helper rather than to independent expressions.
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
|
||||
|
||||
width = AiterMLAHelper.get_fp8_prefill_num_heads(num_heads)
|
||||
assert width % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
|
||||
assert width >= num_heads
|
||||
assert width - num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS
|
||||
# Every count the enable gate accepts must have a usable padded width.
|
||||
assert AiterMLAHelper.is_valid_num_heads(num_heads)
|
||||
|
||||
# The builder must actually size its maps at that width. Assert against the
|
||||
# buffer it allocated rather than recomputing the width here, or the test
|
||||
# passes no matter what _init_fp8_prefill_ps_buffers does.
|
||||
#
|
||||
# fp8_ps_reduce_partial_map is sized by the partial-tile count, which is
|
||||
# gcd-driven and so is a sharp probe of the width used: a lower head
|
||||
# alignment yields *more* tiles. At 24 heads, sizing at the unpadded count
|
||||
# gives 4x the tiles (64 vs 16) and ~2.8x the scratch reservation (193.6
|
||||
# vs 68.6 MiB at batch=1/qlen=512) -- not wrong numerically (the 24-wide
|
||||
# maps still cover the real heads) but the reason to pin the widths
|
||||
# together.
|
||||
from aiter import get_ps_metadata_info_v1
|
||||
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import _FP8_PREFILL_TILE_Q
|
||||
|
||||
def _tiles(nhk: int) -> int:
|
||||
return get_ps_metadata_info_v1(
|
||||
batch_size=1,
|
||||
num_head_k=nhk,
|
||||
max_qlen=512,
|
||||
qlen_granularity=_FP8_PREFILL_TILE_Q,
|
||||
)[5][0]
|
||||
|
||||
_, _, builder = _build_prefill_metadata([512], torch.device("cuda"), num_heads)
|
||||
assert builder.fp8_ps_reduce_partial_map.numel() == _tiles(width)
|
||||
@@ -1,252 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ROCm AITER MLA FP8 support detection."""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_SKIP_UNSUPPORTED_AITER_HARDWARE = True
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
_SKIP_UNSUPPORTED_AITER_HARDWARE = get_cdna_version() <= 2
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_aiter_mla_support_cache(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
|
||||
monkeypatch.setattr(aiter_ops, "_AITER_MLA_SUPPORTS_FP8", None)
|
||||
|
||||
|
||||
def _install_fake_aiter_modules(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
supports_fp8: bool,
|
||||
supports_causal: bool = False,
|
||||
) -> None:
|
||||
aiter_mod: Any = types.ModuleType("aiter")
|
||||
mla_mod: Any = types.ModuleType("aiter.mla")
|
||||
|
||||
if supports_fp8:
|
||||
|
||||
def mla_decode_fwd_with_fp8(
|
||||
q,
|
||||
kv_buffer,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
o,
|
||||
sm_scale,
|
||||
q_scale=None,
|
||||
kv_scale=None,
|
||||
):
|
||||
return None
|
||||
|
||||
mla_decode_fwd: Any = mla_decode_fwd_with_fp8
|
||||
|
||||
else:
|
||||
|
||||
def mla_decode_fwd_without_fp8(
|
||||
q,
|
||||
kv_buffer,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
o,
|
||||
sm_scale,
|
||||
):
|
||||
return None
|
||||
|
||||
mla_decode_fwd = mla_decode_fwd_without_fp8
|
||||
|
||||
if supports_causal:
|
||||
inner = mla_decode_fwd
|
||||
|
||||
def mla_decode_fwd_with_causal(*args, causal=True, **kwargs):
|
||||
return inner(*args, **kwargs)
|
||||
|
||||
mla_decode_fwd = mla_decode_fwd_with_causal
|
||||
|
||||
mla_mod.mla_decode_fwd = mla_decode_fwd
|
||||
aiter_mod.mla = mla_mod
|
||||
|
||||
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
|
||||
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
|
||||
|
||||
|
||||
def test_aiter_mla_fp8_support_detects_fp8_signature(monkeypatch):
|
||||
"""The support check should detect q_scale and kv_scale parameters."""
|
||||
from vllm._aiter_ops import _check_aiter_mla_fp8_support
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
|
||||
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
|
||||
|
||||
def test_aiter_mla_fp8_support_rejects_missing_fp8_signature(monkeypatch):
|
||||
"""The support check should return False when FP8 params are absent."""
|
||||
from vllm._aiter_ops import _check_aiter_mla_fp8_support
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=False)
|
||||
|
||||
assert _check_aiter_mla_fp8_support() is False
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_SKIP_UNSUPPORTED_AITER_HARDWARE,
|
||||
reason="Installed AITER MLA FP8 check requires CDNA 3 or newer",
|
||||
)
|
||||
def test_installed_aiter_mla_supports_fp8():
|
||||
"""Supported ROCm CI must provide AITER with MLA FP8 scaling."""
|
||||
from vllm._aiter_ops import (
|
||||
_check_aiter_mla_fp8_support,
|
||||
is_aiter_found_and_supported,
|
||||
)
|
||||
|
||||
assert is_aiter_found_and_supported(), (
|
||||
"AITER must be installed on supported ROCm hardware"
|
||||
)
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error_type",
|
||||
[ImportError, ModuleNotFoundError, AttributeError, ValueError, TypeError],
|
||||
)
|
||||
def test_aiter_mla_fp8_support_handles_signature_errors(monkeypatch, error_type):
|
||||
"""The support check should fail closed on import or signature problems."""
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
from vllm._aiter_ops import _check_aiter_mla_fp8_support
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
|
||||
|
||||
with patch("inspect.signature", side_effect=error_type("boom")):
|
||||
assert _check_aiter_mla_fp8_support() is False
|
||||
assert aiter_ops._AITER_MLA_SUPPORTS_FP8 is False
|
||||
|
||||
|
||||
def test_aiter_mla_fp8_support_result_is_cached(monkeypatch):
|
||||
"""The support check should reuse the cached result on later calls."""
|
||||
import inspect
|
||||
|
||||
from vllm._aiter_ops import _check_aiter_mla_fp8_support
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
|
||||
|
||||
with patch("inspect.signature", wraps=inspect.signature) as signature_mock:
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
assert signature_mock.call_count == 1
|
||||
|
||||
|
||||
def test_missing_causal_arg_fails_closed(monkeypatch):
|
||||
"""Old aiter wheels have no causal=; do not report that as a valid draft path."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=False)
|
||||
with pytest.raises(RuntimeError, match="causal-only"):
|
||||
rocm_aiter_ops.mla_decode_supports_non_causal()
|
||||
with pytest.raises(RuntimeError, match="causal-only"):
|
||||
AiterMLABackend.supports_non_causal()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_kernels", [False, True])
|
||||
def test_supports_non_causal_requires_arch_kernels(monkeypatch, has_kernels):
|
||||
"""The Python argument alone does not guarantee an arch has the kernel."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_non_causal_asm_kernels",
|
||||
lambda: has_kernels,
|
||||
)
|
||||
assert rocm_aiter_ops.mla_decode_supports_non_causal() is True
|
||||
assert AiterMLABackend.supports_non_causal() is has_kernels
|
||||
|
||||
|
||||
@pytest.mark.parametrize("causal", [True, False])
|
||||
def test_decode_fwd_forwards_causal_to_aiter(monkeypatch, causal):
|
||||
"""The decode op must pass the block mask through; omitting it is causal-only."""
|
||||
seen: list = []
|
||||
|
||||
def mla_decode_fwd(*args, causal=None, **kwargs):
|
||||
seen.append(causal)
|
||||
|
||||
mla_mod: Any = types.ModuleType("aiter.mla")
|
||||
mla_mod.mla_decode_fwd = mla_decode_fwd
|
||||
aiter_mod: Any = types.ModuleType("aiter")
|
||||
aiter_mod.mla = mla_mod
|
||||
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
|
||||
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
|
||||
aiter_ops._rocm_aiter_mla_decode_fwd_impl(
|
||||
torch.zeros(2, 1, 8),
|
||||
torch.zeros(2, 1, 1, 8),
|
||||
torch.zeros(2, 1, 8),
|
||||
torch.zeros(3, dtype=torch.int32),
|
||||
1,
|
||||
causal=causal,
|
||||
)
|
||||
assert seen == [causal]
|
||||
|
||||
|
||||
def _aiter_mla_validate(
|
||||
monkeypatch,
|
||||
*,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
use_non_causal,
|
||||
):
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_non_causal_asm_kernels", lambda: True
|
||||
)
|
||||
return AiterMLABackend.validate_configuration(
|
||||
head_size=576,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
block_size=1,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(9, 5),
|
||||
attn_type=AttentionType.DECODER,
|
||||
use_non_causal=use_non_causal,
|
||||
)
|
||||
|
||||
|
||||
def test_non_causal_fp16_auto_cache_is_rejected(monkeypatch):
|
||||
"""Pinned AITER aborts fp16 Q; auto-select must fall through to Triton."""
|
||||
import torch
|
||||
|
||||
reasons = _aiter_mla_validate(
|
||||
monkeypatch,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="auto",
|
||||
use_non_causal=True,
|
||||
)
|
||||
assert any("non-causal fp16" in reason for reason in reasons)
|
||||
@@ -1,357 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Head padding and kernel selection for the ROCm AITER MLA backend.
|
||||
|
||||
The asm persistent decode requires a 16-aligned head count, so unaligned
|
||||
counts through 128 are tile-padded to the next multiple of 16 and sliced back
|
||||
off the output. Small divisor counts (1/2/4/8) preserve their existing
|
||||
repeat-interleave path and may keep the Gluon kernel on gfx950.
|
||||
"""
|
||||
|
||||
import math
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLADecodeMetadata,
|
||||
AiterMLAHelper,
|
||||
AiterMLAImpl,
|
||||
AiterMLAMetadata,
|
||||
)
|
||||
|
||||
NUM_HEADS = 12
|
||||
KV_LORA_RANK = 512
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
CONTEXT_LEN = 4096
|
||||
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
|
||||
|
||||
# Small non-divisor counts use tile-and-slice; divisors of 16 keep
|
||||
# repeat_interleave. Both pad to exactly 16 and round-trip.
|
||||
NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15]
|
||||
DIVISOR_HEADS = [1, 2, 4, 8]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _disable_native_h24(monkeypatch):
|
||||
"""Exercise the padding fallback unless a test explicitly enables H24."""
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_available() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available()
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not (current_platform.is_rocm() and is_aiter_found()):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
def _expected_tile_pad(q: torch.Tensor, num_heads: int, m: int = 16) -> torch.Tensor:
|
||||
# get_mla_padded_q tiles the heads and slices to m, i.e. head i of the
|
||||
# padded tensor is head (i % num_heads) of the input.
|
||||
idx = [i % num_heads for i in range(m)]
|
||||
return q[:, idx, :]
|
||||
|
||||
|
||||
def _make_h12_decode_metadata(device: torch.device) -> AiterMLAMetadata:
|
||||
page_indices = torch.arange(CONTEXT_LEN, dtype=torch.int32, device=device)
|
||||
decode = AiterMLADecodeMetadata(
|
||||
block_table=page_indices.view(1, -1),
|
||||
seq_lens=torch.tensor([CONTEXT_LEN], dtype=torch.int32, device=device),
|
||||
dcp_tot_seq_lens=None,
|
||||
paged_kv_indptr=torch.tensor(
|
||||
[0, CONTEXT_LEN], dtype=torch.int32, device=device
|
||||
),
|
||||
paged_kv_indices=page_indices,
|
||||
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32, device=device),
|
||||
qo_indptr=torch.tensor([0, 1], dtype=torch.int32, device=device),
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
max_qo_len=1,
|
||||
)
|
||||
return AiterMLAMetadata(
|
||||
num_reqs=1,
|
||||
max_query_len=1,
|
||||
max_seq_len=CONTEXT_LEN,
|
||||
num_actual_tokens=1,
|
||||
query_start_loc=decode.qo_indptr,
|
||||
slot_mapping=torch.tensor([CONTEXT_LEN - 1], dtype=torch.int64, device=device),
|
||||
num_decodes=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefills=0,
|
||||
decode=decode,
|
||||
)
|
||||
|
||||
|
||||
def test_h12_query_is_tile_padded_to_h16():
|
||||
q = torch.arange(2 * 12 * 4, dtype=torch.bfloat16).view(2, 12, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(12, q)
|
||||
|
||||
assert padded_q.shape == (2, 16, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
# The real heads are untouched...
|
||||
torch.testing.assert_close(padded_q[:, :12], q)
|
||||
# ...and the 4 padding heads are the tiled wrap-around (heads 0..3), not
|
||||
# zeros: MLA attention is per-head independent so duplicate query heads are
|
||||
# harmless and get sliced back off the output.
|
||||
torch.testing.assert_close(padded_q[:, 12:], q[:, :4])
|
||||
|
||||
|
||||
def test_h6_tp16_query_is_padded_to_h16():
|
||||
# TP16 puts 6 heads/rank. The old append-only padding produced cat(6, 6) =
|
||||
# 12 heads and broke the asm kernel; tile-and-slice must reach exactly 16.
|
||||
q = torch.arange(2 * 6 * 4, dtype=torch.bfloat16).view(2, 6, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(6, q)
|
||||
|
||||
assert padded_q.shape == (2, 16, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(padded_q, _expected_tile_pad(q, 6))
|
||||
|
||||
|
||||
def test_h12_output_discards_padding_heads():
|
||||
o = torch.arange(2 * 16 * 4, dtype=torch.bfloat16).view(2, 16, 4)
|
||||
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(12, o)
|
||||
|
||||
assert unpadded_o.shape == (2, 12, 4)
|
||||
torch.testing.assert_close(unpadded_o, o[:, :12])
|
||||
|
||||
|
||||
def test_h24_query_is_tile_padded_to_h32():
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
|
||||
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(padded_q[:, :24], q)
|
||||
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
|
||||
|
||||
|
||||
def test_h24_output_discards_h32_padding_heads():
|
||||
o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
|
||||
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o)
|
||||
|
||||
assert unpadded_o.shape == (2, 24, 4)
|
||||
torch.testing.assert_close(unpadded_o, o[:, :24])
|
||||
|
||||
|
||||
def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_native_h24_supported",
|
||||
lambda: (
|
||||
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
|
||||
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
|
||||
),
|
||||
)
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
torch.testing.assert_close(padded_q[:, :24], q)
|
||||
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
|
||||
|
||||
|
||||
def test_native_h24_requires_reducer_and_metadata(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_native_h24_supported",
|
||||
lambda: (
|
||||
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
|
||||
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
|
||||
),
|
||||
)
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24
|
||||
assert AiterMLAHelper.get_mla_padded_q(24, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q
|
||||
|
||||
|
||||
def test_existing_divisor_head_mapping_is_unchanged():
|
||||
q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(8, q)
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(8, padded_q)
|
||||
|
||||
# Divisor counts still use repeat_interleave / strided unpad, unchanged.
|
||||
torch.testing.assert_close(padded_q, q.repeat_interleave(2, dim=1))
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [17, 24, 31])
|
||||
def test_unaligned_head_counts_round_trip_through_h32(num_heads: int):
|
||||
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
|
||||
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS)
|
||||
def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
|
||||
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
|
||||
|
||||
assert padded_q.shape == (2, 16, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
# The real heads survive the pad -> unpad round trip exactly.
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
def test_aligned_h32_is_zero_copy():
|
||||
q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
|
||||
assert AiterMLAHelper.get_mla_padded_q(32, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q
|
||||
|
||||
|
||||
def test_is_valid_num_heads():
|
||||
for n in range(1, 129):
|
||||
assert AiterMLAHelper.is_valid_num_heads(n)
|
||||
assert AiterMLAHelper.is_valid_num_heads(24)
|
||||
assert AiterMLAHelper.is_valid_num_heads(127)
|
||||
# Aligned counts remain valid above the range where padding is supported.
|
||||
assert AiterMLAHelper.is_valid_num_heads(144)
|
||||
assert not AiterMLAHelper.is_valid_num_heads(0)
|
||||
assert not AiterMLAHelper.is_valid_num_heads(129)
|
||||
|
||||
|
||||
def test_nondivisor_and_multitoken_never_use_gluon():
|
||||
# Non-divisor decode always takes the asm path (12 heads/rank at TP8).
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto")
|
||||
# >=16 heads never use Gluon, including unaligned counts padded for asm.
|
||||
assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto")
|
||||
# Multi-token (verify / qlen>1) is never the single-token Gluon decode.
|
||||
assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto")
|
||||
|
||||
|
||||
def test_divisor_gluon_selection_follows_arch():
|
||||
# Divisor head counts keep Gluon only where the kernel exists (gfx950).
|
||||
# On gfx942 / non-ROCm they must route to the asm persistent decode.
|
||||
on_gfx950 = _on_gfx950()
|
||||
assert AiterMLAHelper.use_gluon_decode(8, 1, "auto") is on_gfx950
|
||||
assert AiterMLAHelper.use_gluon_decode(4, 1, "auto") is on_gfx950
|
||||
assert AiterMLAHelper.use_gluon_decode(1, 1, "auto") is on_gfx950
|
||||
|
||||
|
||||
def test_asm_padding_env_default_is_auto(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", raising=False)
|
||||
import vllm.envs as envs
|
||||
|
||||
assert envs.VLLM_ROCM_AITER_MLA_ASM_PADDING == "auto"
|
||||
|
||||
|
||||
def test_asm_padding_env_force_asm_disables_gluon(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "asm")
|
||||
# Forcing the asm path: no small-head count uses Gluon on any arch.
|
||||
for num_heads in (1, 2, 4, 8, 12):
|
||||
assert not AiterMLAHelper.use_gluon_decode(num_heads, 1, "auto")
|
||||
|
||||
|
||||
def test_asm_padding_env_force_gluon_follows_arch(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "gluon")
|
||||
on_gfx950 = _on_gfx950()
|
||||
# Forcing Gluon: any 1..15 single-token decode uses it where a build exists
|
||||
# (gfx950), including non-divisor counts like 12; gfx942/non-ROCm still
|
||||
# falls back to the asm path.
|
||||
for num_heads in (1, 2, 4, 8, 12):
|
||||
assert AiterMLAHelper.use_gluon_decode(num_heads, 1, "auto") is on_gfx950
|
||||
|
||||
|
||||
def test_asm_padding_env_auto_matches_arch_gate(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "auto")
|
||||
on_gfx950 = _on_gfx950()
|
||||
# auto: divisor counts keep Gluon on gfx950, non-divisor counts take asm.
|
||||
assert AiterMLAHelper.use_gluon_decode(8, 1, "auto") is on_gfx950
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _rocm_aiter_available(),
|
||||
reason="12-head AITER MLA asm persistent decode needs ROCm + AITER",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_h12_aiter_mla_decode_matches_reference():
|
||||
"""The 12-head AITER decode output must match an attention reference.
|
||||
|
||||
12 is a non-divisor of 16 so use_gluon_decode is False on every arch; the
|
||||
query is tile-padded to 16 heads and served by the asm persistent decode
|
||||
(the CDNA3 path this PR enables, and unchanged on CDNA4).
|
||||
"""
|
||||
device = torch.device("cuda:0")
|
||||
torch.manual_seed(42)
|
||||
|
||||
q = torch.randn(1, NUM_HEADS, QK_HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
kv_cache = torch.randn(
|
||||
CONTEXT_LEN, 1, QK_HEAD_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = NUM_HEADS
|
||||
impl.dcp_world_size = 1
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
|
||||
impl.scale = SCALE
|
||||
impl.kv_cache_dtype = "auto"
|
||||
|
||||
one = torch.ones(1, dtype=torch.float32, device=device)
|
||||
layer = types.SimpleNamespace(_q_scale=one, _k_scale=one)
|
||||
metadata = _make_h12_decode_metadata(device)
|
||||
|
||||
out, _ = impl.forward_mqa(q, kv_cache, metadata, layer)
|
||||
|
||||
assert out.shape[1] == NUM_HEADS # padding heads sliced back off
|
||||
|
||||
key = kv_cache[:, 0].float().unsqueeze(0)
|
||||
value = key[..., :KV_LORA_RANK]
|
||||
out_ref = F.scaled_dot_product_attention(
|
||||
q[0].float().unsqueeze(1),
|
||||
key,
|
||||
value,
|
||||
scale=SCALE,
|
||||
enable_gqa=True,
|
||||
).squeeze(1)
|
||||
|
||||
assert torch.isfinite(out).all()
|
||||
torch.testing.assert_close(
|
||||
out.float(),
|
||||
out_ref.unsqueeze(0),
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
@@ -1,523 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for ROCm AITER sparse MLA attention sinks."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
Q_HEAD_DIM = 576
|
||||
V_HEAD_DIM = 512
|
||||
SM_SCALE = Q_HEAD_DIM**-0.5
|
||||
|
||||
|
||||
def _gamma_fp32(operations: int) -> float:
|
||||
u = torch.finfo(torch.float32).eps / 2
|
||||
return operations * u / (1 - operations * u)
|
||||
|
||||
|
||||
def _sink_reference(q, keys, sinks, scale):
|
||||
# Reference the stored operands, so input quantization is not kernel error.
|
||||
q, keys, sinks = q.double(), keys.double(), sinks.double()
|
||||
scores = q @ keys.T * scale
|
||||
logits = torch.cat((scores, sinks[:, None]), dim=-1)
|
||||
probabilities = logits.softmax(dim=-1)[:, :-1]
|
||||
values = keys[:, :V_HEAD_DIM]
|
||||
expected = probabilities @ values
|
||||
value_scale = probabilities @ values.abs()
|
||||
# FP32 dot accumulation and conversion/multiplication of the scale.
|
||||
score_error = _gamma_fp32(q.shape[-1] + 2) * (q.abs() @ keys.abs().T) * abs(scale)
|
||||
score_error = score_error.amax(dim=-1) if keys.shape[0] else torch.zeros_like(sinks)
|
||||
return expected, logits.logsumexp(dim=-1), value_scale, score_error
|
||||
|
||||
|
||||
def _assert_sink_output_close(
|
||||
actual, expected, value_scale, score_error, probability_dtype, native
|
||||
):
|
||||
assert actual.shape == expected.shape
|
||||
# Budget the identifiable rounding stages against the absolute value
|
||||
# contributions: output-relative ULPs become unbounded when values cancel.
|
||||
# Rounding P before P@V costs u(P) * sum(P*abs(V)).
|
||||
# AITER rounds output before and after sink scaling; Triton rounds once.
|
||||
u_probability = torch.finfo(probability_dtype).eps / 2
|
||||
u_output = torch.finfo(actual.dtype).eps / 2
|
||||
output_rounds = 2 if native else 1
|
||||
amplification = (1 + u_output) ** output_rounds
|
||||
relative_error = (1 + u_probability) * torch.exp(2 * score_error) - 1
|
||||
allowance = (
|
||||
relative_error.unsqueeze(-1) * value_scale * amplification
|
||||
+ (amplification - 1) * expected.abs()
|
||||
+ output_rounds * torch.finfo(actual.dtype).tiny * u_output
|
||||
)
|
||||
actual = actual.to(device=expected.device, dtype=torch.float64)
|
||||
assert torch.isfinite(actual).all()
|
||||
assert torch.count_nonzero(actual[value_scale == 0]) == 0
|
||||
error = (actual - expected).abs()
|
||||
max_ratio = (error / allowance.clamp_min(1e-300)).max().item()
|
||||
assert max_ratio <= 1, f"exceeded rounding budget by {max_ratio:.3f}x"
|
||||
|
||||
|
||||
def _require_aiter() -> None:
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
if get_cdna_version() not in (3, 4):
|
||||
pytest.skip("AITER MLA requires CDNA 3 or 4")
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip("aiter is required on supported ROCm hardware for this test")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("real_heads", "cache_kind"),
|
||||
[
|
||||
(1, "bf16"),
|
||||
(4, "bf16"),
|
||||
(8, "bf16"),
|
||||
(8, "fp8"),
|
||||
(32, "fp8"),
|
||||
(64, "fp8"),
|
||||
(12, "bf16"),
|
||||
(16, "bf16"),
|
||||
(24, "bf16"),
|
||||
(32, "bf16"),
|
||||
(40, "bf16"),
|
||||
(48, "bf16"),
|
||||
(64, "bf16"),
|
||||
(80, "bf16"),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_sparse_mla_sink_matches_ragged_reference(
|
||||
real_heads: int, cache_kind: str
|
||||
) -> None:
|
||||
"""Exercise ragged sink decode, head padding, and FP8 scale forwarding."""
|
||||
_require_aiter()
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
set_random_seed(real_heads * 17 + (cache_kind == "fp8"))
|
||||
device = torch.device("cuda")
|
||||
seq_lens = [0, 1, 5, 23, 17]
|
||||
batch_size = len(seq_lens)
|
||||
|
||||
q_source = torch.randn(batch_size, real_heads, Q_HEAD_DIM, device=device) * 0.2
|
||||
|
||||
pool_size = sum(seq_lens) + 52
|
||||
kv_source = torch.randn(pool_size, 1, Q_HEAD_DIM, device=device) * 0.2
|
||||
if cache_kind == "fp8":
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
q_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
|
||||
kv_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
|
||||
q_real = (q_source / q_scale).to(fp8_dtype)
|
||||
kv = (kv_source / kv_scale).to(fp8_dtype)
|
||||
q_ref = q_real.float() * q_scale
|
||||
kv_ref = kv.float() * kv_scale
|
||||
else:
|
||||
q_scale = kv_scale = None
|
||||
q_real = q_source.to(torch.bfloat16)
|
||||
kv = kv_source.to(torch.bfloat16)
|
||||
q_ref = q_real.float()
|
||||
kv_ref = kv.float()
|
||||
q = AiterMLAHelper.get_mla_padded_q(real_heads, q_real)
|
||||
|
||||
indices = torch.randperm(pool_size, device=device)[: sum(seq_lens)].to(torch.int32)
|
||||
kv_indptr = torch.tensor(
|
||||
[0] + [sum(seq_lens[:i]) for i in range(1, len(seq_lens) + 1)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Non-None garbage proves the sink path cannot accidentally select the
|
||||
# gfx942 persistent kernel, which has no return-LSE code object.
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
qo_indptr=torch.arange(batch_size + 1, dtype=torch.int32, device=device),
|
||||
paged_kv_indptr=kv_indptr,
|
||||
paged_kv_indices=indices,
|
||||
paged_kv_last_page_len=torch.ones(batch_size, dtype=torch.int32, device=device),
|
||||
work_meta_data=torch.tensor([123], dtype=torch.int32),
|
||||
work_indptr=None,
|
||||
work_info_set=None,
|
||||
reduce_indptr=None,
|
||||
reduce_final_map=None,
|
||||
reduce_partial_map=None,
|
||||
num_prefills=0,
|
||||
num_decodes=batch_size,
|
||||
num_decode_tokens=batch_size,
|
||||
max_query_len=1,
|
||||
max_seq_len=max(seq_lens),
|
||||
topk_tokens=max(seq_lens),
|
||||
)
|
||||
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
|
||||
|
||||
impl = object.__new__(ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = real_heads
|
||||
impl.kv_lora_rank = V_HEAD_DIM
|
||||
impl.kv_cache_dtype = cache_kind
|
||||
impl.scale = SM_SCALE
|
||||
impl.sinks = sinks
|
||||
|
||||
output, lse = impl._forward_mla(
|
||||
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale),
|
||||
q,
|
||||
kv,
|
||||
metadata,
|
||||
)
|
||||
kv_flat = kv_ref[:, 0]
|
||||
references = []
|
||||
start = 0
|
||||
for batch_idx, seq_len in enumerate(seq_lens):
|
||||
rows = kv_flat[indices[start : start + seq_len].long()]
|
||||
references.append(_sink_reference(q_ref[batch_idx], rows, sinks, SM_SCALE))
|
||||
start += seq_len
|
||||
expected, expected_lse, value_scale, score_error = (
|
||||
torch.stack(values) for values in zip(*references)
|
||||
)
|
||||
|
||||
assert output.dtype == torch.bfloat16
|
||||
if lse is not None:
|
||||
assert lse.dtype == torch.float32
|
||||
assert lse.shape == expected_lse.shape
|
||||
lse_error = (lse.double() - expected_lse).abs()
|
||||
rounded_lse = expected_lse.float()
|
||||
positive_inf = torch.full_like(rounded_lse, float("inf"))
|
||||
ulp = torch.maximum(
|
||||
torch.nextafter(rounded_lse, positive_inf) - rounded_lse,
|
||||
rounded_lse - torch.nextafter(rounded_lse, -positive_inf),
|
||||
).double()
|
||||
lse_allowance = score_error + 2 * ulp
|
||||
assert torch.all(lse_error <= lse_allowance)
|
||||
torch.testing.assert_close(lse[0].double(), expected_lse[0], atol=0, rtol=0)
|
||||
else:
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
assert cache_kind == "bf16" and real_heads in (40, 48, 64) and on_gfx942()
|
||||
_assert_sink_output_close(
|
||||
output,
|
||||
expected,
|
||||
value_scale,
|
||||
score_error,
|
||||
q_real.dtype,
|
||||
native=lse is not None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("q_dtype", "kv_dtype"),
|
||||
[
|
||||
(torch.float16, torch.bfloat16),
|
||||
(torch.bfloat16, torch.float16),
|
||||
],
|
||||
)
|
||||
def test_sparse_mla_sink_rejects_unsupported_aiter_dtypes(
|
||||
q_dtype: torch.dtype, kv_dtype: torch.dtype
|
||||
) -> None:
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
impl = object.__new__(ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = 16
|
||||
impl.kv_lora_rank = V_HEAD_DIM
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.scale = SM_SCALE
|
||||
impl.sinks = torch.zeros(16, dtype=torch.float32)
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
num_prefills=0,
|
||||
num_decodes=1,
|
||||
num_decode_tokens=1,
|
||||
max_query_len=1,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="both use BF16 or both use FP8"):
|
||||
impl._forward_mla(
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
torch.empty(1, 16, Q_HEAD_DIM, dtype=q_dtype),
|
||||
torch.empty(1, 1, Q_HEAD_DIM, dtype=kv_dtype),
|
||||
metadata,
|
||||
)
|
||||
|
||||
|
||||
def _make_noncontiguous_sink() -> torch.Tensor:
|
||||
return torch.empty(8, dtype=torch.float32)[::2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sinks", "match"),
|
||||
[
|
||||
(torch.empty(4, dtype=torch.bfloat16), "must be float32"),
|
||||
(torch.empty(2, 2, dtype=torch.float32), "must have shape"),
|
||||
(_make_noncontiguous_sink(), "must be contiguous"),
|
||||
],
|
||||
)
|
||||
def test_sparse_mla_sink_validation(sinks: torch.Tensor, match: str) -> None:
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=match):
|
||||
ROCMAiterMLASparseImpl(
|
||||
num_heads=4,
|
||||
head_size=Q_HEAD_DIM,
|
||||
scale=SM_SCALE,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="bfloat16",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
sinks=sinks,
|
||||
kv_lora_rank=V_HEAD_DIM,
|
||||
)
|
||||
|
||||
|
||||
def test_sparse_mla_backend_reports_sink_support_for_current_hardware() -> None:
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
assert ROCMAiterMLASparseBackend.supports_sink() == (get_cdna_version() in (3, 4))
|
||||
|
||||
|
||||
def test_sparse_mla_backend_rejects_dcp() -> None:
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
|
||||
selector_config = AttentionSelectorConfig(
|
||||
head_size=Q_HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="bfloat16",
|
||||
block_size=16,
|
||||
use_mla=True,
|
||||
has_sink=True,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
attn_type="decoder",
|
||||
use_dcp=True,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="DCP not supported"):
|
||||
RocmPlatform.get_attn_backend_cls(
|
||||
selected_backend=AttentionBackendEnum.ROCM_AITER_MLA_SPARSE,
|
||||
attn_selector_config=selector_config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads,block_size", [(8, 16), (20, 32)])
|
||||
@pytest.mark.parametrize("interleaved_pages", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache(
|
||||
num_heads: int,
|
||||
block_size: int,
|
||||
interleaved_pages: bool,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Preserve head-specific sinks and latent values across ragged cache pages."""
|
||||
_require_aiter()
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
|
||||
|
||||
head_dim, value_dim = 576, 512
|
||||
generator = torch.Generator().manual_seed(421)
|
||||
# The empty row exercises sink-only attention; the last row crosses both
|
||||
# page boundaries and the kernel's 16-key reduction tile.
|
||||
selected_rows = [
|
||||
[],
|
||||
[(2, 7)],
|
||||
[(3, 1), (0, block_size - 1), (1, 0), (2, 4)],
|
||||
[(i % 4, (i * 7) % block_size) for i in range(23)],
|
||||
]
|
||||
q_cpu = torch.randn(
|
||||
len(selected_rows), num_heads, head_dim, generator=generator
|
||||
).to(dtype)
|
||||
cache_cpu = torch.randn(4, block_size, head_dim, generator=generator).to(dtype)
|
||||
sinks_cpu = torch.linspace(-4.0, 6.0, num_heads)
|
||||
|
||||
# Layers can share a backing allocation, leaving unused rows between pages.
|
||||
num_layers = 3 if interleaved_pages else 1
|
||||
backing = torch.full(
|
||||
(4, num_layers, block_size, head_dim),
|
||||
float("nan"),
|
||||
dtype=dtype,
|
||||
device="cuda",
|
||||
)
|
||||
cache = backing[:, num_layers - 1]
|
||||
cache.copy_(cache_cpu)
|
||||
indices = [
|
||||
page * num_layers * block_size + offset
|
||||
for rows in selected_rows
|
||||
for page, offset in rows
|
||||
]
|
||||
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
|
||||
longest_row = max(len(rows) for rows in selected_rows)
|
||||
metadata = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
num_prefills=0,
|
||||
num_decodes=len(selected_rows),
|
||||
num_decode_tokens=len(selected_rows),
|
||||
max_query_len=1,
|
||||
max_seq_len=longest_row,
|
||||
topk_tokens=longest_row,
|
||||
qo_indptr=torch.arange(
|
||||
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
paged_kv_last_page_len=torch.ones(
|
||||
len(selected_rows), dtype=torch.int32, device="cuda"
|
||||
),
|
||||
work_meta_data=None,
|
||||
attn_out_dtype=dtype,
|
||||
paged_kv_indices=torch.tensor(indices, dtype=torch.int32, device="cuda"),
|
||||
paged_kv_indptr=lengths.cumsum(0).to(device="cuda", dtype=torch.int32),
|
||||
)
|
||||
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = num_heads
|
||||
impl.head_size = head_dim
|
||||
impl.kv_lora_rank = value_dim
|
||||
impl.scale = head_dim**-0.5
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.sinks = sinks_cpu.cuda()
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q_cpu.cuda())
|
||||
|
||||
kv_rows, _ = flat_kv_row_view(cache, block_size)
|
||||
actual, _ = impl._forward_mla(
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
padded_q,
|
||||
kv_rows.unsqueeze(1),
|
||||
metadata,
|
||||
)
|
||||
|
||||
references = []
|
||||
for query_idx, rows in enumerate(selected_rows):
|
||||
keys = (
|
||||
torch.stack([cache_cpu[page, offset] for page, offset in rows])
|
||||
if rows
|
||||
else cache_cpu.new_empty((0, head_dim))
|
||||
)
|
||||
references.append(
|
||||
_sink_reference(q_cpu[query_idx], keys, sinks_cpu, impl.scale)
|
||||
)
|
||||
expected, _, value_scale, score_error = (
|
||||
torch.stack(values) for values in zip(*references)
|
||||
)
|
||||
assert actual.shape == (len(selected_rows), num_heads, value_dim)
|
||||
assert actual.dtype == dtype
|
||||
_assert_sink_output_close(
|
||||
actual,
|
||||
expected,
|
||||
value_scale,
|
||||
score_error,
|
||||
dtype,
|
||||
native=dtype == torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC"])
|
||||
def test_sparse_mla_backend_resolves_only_contiguous_layer_layouts(monkeypatch, layout):
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.setenv("VLLM_KV_CACHE_LAYOUT", layout)
|
||||
config = SimpleNamespace(cache_config=CacheConfig())
|
||||
supported = [
|
||||
[x.name for x in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()]
|
||||
]
|
||||
if layout == "BLNHC":
|
||||
with pytest.raises(ValueError, match="does not satisfy"):
|
||||
resolve_kv_cache_layout(config, supported)
|
||||
else:
|
||||
assert resolve_kv_cache_layout(config, supported).name == layout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
"""The public sparse forward joins latent/RoPE queries in the model dtype."""
|
||||
_require_aiter()
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
set_random_seed(412)
|
||||
num_tokens, num_heads, block_size = 2, 8, 16
|
||||
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
|
||||
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
|
||||
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
|
||||
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
|
||||
impl = object.__new__(ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = num_heads
|
||||
impl.kv_lora_rank = V_HEAD_DIM
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.scale = SM_SCALE
|
||||
impl.sinks = sinks
|
||||
impl.topk_indices_buffer = torch.full(
|
||||
(num_tokens, 128), -1, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
impl.topk_indices_buffer[:, : selected.shape[1]] = selected
|
||||
impl.owns_indexer = True
|
||||
impl.index_epoch = 0
|
||||
impl.q_concat_buffer = torch.empty_like(q)
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=dtype,
|
||||
num_actual_tokens=num_tokens,
|
||||
num_prefills=0,
|
||||
num_decodes=num_tokens,
|
||||
num_decode_tokens=num_tokens,
|
||||
max_query_len=1,
|
||||
max_seq_len=kv.shape[0],
|
||||
block_size=block_size,
|
||||
topk_tokens=impl.topk_indices_buffer.shape[1],
|
||||
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32, device="cuda"),
|
||||
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32, device="cuda"),
|
||||
paged_kv_indptr=torch.tensor([0, 3, 6], dtype=torch.int32, device="cuda"),
|
||||
paged_kv_indices=torch.empty(
|
||||
selected.numel(), dtype=torch.int32, device="cuda"
|
||||
),
|
||||
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32, device="cuda"),
|
||||
work_meta_data=None,
|
||||
)
|
||||
actual, _ = impl.forward_mqa(
|
||||
(q[..., :V_HEAD_DIM], q[..., V_HEAD_DIM:]),
|
||||
kv,
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
)
|
||||
references = [
|
||||
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
|
||||
for i in range(num_tokens)
|
||||
]
|
||||
expected, _, value_scale, score_error = (
|
||||
torch.stack(values) for values in zip(*references)
|
||||
)
|
||||
assert actual.dtype == dtype
|
||||
_assert_sink_output_close(
|
||||
actual,
|
||||
expected,
|
||||
value_scale,
|
||||
score_error,
|
||||
dtype,
|
||||
native=dtype == torch.bfloat16,
|
||||
)
|
||||
@@ -1,295 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"ROCm AITER sparse MLA index remap test requires ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip(
|
||||
"ROCm AITER sparse MLA index remap test requires a supported AITER "
|
||||
"installation.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.attention import sparse_mla_attention as sparse_mla_mod
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
|
||||
|
||||
NUM_HEADS = 16
|
||||
KV_LORA_RANK = 512
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
HEAD_SIZE = 576
|
||||
NUM_TOKENS = 2
|
||||
TOPK_TOKENS = 4
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_index_epochs():
|
||||
# The counter is keyed by id(), and a freed test buffer can hand its id to
|
||||
# the next one, so a leftover count would leak across tests.
|
||||
sparse_mod._INDEX_EPOCHS.clear()
|
||||
yield
|
||||
sparse_mod._INDEX_EPOCHS.clear()
|
||||
|
||||
|
||||
def _make_metadata():
|
||||
return SimpleNamespace(
|
||||
num_actual_tokens=NUM_TOKENS,
|
||||
req_id_per_token=torch.zeros(NUM_TOKENS, dtype=torch.int32, device="cpu"),
|
||||
block_table=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
|
||||
paged_kv_indptr=torch.zeros(NUM_TOKENS + 1, dtype=torch.int32, device="cpu"),
|
||||
paged_kv_indices=torch.zeros(
|
||||
NUM_TOKENS * TOPK_TOKENS, dtype=torch.int32, device="cpu"
|
||||
),
|
||||
block_size=1,
|
||||
topk_tokens=TOPK_TOKENS,
|
||||
remapped_buf=None,
|
||||
remapped_epoch=-1,
|
||||
)
|
||||
|
||||
|
||||
def _patch_init_deps(monkeypatch, *, speculative: bool = False):
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper,
|
||||
"check_num_heads_validity",
|
||||
staticmethod(lambda num_heads: None),
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=NUM_TOKENS),
|
||||
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||
speculative_config=SimpleNamespace() if speculative else None,
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
)
|
||||
monkeypatch.setattr(sparse_mod, "get_current_vllm_config", lambda: vllm_config)
|
||||
monkeypatch.setattr(sparse_mla_mod, "get_current_vllm_config", lambda: vllm_config)
|
||||
monkeypatch.setattr(
|
||||
sparse_mla_mod, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"current_workspace_manager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simultaneous=lambda *specs: tuple(
|
||||
torch.zeros(shape, dtype=dtype, device="cpu") for shape, dtype in specs
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_impl(indexer, topk_indices_buffer):
|
||||
return sparse_mod.ROCMAiterMLASparseImpl(
|
||||
num_heads=NUM_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
scale=1.0,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="auto",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
indexer=indexer,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_b_proj=None,
|
||||
)
|
||||
|
||||
|
||||
def _new_buffer():
|
||||
return torch.zeros(NUM_TOKENS, TOPK_TOKENS, dtype=torch.int32, device="cpu")
|
||||
|
||||
|
||||
def _patch_forward_deps(monkeypatch, impl, remap_calls):
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"triton_convert_req_index_to_global_index",
|
||||
lambda *args, **kwargs: remap_calls.append(args),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper,
|
||||
"get_mla_padded_q",
|
||||
staticmethod(lambda num_heads, q: q),
|
||||
)
|
||||
attn_out = torch.zeros(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK, device="cpu")
|
||||
monkeypatch.setattr(
|
||||
type(impl),
|
||||
"_forward_mla",
|
||||
lambda self, layer, q, kv_cache, attn_metadata: (attn_out, None),
|
||||
)
|
||||
return attn_out
|
||||
|
||||
|
||||
def _run_forward(impl, metadata):
|
||||
return impl.forward_mqa(
|
||||
torch.zeros(NUM_TOKENS, NUM_HEADS, HEAD_SIZE, device="cpu"),
|
||||
torch.zeros(4, 1, HEAD_SIZE, device="cpu"),
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_indexer", [True, False])
|
||||
def test_owns_indexer_is_structural(monkeypatch, has_indexer):
|
||||
"""owns_indexer reflects the layer, not the serving config."""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
indexer = SimpleNamespace(topk_indices_buffer=buffer) if has_indexer else None
|
||||
|
||||
impl = _build_impl(indexer, buffer)
|
||||
|
||||
assert impl.owns_indexer is has_indexer
|
||||
assert impl.topk_indices_buffer is buffer
|
||||
|
||||
|
||||
def test_forward_mqa_remaps_once_per_metadata(monkeypatch):
|
||||
"""Skip layers sharing a metadata share its remap; the first one pays.
|
||||
|
||||
All layers read one top-k buffer, so the reuse has to key off the metadata
|
||||
that owns ``paged_kv_indices``, not off the buffer being distinct.
|
||||
"""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
first = _build_impl(None, buffer)
|
||||
second = _build_impl(None, buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
attn_out = _patch_forward_deps(monkeypatch, first, remap_calls)
|
||||
metadata = _make_metadata()
|
||||
|
||||
out, lse = _run_forward(first, metadata)
|
||||
assert out is attn_out
|
||||
assert lse is None
|
||||
assert len(remap_calls) == 1
|
||||
assert metadata.remapped_buf is buffer
|
||||
|
||||
_run_forward(second, metadata)
|
||||
assert len(remap_calls) == 1
|
||||
|
||||
|
||||
def test_forward_mqa_remaps_for_each_metadata(monkeypatch):
|
||||
"""Skip layers on a second metadata must remap its own paged_kv_indices.
|
||||
|
||||
Regression: gating on ``indexer is not None`` skipped every layer in the
|
||||
non-indexer metadata group, leaving its ``paged_kv_indices`` unwritten.
|
||||
"""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
indexer_layer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
skip_layer = _build_impl(None, buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
_patch_forward_deps(monkeypatch, indexer_layer, remap_calls)
|
||||
|
||||
indexer_metadata = _make_metadata()
|
||||
_run_forward(indexer_layer, indexer_metadata)
|
||||
assert len(remap_calls) == 1
|
||||
assert indexer_metadata.remapped_buf is buffer
|
||||
|
||||
# A distinct metadata instance carries its own, still-unwritten indices.
|
||||
skip_metadata = _make_metadata()
|
||||
_run_forward(skip_layer, skip_metadata)
|
||||
assert len(remap_calls) == 2
|
||||
assert skip_metadata.remapped_buf is buffer
|
||||
|
||||
|
||||
def test_forward_mqa_remaps_after_a_later_indexer_write(monkeypatch):
|
||||
"""A skip metadata must be refreshed once per indexer write, not once ever.
|
||||
|
||||
Regression for cross-layer index sharing (``index_topk_freq``). With
|
||||
freq=4 on a 78-layer model the 57 skip layers share one metadata while the
|
||||
21 indexer layers share another, so the skip metadata has to pick up every
|
||||
new selection. Recording only the buffer left layers 4 onwards attending
|
||||
to the first selection of the pass, which collapsed RULER niah_single_2 to
|
||||
0.026 on GLM-5.3 and took GPQA from 0.884 to 0.672.
|
||||
"""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
# Construction order mirrors the layer stack: indexer, three skips, indexer.
|
||||
_build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
first_skip = _build_impl(None, buffer)
|
||||
reuse_skip = _build_impl(None, buffer)
|
||||
_build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
next_skip = _build_impl(None, buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
_patch_forward_deps(monkeypatch, first_skip, remap_calls)
|
||||
skip_metadata = _make_metadata()
|
||||
|
||||
_run_forward(first_skip, skip_metadata)
|
||||
assert len(remap_calls) == 1
|
||||
|
||||
# Same selection, same metadata: reuse.
|
||||
_run_forward(reuse_skip, skip_metadata)
|
||||
assert len(remap_calls) == 1
|
||||
|
||||
# A later indexer layer has since rewritten the buffer in place. The object
|
||||
# is unchanged, so only the epoch can reveal it.
|
||||
_run_forward(next_skip, skip_metadata)
|
||||
assert len(remap_calls) == 2
|
||||
assert skip_metadata.remapped_epoch == next_skip.index_epoch
|
||||
|
||||
|
||||
def test_index_epoch_tracks_the_preceding_indexer_write(monkeypatch):
|
||||
"""Layers between two indexer layers share the earlier one's epoch."""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
first_indexer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
skip_a = _build_impl(None, buffer)
|
||||
skip_b = _build_impl(None, buffer)
|
||||
second_indexer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
skip_c = _build_impl(None, buffer)
|
||||
|
||||
assert skip_a.index_epoch == first_indexer.index_epoch
|
||||
assert skip_b.index_epoch == first_indexer.index_epoch
|
||||
assert second_indexer.index_epoch == first_indexer.index_epoch + 1
|
||||
assert skip_c.index_epoch == second_indexer.index_epoch
|
||||
|
||||
|
||||
def test_forward_mqa_remaps_when_buffer_differs(monkeypatch):
|
||||
"""A layer reading a different buffer cannot reuse the metadata's remap."""
|
||||
_patch_init_deps(monkeypatch)
|
||||
remapped_buffer = _new_buffer()
|
||||
other_buffer = _new_buffer()
|
||||
layer = _build_impl(None, other_buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
_patch_forward_deps(monkeypatch, layer, remap_calls)
|
||||
metadata = _make_metadata()
|
||||
metadata.remapped_buf = remapped_buffer
|
||||
|
||||
_run_forward(layer, metadata)
|
||||
assert len(remap_calls) == 1
|
||||
assert metadata.remapped_buf is other_buffer
|
||||
|
||||
|
||||
def test_forward_mqa_indexer_layer_always_remaps(monkeypatch):
|
||||
"""The indexer rewrites its buffer in place, so its remap is never stale."""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
impl = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
_patch_forward_deps(monkeypatch, impl, remap_calls)
|
||||
metadata = _make_metadata()
|
||||
metadata.remapped_buf = buffer
|
||||
metadata.remapped_epoch = impl.index_epoch
|
||||
|
||||
_run_forward(impl, metadata)
|
||||
assert len(remap_calls) == 1
|
||||
@@ -1,751 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"ROCm AITER sparse MLA metadata sync test requires ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip(
|
||||
"ROCm AITER sparse MLA metadata sync test requires a supported AITER "
|
||||
"installation.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
|
||||
|
||||
|
||||
class _FakeAiter(ModuleType):
|
||||
get_mla_metadata_v1: Mock
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [16, 64, 128])
|
||||
@pytest.mark.parametrize("topk_tokens", [128, 256])
|
||||
def test_rocm_index_conversion_matrix(block_size, topk_tokens):
|
||||
"""Convert valid, masked, and out-of-range sparse indices exactly."""
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 4
|
||||
max_blocks = 4
|
||||
req_ids = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor(
|
||||
[[3, 5, 7, 9], [2, 4, 6, 8]], dtype=torch.int32, device=device
|
||||
)
|
||||
token_indices = torch.arange(
|
||||
num_tokens * topk_tokens, dtype=torch.int32, device=device
|
||||
).view(num_tokens, topk_tokens)
|
||||
token_indices %= max_blocks * block_size
|
||||
token_indices[0, 0] = -1
|
||||
token_indices[1, 1] = max_blocks * block_size
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
(num_tokens + 1) * topk_tokens,
|
||||
topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
output = torch.full(
|
||||
(num_tokens * topk_tokens,), -99, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
token_indices,
|
||||
cu_seqlens,
|
||||
output,
|
||||
BLOCK_STRIDE_ROWS=block_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=topk_tokens,
|
||||
)
|
||||
|
||||
block_ids = token_indices // block_size
|
||||
valid = (token_indices >= 0) & (block_ids < max_blocks)
|
||||
safe_blocks = block_ids.clamp(0, max_blocks - 1)
|
||||
bases = block_table[req_ids[:, None].long(), safe_blocks.long()]
|
||||
expected = torch.where(valid, bases * block_size + token_indices % block_size, 0)
|
||||
torch.testing.assert_close(output.view_as(expected), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_rocm_index_conversion_rejects_mismatched_token_extents():
|
||||
"""Reject full-batch request IDs paired with decode-subset indices."""
|
||||
with pytest.raises(AssertionError, match="must cover the same tokens"):
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
torch.zeros(3, dtype=torch.int32),
|
||||
torch.zeros(1, 4, dtype=torch.int32),
|
||||
torch.zeros(2, 128, dtype=torch.int32),
|
||||
torch.arange(3, dtype=torch.int32),
|
||||
torch.zeros(256, dtype=torch.int32),
|
||||
BLOCK_STRIDE_ROWS=64,
|
||||
NUM_TOPK_TOKENS=128,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_index_conversion_respects_packed_valid_counts():
|
||||
"""ROCm's indptr contract packs only each token's valid top-k prefix."""
|
||||
device = torch.device("cuda")
|
||||
topk_tokens = 128
|
||||
valid_counts = torch.tensor([1, 17, 128], dtype=torch.int32, device=device)
|
||||
cu_seqlens = torch.zeros(4, dtype=torch.int32, device=device)
|
||||
torch.cumsum(valid_counts, dim=0, out=cu_seqlens[1:])
|
||||
req_ids = torch.tensor([0, 0, 0], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor([[2, 3, 4]], dtype=torch.int32, device=device)
|
||||
token_indices = torch.arange(
|
||||
3 * topk_tokens, dtype=torch.int32, device=device
|
||||
).view(3, topk_tokens)
|
||||
token_indices %= 3 * 64
|
||||
output = torch.full((int(cu_seqlens[-1]),), -99, dtype=torch.int32, device=device)
|
||||
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
token_indices,
|
||||
cu_seqlens,
|
||||
output,
|
||||
BLOCK_STRIDE_ROWS=64,
|
||||
BLOCK_SIZE=64,
|
||||
NUM_TOPK_TOKENS=topk_tokens,
|
||||
)
|
||||
|
||||
expected_parts = []
|
||||
for token_id, count in enumerate(valid_counts.tolist()):
|
||||
local = token_indices[token_id, :count]
|
||||
expected_parts.append(block_table[0, (local // 64).long()] * 64 + local % 64)
|
||||
expected = torch.cat(expected_parts)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _make_builder():
|
||||
builder = object.__new__(sparse_mod.ROCMAiterMLASparseMetadataBuilder)
|
||||
max_num_batched_tokens = 8
|
||||
topk_tokens = 4
|
||||
|
||||
builder.device = torch.device("cpu")
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=1)
|
||||
builder.model_dtype = torch.bfloat16
|
||||
builder.kv_cache_dtype = "fp8"
|
||||
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
|
||||
builder.topk_tokens = topk_tokens
|
||||
# SparseMLACommonMetadataBuilder state consumed by its build() method.
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.use_pcp = False
|
||||
builder._prefill_backend = None
|
||||
|
||||
def cpu_token_to_req(metadata):
|
||||
return torch.repeat_interleave(
|
||||
torch.arange(metadata.num_reqs, dtype=torch.int32),
|
||||
metadata.query_start_loc_cpu.diff(),
|
||||
)
|
||||
|
||||
builder._build_req_id_per_token = cpu_token_to_req # type: ignore[method-assign]
|
||||
builder.req_id_per_token_buffer = torch.zeros(
|
||||
max_num_batched_tokens, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
builder.qo_indptr = torch.arange(
|
||||
max_num_batched_tokens + 1, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
builder.paged_kv_last_page_len = torch.ones(
|
||||
max_num_batched_tokens, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
builder.paged_kv_indices = torch.zeros(
|
||||
max_num_batched_tokens * topk_tokens, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
builder.paged_kv_indptr = torch.zeros(
|
||||
max_num_batched_tokens + 1, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
builder._use_persistent_metadata = True
|
||||
builder._num_attention_heads = 16
|
||||
builder._num_compute_units = current_platform.num_compute_units()
|
||||
builder._mla_q_dtype = torch.bfloat16
|
||||
builder._mla_kv_dtype = torch.bfloat16
|
||||
builder._mla_work_meta_data = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_work_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_work_info_set = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_reduce_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_reduce_final_map = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_reduce_partial_map = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._prev_req_extent = 0
|
||||
builder._prev_indices_extent = 0
|
||||
builder._prev_metadata_key = None
|
||||
return builder
|
||||
|
||||
|
||||
def _make_common_metadata():
|
||||
query_start_loc = torch.tensor([0, 1, 2], dtype=torch.int32, device="cpu")
|
||||
seq_lens = torch.tensor([16, 8], dtype=torch.int32, device="cpu")
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens,
|
||||
num_reqs=2,
|
||||
num_actual_tokens=2,
|
||||
max_query_len=1,
|
||||
max_seq_len=16,
|
||||
block_table_tensor=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
|
||||
slot_mapping=torch.arange(2, dtype=torch.int64, device="cpu"),
|
||||
)
|
||||
|
||||
|
||||
def _make_mixed_common_metadata():
|
||||
# req0: query_len 1 (decode), req1: query_len 4 (prefill) -> decode-first
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32, device="cpu")
|
||||
seq_lens = torch.tensor([16, 10], dtype=torch.int32, device="cpu")
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens,
|
||||
num_reqs=2,
|
||||
num_actual_tokens=5,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
block_table_tensor=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
|
||||
slot_mapping=torch.arange(5, dtype=torch.int64, device="cpu"),
|
||||
)
|
||||
|
||||
|
||||
def _patch_build_deps(monkeypatch, events=None):
|
||||
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
|
||||
on CPU."""
|
||||
|
||||
def fake_generate_sparse_seqlen_triton(
|
||||
seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
|
||||
):
|
||||
return torch.zeros(num_tokens, dtype=torch.int32, device="cpu")
|
||||
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = Mock(side_effect=lambda *a, **k: None)
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen_triton
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(
|
||||
synchronize=lambda: events.append("sync") if events is not None else None
|
||||
),
|
||||
)
|
||||
return fake_aiter
|
||||
|
||||
|
||||
def test_build_populates_decode_only_split_fields(monkeypatch):
|
||||
"""Decode-only batch: all reqs count as decodes, prefill fields default."""
|
||||
builder = _make_builder()
|
||||
_patch_build_deps(monkeypatch)
|
||||
|
||||
md = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
|
||||
)
|
||||
|
||||
assert md.num_decodes == 2
|
||||
assert md.num_prefills == 0
|
||||
assert md.num_decode_tokens == 2
|
||||
assert md.prefill_max_seq_len == 0
|
||||
assert md.prefill is None
|
||||
|
||||
|
||||
def test_build_populates_mixed_split_fields(monkeypatch):
|
||||
"""Mixed decode+prefill batch: split is reported, prefill fields stay
|
||||
default because this impl always runs the MQA path."""
|
||||
builder = _make_builder()
|
||||
_patch_build_deps(monkeypatch)
|
||||
|
||||
md = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=_make_mixed_common_metadata()
|
||||
)
|
||||
|
||||
assert md.num_decodes == 1
|
||||
assert md.num_prefills == 1
|
||||
assert md.num_decode_tokens == 1
|
||||
assert md.prefill_max_seq_len == 0
|
||||
assert md.prefill is None
|
||||
|
||||
|
||||
def test_build_fills_paged_kv_indptr_tail(monkeypatch):
|
||||
"""Rows past num_tokens must read as empty: the tail repeats the total."""
|
||||
builder = _make_builder()
|
||||
_patch_build_deps(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"generate_sparse_seqlen_triton",
|
||||
lambda *args, **kwargs: torch.tensor([3, 5], dtype=torch.int32),
|
||||
)
|
||||
builder.paged_kv_indptr[1:].fill_(-1)
|
||||
|
||||
md = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
|
||||
)
|
||||
|
||||
assert md.paged_kv_indptr.tolist() == [0, 3, 8]
|
||||
assert builder.paged_kv_indptr.tolist() == [0, 3, 8, 8, 8, 8, 8, 8, 8]
|
||||
|
||||
|
||||
def test_sink_build_skips_persistent_metadata(monkeypatch):
|
||||
builder = _make_builder()
|
||||
builder._use_persistent_metadata = False
|
||||
fake_aiter = _patch_build_deps(monkeypatch)
|
||||
|
||||
md = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
|
||||
)
|
||||
|
||||
fake_aiter.get_mla_metadata_v1.assert_not_called()
|
||||
assert md.work_meta_data is None
|
||||
assert md.work_indptr is None
|
||||
assert md.work_info_set is None
|
||||
assert md.reduce_indptr is None
|
||||
assert md.reduce_final_map is None
|
||||
assert md.reduce_partial_map is None
|
||||
assert builder._prev_metadata_key is None
|
||||
|
||||
|
||||
def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
|
||||
builder = _make_builder()
|
||||
common_metadata = _make_common_metadata()
|
||||
events: list[str] = []
|
||||
|
||||
def fake_generate_sparse_seqlen_triton(*args, **kwargs):
|
||||
return torch.tensor([1, 2], dtype=torch.int32, device="cpu")
|
||||
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
|
||||
def fake_get_mla_metadata_v1(*args, **kwargs):
|
||||
events.append("metadata")
|
||||
|
||||
fake_get_mla_metadata_v1_mock = Mock(side_effect=fake_get_mla_metadata_v1)
|
||||
fake_aiter.get_mla_metadata_v1 = fake_get_mla_metadata_v1_mock
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen_triton
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: events.append("sync")),
|
||||
)
|
||||
|
||||
builder.build(common_prefix_len=0, common_attn_metadata=common_metadata)
|
||||
|
||||
assert events == ["metadata", "sync"]
|
||||
assert fake_get_mla_metadata_v1_mock.call_count == 1
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["max_split_per_batch"] == 1
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_q"] == torch.bfloat16
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_kv"] == torch.bfloat16
|
||||
|
||||
events.clear()
|
||||
|
||||
builder.build(common_prefix_len=0, common_attn_metadata=common_metadata)
|
||||
|
||||
assert events == []
|
||||
assert fake_get_mla_metadata_v1_mock.call_count == 1
|
||||
|
||||
|
||||
def test_dense_prefill_builds_sparse_metadata_for_decode_subset(monkeypatch):
|
||||
"""Dense prefill must exclude prefill tokens from AITER sparse metadata.
|
||||
|
||||
A mixed batch is routed as sparse MQA for its leading decode requests and
|
||||
dense MHA for its trailing short prefill requests. The AITER metadata
|
||||
buffers therefore must describe only the decode subset; including prefill
|
||||
tokens would mismatch the query passed to ``forward_mqa``.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16, 4], dtype=torch.int32)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
num_actual_tokens=5,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(5),
|
||||
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
req_id_per_token=torch.tensor([0, 1, 1, 1, 1], dtype=torch.int32),
|
||||
topk_tokens=4,
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=4,
|
||||
prefill=SimpleNamespace(use_dense_mha=True),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
sparse_call: dict[str, torch.Tensor] = {}
|
||||
|
||||
def fake_generate_sparse_seqlen(
|
||||
sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
|
||||
):
|
||||
sparse_call.update(
|
||||
query_lens=torch.diff(sparse_query_start_loc),
|
||||
seq_lens=sparse_seq_lens.clone(),
|
||||
query_start_loc=sparse_query_start_loc.clone(),
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
return torch.ones(num_tokens, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
|
||||
)
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = Mock()
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
assert result.prefill is metadata.prefill
|
||||
assert result.num_prefills == 1
|
||||
assert result.req_id_per_token.tolist() == [0]
|
||||
assert result.qo_indptr.numel() == 2
|
||||
assert result.paged_kv_indptr.numel() == 2
|
||||
assert sparse_call["num_tokens"] == 1
|
||||
assert sparse_call["query_lens"].tolist() == [1]
|
||||
assert sparse_call["seq_lens"].tolist() == [16]
|
||||
assert sparse_call["query_start_loc"].tolist() == [0, 1]
|
||||
|
||||
|
||||
def test_long_prefill_shares_full_sparse_metadata_with_decode(monkeypatch):
|
||||
"""A prefill longer than top-k must remain in shared sparse metadata.
|
||||
|
||||
Dense MHA is only selected when the prefill sequence fits within the
|
||||
sparse top-k window. Once it exceeds that window, both decode and prefill
|
||||
are handled by sparse MQA, so every query token must be represented by the
|
||||
same sparse paging metadata.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
|
||||
# One decode token followed by a five-token prefill. topk_tokens is four,
|
||||
# so the prefill must not be split onto the dense-MHA path.
|
||||
query_start_loc = torch.tensor([0, 1, 6], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16, 5], dtype=torch.int32)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=5,
|
||||
max_seq_len=16,
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=5,
|
||||
max_seq_len=16,
|
||||
num_actual_tokens=6,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(6),
|
||||
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
req_id_per_token=torch.tensor([0, 1, 1, 1, 1, 1], dtype=torch.int32),
|
||||
topk_tokens=4,
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=5,
|
||||
prefill=SimpleNamespace(use_dense_mha=False),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
sparse_call: dict[str, torch.Tensor] = {}
|
||||
|
||||
def fake_generate_sparse_seqlen(
|
||||
sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
|
||||
):
|
||||
sparse_call.update(
|
||||
query_lens=torch.diff(sparse_query_start_loc),
|
||||
seq_lens=sparse_seq_lens.clone(),
|
||||
query_start_loc=sparse_query_start_loc.clone(),
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
return torch.ones(num_tokens, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
|
||||
)
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
get_persistent_metadata = Mock()
|
||||
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
assert result.num_decodes == 1
|
||||
assert result.num_prefills == 1
|
||||
assert result.req_id_per_token.tolist() == [0, 1, 1, 1, 1, 1]
|
||||
assert result.qo_indptr.numel() == 7
|
||||
assert result.paged_kv_indptr.numel() == 7
|
||||
assert result.paged_kv_indices.numel() == 6 * result.topk_tokens
|
||||
assert sparse_call["num_tokens"] == 6
|
||||
assert sparse_call["query_lens"].tolist() == [1, 5]
|
||||
assert sparse_call["seq_lens"].tolist() == [16, 5]
|
||||
assert sparse_call["query_start_loc"].tolist() == [0, 1, 6]
|
||||
get_persistent_metadata.assert_called_once()
|
||||
assert result.work_meta_data is builder._mla_work_meta_data
|
||||
assert result.reduce_indptr is builder._mla_reduce_indptr
|
||||
|
||||
|
||||
def test_forward_mqa_slices_full_batch_metadata_to_query_tokens(monkeypatch):
|
||||
"""The sparse kernel inputs must have the same token extent as its query.
|
||||
|
||||
In a mixed dense-prefill batch, ``forward_mqa`` receives only decode query
|
||||
tokens while some metadata buffers may represent the full batch. This
|
||||
guards the slicing that prevents the index conversion from writing beyond
|
||||
the decode-sized output.
|
||||
"""
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.num_heads = 4
|
||||
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
|
||||
impl.owns_indexer = True
|
||||
impl.index_epoch = 0
|
||||
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
|
||||
forward_mla = Mock(return_value=(expected_output, None))
|
||||
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
|
||||
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
|
||||
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
|
||||
block_size=1,
|
||||
topk_tokens=4,
|
||||
)
|
||||
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
|
||||
captured = {}
|
||||
|
||||
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
|
||||
captured["req_ids"] = req_ids.clone()
|
||||
captured["token_indices"] = token_indices.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
|
||||
)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
torch.zeros(1, 1, 576, dtype=torch.bfloat16),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert output is expected_output
|
||||
assert lse is None
|
||||
assert captured["req_ids"].tolist() == [0, 1]
|
||||
assert captured["token_indices"].shape == (2, 4)
|
||||
forward_mla.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_forward_mqa_slices_decode_subset_for_bf16_and_fp8(monkeypatch, kv_cache_dtype):
|
||||
"""Both cache dtypes must slice mixed-batch metadata to decode queries."""
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.kv_cache_dtype = kv_cache_dtype
|
||||
impl.num_heads = 4
|
||||
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
|
||||
impl.owns_indexer = True
|
||||
impl.index_epoch = 0
|
||||
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
|
||||
forward_mla = Mock(return_value=(expected_output, None))
|
||||
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
|
||||
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
|
||||
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
|
||||
block_size=1,
|
||||
topk_tokens=4,
|
||||
)
|
||||
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.zeros(1, 1, 576, dtype=torch.uint8)
|
||||
captured = {}
|
||||
|
||||
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
|
||||
captured["req_ids"] = req_ids.clone()
|
||||
captured["token_indices"] = token_indices.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
|
||||
)
|
||||
if kv_cache_dtype == "fp8":
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.ops,
|
||||
"scaled_fp8_quant",
|
||||
lambda q, scale: (q.to(fp8_dtype), None),
|
||||
)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
kv_cache,
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale=torch.tensor(1.0)),
|
||||
)
|
||||
|
||||
assert output is expected_output
|
||||
assert lse is None
|
||||
assert captured["req_ids"].tolist() == [0, 1]
|
||||
assert captured["token_indices"].shape == (2, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("length_relation", ["short", "long"])
|
||||
@pytest.mark.parametrize("workload", ["prefill", "mixed", "decode"])
|
||||
def test_sparse_metadata_workload_length_dtype_matrix(
|
||||
monkeypatch, workload, length_relation, kv_cache_dtype
|
||||
):
|
||||
"""Cover routing and persistent metadata across the ROCm support matrix.
|
||||
|
||||
The matrix crosses pure prefill, mixed prefill/decode, and decode-only
|
||||
workloads with sequence lengths below/above top-k and BF16/FP8 KV cache.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
if kv_cache_dtype == "fp8":
|
||||
builder._mla_q_dtype = current_platform.fp8_dtype()
|
||||
builder._mla_kv_dtype = current_platform.fp8_dtype()
|
||||
|
||||
is_long = length_relation == "long"
|
||||
prefill_len = 5 if is_long else 3
|
||||
decode_seq_len = 8 if is_long else 3
|
||||
if workload == "prefill":
|
||||
query_lens = [prefill_len]
|
||||
seq_lens_list = [prefill_len]
|
||||
num_decodes, num_prefills = 0, 1
|
||||
elif workload == "mixed":
|
||||
query_lens = [1, prefill_len]
|
||||
seq_lens_list = [decode_seq_len, prefill_len]
|
||||
num_decodes, num_prefills = 1, 1
|
||||
else:
|
||||
query_lens = [1, 1]
|
||||
seq_lens_list = [decode_seq_len, decode_seq_len + 1]
|
||||
num_decodes, num_prefills = 2, 0
|
||||
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32)
|
||||
num_tokens = sum(query_lens)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=len(query_lens),
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(num_tokens),
|
||||
block_table=torch.arange(len(query_lens) * 8, dtype=torch.int32).view(
|
||||
len(query_lens), 8
|
||||
),
|
||||
req_id_per_token=torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32),
|
||||
torch.tensor(query_lens),
|
||||
),
|
||||
topk_tokens=4,
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decodes,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=prefill_len if num_prefills else 0,
|
||||
prefill=SimpleNamespace(use_dense_mha=not is_long) if num_prefills else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"generate_sparse_seqlen_triton",
|
||||
lambda *args, **kwargs: torch.ones(args[3], dtype=torch.int32),
|
||||
)
|
||||
get_persistent_metadata = Mock()
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
uses_dense_prefill = workload in ("prefill", "mixed") and not is_long
|
||||
expected_sparse_tokens = num_decodes if uses_dense_prefill else num_tokens
|
||||
assert result.req_id_per_token.numel() == expected_sparse_tokens
|
||||
assert result.qo_indptr.numel() == expected_sparse_tokens + 1
|
||||
assert result.paged_kv_indptr.numel() == expected_sparse_tokens + 1
|
||||
|
||||
uses_fallback = workload == "prefill" and is_long and kv_cache_dtype == "auto"
|
||||
if expected_sparse_tokens == 0 or uses_fallback:
|
||||
get_persistent_metadata.assert_not_called()
|
||||
else:
|
||||
get_persistent_metadata.assert_called_once()
|
||||
if uses_fallback or expected_sparse_tokens == 0:
|
||||
assert result.work_meta_data is None
|
||||
assert result.reduce_indptr is None
|
||||
else:
|
||||
assert result.work_meta_data is builder._mla_work_meta_data
|
||||
assert result.reduce_indptr is builder._mla_reduce_indptr
|
||||
@@ -1,600 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4.1's ROCm MXFP4 sparse indexer against a torch reference.
|
||||
|
||||
The indexer K cache is a view into vLLM's block-major KV pool: its pages are
|
||||
a block stride apart with other layers' pages in between, so the tests put
|
||||
it there. The reference reads the keys back from a second cache the same
|
||||
writer fills in natural order, so it shares none of the preshuffled
|
||||
addressing it checks.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
rocm_mxfp4_indexer_k_store,
|
||||
rocm_mxfp4_indexer_q_quant,
|
||||
)
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerPrefillChunkMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
|
||||
RocmMxfp4IndexerMetadata,
|
||||
native_decode,
|
||||
plan_gather_launches,
|
||||
plan_prefill_chunks,
|
||||
)
|
||||
from vllm.v1.attention.ops import rocm_paged_mxfp4_indexer as ops
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
if get_cdna_version() != 4:
|
||||
pytest.skip("CDNA4-only", allow_module_level=True)
|
||||
|
||||
HEADS, HEAD_DIM = 32, 128
|
||||
WIDTH = HEAD_DIM // 2 + HEAD_DIM // 32
|
||||
# V4.1's 8-token blocks, with the pool (2048 blocks) and top-k (512) scaled
|
||||
# down so short test contexts still select.
|
||||
CAND_BLOCK, POOL_BLOCKS, TOPK = 8, 16, 32
|
||||
MAX_LEN = 4096
|
||||
DEVICE = "cuda"
|
||||
|
||||
_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
|
||||
E2M1 = torch.cat([_E2M1, -_E2M1])
|
||||
|
||||
|
||||
def _dequant(packed, e8m0):
|
||||
"""[..., D/2] e2m1 pairs and [..., D/32] e8m0 to [..., D] fp32."""
|
||||
table = E2M1.to(packed.device)
|
||||
vals = torch.stack([table[(packed & 0xF).long()], table[(packed >> 4).long()]], -1)
|
||||
scale = torch.pow(2.0, e8m0.float() - 127)
|
||||
return (vals.flatten(-2).unflatten(-1, (-1, 32)) * scale[..., None]).flatten(-2)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _workspace():
|
||||
init_workspace_manager(torch.device(DEVICE))
|
||||
yield
|
||||
reset_workspace_manager()
|
||||
|
||||
|
||||
class _Case:
|
||||
"""Requests with their keys written into a block-major pool (ratio-2 and
|
||||
ratio-1 pages side by side in every block) and into natural-order caches
|
||||
the reference reads. The bf16 inputs are kept for the end-to-end test."""
|
||||
|
||||
def __init__(self, seq_lens, block):
|
||||
from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import (
|
||||
k_norm_rope_mxfp4_cache,
|
||||
)
|
||||
|
||||
self.seq_lens = seq_lens
|
||||
num_blocks = sum(cdiv(n, block) for n in seq_lens) + 3
|
||||
perm = torch.randperm(num_blocks)
|
||||
width = max(cdiv(n, block) for n in seq_lens)
|
||||
self.block_table = torch.zeros(len(seq_lens), width, dtype=torch.int32)
|
||||
used = 0
|
||||
for req, n in enumerate(seq_lens):
|
||||
self.block_table[req, : cdiv(n, block)] = perm[used : used + cdiv(n, block)]
|
||||
used += cdiv(n, block)
|
||||
self.block_table = self.block_table.to(DEVICE)
|
||||
|
||||
def align(n):
|
||||
return cdiv(n, 576) * 576
|
||||
|
||||
page2, page1 = block // 2 * WIDTH, block * WIDTH
|
||||
stride = align(page2) + align(page1) + 576
|
||||
self.pool = torch.zeros(num_blocks * stride, dtype=torch.uint8, device=DEVICE)
|
||||
self.page_bytes = torch.zeros_like(self.pool, dtype=torch.bool)
|
||||
for i in range(num_blocks):
|
||||
base = i * stride
|
||||
self.page_bytes[base : base + page2] = True
|
||||
base += align(page2)
|
||||
self.page_bytes[base : base + page1] = True
|
||||
self.cache = {
|
||||
2: torch.as_strided(
|
||||
self.pool, (num_blocks, block // 2, WIDTH), (stride, WIDTH, 1), 0
|
||||
),
|
||||
1: torch.as_strided(
|
||||
self.pool,
|
||||
(num_blocks, block, WIDTH),
|
||||
(stride, WIDTH, 1),
|
||||
align(page2),
|
||||
),
|
||||
}
|
||||
self.offsets = [0]
|
||||
for n in seq_lens:
|
||||
self.offsets.append(self.offsets[-1] + n)
|
||||
# the same keys in natural order, one entry per page
|
||||
self.natural = {
|
||||
r: torch.zeros(self.offsets[-1], 1, WIDTH, dtype=torch.uint8, device=DEVICE)
|
||||
for r in (1, 2)
|
||||
}
|
||||
self.cos_sin = cos_sin = torch.randn(MAX_LEN, 64, device=DEVICE)
|
||||
self.norm = norm = torch.rand(HEAD_DIM, device=DEVICE) + 0.5
|
||||
self.k_pre = {}
|
||||
for req, n in enumerate(seq_lens):
|
||||
if n == 0:
|
||||
continue
|
||||
k_pre = torch.randn(n, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
|
||||
self.k_pre[req] = k_pre
|
||||
pos = torch.arange(n, device=DEVICE)
|
||||
for ratio, cache in self.cache.items():
|
||||
comp = pos // ratio
|
||||
boundary = (pos + 1) % ratio == 0
|
||||
entries = cache.shape[1]
|
||||
page = self.block_table[req, comp // entries].long()
|
||||
slot = torch.where(boundary, page * entries + comp % entries, -1)
|
||||
nat = torch.where(boundary, self.offsets[req] + comp, -1)
|
||||
rocm_mxfp4_indexer_k_store(
|
||||
k_pre,
|
||||
pos,
|
||||
cos_sin,
|
||||
norm,
|
||||
1e-6,
|
||||
cache,
|
||||
slot,
|
||||
ratio,
|
||||
True,
|
||||
num_heads=HEADS,
|
||||
)
|
||||
k_norm_rope_mxfp4_cache(
|
||||
k_pre, pos, cos_sin, norm, 1e-6, self.natural[ratio], nat, ratio
|
||||
)
|
||||
|
||||
def keys(self, ratio, req, n):
|
||||
nat = self.natural[ratio][self.offsets[req] : self.offsets[req] + n, 0]
|
||||
return _dequant(nat[:, : HEAD_DIM // 2], nat[:, HEAD_DIM // 2 :])
|
||||
|
||||
|
||||
def _queries(num_rows):
|
||||
q = torch.randint(0, 256, (num_rows, HEADS, HEAD_DIM // 2), dtype=torch.uint8)
|
||||
scales = torch.randint(122, 130, (num_rows, HEADS, HEAD_DIM // 32))
|
||||
q, scales = q.to(DEVICE), scales.to(torch.uint8).to(DEVICE)
|
||||
weights = torch.randn(num_rows, HEADS, device=DEVICE)
|
||||
# the fused Q kernel returns the scales as one int32 per head
|
||||
return q, scales.view(torch.int32).squeeze(-1), weights, _dequant(q, scales)
|
||||
|
||||
|
||||
def _scores(q, w, keys):
|
||||
return (torch.relu(q @ keys.T) * w[:, None]).sum(0)
|
||||
|
||||
|
||||
def _assert_topk(row, scores, k):
|
||||
"""``row`` is a top-k of the entries of ``scores`` above -inf, up to the
|
||||
fp32 reduce order."""
|
||||
valid = scores > float("-inf")
|
||||
want = min(k, int(valid.sum()))
|
||||
sel = row[row >= 0].long()
|
||||
assert sel.numel() == want and sel.unique().numel() == want
|
||||
if want:
|
||||
assert bool(valid[sel].all())
|
||||
kth = scores[valid].topk(want).values[-1]
|
||||
finite = scores[torch.isfinite(scores)]
|
||||
tol = 1e-4 * float(finite.abs().max()) if finite.numel() else 0.0
|
||||
assert bool((scores[sel] >= kth - tol).all())
|
||||
|
||||
|
||||
def _block_maxima(scores):
|
||||
end = scores.numel()
|
||||
if end == 0:
|
||||
return scores
|
||||
padded = torch.full((cdiv(end, CAND_BLOCK) * CAND_BLOCK,), float("-inf"))
|
||||
padded[:end] = scores.cpu()
|
||||
out = padded.view(-1, CAND_BLOCK).amax(1)
|
||||
out[(end - 1) // CAND_BLOCK] = float("inf")
|
||||
return out.to(scores.device)
|
||||
|
||||
|
||||
def _forward_context(monkeypatch, metadata):
|
||||
context = types.SimpleNamespace(
|
||||
attn_metadata={"indexer": metadata},
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
monkeypatch.setattr(ops, "get_forward_context", lambda: context)
|
||||
|
||||
|
||||
def _decode_metadata(case, rows, ratio, query_lens):
|
||||
lens = torch.tensor([(p + 1) // ratio for _, p in rows], dtype=torch.int32)
|
||||
lens = lens.to(DEVICE)
|
||||
block_table = case.block_table[[req for req, _ in rows]].contiguous()
|
||||
entries = case.cache[ratio].shape[1]
|
||||
next_n = max(query_lens)
|
||||
context_lens = (torch.tensor(case.seq_lens, dtype=torch.int32) // ratio).to(DEVICE)
|
||||
native = native_decode(lens, block_table, query_lens, next_n, context_lens)
|
||||
schedule = torch.empty(
|
||||
ops.rocm_mxfp4_decode_schedule_words(HEADS, HEAD_DIM, entries, next_n),
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
return RocmMxfp4IndexerMetadata(
|
||||
seq_lens=None,
|
||||
max_seq_len=max(case.seq_lens),
|
||||
slot_mapping=None,
|
||||
num_decodes=len(case.seq_lens),
|
||||
num_decode_tokens=len(rows),
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]),
|
||||
decode_row_lens=lens,
|
||||
decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK,
|
||||
decode_native=native,
|
||||
decode_schedule=ops.build_rocm_mxfp4_decode_schedule(
|
||||
lens, HEADS, HEAD_DIM, entries, schedule, MAX_LEN // ratio, native
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _prefill_metadata(
|
||||
case, rows, ratio, chunk_bounds, query_start_loc, gather_rows, gather
|
||||
):
|
||||
chunks = []
|
||||
for req_lo, req_hi, t0, t1 in chunk_bounds:
|
||||
ends = torch.tensor(
|
||||
[(rows[t][1] + 1) // ratio for t in range(t0, t1)], dtype=torch.int32
|
||||
).to(DEVICE)
|
||||
chunks.append(
|
||||
DeepseekV32IndexerPrefillChunkMetadata(
|
||||
block_table=case.block_table[req_lo:req_hi],
|
||||
cu_seqlen_ks=torch.zeros_like(ends),
|
||||
cu_seqlen_ke=ends,
|
||||
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
total_seq_lens=1,
|
||||
max_seqlen_k=int(ends.max()),
|
||||
token_start=t0,
|
||||
token_end=t1,
|
||||
num_reqs=req_hi - req_lo,
|
||||
)
|
||||
)
|
||||
seq_lens = torch.tensor(case.seq_lens)
|
||||
context_lens = (seq_lens // ratio).int().to(DEVICE)
|
||||
plans = plan_prefill_chunks(
|
||||
chunks,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
context_lens,
|
||||
ratio,
|
||||
CAND_BLOCK,
|
||||
0.0 if gather else None,
|
||||
torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
return RocmMxfp4IndexerMetadata(
|
||||
seq_lens=None,
|
||||
max_seq_len=max(case.seq_lens),
|
||||
slot_mapping=None,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_prefills=len(case.seq_lens),
|
||||
num_prefill_tokens=len(rows),
|
||||
prefill=types.SimpleNamespace(chunks=chunks),
|
||||
prefill_plans=plans,
|
||||
gather_launches=plan_gather_launches(chunks, plans, gather_rows),
|
||||
)
|
||||
|
||||
|
||||
# The kernel prefers 128-token pages.
|
||||
BLOCKS = pytest.mark.parametrize("block", [128, 64])
|
||||
|
||||
|
||||
@BLOCKS
|
||||
def test_k_store_stays_inside_its_pages(block):
|
||||
"""The writer touches only its pages of the block-major pool. What they hold
|
||||
is checked by the layer tests, which read them back through aiter's kernel."""
|
||||
set_random_seed(0)
|
||||
case = _Case([700, 333], block)
|
||||
assert int(case.pool[~case.page_bytes].count_nonzero()) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rows, width, k",
|
||||
# vLLM's kernel below 128 rows, aiter's from 128 rows and on every k=2048
|
||||
[(8, 4096, 512), (128, 4096, 512), (8, 16384, 2048)],
|
||||
)
|
||||
def test_topk_matches_torch_on_both_kernels(rows, width, k):
|
||||
set_random_seed(0)
|
||||
logits = torch.randn(rows, width, device=DEVICE)
|
||||
lengths = torch.randint(1, width + 1, (rows,), dtype=torch.int32, device=DEVICE)
|
||||
lengths[0] = k // 2 # shorter than k, so the rest of the row pads with -1
|
||||
out = torch.full((rows, k), 7, dtype=torch.int32, device=DEVICE)
|
||||
ops._topk(logits, lengths, out, k)
|
||||
for r in range(rows):
|
||||
n = min(k, int(lengths[r]))
|
||||
want = torch.topk(logits[r, : int(lengths[r])], n).indices.sort().values
|
||||
got = out[r]
|
||||
assert torch.equal(got[got >= 0].sort().values, want.int())
|
||||
assert int((got == -1).sum()) == k - n
|
||||
|
||||
|
||||
def _run_layers(monkeypatch, case, rows, metadata):
|
||||
"""Ratio-2 dense layer, the candidate source, and a consumer both ways;
|
||||
each checked against the torch reference."""
|
||||
num_rows = len(rows)
|
||||
q, q_scale, weights, q_ref = _queries(num_rows)
|
||||
hidden = torch.empty(num_rows, 1, device=DEVICE)
|
||||
|
||||
def topk_buffer():
|
||||
return torch.full((num_rows + 2, TOPK), 7, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
_forward_context(monkeypatch, metadata(2))
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[2],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN // 2,
|
||||
out,
|
||||
compress_ratio=2,
|
||||
)
|
||||
for i, (req, pos) in enumerate(rows):
|
||||
end = (pos + 1) // 2
|
||||
keys = case.keys(2, req, end)
|
||||
_assert_topk(out[i], _scores(q_ref[i], weights[i], keys), TOPK)
|
||||
|
||||
source = metadata(1)
|
||||
_forward_context(monkeypatch, source)
|
||||
pool = torch.full((num_rows + 2, POOL_BLOCKS), 5, dtype=torch.int32, device=DEVICE)
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[1],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN,
|
||||
out,
|
||||
compress_ratio=1,
|
||||
candidate_blocks=pool,
|
||||
candidate_block_size=CAND_BLOCK,
|
||||
candidate_write=True,
|
||||
)
|
||||
allowed = []
|
||||
for i, (req, pos) in enumerate(rows):
|
||||
scores = _scores(q_ref[i], weights[i], case.keys(1, req, pos + 1))
|
||||
_assert_topk(out[i], scores, TOPK)
|
||||
_assert_topk(pool[i], _block_maxima(scores), POOL_BLOCKS)
|
||||
mask = torch.zeros_like(scores, dtype=torch.bool)
|
||||
for block in pool[i][pool[i] >= 0].tolist():
|
||||
mask[block * CAND_BLOCK : (block + 1) * CAND_BLOCK] = True
|
||||
allowed.append(torch.where(mask, scores, float("-inf")))
|
||||
|
||||
pa = ops._aiter()
|
||||
build = pa.build_candidate_gather
|
||||
builds = []
|
||||
|
||||
def counting_build(*args, **kwargs):
|
||||
builds.append(1)
|
||||
return build(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(pa, "build_candidate_gather", counting_build)
|
||||
for gather in (True, False):
|
||||
consumer = metadata(1, gather)
|
||||
consumer.decode_use_gather = gather
|
||||
_forward_context(monkeypatch, consumer)
|
||||
builds.clear()
|
||||
# Two consumer layers of one step: the second reuses the first's pools.
|
||||
for _ in range(2):
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_mqa_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[1],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN,
|
||||
out,
|
||||
1,
|
||||
pool,
|
||||
CAND_BLOCK,
|
||||
POOL_BLOCKS * CAND_BLOCK,
|
||||
)
|
||||
for i in range(num_rows):
|
||||
_assert_topk(out[i], allowed[i], TOPK)
|
||||
if gather:
|
||||
launches = len(consumer.gather_launches) + (consumer.decode is not None)
|
||||
assert len(builds) == launches
|
||||
|
||||
|
||||
@BLOCKS
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens",
|
||||
# Uniform steps launch the dense layers on next_n-row sequences, also with
|
||||
# cudagraph padding (query length 0) after them; a ragged step keeps a row
|
||||
# per token.
|
||||
[[6, 6, 6, 6], [2, 2, 2], [2, 2, 2, 0], [2, 1, 2]],
|
||||
ids=["native6", "native2", "padded", "ragged"],
|
||||
)
|
||||
def test_decode_layers_match_reference(monkeypatch, block, query_lens):
|
||||
set_random_seed(0)
|
||||
next_n = max(query_lens)
|
||||
case = _Case([q and n for q, n in zip(query_lens, [900, 333, 610, 1200])], block)
|
||||
# a padding request has next_n rows of its own and no context
|
||||
rows = [
|
||||
(req, n - q + j if q else -1)
|
||||
for req, (n, q) in enumerate(zip(case.seq_lens, query_lens))
|
||||
for j in range(q or next_n)
|
||||
]
|
||||
_run_layers(
|
||||
monkeypatch,
|
||||
case,
|
||||
rows,
|
||||
lambda r, gather=False: _decode_metadata(case, rows, r, query_lens),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,new_tokens,chunks",
|
||||
[
|
||||
# one request past a cached prefix, then one sliced into two chunks
|
||||
([760, 500], [300, 500], [(0, 1, 0, 300), (1, 2, 300, 560), (1, 2, 560, 800)]),
|
||||
# three requests in one chunk, the first two with equal rows
|
||||
([600, 450, 900], [200, 200, 120], [(0, 3, 0, 520)]),
|
||||
# four requests with four different query lengths in one chunk
|
||||
([900, 600, 450, 700], [150, 90, 200, 60], [(0, 4, 0, 500)]),
|
||||
],
|
||||
ids=["sliced", "batched", "ragged"],
|
||||
)
|
||||
# The consumers rejoin a request's rows across chunks, then cut them at most
|
||||
# gather_rows a launch.
|
||||
@pytest.mark.parametrize("gather_rows", [1 << 20, 128], ids=["joined", "cut"])
|
||||
@BLOCKS
|
||||
def test_prefill_layers_match_reference(
|
||||
monkeypatch, seq_lens, new_tokens, chunks, gather_rows, block
|
||||
):
|
||||
set_random_seed(0)
|
||||
case = _Case(seq_lens, block)
|
||||
rows = [
|
||||
(req, n - q + i)
|
||||
for req, (n, q) in enumerate(zip(seq_lens, new_tokens))
|
||||
for i in range(q)
|
||||
]
|
||||
query_start_loc = [0, *itertools.accumulate(new_tokens)]
|
||||
_run_layers(
|
||||
monkeypatch,
|
||||
case,
|
||||
rows,
|
||||
lambda r, gather=True: _prefill_metadata(
|
||||
case, rows, r, chunks, query_start_loc, gather_rows, gather
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _rope_gptj(x, cos_sin, positions):
|
||||
"""GPT-J RoPE on the last 64 dims, as the indexer rotates Q and K: bf16 in
|
||||
and out."""
|
||||
x = x.bfloat16().float()
|
||||
cos, sin = cos_sin[positions].chunk(2, dim=-1)
|
||||
if x.dim() == 3:
|
||||
cos, sin = cos[:, None], sin[:, None]
|
||||
rot = x[..., -64:]
|
||||
even, odd = rot[..., 0::2].clone(), rot[..., 1::2].clone()
|
||||
rot[..., 0::2] = even * cos - odd * sin
|
||||
rot[..., 1::2] = odd * cos + even * sin
|
||||
return x.bfloat16()
|
||||
|
||||
|
||||
def _mxfp4_round_trip(x):
|
||||
"""MXFP4 round trip of x, rounded as the writers do: a scale of
|
||||
2^ceil(log2(amax / 6)) per 32 values, then the nearest e2m1 value, ties to
|
||||
even."""
|
||||
blocks = x.float().unflatten(-1, (-1, 32))
|
||||
amax = blocks.abs().amax(-1, keepdim=True).clamp(min=2**-126)
|
||||
scale = torch.exp2(torch.ceil(torch.log2(amax / 6)))
|
||||
y = (blocks / scale).clamp(-6, 6)
|
||||
# Even codes first: argmin keeps the first of two equal distances.
|
||||
grid = torch.cat([_E2M1[0::2], _E2M1[1::2]]).to(x.device)
|
||||
idx = (y.abs()[..., None] - grid).abs().argmin(-1)
|
||||
return (grid[idx] * y.sign() * scale).flatten(-2)
|
||||
|
||||
|
||||
def _torch_keys(case, ratio, req):
|
||||
"""A request's compressed keys in torch: k_norm of each group's last
|
||||
token, rotated at the group's first position, through MXFP4."""
|
||||
k = case.k_pre[req][ratio - 1 :: ratio].float()
|
||||
k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) * case.norm
|
||||
pos = torch.arange(k.shape[0], device=DEVICE) * ratio
|
||||
return _mxfp4_round_trip(_rope_gptj(k, case.cos_sin, pos))
|
||||
|
||||
|
||||
def _topk_indices(q, weights, keys, ends, k):
|
||||
"""Top-k of sum_h relu(q_h . key) * weights_h over each row's [0, end)."""
|
||||
scores = (torch.einsum("thd,cd->thc", q, keys).relu() * weights[..., None]).sum(1)
|
||||
cols = torch.arange(keys.shape[0], device=DEVICE)
|
||||
return scores.masked_fill(cols >= ends[:, None], float("-inf")).topk(k).indices
|
||||
|
||||
|
||||
def _recall(selected, reference):
|
||||
"""Per row, the fraction of the reference top-k that ``selected`` picked."""
|
||||
return (selected[:, :, None] == reference[:, None, :]).any(1).float().mean(1)
|
||||
|
||||
|
||||
@BLOCKS
|
||||
@pytest.mark.parametrize("ratio", [2, 1])
|
||||
@pytest.mark.parametrize("step", ["decode", "prefill"])
|
||||
def test_selection_from_bf16_matches_torch(monkeypatch, step, ratio, block):
|
||||
"""End to end from bf16 K and Q: vLLM's MXFP4 K store and Q quant, the
|
||||
paged pool, metadata from the planning functions the builder calls, and
|
||||
the indexer op pick what the same MXFP4 math picks in torch."""
|
||||
set_random_seed(0)
|
||||
if step == "decode":
|
||||
case = _Case([3000, 2600, 1900, 3500, 1200, 2200, 3900, 1600], block)
|
||||
rows = [(req, n - 1) for req, n in enumerate(case.seq_lens)]
|
||||
metadata = _decode_metadata(case, rows, ratio, [1] * len(rows))
|
||||
else:
|
||||
new_tokens = [300, 200, 250]
|
||||
case = _Case([1500, 900, 2000], block)
|
||||
rows = [
|
||||
(req, n - q + i)
|
||||
for req, (n, q) in enumerate(zip(case.seq_lens, new_tokens))
|
||||
for i in range(q)
|
||||
]
|
||||
metadata = _prefill_metadata(
|
||||
case,
|
||||
rows,
|
||||
ratio,
|
||||
[(0, len(new_tokens), 0, len(rows))],
|
||||
[0, *itertools.accumulate(new_tokens)],
|
||||
1 << 20,
|
||||
gather=False,
|
||||
)
|
||||
num_rows = len(rows)
|
||||
q_bf16 = torch.randn(num_rows, HEADS, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
|
||||
w = torch.randn(num_rows, HEADS, device=DEVICE)
|
||||
pos = torch.tensor([p for _, p in rows], device=DEVICE)
|
||||
scales = (HEAD_DIM**-0.5, HEADS**-0.5)
|
||||
|
||||
(q, q_scale), weights = rocm_mxfp4_indexer_q_quant(
|
||||
pos, q_bf16, case.cos_sin, w, *scales
|
||||
)
|
||||
_forward_context(monkeypatch, metadata)
|
||||
out = torch.full((num_rows, TOPK), 7, dtype=torch.int32, device=DEVICE)
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
torch.empty(num_rows, 1, device=DEVICE),
|
||||
"indexer",
|
||||
case.cache[ratio],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN // ratio,
|
||||
out,
|
||||
compress_ratio=ratio,
|
||||
)
|
||||
ends = (pos + 1) // ratio
|
||||
assert bool(((out >= 0) & (out < ends[:, None])).all())
|
||||
assert all(row.unique().numel() == TOPK for row in out)
|
||||
|
||||
q_ref = _mxfp4_round_trip(_rope_gptj(q_bf16, case.cos_sin, pos))
|
||||
w_ref = w * scales[0] * scales[1]
|
||||
agree = []
|
||||
for req in range(len(case.seq_lens)):
|
||||
mine = torch.tensor([r == req for r, _ in rows], device=DEVICE)
|
||||
ref = _topk_indices(
|
||||
q_ref[mine], w_ref[mine], _torch_keys(case, ratio, req), ends[mine], TOPK
|
||||
)
|
||||
agree.append(_recall(out[mine], ref))
|
||||
assert torch.cat(agree).mean().item() >= 0.99
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,215 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the multi-token decode flatten in ``TritonMLAImpl.forward_mqa``.
|
||||
|
||||
``decode_attention_fwd`` has no causal flag: it launches one program per row of
|
||||
``q`` and reads that row's KV extent from ``B_Seqlen``. So a ``query_len``-token
|
||||
decode block is flattened to one row per query token, and intra-block causality
|
||||
has to be expressed through the per-row sequence lengths -- row ``t`` gets
|
||||
``seq_len - (query_len - 1) + t``, i.e. the committed prefix plus block tokens
|
||||
``0..t``. A non-causal draft block instead gives every row the same full extent.
|
||||
|
||||
The kernel is replaced by a spy, so these tests assert on the ``block_table``
|
||||
and ``seq_lens`` the backend actually submits and need no GPU or Triton runtime.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.attention.mla_attention import QueryLenSupport
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.mla.triton_mla import (
|
||||
TritonMLAImpl,
|
||||
TritonMLAMetadataBuilder,
|
||||
)
|
||||
|
||||
NUM_HEADS = 16
|
||||
KV_LORA_RANK = 512
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
PAGE_SIZE = 16
|
||||
|
||||
# Committed context per request, including a fresh request at zero context
|
||||
# where the whole KV extent is the block itself. A decode block's seq_len
|
||||
# already counts the block, so seq_len = context + query_len.
|
||||
CONTEXT_LENS = [0, 1, 37, 512]
|
||||
|
||||
|
||||
def _seq_lens(query_len: int) -> list[int]:
|
||||
return [c + query_len for c in CONTEXT_LENS]
|
||||
|
||||
|
||||
def _make_impl(dcp_world_size: int = 1) -> TritonMLAImpl:
|
||||
"""Only the attributes ``forward_mqa`` reads; __init__ wants a full config."""
|
||||
impl = object.__new__(TritonMLAImpl)
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.scale = HEAD_SIZE**-0.5
|
||||
impl._sm_count = 304
|
||||
impl.dcp_world_size = dcp_world_size
|
||||
return impl
|
||||
|
||||
|
||||
def _make_metadata(seq_lens: list[int], query_len: int, causal: bool):
|
||||
num_decodes = len(seq_lens)
|
||||
return SimpleNamespace(
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decodes * query_len,
|
||||
max_seq_len=max(seq_lens),
|
||||
causal=causal,
|
||||
decode=SimpleNamespace(
|
||||
# arange page ids so each request's window is an identifiable slice.
|
||||
block_table=torch.arange(num_decodes * 64, dtype=torch.int32).reshape(
|
||||
num_decodes, 64
|
||||
),
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int32),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _run_forward_mqa(
|
||||
seq_lens: list[int],
|
||||
query_len: int,
|
||||
causal: bool,
|
||||
q_rows=None,
|
||||
dcp_world_size: int = 1,
|
||||
):
|
||||
"""Drive forward_mqa with the decode kernel spied out.
|
||||
|
||||
Returns the ``(block_table, seq_lens)`` handed to ``decode_attention_fwd``.
|
||||
"""
|
||||
captured: dict = {}
|
||||
|
||||
def spy(q, kv_cache, kv_c_cache, o, lse, block_table, b_seq_len, *args, **kwargs):
|
||||
captured["block_table"] = block_table.detach().clone()
|
||||
captured["seq_lens"] = b_seq_len.detach().clone()
|
||||
|
||||
metadata = _make_metadata(seq_lens, query_len, causal)
|
||||
rows = metadata.num_decode_tokens if q_rows is None else q_rows
|
||||
q = torch.zeros(rows, NUM_HEADS, HEAD_SIZE, dtype=torch.bfloat16)
|
||||
kv_cache = torch.zeros(
|
||||
len(seq_lens) * 64, PAGE_SIZE, HEAD_SIZE, dtype=torch.bfloat16
|
||||
)
|
||||
layer = SimpleNamespace(_k_scale=torch.ones(1), layer_name="test")
|
||||
|
||||
with patch("vllm.v1.attention.backends.mla.triton_mla.decode_attention_fwd", spy):
|
||||
_make_impl(dcp_world_size).forward_mqa(q, kv_cache, metadata, layer)
|
||||
|
||||
assert captured, "forward_mqa did not reach the decode kernel"
|
||||
return captured
|
||||
|
||||
|
||||
def test_multi_token_decode_flags_are_declared_together():
|
||||
"""Capture probes max_query_len > 1, which only clears the assert in
|
||||
build_for_cudagraph_capture once query_len_support raised the threshold."""
|
||||
assert (
|
||||
TritonMLAMetadataBuilder._cudagraph_support == AttentionCGSupport.UNIFORM_BATCH
|
||||
)
|
||||
assert TritonMLAMetadataBuilder.query_len_support == QueryLenSupport.UNIFORM
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query_len", [2, 3, 4, 5, 8])
|
||||
def test_causal_block_rows_see_prefix_plus_own_position(query_len):
|
||||
"""Causal verify: row t sees seq_len - (query_len - 1) + t KV entries.
|
||||
|
||||
Regression guard: the causal branch used to be skipped entirely, so a verify
|
||||
token could attend the draft siblings it is supposed to be checking. Fails
|
||||
on unmodified upstream, where no flatten happens at all for causal blocks.
|
||||
"""
|
||||
seq_lens = _seq_lens(query_len)
|
||||
captured = _run_forward_mqa(seq_lens, query_len, causal=True)
|
||||
|
||||
got = captured["seq_lens"].tolist()
|
||||
want = [s - (query_len - 1) + t for s in seq_lens for t in range(query_len)]
|
||||
assert got == want, (
|
||||
"causal flatten is not causal: verify row r*query_len+t must get "
|
||||
f"seq_len_r - {query_len - 1} + t entries.\n"
|
||||
f" seq_lens {seq_lens}\n got {got}\n expected {want}"
|
||||
)
|
||||
|
||||
# Per-request invariants the arithmetic must hold regardless of query_len.
|
||||
for r, seq_len in enumerate(seq_lens):
|
||||
rows = got[r * query_len : (r + 1) * query_len]
|
||||
assert rows == sorted(rows) and len(set(rows)) == query_len, (
|
||||
f"request {r} rows must strictly increase, got {rows}"
|
||||
)
|
||||
assert rows[-1] == seq_len, "last row must see the full sequence"
|
||||
assert rows[0] == seq_len - (query_len - 1)
|
||||
assert all(n > 0 for n in rows), f"non-positive KV extent in {rows}"
|
||||
|
||||
|
||||
def test_cudagraph_padding_rows_present_no_kv_extent():
|
||||
"""Causal flattening pins cudagraph padding rows (seq_len 0) to exactly
|
||||
zero KV extent, keeping them in the kernel's skipped branch."""
|
||||
query_len = 4
|
||||
seq_lens = _seq_lens(query_len) + [0]
|
||||
captured = _run_forward_mqa(seq_lens, query_len, causal=True)
|
||||
|
||||
padding = captured["seq_lens"].tolist()[-query_len:]
|
||||
assert padding == [0] * query_len, (
|
||||
f"padding rows must present exactly zero KV extent, got {padding}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query_len", [2, 4])
|
||||
def test_non_causal_block_rows_all_see_the_full_prefix(query_len):
|
||||
"""The DSpark draft block is generated in one pass, so no row is masked."""
|
||||
seq_lens = _seq_lens(query_len)
|
||||
captured = _run_forward_mqa(seq_lens, query_len, causal=False)
|
||||
|
||||
want = [s for s in seq_lens for _ in range(query_len)]
|
||||
assert captured["seq_lens"].tolist() == want
|
||||
|
||||
|
||||
@pytest.mark.parametrize("causal", [True, False])
|
||||
def test_block_table_is_expanded_to_match_q_rows(causal):
|
||||
"""The kernel indexes block_table by program_id, so it needs one row per
|
||||
query token -- a short block_table is read out of bounds."""
|
||||
query_len = 4
|
||||
captured = _run_forward_mqa(_seq_lens(query_len), query_len, causal=causal)
|
||||
|
||||
num_rows = len(CONTEXT_LENS) * query_len
|
||||
assert captured["block_table"].shape[0] == num_rows
|
||||
assert captured["seq_lens"].shape[0] == num_rows
|
||||
# repeat_interleave, not repeat: a request's rows must stay adjacent.
|
||||
for r in range(len(CONTEXT_LENS)):
|
||||
block = captured["block_table"][r * query_len : (r + 1) * query_len]
|
||||
assert torch.equal(block, block[:1].expand_as(block))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("causal", [True, False])
|
||||
def test_single_token_decode_is_untouched(causal):
|
||||
"""query_len == 1 must short-circuit: ordinary decode sees no expansion."""
|
||||
seq_lens = _seq_lens(1)
|
||||
captured = _run_forward_mqa(seq_lens, query_len=1, causal=causal)
|
||||
|
||||
assert captured["seq_lens"].tolist() == seq_lens
|
||||
assert captured["block_table"].shape[0] == len(seq_lens)
|
||||
|
||||
|
||||
def test_expansion_factor_follows_the_q_row_count():
|
||||
"""The kernel indexes block_table and seq_lens by program_id, so the
|
||||
expansion must follow q rows, not the token/request counts."""
|
||||
num_rows = len(CONTEXT_LENS) * 4
|
||||
captured = _run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=num_rows)
|
||||
|
||||
assert captured["block_table"].shape[0] == num_rows
|
||||
assert captured["seq_lens"].shape[0] == num_rows
|
||||
|
||||
|
||||
def test_non_uniform_block_is_rejected():
|
||||
"""Q rows must divide evenly across the decode requests."""
|
||||
with pytest.raises(AssertionError, match="non-uniform decode block"):
|
||||
_run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=6)
|
||||
|
||||
|
||||
def test_causal_multi_token_decode_is_rejected_under_dcp():
|
||||
"""Per-row extents offset the global seq_len, but DCP passes the rank-local
|
||||
slice, so the causal arithmetic would silently address the wrong KV."""
|
||||
with pytest.raises(AssertionError, match="not supported with DCP"):
|
||||
_run_forward_mqa(_seq_lens(4), query_len=4, causal=True, dcp_world_size=2)
|
||||
|
||||
# Non-causal rows all take the same extent, which stays correct when local.
|
||||
_run_forward_mqa(_seq_lens(4), query_len=4, causal=False, dcp_world_size=2)
|
||||
@@ -1,420 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant.
|
||||
|
||||
Tests both contiguous and non-contiguous (cross-layer unified) KV cache
|
||||
layouts against a pure-PyTorch reference implementation.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"trtllm kvfp8 dequant is not supported on ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
NUM_BLOCKS = 128
|
||||
|
||||
|
||||
def to_float8(x, dtype=None):
|
||||
if dtype is None:
|
||||
dtype = FP8_DTYPE
|
||||
finfo = torch.finfo(dtype)
|
||||
min_val, max_val = x.aminmax()
|
||||
amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
|
||||
scale = finfo.max / amax * 0.1
|
||||
x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max)
|
||||
return x_scl_sat.to(dtype), scale.float().reciprocal()
|
||||
|
||||
|
||||
def make_random_kv_cache(
|
||||
num_blocks,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
layout=KVCacheLayout.LBHNC,
|
||||
num_layers=None,
|
||||
):
|
||||
"""Create a random fp8 KV cache mimicking the production allocation.
|
||||
|
||||
Physical layout: [L, B, H, N, 2 * hs] permuted by ``layout.stride_order``
|
||||
Returned view: (num_blocks, 2, num_kv_heads, block_size, head_size)
|
||||
with non-contiguous strides under cross-layer layouts (they skip over
|
||||
num_layers), matching the actual forward path.
|
||||
"""
|
||||
logical_4d = (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
if num_layers is None:
|
||||
num_layers = 1 if layout.is_layer_compact else 2
|
||||
logical_5d = (num_layers, *logical_4d)
|
||||
physical_5d = tuple(logical_5d[i] for i in layout.stride_order)
|
||||
inv_order = [layout.stride_order.index(i) for i in range(5)]
|
||||
|
||||
raw_phys = torch.randn(*physical_5d, dtype=torch.bfloat16, device="cuda")
|
||||
fp8_phys, scale = to_float8(raw_phys)
|
||||
fp8_4d = fp8_phys.permute(*inv_order)[0]
|
||||
kv_5d = fp8_4d.view(
|
||||
num_blocks,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
2,
|
||||
head_size,
|
||||
).permute(0, 3, 1, 2, 4)
|
||||
return kv_5d, scale
|
||||
|
||||
|
||||
def ref_dequant(kv_cache, block_tables, k_scale, v_scale, dequant_dtype):
|
||||
"""Pure PyTorch reference: gather pages and dequantize fp8 -> dequant_dtype."""
|
||||
batch_size, num_pages_per_seq = block_tables.shape
|
||||
s = kv_cache.shape
|
||||
out = torch.zeros(
|
||||
batch_size * num_pages_per_seq + 1,
|
||||
s[1],
|
||||
s[2],
|
||||
s[3],
|
||||
s[4],
|
||||
dtype=dequant_dtype,
|
||||
device=kv_cache.device,
|
||||
)
|
||||
for b in range(batch_size):
|
||||
for p in range(num_pages_per_seq):
|
||||
page_idx = block_tables[b, p].item()
|
||||
if page_idx <= 0:
|
||||
continue
|
||||
mock_idx = b * num_pages_per_seq + p + 1
|
||||
out[mock_idx, 0] = (kv_cache[page_idx, 0].float() * k_scale.item()).to(
|
||||
dequant_dtype
|
||||
)
|
||||
out[mock_idx, 1] = (kv_cache[page_idx, 1].float() * v_scale.item()).to(
|
||||
dequant_dtype
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 8])
|
||||
@pytest.mark.parametrize("head_size", [64, 128])
|
||||
@pytest.mark.parametrize("block_size", [16, 32])
|
||||
@pytest.mark.parametrize("batch_size", [1, 4])
|
||||
@pytest.mark.parametrize("num_pages_per_seq", [3, 8])
|
||||
@pytest.mark.parametrize("layout", list(KVCacheLayout))
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_kvfp8_dequant(
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
batch_size: int,
|
||||
num_pages_per_seq: int,
|
||||
layout: KVCacheLayout,
|
||||
):
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
layout=layout,
|
||||
)
|
||||
|
||||
k_scale = scale.clone()
|
||||
v_scale = scale.clone()
|
||||
|
||||
block_tables = torch.randint(
|
||||
1,
|
||||
NUM_BLOCKS,
|
||||
(batch_size, num_pages_per_seq),
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache,
|
||||
block_tables,
|
||||
k_scale,
|
||||
v_scale,
|
||||
torch.bfloat16,
|
||||
)
|
||||
|
||||
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
|
||||
|
||||
expected_bt = torch.arange(
|
||||
1,
|
||||
batch_size * num_pages_per_seq + 1,
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
).reshape(batch_size, num_pages_per_seq)
|
||||
torch.testing.assert_close(mock_block_table, expected_bt)
|
||||
|
||||
# Page 0 is padding (never written), compare only pages 1+
|
||||
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_block_tables_with_zero_pages():
|
||||
"""Pages with index <= 0 must be skipped (early return in kernel)."""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 64
|
||||
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
)
|
||||
k_scale = v_scale = scale.clone()
|
||||
|
||||
# Mix of valid pages and zeros (padding)
|
||||
block_tables = torch.tensor(
|
||||
[[5, 0, 10], [0, 0, 0], [3, 7, 0]],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache,
|
||||
block_tables,
|
||||
k_scale,
|
||||
v_scale,
|
||||
torch.bfloat16,
|
||||
)
|
||||
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
|
||||
|
||||
# Only compare pages that were actually written (non-zero page indices)
|
||||
for b in range(block_tables.shape[0]):
|
||||
for p in range(block_tables.shape[1]):
|
||||
if block_tables[b, p].item() > 0:
|
||||
idx = b * block_tables.shape[1] + p + 1
|
||||
torch.testing.assert_close(
|
||||
mock_kv_cache[idx],
|
||||
ref[idx],
|
||||
atol=1e-3,
|
||||
rtol=1e-3,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_all_zero_block_tables():
|
||||
"""All-zero block_tables: kernel should write nothing."""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 4, 16, 64
|
||||
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
)
|
||||
k_scale = v_scale = scale.clone()
|
||||
|
||||
block_tables = torch.zeros(2, 4, dtype=torch.int32, device="cuda")
|
||||
|
||||
# Should not crash even though no pages are valid
|
||||
mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache,
|
||||
block_tables,
|
||||
k_scale,
|
||||
v_scale,
|
||||
torch.bfloat16,
|
||||
)
|
||||
assert mock_kv_cache.shape[0] == 2 * 4 + 1
|
||||
assert mock_block_table.shape == (2, 4)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_different_k_v_scales():
|
||||
"""Verify K and V are dequantized with independent scales."""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 64
|
||||
|
||||
kv_cache, _ = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
)
|
||||
k_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
v_scale = torch.tensor([2.0], dtype=torch.float32, device="cuda")
|
||||
|
||||
block_tables = torch.tensor([[1, 2]], dtype=torch.int32, device="cuda")
|
||||
|
||||
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache,
|
||||
block_tables,
|
||||
k_scale,
|
||||
v_scale,
|
||||
torch.bfloat16,
|
||||
)
|
||||
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
|
||||
|
||||
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_single_page_per_seq():
|
||||
"""Minimum grid dim 1 = 1 page per sequence."""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 128
|
||||
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
)
|
||||
k_scale = v_scale = scale.clone()
|
||||
|
||||
block_tables = torch.tensor([[5], [10], [20]], dtype=torch.int32, device="cuda")
|
||||
|
||||
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache,
|
||||
block_tables,
|
||||
k_scale,
|
||||
v_scale,
|
||||
torch.bfloat16,
|
||||
)
|
||||
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
|
||||
|
||||
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_large_page_indices():
|
||||
"""Page indices near the top of the buffer stress offset arithmetic."""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 128
|
||||
large_num_blocks = 32768
|
||||
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
large_num_blocks,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
)
|
||||
k_scale = v_scale = scale.clone()
|
||||
|
||||
# Use page indices near the top of the buffer
|
||||
block_tables = torch.tensor(
|
||||
[[large_num_blocks - 1, large_num_blocks - 2, 1]],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache,
|
||||
block_tables,
|
||||
k_scale,
|
||||
v_scale,
|
||||
torch.bfloat16,
|
||||
)
|
||||
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
|
||||
|
||||
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_large_block_size():
|
||||
"""block_size=64 -> HEAD_STRIDE=8192, large tl.arange per thread block."""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 4, 64, 128
|
||||
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
)
|
||||
k_scale = v_scale = scale.clone()
|
||||
|
||||
block_tables = torch.randint(
|
||||
1,
|
||||
NUM_BLOCKS,
|
||||
(2, 4),
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache,
|
||||
block_tables,
|
||||
k_scale,
|
||||
v_scale,
|
||||
torch.bfloat16,
|
||||
)
|
||||
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
|
||||
|
||||
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_cross_layer_many_layers():
|
||||
"""Non-contiguous with 36 layers -- matches real gpt-oss-120b.
|
||||
Cross-layer (BLHNC) strides are far from contiguous (factor of 36 in the gaps).
|
||||
"""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 64
|
||||
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
layout=KVCacheLayout.BLHNC,
|
||||
num_layers=36,
|
||||
)
|
||||
k_scale = v_scale = scale.clone()
|
||||
|
||||
block_tables = torch.randint(
|
||||
1,
|
||||
NUM_BLOCKS,
|
||||
(4, 6),
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache,
|
||||
block_tables,
|
||||
k_scale,
|
||||
v_scale,
|
||||
torch.bfloat16,
|
||||
)
|
||||
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
|
||||
|
||||
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
|
||||
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the replicated-embedding fused gather/norm kernels
|
||||
(``vllm.model_executor.layers.fused_embed_norm``).
|
||||
|
||||
The guarantee: enabling ``VLLM_REPLICATE_EMBED`` (replicated table + fused
|
||||
kernels) must not change model outputs. The gathered residual is bit-exact and
|
||||
the fused norms match the unfused reference.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_embed_norm import (
|
||||
fused_embed_eh_norm,
|
||||
fused_embed_norm,
|
||||
)
|
||||
|
||||
# The model-local (untouched) eh-norm the replicate path must match.
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPE = torch.bfloat16
|
||||
VOCAB, HIDDEN, NUM_TOKENS, EPS = 8192, 4096, 129, 1e-6
|
||||
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fused embed/norm Triton kernels require a CUDA/ROCm device",
|
||||
)
|
||||
|
||||
|
||||
def _rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
# Full-precision (fp32) reference RMSNorm.
|
||||
var = x.float().pow(2).mean(dim=-1, keepdim=True)
|
||||
return x.float() * torch.rsqrt(var + eps) * w.float()
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@torch.inference_mode()
|
||||
def test_fused_embed_norm_matches_reference():
|
||||
"""Main-model fusion: the residual is the exact gather and the second output
|
||||
is a correct RMSNorm. The norm matches a full-precision reference to ~2 bf16
|
||||
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel."""
|
||||
set_random_seed(13)
|
||||
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
||||
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
||||
weight = torch.empty(HIDDEN, dtype=DTYPE, device="cuda").normal_(1.0, 0.1)
|
||||
|
||||
residual, normed = fused_embed_norm(ids, table, chain_weight=weight, eps=EPS)
|
||||
|
||||
embeds = table[ids.long()]
|
||||
torch.testing.assert_close(residual, embeds, atol=0.0, rtol=0.0)
|
||||
torch.testing.assert_close(
|
||||
normed.float(), _rmsnorm(embeds, weight, EPS), atol=1e-3, rtol=1e-2
|
||||
)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@torch.inference_mode()
|
||||
def test_fused_embed_eh_norm_matches_reference():
|
||||
"""MTP fusion (folded gather) is bit-exact vs gathering the embeds and
|
||||
feeding the untouched model-local ``fused_eh_norm``."""
|
||||
set_random_seed(13)
|
||||
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
||||
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
||||
prev = torch.randn(NUM_TOKENS, HIDDEN, dtype=DTYPE, device="cuda")
|
||||
enorm_w = torch.randn(HIDDEN, dtype=DTYPE, device="cuda")
|
||||
hnorm_w = torch.randn(HIDDEN, dtype=DTYPE, device="cuda")
|
||||
positions = torch.arange(NUM_TOKENS, device="cuda") # includes pos 0
|
||||
|
||||
fused = fused_embed_eh_norm(positions, ids, table, prev, enorm_w, hnorm_w, EPS)
|
||||
ref = fused_eh_norm(positions, table[ids.long()], prev, enorm_w, hnorm_w, EPS)
|
||||
|
||||
torch.testing.assert_close(fused, ref, atol=0.0, rtol=0.0)
|
||||
@@ -1,303 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness + large-token-count launch tests for fused_q_kv_rmsnorm.
|
||||
|
||||
Before the grid-dim fix the kernel used grid ``(2, num_tokens)``, which hit
|
||||
CUDA's 65535 grid-y cap for ``num_tokens >= 65536`` and failed with
|
||||
``Triton Error [CUDA]: invalid argument`` at every large chunked-prefill
|
||||
profile run. These tests pin the new grid layout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.common.ops.fused_qk_rmsnorm import (
|
||||
fused_q_kv_rmsnorm,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fused_q_kv_rmsnorm requires a CUDA/ROCm device",
|
||||
)
|
||||
|
||||
|
||||
def _ref_rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
x_f32 = x.to(torch.float32)
|
||||
variance = x_f32.pow(2).mean(dim=-1, keepdim=True)
|
||||
y = x_f32 * torch.rsqrt(variance + eps) * w.to(torch.float32)
|
||||
return y.to(x.dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 1024, 8192])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_fused_q_kv_rmsnorm_correctness(num_tokens: int, dtype: torch.dtype):
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
q_size, kv_size = 192, 576
|
||||
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
|
||||
qw = torch.randn(q_size, dtype=dtype, device=device)
|
||||
kvw = torch.randn(kv_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
|
||||
|
||||
qr_ref = _ref_rmsnorm(qr, qw, eps)
|
||||
kv_ref = _ref_rmsnorm(kv, kvw, eps)
|
||||
|
||||
tol = dict(rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(qr_out, qr_ref, **tol)
|
||||
torch.testing.assert_close(kv_out, kv_ref, **tol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16])
|
||||
def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
|
||||
"""Regression guard: the outputs must be packed row-major even when the
|
||||
inputs are column slices of a wider fused-projection buffer.
|
||||
|
||||
empty_like preserves the strides of size-1 dims, so with num_tokens == 1
|
||||
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
|
||||
qr_out with row stride q_size + kv_size. Downstream dispatchers that
|
||||
require packed row-major inputs then reject the tensor and silently fall
|
||||
back to a slower GEMM path on every decode step."""
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
q_size, kv_size = 192, 576
|
||||
fused = torch.randn(num_tokens, q_size + kv_size, dtype=dtype, device=device)
|
||||
qr, kv = fused.split([q_size, kv_size], dim=-1)
|
||||
qw = torch.randn(q_size, dtype=dtype, device=device)
|
||||
kvw = torch.randn(kv_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
|
||||
|
||||
assert qr_out.stride() == (q_size, 1)
|
||||
assert kv_out.stride() == (kv_size, 1)
|
||||
|
||||
qr_ref = _ref_rmsnorm(qr, qw, eps)
|
||||
kv_ref = _ref_rmsnorm(kv, kvw, eps)
|
||||
tol = dict(rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(qr_out, qr_ref, **tol)
|
||||
torch.testing.assert_close(kv_out, kv_ref, **tol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [65535, 65536, 131072])
|
||||
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
|
||||
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
|
||||
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
|
||||
lifts that bound to 2**31-1."""
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
q_size, kv_size = 192, 576
|
||||
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
|
||||
qw = torch.randn(q_size, dtype=dtype, device=device)
|
||||
kvw = torch.randn(kv_size, dtype=dtype, device=device)
|
||||
|
||||
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, 1e-6)
|
||||
# spot-check a couple of rows against the torch reference
|
||||
for row in (0, num_tokens // 2, num_tokens - 1):
|
||||
torch.testing.assert_close(
|
||||
qr_out[row],
|
||||
_ref_rmsnorm(qr[row : row + 1], qw, 1e-6)[0],
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
kv_out[row],
|
||||
_ref_rmsnorm(kv[row : row + 1], kvw, 1e-6)[0],
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(100), reason="MXFP8 needs Blackwell"
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 17, 128, 129, 1024])
|
||||
@pytest.mark.parametrize("q_size", [384, 1280, 1344])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_fused_q_kv_rmsnorm_quant_matches_separate(num_tokens, q_size, dtype):
|
||||
"""Preserve native MXFP8 rounding and all swizzled scale padding bytes."""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.query_quant import (
|
||||
fused_q_kv_rmsnorm_quant,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
kv_size = 512
|
||||
x = torch.randn(num_tokens, q_size + kv_size, device="cuda", dtype=dtype)
|
||||
x[:1] = 0
|
||||
x[1:2] *= 1e-30
|
||||
qr, kv = x.split([q_size, kv_size], -1)
|
||||
qw = torch.randn(q_size, device="cuda", dtype=x.dtype)
|
||||
kvw = torch.randn(kv_size, device="cuda", dtype=x.dtype)
|
||||
eps = 1e-20
|
||||
result = fused_q_kv_rmsnorm_quant(qr, kv, qw, kvw, eps)
|
||||
assert result[0].orig_dtype == dtype
|
||||
assert result[0].orig_shape == qr.shape
|
||||
if not num_tokens:
|
||||
assert result[0].data.shape == qr.shape
|
||||
assert result[0].scale.numel() == 0
|
||||
assert result[1].shape == kv.shape
|
||||
return
|
||||
qr_ref, kv_ref = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
|
||||
q_ref, scale_ref = mxfp8_e4m3_quantize(qr_ref, is_sf_swizzled_layout=True)
|
||||
for replay in (False, True):
|
||||
if replay:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
result = fused_q_kv_rmsnorm_quant(qr, kv, qw, kvw, eps)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
result[0].data.view(torch.uint8), q_ref.view(torch.uint8), rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(result[0].scale, scale_ref, rtol=0, atol=0)
|
||||
torch.testing.assert_close(result[1], kv_ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(100), reason="MXFP8 needs Blackwell"
|
||||
)
|
||||
@pytest.mark.parametrize("backend", ["cute-dsl", "cutlass"])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 256])
|
||||
@pytest.mark.parametrize("projection", ["attention", "indexer"])
|
||||
@pytest.mark.parametrize("bias", [False, True])
|
||||
def test_shared_query_quant_preserves_projection(
|
||||
num_tokens, backend, projection, bias, monkeypatch
|
||||
):
|
||||
"""The native-checkpoint projection keeps identical GEMM outputs."""
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8 import Mxfp8LinearLayerConfig
|
||||
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
FlashInferCutlassMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
expose_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
|
||||
from vllm.models.deepseek_v41.attention import (
|
||||
DeepseekV4Attention,
|
||||
DeepseekV4Indexer,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.query_quant import (
|
||||
can_fuse_query_quant,
|
||||
fused_q_kv_rmsnorm_quant,
|
||||
)
|
||||
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
|
||||
|
||||
cls: (
|
||||
type[FlashInferCutedslMxfp8LinearKernel]
|
||||
| type[FlashInferCutlassMxfp8LinearKernel]
|
||||
)
|
||||
if backend == "cute-dsl":
|
||||
cls = FlashInferCutedslMxfp8LinearKernel
|
||||
else:
|
||||
cls = FlashInferCutlassMxfp8LinearKernel
|
||||
supported, reason = cls.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
torch.manual_seed(0)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(compilation_config=CompilationConfig(mode=CompilationMode.NONE))
|
||||
):
|
||||
# Match the checkpoint's TP4 attention and replicated indexer widths.
|
||||
if projection == "attention":
|
||||
linear = ColumnParallelLinear.__new__(ColumnParallelLinear)
|
||||
output_size = 8192
|
||||
else:
|
||||
linear = ReplicatedLinear.__new__(ReplicatedLinear)
|
||||
output_size = 4096
|
||||
torch.nn.Module.__init__(linear)
|
||||
linear.bias = (
|
||||
torch.nn.Parameter(
|
||||
torch.randn(output_size, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
if bias
|
||||
else None
|
||||
)
|
||||
linear.skip_bias_add = False
|
||||
linear.return_bias = projection == "indexer"
|
||||
linear.gather_output = False
|
||||
w, ws = mxfp8_e4m3_quantize(
|
||||
torch.randn(output_size, 1280, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
linear.weight = torch.nn.Parameter(w, requires_grad=False)
|
||||
linear.weight_scale = torch.nn.Parameter(ws, requires_grad=False)
|
||||
kernel = cls(Mxfp8LinearLayerConfig(weight_shape=w.shape))
|
||||
quant_config = DeepseekV4FP8Config.from_config(
|
||||
{
|
||||
"quant_method": "fp8",
|
||||
"activation_scheme": "dynamic",
|
||||
"weight_block_size": [32, 32],
|
||||
"scale_fmt": "ue8m0",
|
||||
"expert_dtype": "fp4",
|
||||
}
|
||||
)
|
||||
method = quant_config.get_quant_method(linear, "model.layers.2.attn.wq_b")
|
||||
assert type(method) is ModelOptLinearMethod
|
||||
method.out_dtype = torch.bfloat16
|
||||
method.kernel = kernel
|
||||
linear.quant_method = method
|
||||
kernel.process_weights_after_loading(linear)
|
||||
expose_input_quant_key(linear, kernel)
|
||||
assert can_fuse_query_quant([linear, linear])
|
||||
x = torch.randn(num_tokens, 1792, device="cuda", dtype=torch.bfloat16)
|
||||
qr, kv = x.split([1280, 512], -1)
|
||||
qw = torch.randn(1280, device="cuda", dtype=x.dtype)
|
||||
kvw = torch.randn(512, device="cuda", dtype=x.dtype)
|
||||
qr_ref, _ = fused_q_kv_rmsnorm(qr, kv, qw, kvw, 1e-20)
|
||||
q, _ = fused_q_kv_rmsnorm_quant(qr, kv, qw, kvw, 1e-20)
|
||||
owner = cast(Any, SimpleNamespace(wq_b=linear))
|
||||
project = (
|
||||
DeepseekV4Attention._wq_b_proj
|
||||
if projection == "attention"
|
||||
else DeepseekV4Indexer._wq_b_proj
|
||||
)
|
||||
expected = project(owner, qr_ref)
|
||||
|
||||
def reject_requantization(*args, **kwargs):
|
||||
raise AssertionError("pre-quantized input must skip quantization")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mxfp8.flashinfer.mxfp8_e4m3_quantize",
|
||||
reject_requantization,
|
||||
)
|
||||
actual = project(owner, q)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = project(owner, q)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
assert not can_fuse_query_quant(
|
||||
[linear, torch.nn.Linear(1280, 256, bias=False)]
|
||||
)
|
||||
reshaped = project(
|
||||
owner, replace(q, orig_shape=torch.Size((1, num_tokens, 1280)))
|
||||
)
|
||||
torch.testing.assert_close(reshaped, expected.unsqueeze(0), rtol=0, atol=0)
|
||||
with pytest.raises(AssertionError, match="consumer kernel"):
|
||||
linear(replace(q, quant_key=kNvfp4Dynamic))
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Accuracy tests for the fused Triton bilinear position-embedding kernel.
|
||||
|
||||
Compares ``triton_pos_embed_interpolate`` against the pure-PyTorch
|
||||
``pos_embed_interpolate_native`` across a variety of grid shapes and dtypes.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
if HAS_TRITON:
|
||||
from vllm.model_executor.models.qwen3_vl import (
|
||||
pos_embed_interpolate_native,
|
||||
triton_pos_embed_interpolate,
|
||||
)
|
||||
|
||||
|
||||
DTYPES = [torch.float32, torch.bfloat16]
|
||||
# Qwen3-VL default
|
||||
NUM_GRID_PER_SIDE = 48
|
||||
SPATIAL_MERGE_SIZE = 2
|
||||
HIDDEN_DIM = 1152
|
||||
|
||||
# 4 square + 4 non-square grids (h, w divisible by spatial_merge_size=2)
|
||||
SQUARE_GRIDS = [(1, 4, 4), (1, 16, 16), (1, 32, 32), (1, 48, 48)]
|
||||
NON_SQUARE_GRIDS = [(1, 8, 16), (1, 14, 20), (1, 32, 48), (1, 60, 80)]
|
||||
ALL_GRIDS = SQUARE_GRIDS + NON_SQUARE_GRIDS
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
|
||||
@pytest.mark.parametrize("dtype", DTYPES, ids=lambda d: str(d).split(".")[-1])
|
||||
@pytest.mark.parametrize(
|
||||
"grid_thw",
|
||||
ALL_GRIDS,
|
||||
ids=[f"{t}x{h}x{w}" for t, h, w in ALL_GRIDS],
|
||||
)
|
||||
def test_triton_matches_native(
|
||||
grid_thw: tuple[int, int, int],
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Triton kernel output must match the native PyTorch implementation."""
|
||||
t, h, w = grid_thw
|
||||
device = "cuda"
|
||||
|
||||
# Scale to match real Qwen3-VL pos_embed weight distribution (std~0.23).
|
||||
torch.manual_seed(42)
|
||||
embed_weight = (
|
||||
torch.randn(
|
||||
NUM_GRID_PER_SIDE * NUM_GRID_PER_SIDE,
|
||||
HIDDEN_DIM,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
* 0.25
|
||||
)
|
||||
|
||||
native_out = pos_embed_interpolate_native(
|
||||
embed_weight, t, h, w, NUM_GRID_PER_SIDE, SPATIAL_MERGE_SIZE, dtype
|
||||
)
|
||||
triton_out = triton_pos_embed_interpolate(
|
||||
embed_weight, t, h, w, NUM_GRID_PER_SIDE, SPATIAL_MERGE_SIZE, dtype
|
||||
)
|
||||
|
||||
assert native_out.shape == triton_out.shape, (
|
||||
f"Shape mismatch: native {native_out.shape} vs triton {triton_out.shape}"
|
||||
)
|
||||
|
||||
# Small numerical differences arise from the precomputed h/w_scale
|
||||
# in the triton kernel vs torch.linspace in the native path, which can
|
||||
# cause single-ULP output differences
|
||||
# in a handful of elements.
|
||||
atol = {torch.float32: 5e-5, torch.bfloat16: 1e-2}[dtype]
|
||||
rtol = {torch.float32: 1e-5, torch.bfloat16: 1e-2}[dtype]
|
||||
torch.testing.assert_close(triton_out, native_out, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
|
||||
@pytest.mark.parametrize("dtype", DTYPES, ids=lambda d: str(d).split(".")[-1])
|
||||
def test_temporal_repeat(dtype: torch.dtype) -> None:
|
||||
"""Verify temporal dimension t > 1 correctly repeats the spatial pattern."""
|
||||
device = "cuda"
|
||||
h, w = 16, 16
|
||||
t_single, t_multi = 1, 3
|
||||
|
||||
# Scale to match real Qwen3-VL pos_embed weight distribution (std~0.23).
|
||||
torch.manual_seed(42)
|
||||
embed_weight = (
|
||||
torch.randn(
|
||||
NUM_GRID_PER_SIDE * NUM_GRID_PER_SIDE,
|
||||
HIDDEN_DIM,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
* 0.25
|
||||
)
|
||||
|
||||
out_single = triton_pos_embed_interpolate(
|
||||
embed_weight,
|
||||
t_single,
|
||||
h,
|
||||
w,
|
||||
NUM_GRID_PER_SIDE,
|
||||
SPATIAL_MERGE_SIZE,
|
||||
dtype,
|
||||
)
|
||||
out_multi = triton_pos_embed_interpolate(
|
||||
embed_weight,
|
||||
t_multi,
|
||||
h,
|
||||
w,
|
||||
NUM_GRID_PER_SIDE,
|
||||
SPATIAL_MERGE_SIZE,
|
||||
dtype,
|
||||
)
|
||||
|
||||
expected = out_single.repeat(t_multi, 1)
|
||||
torch.testing.assert_close(out_multi, expected, atol=0, rtol=0)
|
||||
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the fused_qk_norm_rope helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_fused_qk_norm_rope.py`.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from tests.kernels.helion.utils import skip_if_platform_unsupported
|
||||
from vllm.benchmarks.lib.utils import default_vllm_config
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.ops.fused_qk_norm_rope import (
|
||||
_pick_cache,
|
||||
baseline,
|
||||
fused_qk_norm_rope,
|
||||
pick_config,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
@default_vllm_config()
|
||||
def _generate_fake_input(
|
||||
num_tokens: int, num_q_heads: int, num_kv_heads: int
|
||||
) -> tuple[Any, ...]:
|
||||
with FakeTensorMode():
|
||||
head_dim = 128
|
||||
eps = 1e-6
|
||||
is_neox = True
|
||||
rotary_ratio = 1.0
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim
|
||||
qkv = torch.randn(num_tokens, total_dim, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
|
||||
q_weight = torch.normal(
|
||||
mean=1.0,
|
||||
std=1.0,
|
||||
size=(head_dim,),
|
||||
dtype=qkv.dtype,
|
||||
device=device,
|
||||
)
|
||||
k_weight = torch.normal(
|
||||
mean=1.0,
|
||||
std=1.0,
|
||||
size=(head_dim,),
|
||||
dtype=qkv.dtype,
|
||||
device=device,
|
||||
)
|
||||
rotary_dim = int(head_dim * rotary_ratio)
|
||||
rope = RotaryEmbedding(
|
||||
head_size=head_dim,
|
||||
rotary_dim=rotary_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000.0,
|
||||
is_neox_style=is_neox,
|
||||
dtype=dtype,
|
||||
).to(device)
|
||||
args = (
|
||||
qkv,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
q_weight,
|
||||
k_weight,
|
||||
rope.cos_sin_cache,
|
||||
is_neox,
|
||||
positions.view(-1),
|
||||
)
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_config_manager_singleton():
|
||||
ConfigManager.reset_instance()
|
||||
ConfigManager()
|
||||
yield
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
|
||||
class TestFusedQkNormRopeConfigPicker:
|
||||
def setup_method(self):
|
||||
_pick_cache.clear()
|
||||
|
||||
def test_config_picker_exact_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}
|
||||
)
|
||||
|
||||
def test_config_picker_closest_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}),
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(20, 3000, 70)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key is None
|
||||
|
||||
def test_config_picker_fallback_to_largest(self):
|
||||
config_keys = [
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}),
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
|
||||
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(64, 8192, 256)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}
|
||||
)
|
||||
|
||||
|
||||
class TestFusedQkNormRopeCorrectness:
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads, num_kv_heads, head_dim", [(16, 4, 128), (64, 8, 128)]
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 1024, 1025])
|
||||
@pytest.mark.parametrize("is_neox", [False, True])
|
||||
@pytest.mark.parametrize("rotary_ratio", [1.0, 0.5, 0.25])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@default_vllm_config()
|
||||
def test_fused_qk_norm_rope(
|
||||
self,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
num_tokens: int,
|
||||
is_neox: bool,
|
||||
rotary_ratio: float,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
skip_if_platform_unsupported("fused_qk_norm_rope")
|
||||
|
||||
torch.manual_seed(42)
|
||||
eps = 1e-6
|
||||
device = "cuda"
|
||||
total_dim = (num_heads + 2 * num_kv_heads) * head_dim
|
||||
ref_qkv = torch.empty(
|
||||
num_tokens, total_dim, dtype=dtype, device=device
|
||||
).uniform_(-0.1, 0.1)
|
||||
ops_qkv = ref_qkv.clone()
|
||||
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
|
||||
q_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2)
|
||||
k_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2)
|
||||
rotary_dim = int(head_dim * rotary_ratio)
|
||||
rope = RotaryEmbedding(
|
||||
head_size=head_dim,
|
||||
rotary_dim=rotary_dim,
|
||||
max_position_embeddings=40960,
|
||||
base=10000.0,
|
||||
is_neox_style=is_neox,
|
||||
dtype=dtype,
|
||||
).to(device)
|
||||
|
||||
baseline(
|
||||
ref_qkv,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
q_weight,
|
||||
k_weight,
|
||||
rope.cos_sin_cache,
|
||||
is_neox,
|
||||
positions.view(-1),
|
||||
)
|
||||
|
||||
fused_qk_norm_rope(
|
||||
ops_qkv,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
q_weight,
|
||||
k_weight,
|
||||
rope.cos_sin_cache,
|
||||
is_neox,
|
||||
positions.view(-1),
|
||||
)
|
||||
|
||||
if dtype == torch.bfloat16:
|
||||
atol = 5e-2
|
||||
rtol = 5e-2
|
||||
else:
|
||||
atol = 1e-2
|
||||
rtol = 1e-2
|
||||
|
||||
torch.testing.assert_close(
|
||||
ref_qkv,
|
||||
ops_qkv,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
)
|
||||
|
||||
|
||||
class TestFusedQkNormRopeIntegration:
|
||||
def test_kernel_registration_integration(self):
|
||||
from vllm.kernels.helion.register import get_registered_kernels
|
||||
|
||||
registered_kernels = get_registered_kernels()
|
||||
assert "fused_qk_norm_rope" in registered_kernels
|
||||
|
||||
kernel_wrapper = registered_kernels["fused_qk_norm_rope"]
|
||||
assert kernel_wrapper.op_name == "fused_qk_norm_rope"
|
||||
assert kernel_wrapper._config_picker is not None
|
||||
assert kernel_wrapper._mutates_args == ["qkv"]
|
||||
@@ -1,302 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Integration test for the non-spec decode split in
|
||||
``GatedDeltaNet._forward_core``.
|
||||
|
||||
On a pure non-spec batch that mixes prefills with 1-token decodes, the layer
|
||||
peels the decodes (the contiguous decode-first front slice) off to
|
||||
``fused_sigmoid_gating_delta_rule_update`` -- the same recurrent update kernel
|
||||
the spec-decode path uses -- and runs only the prefill tail through
|
||||
``chunk_gated_delta_rule``. This must produce the same core-attention output and
|
||||
the same ssm-state pool update as running *everything* through
|
||||
``chunk_gated_delta_rule`` (the previous behavior).
|
||||
|
||||
Both paths are exercised through the REAL ``_forward_core``:
|
||||
|
||||
* ``meta_split`` is built by the real ``GDNAttentionMetadataBuilder`` for a
|
||||
mixed batch, so ``num_decodes > 0`` triggers the peel (and the builder rebases
|
||||
``chunk_indices``/``chunk_offsets`` to the prefill-only tail).
|
||||
* ``meta_unified`` is the same metadata with the decodes reclassified as
|
||||
prefills and full-batch chunk metadata, which forces ``_forward_core`` through
|
||||
the existing chunk-only path on identical inputs (the conv is unified over all
|
||||
non-spec tokens in both paths, so it cancels out and only the recurrent split
|
||||
is compared).
|
||||
|
||||
The Triton/FLA chunk backend is forced so the prefill-only ``chunk_indices``
|
||||
must stay consistent with the rebased ``cu_seqlens`` (a stringent, backend
|
||||
portable check of the split wiring).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip(
|
||||
reason="GDN _forward_core split test uses the CuteDSL prefill backend "
|
||||
"(requires CUDA SM10x).",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from tests.v1.attention.utils import ( # noqa: E402
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import set_current_vllm_config # noqa: E402
|
||||
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn # noqa: E402
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( # noqa: E402
|
||||
ChunkGatedDeltaRule,
|
||||
QwenGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import ( # noqa: E402
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.index import ( # noqa: E402
|
||||
prepare_chunk_indices,
|
||||
prepare_chunk_offsets,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import ( # noqa: E402
|
||||
FLA_CHUNK_SIZE,
|
||||
)
|
||||
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402
|
||||
|
||||
# Small GDN dims; head_k_dim/head_v_dim=128 keeps the chunk/update kernels happy.
|
||||
H = 4 # num key heads
|
||||
HV = 8 # num value heads
|
||||
K = 128 # head_k_dim
|
||||
V = 128 # head_v_dim
|
||||
CONV_KERNEL = 4
|
||||
KEY_DIM = H * K
|
||||
VALUE_DIM = HV * V
|
||||
CONV_DIM = 2 * KEY_DIM + VALUE_DIM
|
||||
BLOCK_SIZE = 16
|
||||
PREFIX = "model.layers.0.linear_attn"
|
||||
|
||||
|
||||
def _make_vllm_config():
|
||||
# A small, ungated GDN model whose config is cached locally; only the config
|
||||
# (scheduler/cache/compilation/hf) is used here, never the weights. Inject
|
||||
# linear_key_head_dim=128 and request the CuteDSL prefill backend -- the
|
||||
# supported GDN chunk kernel on Blackwell (the Triton/FLA chunk kernel is
|
||||
# unsupported on SM10x). CuteDSL consumes chunk_indices/chunk_offsets, so
|
||||
# this also exercises the prefill-only chunk-metadata wiring.
|
||||
cfg = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
hf_config_override={"linear_key_head_dim": K},
|
||||
)
|
||||
cfg.additional_config = {"gdn_prefill_backend": "cutedsl"}
|
||||
return cfg
|
||||
|
||||
|
||||
def _build_layer(
|
||||
vllm_config, conv_state, ssm_state, A_log, dt_bias, conv_weight, conv_bias
|
||||
):
|
||||
"""A minimal object that runs the real ``_forward_core`` bound to it."""
|
||||
layer = types.SimpleNamespace()
|
||||
layer.prefix = PREFIX
|
||||
layer.enable_packed_recurrent_decode = False
|
||||
layer.tp_size = 1
|
||||
layer.num_k_heads = H
|
||||
layer.num_v_heads = HV
|
||||
layer.head_k_dim = K
|
||||
layer.head_v_dim = V
|
||||
layer.key_dim = KEY_DIM
|
||||
layer.value_dim = VALUE_DIM
|
||||
layer.activation = "silu"
|
||||
layer.A_log = A_log
|
||||
layer.dt_bias = dt_bias
|
||||
layer.conv1d = types.SimpleNamespace(weight=conv_weight, bias=conv_bias)
|
||||
layer.kv_cache = (conv_state, ssm_state)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer.chunk_gated_delta_rule = ChunkGatedDeltaRule()
|
||||
for name in (
|
||||
"rearrange_mixed_qkv",
|
||||
"_forward_core",
|
||||
):
|
||||
setattr(
|
||||
layer,
|
||||
name,
|
||||
types.MethodType(getattr(QwenGatedDeltaNetAttention, name), layer),
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def _run_forward_core(layer, meta, mixed_qkv, b, a, num_tokens):
|
||||
core_attn_out = torch.zeros(
|
||||
num_tokens, HV, V, dtype=mixed_qkv.dtype, device=mixed_qkv.device
|
||||
)
|
||||
ctx = types.SimpleNamespace(attn_metadata={PREFIX: meta})
|
||||
with patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=ctx):
|
||||
layer._forward_core(
|
||||
mixed_qkv=mixed_qkv.clone(),
|
||||
b=b.clone(),
|
||||
a=a.clone(),
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
return core_attn_out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state_dtype", [torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("num_decodes,prefill_lens", [(3, [512, 300]), (4, [64, 5])])
|
||||
@pytest.mark.parametrize("fresh_prefill", [False, True])
|
||||
def test_forward_core_split_matches_unified(
|
||||
state_dtype: torch.dtype,
|
||||
num_decodes: int,
|
||||
prefill_lens: list[int],
|
||||
fresh_prefill: bool,
|
||||
) -> None:
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
vllm_config = _make_vllm_config()
|
||||
|
||||
# Decode-first batch: D 1-token decodes (with context), then the prefills.
|
||||
decode_seq_lens = [64] * num_decodes
|
||||
prefill_seq_lens = [
|
||||
pl if (fresh_prefill and i == 0) else pl + 37
|
||||
for i, pl in enumerate(prefill_lens)
|
||||
]
|
||||
seq_lens = decode_seq_lens + prefill_seq_lens
|
||||
query_lens = [1] * num_decodes + list(prefill_lens)
|
||||
batch = BatchSpec(seq_lens=seq_lens, query_lens=query_lens)
|
||||
|
||||
builder = GDNAttentionMetadataBuilder(
|
||||
# GDNAttentionMetadataBuilder declares kv_cache_spec as AttentionSpec but
|
||||
# asserts isinstance(kv_cache_spec, MambaSpec). MambaSpec is correct here.
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,)
|
||||
),
|
||||
layer_names=[PREFIX],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
meta_split = builder.build(common_prefix_len=0, common_attn_metadata=common)
|
||||
|
||||
assert meta_split.spec_sequence_masks is None
|
||||
assert meta_split.num_decodes == num_decodes
|
||||
assert meta_split.num_prefills == len(prefill_lens)
|
||||
assert meta_split.num_decode_tokens == num_decodes
|
||||
assert builder.gdn_prefill_backend == "cutedsl"
|
||||
|
||||
num_tokens = sum(query_lens)
|
||||
|
||||
# Full-batch chunk metadata for the unified reference path, built the same
|
||||
# way the builder would for a non-split batch (backend-matched).
|
||||
cu_full = meta_split.non_spec_query_start_loc
|
||||
assert cu_full is not None
|
||||
if builder.gdn_prefill_backend == "cutedsl":
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
prepare_metadata_cutedsl,
|
||||
)
|
||||
|
||||
full_ci, full_co = prepare_metadata_cutedsl(
|
||||
cu_full, int(cu_full[-1].item()), FLA_CHUNK_SIZE
|
||||
)
|
||||
else:
|
||||
cu_full_cpu = cu_full.cpu()
|
||||
full_ci = prepare_chunk_indices(cu_full_cpu, FLA_CHUNK_SIZE).to(device)
|
||||
full_co = prepare_chunk_offsets(cu_full_cpu, FLA_CHUNK_SIZE).to(device)
|
||||
meta_unified = dataclasses.replace(
|
||||
meta_split,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_prefills=meta_split.num_decodes + meta_split.num_prefills,
|
||||
num_prefill_tokens=(
|
||||
meta_split.num_decode_tokens + meta_split.num_prefill_tokens
|
||||
),
|
||||
chunk_indices=full_ci,
|
||||
chunk_offsets=full_co,
|
||||
# Unified path: the chunk kernel processes the full non-spec batch.
|
||||
prefill_query_start_loc=meta_split.non_spec_query_start_loc,
|
||||
prefill_state_indices=meta_split.non_spec_state_indices_tensor,
|
||||
prefill_has_initial_state=meta_split.has_initial_state,
|
||||
)
|
||||
|
||||
# Size the state pools from the indices the builder actually produced.
|
||||
assert meta_split.non_spec_state_indices_tensor is not None
|
||||
pool_size = int(meta_split.non_spec_state_indices_tensor.max().item()) + 1
|
||||
conv_state_shape, temporal_state_shape = (
|
||||
MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
1, H, HV, K, V, CONV_KERNEL, num_spec=0
|
||||
)
|
||||
)
|
||||
conv_state0 = (
|
||||
torch.randn(pool_size, *conv_state_shape, dtype=torch.bfloat16, device=device)
|
||||
* 0.05
|
||||
)
|
||||
ssm_state0 = (
|
||||
torch.randn(pool_size, *temporal_state_shape, dtype=state_dtype, device=device)
|
||||
* 0.05
|
||||
)
|
||||
|
||||
A_log = torch.randn(HV, dtype=torch.float32, device=device) * 0.1
|
||||
dt_bias = torch.randn(HV, dtype=torch.float32, device=device) * 0.1
|
||||
conv_weight = (
|
||||
torch.randn(CONV_DIM, 1, CONV_KERNEL, dtype=torch.bfloat16, device=device) * 0.1
|
||||
)
|
||||
conv_bias = torch.randn(CONV_DIM, dtype=torch.bfloat16, device=device) * 0.1
|
||||
|
||||
mixed_qkv = (
|
||||
torch.randn(num_tokens, CONV_DIM, dtype=torch.bfloat16, device=device) * 0.1
|
||||
)
|
||||
a = torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device) * 0.1
|
||||
b = torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device) * 0.1
|
||||
|
||||
# ---- Split path (real _forward_core, meta_split) ----
|
||||
conv_state_split = conv_state0.clone()
|
||||
ssm_state_split = ssm_state0.clone()
|
||||
layer_split = _build_layer(
|
||||
vllm_config,
|
||||
conv_state_split,
|
||||
ssm_state_split,
|
||||
A_log,
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
conv_bias,
|
||||
)
|
||||
out_split = _run_forward_core(layer_split, meta_split, mixed_qkv, b, a, num_tokens)
|
||||
|
||||
# ---- Unified path (real _forward_core, meta_unified) ----
|
||||
conv_state_unified = conv_state0.clone()
|
||||
ssm_state_unified = ssm_state0.clone()
|
||||
layer_unified = _build_layer(
|
||||
vllm_config,
|
||||
conv_state_unified,
|
||||
ssm_state_unified,
|
||||
A_log,
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
conv_bias,
|
||||
)
|
||||
out_unified = _run_forward_core(
|
||||
layer_unified, meta_unified, mixed_qkv, b, a, num_tokens
|
||||
)
|
||||
|
||||
# Conv is unified in both paths, so the conv-state update must be identical.
|
||||
torch.testing.assert_close(conv_state_split, conv_state_unified, atol=0, rtol=0)
|
||||
|
||||
# Chunk vs. recurrent update accumulate in different orders; mirror the
|
||||
# tolerances used by the kernel-level parity test.
|
||||
if state_dtype == torch.float32:
|
||||
atol = rtol = 2e-2
|
||||
else:
|
||||
atol = rtol = 6e-2
|
||||
torch.testing.assert_close(out_split, out_unified, atol=atol, rtol=rtol)
|
||||
torch.testing.assert_close(ssm_state_split, ssm_state_unified, atol=atol, rtol=rtol)
|
||||
@@ -1,277 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip(
|
||||
reason="GDN CuteDSL prefill requires CUDA SM10x.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import ( # noqa: E402
|
||||
chunk_gated_delta_rule_cutedsl,
|
||||
prepare_metadata_cutedsl,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops import ( # noqa: E402
|
||||
chunk_gated_delta_rule,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.index import ( # noqa: E402
|
||||
prepare_chunk_indices,
|
||||
prepare_chunk_offsets,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_seqs", [1, 5, 257])
|
||||
@pytest.mark.parametrize("state_dtype", [torch.bfloat16, torch.float32])
|
||||
def test_gdn_chunk_cutedsl_correctness(num_seqs: int, state_dtype: torch.dtype):
|
||||
rng_cpu = torch.Generator("cpu").manual_seed(1234)
|
||||
rng = torch.Generator("cuda").manual_seed(2345)
|
||||
|
||||
seq_lens = torch.randint(1, 130, (num_seqs,), dtype=torch.int32, generator=rng_cpu)
|
||||
cu_seqlens = torch.zeros(num_seqs + 1, device="cuda", dtype=torch.int32)
|
||||
cu_seqlens[1:] = seq_lens.to(device="cuda").cumsum(0)
|
||||
total_tokens = int(cu_seqlens[-1].item())
|
||||
|
||||
num_k_heads = 4
|
||||
num_v_heads = 8
|
||||
head_k_dim = 128
|
||||
head_v_dim = 128
|
||||
dtype = torch.bfloat16
|
||||
|
||||
q = torch.randn(
|
||||
1,
|
||||
total_tokens,
|
||||
num_k_heads,
|
||||
head_k_dim,
|
||||
device="cuda",
|
||||
dtype=dtype,
|
||||
generator=rng,
|
||||
)
|
||||
k = torch.randn_like(q, generator=rng)
|
||||
v = torch.randn(
|
||||
1,
|
||||
total_tokens,
|
||||
num_v_heads,
|
||||
head_v_dim,
|
||||
device="cuda",
|
||||
dtype=dtype,
|
||||
generator=rng,
|
||||
)
|
||||
q = F.normalize(q.float(), p=2, dim=-1).to(dtype)
|
||||
k = F.normalize(k.float(), p=2, dim=-1).to(dtype)
|
||||
a = torch.randn(
|
||||
1, total_tokens, num_v_heads, device="cuda", dtype=dtype, generator=rng
|
||||
)
|
||||
b = torch.randn(
|
||||
1, total_tokens, num_v_heads, device="cuda", dtype=dtype, generator=rng
|
||||
)
|
||||
# Match upstream FLA GatedDeltaNet synthetic initialization:
|
||||
# https://github.com/fla-org/flash-linear-attention/blob/main/fla/layers/gated_deltanet.py
|
||||
A = torch.empty(num_v_heads, device="cuda", dtype=torch.float32).uniform_(
|
||||
0, 16, generator=rng
|
||||
)
|
||||
A_log = torch.log(A)
|
||||
dt = torch.exp(
|
||||
torch.rand(num_v_heads, device="cuda", dtype=torch.float32, generator=rng)
|
||||
* (math.log(0.1) - math.log(0.001))
|
||||
+ math.log(0.001)
|
||||
)
|
||||
dt = torch.clamp(dt, min=1e-4)
|
||||
dt_bias = dt + torch.log(-torch.expm1(-dt))
|
||||
g = -A_log.exp().view(1, 1, num_v_heads) * F.softplus(
|
||||
a.float() + dt_bias.view(1, 1, num_v_heads)
|
||||
)
|
||||
beta = torch.sigmoid(b.float())
|
||||
initial_state = (
|
||||
torch.randn(
|
||||
num_seqs,
|
||||
num_v_heads,
|
||||
head_v_dim,
|
||||
head_k_dim,
|
||||
device="cuda",
|
||||
dtype=state_dtype,
|
||||
generator=rng,
|
||||
)
|
||||
* 0.05
|
||||
)
|
||||
|
||||
# check metadata kernel
|
||||
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected_indices = prepare_chunk_indices(cu_seqlens, 64)
|
||||
expected_offsets = prepare_chunk_offsets(cu_seqlens, 64)
|
||||
total_chunks = int(expected_offsets[-1].item())
|
||||
|
||||
torch.testing.assert_close(chunk_offsets, expected_offsets.to(torch.int32))
|
||||
torch.testing.assert_close(
|
||||
chunk_indices[:total_chunks],
|
||||
expected_indices,
|
||||
)
|
||||
|
||||
ref_o, ref_state = chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
actual_core_attn_out = torch.empty(
|
||||
total_tokens,
|
||||
num_v_heads,
|
||||
head_v_dim,
|
||||
device="cuda",
|
||||
dtype=dtype,
|
||||
)
|
||||
actual_o, actual_state = chunk_gated_delta_rule_cutedsl(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
core_attn_out=actual_core_attn_out,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# check main kernel
|
||||
o_error = (actual_o.float() - ref_o.float()).abs()
|
||||
state_error = (
|
||||
actual_state.float() - ref_state.to(actual_state.dtype).float()
|
||||
).abs()
|
||||
assert o_error.max().item() < 2e-3
|
||||
assert o_error.mean().item() < 6e-5
|
||||
assert state_error.max().item() < 2e-2
|
||||
assert state_error.mean().item() < 6e-4
|
||||
core_attn_out_error = (
|
||||
actual_core_attn_out.float() - actual_o.squeeze(0).float()
|
||||
).abs()
|
||||
assert core_attn_out_error.max().item() == 0
|
||||
|
||||
# check main kernel when core_attn_out is not passed
|
||||
no_buffer_o, no_buffer_state = chunk_gated_delta_rule_cutedsl(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
no_buffer_o_error = (no_buffer_o.float() - ref_o.float()).abs()
|
||||
no_buffer_state_error = (
|
||||
no_buffer_state.float() - ref_state.to(no_buffer_state.dtype).float()
|
||||
).abs()
|
||||
buffer_o_error = (no_buffer_o.float() - actual_o.float()).abs()
|
||||
buffer_state_error = (
|
||||
no_buffer_state.float() - actual_state.to(no_buffer_state.dtype).float()
|
||||
).abs()
|
||||
assert no_buffer_o_error.max().item() < 2e-3
|
||||
assert no_buffer_o_error.mean().item() < 6e-5
|
||||
assert no_buffer_state_error.max().item() < 2e-2
|
||||
assert no_buffer_state_error.mean().item() < 6e-4
|
||||
assert buffer_o_error.max().item() == 0
|
||||
assert buffer_state_error.max().item() == 0
|
||||
|
||||
|
||||
def test_gdn_chunk_cutedsl_correlated_keys_stability():
|
||||
"""Cover a KKT block where three BF16 Newton rounds diverge."""
|
||||
total_tokens = 64
|
||||
num_k_heads = 4
|
||||
num_v_heads = 8
|
||||
head_dim = 128
|
||||
rng = torch.Generator("cuda").manual_seed(3456)
|
||||
|
||||
q = torch.zeros(
|
||||
1,
|
||||
total_tokens,
|
||||
num_k_heads,
|
||||
head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k = torch.zeros_like(q)
|
||||
q[..., 0] = 1
|
||||
k[..., 0] = 1
|
||||
v = torch.randn(
|
||||
1,
|
||||
total_tokens,
|
||||
num_v_heads,
|
||||
head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
g = torch.zeros(
|
||||
1,
|
||||
total_tokens,
|
||||
num_v_heads,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
beta = torch.full_like(g, 0.9)
|
||||
initial_state = torch.zeros(
|
||||
1,
|
||||
num_v_heads,
|
||||
head_dim,
|
||||
head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
cu_seqlens = torch.tensor([0, total_tokens], device="cuda", dtype=torch.int32)
|
||||
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens)
|
||||
|
||||
ref_o, ref_state = chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
actual_o, actual_state = chunk_gated_delta_rule_cutedsl(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.isfinite(actual_o).all()
|
||||
assert torch.isfinite(actual_state).all()
|
||||
output_error = (actual_o.float() - ref_o.float()).abs()
|
||||
state_error = (
|
||||
actual_state.float() - ref_state.to(actual_state.dtype).float()
|
||||
).abs()
|
||||
assert output_error.max().item() < 1e-1
|
||||
assert output_error.mean().item() < 2e-3
|
||||
assert state_error.max().item() < 2e-1
|
||||
assert state_error.mean().item() < 6e-4
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,422 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
|
||||
ProcessGroupInfo,
|
||||
parallel_launch_with_config,
|
||||
)
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.expert_substitution import (
|
||||
ConstantExpertSubstitution,
|
||||
SubstitutedRoutedExperts,
|
||||
get_expert_substitution_spec,
|
||||
make_expert_substitution,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
from vllm.model_executor.models.mixtral import MixtralMoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
MOE_PREFIX = "model.layers.1.mlp.experts"
|
||||
|
||||
|
||||
def _model_config(approximate_experts, **kwargs) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
hf_config=SimpleNamespace(approximate_experts=approximate_experts),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _expert_substitution(targets: dict[str, dict[int, str]]) -> dict:
|
||||
"""The ``compression_config.transform_config.expert_substitution`` format."""
|
||||
return {
|
||||
"version": 1,
|
||||
"router_semantics": {
|
||||
"preserve_logical_expert_ids": True,
|
||||
"preserve_router_weights": True,
|
||||
"renormalize_after_substitution": False,
|
||||
},
|
||||
"targets": {
|
||||
path: {
|
||||
"num_logical_experts": 5,
|
||||
"weight_layout": "compact_retained_experts",
|
||||
"replacements": {
|
||||
str(expert_id): {
|
||||
"format": "constant-v1",
|
||||
"tensors": {"value": name},
|
||||
}
|
||||
for expert_id, name in replacements.items()
|
||||
},
|
||||
}
|
||||
for path, replacements in targets.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _compressed_model_config(expert_substitution: dict, **kwargs) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
compression_config={
|
||||
"transform_config": {"expert_substitution": expert_substitution}
|
||||
}
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _substitution(hidden_size: int, dtype=torch.float32):
|
||||
# Experts 1 and 3 of 5 are substituted; 0, 2, 4 map to physical 0, 1, 2.
|
||||
return ConstantExpertSubstitution(1, 5, (1, 3), hidden_size, dtype)
|
||||
|
||||
|
||||
def test_parse_approximate_experts():
|
||||
assert get_expert_substitution_spec(None) is None
|
||||
assert get_expert_substitution_spec(_model_config(None)) is None
|
||||
spec = get_expert_substitution_spec(_model_config({"1": [3, 1], "2": [], 5: (0,)}))
|
||||
assert spec.experts == {1: (1, 3), 5: (0,)}
|
||||
assert spec.value_names == {}
|
||||
text_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(approximate_experts={"0": [2]}),
|
||||
hf_config=SimpleNamespace(),
|
||||
)
|
||||
assert get_expert_substitution_spec(text_config).experts == {0: (2,)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"approximate_experts",
|
||||
[[1, 2], {"layer": [1]}, {"1": [1, 1]}, {"1": [-1]}, {"1": ["x"]}],
|
||||
)
|
||||
def test_parse_approximate_experts_rejects_invalid_entries(approximate_experts):
|
||||
with pytest.raises(ValueError, match="approximate_experts"):
|
||||
get_expert_substitution_spec(_model_config(approximate_experts))
|
||||
|
||||
|
||||
def test_parse_expert_substitution_with_explicit_value_names():
|
||||
spec = get_expert_substitution_spec(
|
||||
_compressed_model_config(
|
||||
_expert_substitution(
|
||||
{
|
||||
MOE_PREFIX: {3: "shared", 1: "layer_1.expert_1"},
|
||||
"model.layers.2.mlp.experts": {0: "shared"},
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
assert spec.experts == {1: (1, 3), 2: (0,)}
|
||||
assert spec.value_names == {
|
||||
"shared": ((1, 3), (2, 0)),
|
||||
"layer_1.expert_1": ((1, 1),),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate",
|
||||
[
|
||||
lambda raw: raw.update(version=2),
|
||||
lambda raw: raw["router_semantics"].update(renormalize_after_substitution=True),
|
||||
lambda raw: raw["targets"].update({"mlp.experts": raw["targets"][MOE_PREFIX]}),
|
||||
lambda raw: raw["targets"][MOE_PREFIX].update(weight_layout="dense"),
|
||||
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].update(
|
||||
format="linear-v1"
|
||||
),
|
||||
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].pop("tensors"),
|
||||
],
|
||||
)
|
||||
def test_parse_expert_substitution_rejects_unsupported_metadata(mutate):
|
||||
raw = _expert_substitution({MOE_PREFIX: {1: "value"}})
|
||||
mutate(raw)
|
||||
with pytest.raises(ValueError, match="invalid expert_substitution metadata"):
|
||||
get_expert_substitution_spec(_compressed_model_config(raw))
|
||||
|
||||
|
||||
def test_parse_rejects_both_formats():
|
||||
config = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
|
||||
config.hf_config.approximate_experts = {"1": [1]}
|
||||
with pytest.raises(ValueError, match="both"):
|
||||
get_expert_substitution_spec(config)
|
||||
|
||||
|
||||
def test_make_expert_substitution_matches_layer_index():
|
||||
config = _model_config({"1": [3, 1]})
|
||||
substitution = make_expert_substitution(config, MOE_PREFIX, 5, 16, torch.bfloat16)
|
||||
|
||||
assert substitution is not None
|
||||
assert substitution.layer_idx == 1
|
||||
assert substitution.substituted_expert_ids == (1, 3)
|
||||
assert substitution.num_compute_experts == 3
|
||||
assert substitution.values.dtype == torch.bfloat16
|
||||
assert [substitution.physical_expert_id(i) for i in range(5)] == [0, -1, 1, -1, 2]
|
||||
assert (
|
||||
make_expert_substitution(config, "model.layers.2.mlp.experts", 5, 16, None)
|
||||
is None
|
||||
)
|
||||
assert make_expert_substitution(None, MOE_PREFIX, 5, 16, None) is None
|
||||
explicit = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
|
||||
substitution = make_expert_substitution(explicit, MOE_PREFIX, 5, 16, None)
|
||||
assert substitution is not None
|
||||
assert substitution.substituted_expert_ids == (1,)
|
||||
with pytest.raises(ValueError, match="exactly one layer index"):
|
||||
make_expert_substitution(config, "mlp.experts", 5, 16, None)
|
||||
with pytest.raises(ValueError, match="within"):
|
||||
make_expert_substitution(config, MOE_PREFIX, 3, 16, None)
|
||||
with pytest.raises(ValueError, match="retain at least one expert"):
|
||||
make_expert_substitution(_model_config({"1": [0, 1]}), MOE_PREFIX, 2, 16, None)
|
||||
|
||||
|
||||
def test_constant_substitution_loads_values():
|
||||
substitution = _substitution(2)
|
||||
substitution.weight_loader(substitution.values, torch.tensor([1.0, 2.0]), 3)
|
||||
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
|
||||
substitution.validate_loaded_values("layer")
|
||||
substitution.weight_loader(substitution.values, torch.tensor([3.0, 4.0]), 1)
|
||||
substitution.validate_loaded_values("layer")
|
||||
torch.testing.assert_close(
|
||||
substitution.values, torch.tensor([[3.0, 4.0], [1.0, 2.0]])
|
||||
)
|
||||
with pytest.raises(ValueError, match="not listed"):
|
||||
substitution.weight_loader(substitution.values, torch.ones(2), 2)
|
||||
with pytest.raises(ValueError, match="has shape"):
|
||||
substitution.weight_loader(substitution.values, torch.ones(3), 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 3, 7])
|
||||
@pytest.mark.parametrize("hidden_size", [2, 4])
|
||||
def test_constant_substitution_transforms_routes_and_computes_side_output(
|
||||
num_tokens, hidden_size
|
||||
):
|
||||
substitution = _substitution(2)
|
||||
substitution.values.data.copy_(torch.tensor([[10.0, 20.0], [30.0, 40.0]]))
|
||||
|
||||
rows = torch.arange(num_tokens) % 3
|
||||
topk_ids = torch.tensor([[1, 2], [4, 3], [99, -1]])[rows]
|
||||
topk_weights = torch.tensor([[0.25, 0.75], [0.6, 0.4], [0.5, 0.5]])[rows]
|
||||
constant_output = substitution.transform_routes(
|
||||
torch.zeros(num_tokens, hidden_size), topk_weights, topk_ids
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
topk_weights, torch.tensor([[0.0, 0.75], [0.6, 0.0], [0.0, 0.0]])[rows]
|
||||
)
|
||||
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1], [2, 0], [0, 0]])[rows])
|
||||
expected = torch.zeros(num_tokens, hidden_size)
|
||||
expected[:, :2] = torch.tensor([[2.5, 5.0], [12.0, 16.0], [0.0, 0.0]])[rows]
|
||||
torch.testing.assert_close(constant_output, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[
|
||||
"cpu",
|
||||
pytest.param(
|
||||
"cuda",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="requires a CUDA-like platform",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_constant_substitution_preserves_router_weight_precision(dtype, device):
|
||||
substitution = _substitution(1, dtype).to(device)
|
||||
substitution.values.data.copy_(
|
||||
torch.tensor([[60000.0], [-60000.0]], dtype=dtype, device=device)
|
||||
)
|
||||
topk_weights = torch.tensor([[0.5001, 0.4999]], dtype=torch.float32, device=device)
|
||||
topk_ids = torch.tensor([[1, 3]], device=device)
|
||||
# Nearly cancelling constants must retain the difference between FP32 weights.
|
||||
expected = (topk_weights.double() @ substitution.values.double()).to(dtype)
|
||||
|
||||
actual = substitution.transform_routes(
|
||||
torch.zeros(1, 1, dtype=dtype, device=device), topk_weights, topk_ids
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _bare_substituted_routed_experts(**attrs) -> SubstitutedRoutedExperts:
|
||||
layer = SubstitutedRoutedExperts.__new__(SubstitutedRoutedExperts)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.expert_substitution = _substitution(2)
|
||||
for name, value in attrs.items():
|
||||
setattr(layer, name, value)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize("padded_input", [False, True])
|
||||
def test_substituted_routed_experts_adds_constant_on_one_tp_rank(tp_rank, padded_input):
|
||||
def apply(*, layer, x, topk_weights, topk_ids, **kwargs):
|
||||
torch.testing.assert_close(topk_weights, torch.tensor([[0.0, 0.75]]))
|
||||
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1]]))
|
||||
return torch.ones(x.shape[0], 2)
|
||||
|
||||
layer = _bare_substituted_routed_experts(
|
||||
moe_config=SimpleNamespace(tp_rank=tp_rank),
|
||||
quant_method=SimpleNamespace(is_monolithic=False, apply=apply),
|
||||
)
|
||||
layer.expert_substitution.values.data.copy_(
|
||||
torch.tensor([[10.0, 20.0], [30.0, 40.0]])
|
||||
)
|
||||
actual = layer.forward_modular(
|
||||
torch.zeros(1, 4 if padded_input else 2),
|
||||
torch.tensor([[0.25, 0.75]]),
|
||||
torch.tensor([[1, 2]]),
|
||||
)
|
||||
expected = torch.ones(1, 2)
|
||||
if tp_rank == 0:
|
||||
expected += torch.tensor([[2.5, 5.0]])
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
|
||||
)
|
||||
def test_standard_fused_moe_model_discovers_substitution(dist_init):
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1]}, dtype=torch.bfloat16, is_moe=True
|
||||
)
|
||||
vllm_config.kernel_config.moe_backend = "triton"
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
moe = MixtralMoE(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=256,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.block_sparse_moe",
|
||||
)
|
||||
|
||||
routed_experts = moe.experts.routed_experts
|
||||
assert isinstance(routed_experts, SubstitutedRoutedExperts)
|
||||
assert routed_experts.expert_substitution.substituted_expert_ids == (1,)
|
||||
assert routed_experts.w13_weight.shape[0] == 3
|
||||
assert moe.experts.moe_config.require_decomposed_backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"moe_backend",
|
||||
[
|
||||
pytest.param("triton", id="triton"),
|
||||
pytest.param(
|
||||
"flashinfer_cutlass",
|
||||
id="flashinfer-cutlass",
|
||||
marks=pytest.mark.skipif(
|
||||
not FlashInferExperts._supports_current_device(),
|
||||
reason="FlashInfer CUTLASS is unavailable on this platform",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_fused_moe_combines_compact_experts_and_substitution(dist_init, moe_backend):
|
||||
hidden_size = 256
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1, 3]}, dtype=torch.bfloat16, is_moe=True
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context = {}
|
||||
vllm_config.kernel_config.moe_backend = moe_backend
|
||||
|
||||
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
|
||||
init_workspace_manager(torch.accelerator.current_device_index())
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
renormalize=False,
|
||||
).cuda()
|
||||
substitution = layer.routed_experts.expert_substitution
|
||||
assert layer.routed_experts.w13_weight.shape[0] == 2
|
||||
with torch.no_grad():
|
||||
layer.routed_experts.w13_weight.normal_(0, 0.01)
|
||||
layer.routed_experts.w2_weight.normal_(0, 0.01)
|
||||
substitution.values.normal_(0, 0.01)
|
||||
layer._quant_method.process_weights_after_loading(layer.routed_experts)
|
||||
|
||||
hidden_states = torch.randn(8, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
router_logits = torch.randn(8, 4, dtype=torch.float32, device="cuda")
|
||||
topk_weights, topk_ids = layer.router.select_experts(
|
||||
hidden_states.clone(), router_logits
|
||||
)
|
||||
expected = substitution.transform_routes(hidden_states, topk_weights, topk_ids)
|
||||
expected += layer._quant_method.apply(
|
||||
layer=layer.routed_experts,
|
||||
x=hidden_states.clone(),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
shared_experts=None,
|
||||
shared_experts_input=None,
|
||||
)
|
||||
|
||||
get_forward_context().all_moe_layers = None
|
||||
actual = layer(hidden_states.clone(), router_logits)
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _tp_substitution_worker(
|
||||
process_group: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
_cpu_group,
|
||||
) -> None:
|
||||
with set_forward_context(None, vllm_config):
|
||||
init_workspace_manager(process_group.local_rank)
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
renormalize=True,
|
||||
)
|
||||
with torch.no_grad():
|
||||
layer.routed_experts.w13_weight.zero_()
|
||||
layer.routed_experts.w2_weight.zero_()
|
||||
layer.routed_experts.expert_substitution.values.fill_(0.25)
|
||||
layer._quant_method.process_weights_after_loading(layer.routed_experts)
|
||||
|
||||
hidden_states = torch.zeros(2, 256, dtype=torch.bfloat16)
|
||||
router_logits = torch.tensor(
|
||||
[[-100.0, 100.0, -100.0, -100.0]] * 2,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
get_forward_context().all_moe_layers = None
|
||||
actual = layer(hidden_states, router_logits)
|
||||
torch.testing.assert_close(
|
||||
actual, torch.full_like(actual, 0.25), atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike() or current_platform.device_count() < 2,
|
||||
reason="requires two CUDA-like devices",
|
||||
)
|
||||
def test_constant_substitution_is_added_once_with_tp2():
|
||||
vllm_config = VllmConfig(parallel_config=ParallelConfig(tensor_parallel_size=2))
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1]},
|
||||
dtype=torch.bfloat16,
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context = {}
|
||||
vllm_config.kernel_config.moe_backend = "triton"
|
||||
|
||||
parallel_launch_with_config(2, _tp_substitution_worker, vllm_config, None)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,57 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.gemma4 import (
|
||||
gemma4_fused_routing_kernel_triton,
|
||||
gemma4_routing_function_torch,
|
||||
)
|
||||
|
||||
|
||||
def sort_by_id(w, ids):
|
||||
order = ids.argsort(dim=-1)
|
||||
return w.gather(1, order), ids.gather(1, order)
|
||||
|
||||
|
||||
# Gemma4 MoE Model has context length of 250K
|
||||
# the minus 1 is to ensure that edge cases are tested
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 2048, 250000])
|
||||
@pytest.mark.parametrize("num_experts", [128]) # gemma4 moe experts
|
||||
@pytest.mark.parametrize("topk", [8]) # gemma4 topk
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_gemma4_routing_kernel_triton(
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
|
||||
gating = torch.randn(num_tokens, num_experts, dtype=dtype, device="cuda")
|
||||
scales = torch.rand(num_experts, dtype=torch.float32, device="cuda")
|
||||
|
||||
ref_w, ref_ids = gemma4_routing_function_torch(gating, topk, scales)
|
||||
tri_w, tri_ids = gemma4_fused_routing_kernel_triton(gating, topk, scales)
|
||||
|
||||
# Sort by expert id — to remove tie-breaking differences
|
||||
ref_ws, ref_is = sort_by_id(ref_w, ref_ids)
|
||||
tri_ws, tri_is = sort_by_id(tri_w, tri_ids)
|
||||
|
||||
ids_match = (ref_is == tri_is).all().item()
|
||||
weights_match = torch.allclose(ref_ws, tri_ws, atol=1e-2, rtol=1e-2)
|
||||
all_match = ids_match and weights_match
|
||||
max_err = (ref_ws - tri_ws).abs().max().item()
|
||||
print(
|
||||
f"T={num_tokens:5d} E={num_experts:4d} K={topk} "
|
||||
f"{str(dtype).split('.')[-1]:7s} ids={ids_match} max_Δweight={max_err:.2e}"
|
||||
)
|
||||
if not all_match:
|
||||
bad = (ref_is != tri_is).any(dim=-1).nonzero(as_tuple=True)[0]
|
||||
if len(bad):
|
||||
r = bad[0].item()
|
||||
print(
|
||||
f" first bad row {r}: ref_ids={ref_ids[r].tolist()} "
|
||||
f"tri_ids={tri_ids[r].tolist()}"
|
||||
)
|
||||
assert all_match
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,541 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm-specific tests for FP8 quantization and kernels."""
|
||||
|
||||
import importlib
|
||||
import math
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.quant_utils import (
|
||||
ref_dynamic_per_tensor_fp8_quant,
|
||||
ref_dynamic_per_token_quant,
|
||||
)
|
||||
from tests.kernels.utils import _assert_deterministic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
DEVICE = "cuda"
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
fp8_only = pytest.mark.skipif(
|
||||
not current_platform.supports_fp8(), reason="ROCm FP8 only"
|
||||
)
|
||||
|
||||
ENV_DEFAULTS = {
|
||||
"VLLM_ROCM_FP8_PADDING": True,
|
||||
"VLLM_ROCM_USE_AITER_FP8BMM": False,
|
||||
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": False,
|
||||
}
|
||||
|
||||
SCALED_MM_CASES = [
|
||||
pytest.param((1, 128, 128), 0.05, 0.15, 0.95, id="1x128x128"),
|
||||
pytest.param((4, 256, 512), 0.10, 0.30, 0.96, id="4x256x512"),
|
||||
pytest.param((8, 512, 1024), 0.20, 0.50, 0.99, id="8x512x1024"),
|
||||
]
|
||||
|
||||
|
||||
def _reload_envs():
|
||||
import vllm.envs as envs
|
||||
|
||||
return importlib.reload(envs)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_rocm_env_state():
|
||||
"""Restore global env + AITER flag state after every test.
|
||||
|
||||
Tests here reload ``vllm.envs`` (a module singleton) and refresh the
|
||||
cached ``rocm_aiter_ops`` env flags. Restoring in a fixture teardown
|
||||
guarantees cleanup even when an assertion fails mid-test, preventing
|
||||
state from leaking into other tests in the same worker.
|
||||
"""
|
||||
yield
|
||||
_reload_envs()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
def _quantile(flat: torch.Tensor, q: float) -> float:
|
||||
if flat.numel() == 1:
|
||||
return flat.item()
|
||||
return torch.quantile(flat, q).item()
|
||||
|
||||
|
||||
def _format_observed_rate(count: int, total: int) -> str:
|
||||
return f"{count / total:.4%} ({count}/{total})"
|
||||
|
||||
|
||||
def _format_allowed_rate(rate: float, total: int) -> str:
|
||||
allowed_count = math.floor(rate * total)
|
||||
return f"{rate:.4%} (<= {allowed_count}/{total})"
|
||||
|
||||
|
||||
def _assert_abs_error_budget(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
label: str,
|
||||
tight_atol: float,
|
||||
max_atol: float,
|
||||
pass_rate: float,
|
||||
) -> None:
|
||||
diff = (actual.float() - expected.float()).abs().flatten()
|
||||
total = diff.numel()
|
||||
within_tight_count = int((diff <= tight_atol).sum().item())
|
||||
fail_count = total - within_tight_count
|
||||
above_max_count = int((diff > max_atol).sum().item())
|
||||
within_tight = within_tight_count / total
|
||||
allowed_fail_rate = 1.0 - pass_rate
|
||||
above_max = above_max_count / total
|
||||
max_fail_rate = 0.0
|
||||
max_diff = diff.max().item()
|
||||
mean_diff = diff.mean().item()
|
||||
p99 = _quantile(diff, 0.99)
|
||||
p999 = _quantile(diff, 0.999)
|
||||
msg = (
|
||||
f"[rocm_fp8] {label}: abs<={tight_atol} pass={within_tight:.4%} "
|
||||
f"({within_tight_count}/{total}) "
|
||||
f"fail={_format_observed_rate(fail_count, total)} "
|
||||
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
|
||||
f"abs>{max_atol}={_format_observed_rate(above_max_count, total)} "
|
||||
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
|
||||
f"max_diff={max_diff:.6g} mean_diff={mean_diff:.6g} "
|
||||
f"p99={p99:.6g} p999={p999:.6g}"
|
||||
)
|
||||
print(msg)
|
||||
if within_tight < 1.0:
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
assert within_tight >= pass_rate, msg
|
||||
assert above_max <= max_fail_rate, msg
|
||||
|
||||
|
||||
def _assert_rel_error_budget(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
label: str,
|
||||
tight_rtol: float,
|
||||
max_rtol: float,
|
||||
pass_rate: float,
|
||||
max_fail_rate: float,
|
||||
) -> None:
|
||||
rel = (
|
||||
(actual.float() - expected.float()).abs()
|
||||
/ expected.float().abs().clamp_min(1e-5)
|
||||
).flatten()
|
||||
total = rel.numel()
|
||||
within_tight_count = int((rel <= tight_rtol).sum().item())
|
||||
fail_count = total - within_tight_count
|
||||
above_max_count = int((rel > max_rtol).sum().item())
|
||||
within_tight = within_tight_count / total
|
||||
allowed_fail_rate = 1.0 - pass_rate
|
||||
above_max = above_max_count / total
|
||||
max_rel = rel.max().item()
|
||||
mean_rel = rel.mean().item()
|
||||
p99 = _quantile(rel, 0.99)
|
||||
p999 = _quantile(rel, 0.999)
|
||||
msg = (
|
||||
f"[rocm_fp8] {label}: rel<={tight_rtol} pass={within_tight:.4%} "
|
||||
f"({within_tight_count}/{total}) "
|
||||
f"fail={_format_observed_rate(fail_count, total)} "
|
||||
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
|
||||
f"rel>{max_rtol}={_format_observed_rate(above_max_count, total)} "
|
||||
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
|
||||
f"max_rel={max_rel:.6g} mean_rel={mean_rel:.6g} "
|
||||
f"p99={p99:.6g} p999={p999:.6g}"
|
||||
)
|
||||
print(msg)
|
||||
if within_tight < 1.0:
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
assert within_tight >= pass_rate, msg
|
||||
assert above_max <= max_fail_rate, msg
|
||||
|
||||
|
||||
def _ref_paged_attention(
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
*,
|
||||
query_lens: list[int],
|
||||
kv_lens: list[int],
|
||||
block_tables: torch.Tensor,
|
||||
scale: float,
|
||||
) -> torch.Tensor:
|
||||
block_tables_np = block_tables.cpu().numpy()
|
||||
_, block_size, num_kv_heads, head_size = key_cache.shape
|
||||
|
||||
outputs = []
|
||||
start_idx = 0
|
||||
for i, query_len in enumerate(query_lens):
|
||||
kv_len = kv_lens[i]
|
||||
q = query[start_idx : start_idx + query_len] * scale
|
||||
|
||||
num_kv_blocks = (kv_len + block_size - 1) // block_size
|
||||
block_indices = block_tables_np[i, :num_kv_blocks]
|
||||
|
||||
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
|
||||
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
|
||||
|
||||
if q.shape[1] != k.shape[1]:
|
||||
repeat = q.shape[1] // k.shape[1]
|
||||
k = torch.repeat_interleave(k, repeat, dim=1)
|
||||
v = torch.repeat_interleave(v, repeat, dim=1)
|
||||
|
||||
attn = torch.einsum("qhd,khd->hqk", q, k).float()
|
||||
mask = torch.triu(
|
||||
torch.ones(query_len, kv_len, device=q.device),
|
||||
diagonal=kv_len - query_len + 1,
|
||||
).bool()
|
||||
attn.masked_fill_(mask, float("-inf"))
|
||||
attn = torch.softmax(attn, dim=-1).to(v.dtype)
|
||||
outputs.append(torch.einsum("hqk,khd->qhd", attn, v))
|
||||
start_idx += query_len
|
||||
|
||||
return torch.cat(outputs, dim=0)
|
||||
|
||||
|
||||
def test_rocm_fp8_env_defaults(monkeypatch):
|
||||
with monkeypatch.context() as mp:
|
||||
for env_name in ENV_DEFAULTS:
|
||||
mp.delenv(env_name, raising=False)
|
||||
envs = _reload_envs()
|
||||
for env_name, expected in ENV_DEFAULTS.items():
|
||||
assert getattr(envs, env_name) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env_name", "raw_value", "expected"),
|
||||
[
|
||||
("VLLM_ROCM_FP8_PADDING", "0", False),
|
||||
("VLLM_ROCM_FP8_PADDING", "1", True),
|
||||
("VLLM_ROCM_USE_AITER_FP8BMM", "0", False),
|
||||
("VLLM_ROCM_USE_AITER_FP8BMM", "1", True),
|
||||
("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0", False),
|
||||
("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "1", True),
|
||||
],
|
||||
)
|
||||
def test_rocm_fp8_env_overrides(env_name, raw_value, expected, monkeypatch):
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv(env_name, raw_value)
|
||||
envs = _reload_envs()
|
||||
assert getattr(envs, env_name) is expected
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_maybe_pad_fp8_weight_respects_env(monkeypatch):
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_maybe_pad_fp8_weight,
|
||||
)
|
||||
|
||||
weight = torch.randn(64, 512, device=DEVICE).to(FP8_DTYPE)
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_FP8_PADDING", "1")
|
||||
_reload_envs()
|
||||
padded = _maybe_pad_fp8_weight(weight)
|
||||
assert padded.shape == weight.shape
|
||||
assert padded.dtype == weight.dtype
|
||||
assert padded.data_ptr() != weight.data_ptr()
|
||||
torch.testing.assert_close(padded.float(), weight.float())
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_FP8_PADDING", "0")
|
||||
_reload_envs()
|
||||
unpadded = _maybe_pad_fp8_weight(weight)
|
||||
assert unpadded.data_ptr() == weight.data_ptr()
|
||||
|
||||
|
||||
@fp8_only
|
||||
@pytest.mark.parametrize(
|
||||
("use_aiter", "fp8bmm_enabled", "expected"),
|
||||
[
|
||||
(True, True, True),
|
||||
(True, False, False),
|
||||
(False, True, False),
|
||||
],
|
||||
)
|
||||
def test_aiter_fp8bmm_enabled_api_respects_env(
|
||||
use_aiter, fp8bmm_enabled, expected, monkeypatch
|
||||
):
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
|
||||
assert is_aiter_found_and_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_FP8BMM", "1" if fp8bmm_enabled else "0")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
assert rocm_aiter_ops.is_fp8bmm_enabled() is expected
|
||||
|
||||
|
||||
@fp8_only
|
||||
@pytest.mark.parametrize(
|
||||
("shape", "tight_atol", "max_atol", "pass_rate"), SCALED_MM_CASES
|
||||
)
|
||||
def test_fp8_scaled_mm_matches_dequantized_reference(
|
||||
shape, tight_atol, max_atol, pass_rate
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
|
||||
batch, m_dim, k_dim = shape
|
||||
n_dim = k_dim
|
||||
a = torch.randn(batch * m_dim, k_dim, device=DEVICE)
|
||||
b = torch.randn(n_dim, k_dim, device=DEVICE)
|
||||
|
||||
a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a)
|
||||
b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b)
|
||||
out = torch._scaled_mm(
|
||||
a_fp8,
|
||||
b_fp8.t(),
|
||||
out_dtype=torch.bfloat16,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
)
|
||||
|
||||
a_dq = a_fp8.float() * scale_a.float()
|
||||
b_dq = b_fp8.float() * scale_b.float()
|
||||
ref = torch.mm(a_dq, b_dq.t())
|
||||
|
||||
assert out.shape == (batch * m_dim, n_dim)
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert not torch.isnan(out).any()
|
||||
|
||||
_assert_abs_error_budget(
|
||||
out.float(),
|
||||
ref,
|
||||
label=f"scaled_mm shape={shape}",
|
||||
tight_atol=tight_atol,
|
||||
max_atol=max_atol,
|
||||
pass_rate=pass_rate,
|
||||
)
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_fp8_scaled_mm_determinism():
|
||||
torch.manual_seed(5)
|
||||
|
||||
m_dim, k_dim, n_dim = 64, 128, 128
|
||||
a = torch.randn(m_dim, k_dim, device=DEVICE)
|
||||
b = torch.randn(n_dim, k_dim, device=DEVICE)
|
||||
a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a)
|
||||
b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b)
|
||||
|
||||
def run_gemm():
|
||||
return torch._scaled_mm(
|
||||
a_fp8,
|
||||
b_fp8.t(),
|
||||
out_dtype=torch.bfloat16,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
)
|
||||
|
||||
_assert_deterministic(run_gemm, n_runs=4)
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_aiter_fp8_kv_attention_matches_bf16_reference():
|
||||
import aiter
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import cp_mha_gather_cache
|
||||
|
||||
assert is_aiter_found_and_supported()
|
||||
|
||||
set_random_seed(10)
|
||||
|
||||
num_q_heads, num_kv_heads = 8, 8
|
||||
head_size = 128
|
||||
query_len, kv_len = 4, 128
|
||||
block_size = 16
|
||||
num_blocks = 2048
|
||||
scale = head_size**-0.5
|
||||
|
||||
query = torch.randn(
|
||||
query_len, num_q_heads, head_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
key_cache_fp8 = torch.clamp(
|
||||
torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE),
|
||||
min=-1.0,
|
||||
max=1.0,
|
||||
).to(FP8_DTYPE)
|
||||
value_cache_fp8 = torch.clamp(
|
||||
torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE),
|
||||
min=-1.0,
|
||||
max=1.0,
|
||||
).to(FP8_DTYPE)
|
||||
|
||||
cu_query_lens = torch.tensor([0, query_len], dtype=torch.int32, device=DEVICE)
|
||||
cu_seq_lens = torch.tensor([0, kv_len], dtype=torch.int32, device=DEVICE)
|
||||
block_tables = torch.randint(
|
||||
0,
|
||||
num_blocks,
|
||||
(1, (kv_len + block_size - 1) // block_size),
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
token_to_batch = torch.zeros(kv_len, dtype=torch.int32, device=DEVICE)
|
||||
seq_starts = torch.zeros(1, dtype=torch.int32, device=DEVICE)
|
||||
gathered_key = torch.empty(
|
||||
kv_len, num_kv_heads, head_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
gathered_value = torch.empty_like(gathered_key)
|
||||
|
||||
cp_mha_gather_cache(
|
||||
key_cache=key_cache_fp8,
|
||||
value_cache=value_cache_fp8,
|
||||
key=gathered_key,
|
||||
value=gathered_value,
|
||||
block_tables=block_tables,
|
||||
k_scales=torch.ones(1, dtype=torch.float32, device=DEVICE),
|
||||
v_scales=torch.ones(1, dtype=torch.float32, device=DEVICE),
|
||||
cu_seqlens_kv=cu_seq_lens,
|
||||
token_to_batch=token_to_batch,
|
||||
seq_starts=seq_starts,
|
||||
dequant=True,
|
||||
kv_cache_layout="NHD",
|
||||
total_tokens=kv_len,
|
||||
)
|
||||
|
||||
output = torch.empty_like(query)
|
||||
aiter.flash_attn_varlen_func(
|
||||
q=query,
|
||||
k=gathered_key,
|
||||
v=gathered_value,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
cu_seqlens_k=cu_seq_lens,
|
||||
max_seqlen_q=query_len,
|
||||
max_seqlen_k=kv_len,
|
||||
min_seqlen_q=1,
|
||||
dropout_p=0.0,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=(-1, -1),
|
||||
alibi_slopes=None,
|
||||
return_lse=False,
|
||||
out=output,
|
||||
)
|
||||
|
||||
ref = _ref_paged_attention(
|
||||
query=query,
|
||||
key_cache=key_cache_fp8.to(torch.bfloat16),
|
||||
value_cache=value_cache_fp8.to(torch.bfloat16),
|
||||
query_lens=[query_len],
|
||||
kv_lens=[kv_len],
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
)
|
||||
|
||||
_assert_abs_error_budget(
|
||||
output.float(),
|
||||
ref.float(),
|
||||
label="aiter_fp8_kv_attention",
|
||||
tight_atol=0.15,
|
||||
max_atol=0.25,
|
||||
pass_rate=0.98,
|
||||
)
|
||||
|
||||
|
||||
@fp8_only
|
||||
@pytest.mark.parametrize("shape", [(128, 256), (512, 1024), (1024, 4096)])
|
||||
def test_fp8_per_tensor_quant_matches_reference(shape):
|
||||
torch.manual_seed(0)
|
||||
|
||||
x = torch.randn(*shape, dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
out, scale = ops.scaled_fp8_quant(x)
|
||||
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == FP8_DTYPE
|
||||
|
||||
torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4)
|
||||
code_match_rate = (
|
||||
(out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item()
|
||||
)
|
||||
assert code_match_rate >= 0.999, (
|
||||
f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)"
|
||||
)
|
||||
|
||||
dequant = out.float() * scale.float()
|
||||
_assert_rel_error_budget(
|
||||
dequant,
|
||||
x.float(),
|
||||
label=f"per_tensor_quant shape={shape}",
|
||||
tight_rtol=0.06,
|
||||
max_rtol=0.60,
|
||||
pass_rate=0.99,
|
||||
max_fail_rate=1e-4,
|
||||
)
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_aiter_group_fp8_quant_roundtrip():
|
||||
from aiter import dtypes
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
|
||||
assert is_aiter_found_and_supported()
|
||||
|
||||
torch.manual_seed(0)
|
||||
|
||||
m_dim, n_dim = 64, 4096
|
||||
group_size = 128
|
||||
x = torch.randn(m_dim, n_dim, dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
out, scales = rocm_aiter_ops.group_fp8_quant(x, group_size)
|
||||
assert out.dtype == dtypes.fp8
|
||||
assert scales.dtype == torch.float32
|
||||
|
||||
scales_expanded = scales.repeat_interleave(group_size, dim=1)[:, :n_dim]
|
||||
dequant = out.float() * scales_expanded
|
||||
_assert_rel_error_budget(
|
||||
dequant,
|
||||
x.float(),
|
||||
label="aiter_group_fp8_quant",
|
||||
tight_rtol=0.06,
|
||||
max_rtol=0.30,
|
||||
pass_rate=0.99,
|
||||
max_fail_rate=1e-4,
|
||||
)
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_fp8_per_token_quant_matches_reference():
|
||||
torch.manual_seed(0)
|
||||
|
||||
x = torch.randn(32, 4096, dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
out, scale = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
|
||||
ref_out, ref_scale = ref_dynamic_per_token_quant(x, FP8_DTYPE)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == FP8_DTYPE
|
||||
assert scale.shape == (x.shape[0], 1)
|
||||
|
||||
torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4)
|
||||
code_match_rate = (
|
||||
(out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item()
|
||||
)
|
||||
assert code_match_rate >= 0.999, (
|
||||
f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)"
|
||||
)
|
||||
|
||||
dequant = out.float() * scale.float()
|
||||
_assert_rel_error_budget(
|
||||
dequant,
|
||||
x.float(),
|
||||
label="per_token_quant",
|
||||
tight_rtol=0.06,
|
||||
max_rtol=0.30,
|
||||
pass_rate=0.99,
|
||||
max_fail_rate=1e-4,
|
||||
)
|
||||
@@ -1,60 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Check that DeepSeek V4 and the JIT monitor leave the HIP symbols alone.
|
||||
|
||||
TileLang ships a libhip_stub.so that takes over `hipFree` in the global symbol
|
||||
table once it is loaded, so neither the model import nor the JIT monitor may
|
||||
pull TileLang in on ROCm. Both claims are about process-global state, which is
|
||||
why this runs in an interpreter of its own.
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import sys
|
||||
|
||||
from vllm.model_executor.layers import mhc # noqa: F401
|
||||
from vllm.models import deepseek_v4 # noqa: F401
|
||||
from vllm.utils import jit_monitor
|
||||
|
||||
|
||||
class DlInfo(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("dli_fname", ctypes.c_char_p),
|
||||
("dli_fbase", ctypes.c_void_p),
|
||||
("dli_sname", ctypes.c_char_p),
|
||||
("dli_saddr", ctypes.c_void_p),
|
||||
]
|
||||
|
||||
|
||||
def _must_not_run() -> None:
|
||||
raise AssertionError("TileLang JIT monitor must not run on ROCm")
|
||||
|
||||
|
||||
jit_monitor._active = False
|
||||
jit_monitor._setup_triton_autotuning_print = lambda: None
|
||||
jit_monitor._setup_triton_jit_hook = lambda: None
|
||||
jit_monitor._setup_cutedsl_jit_hook = lambda: None
|
||||
jit_monitor._setup_tilelang_jit_hook = _must_not_run
|
||||
jit_monitor.activate()
|
||||
|
||||
imported = [
|
||||
name for name in sys.modules if name == "tilelang" or name.startswith("tilelang.")
|
||||
]
|
||||
assert not imported, f"TileLang was imported: {imported}"
|
||||
|
||||
libdl = ctypes.CDLL("libdl.so.2")
|
||||
dlsym = libdl.dlsym
|
||||
dlsym.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
|
||||
dlsym.restype = ctypes.c_void_p
|
||||
dladdr = libdl.dladdr
|
||||
dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(DlInfo)]
|
||||
dladdr.restype = ctypes.c_int
|
||||
|
||||
address = dlsym(None, b"hipFree")
|
||||
assert address is not None, "hipFree is not available in the global symbol table"
|
||||
info = DlInfo()
|
||||
assert dladdr(address, ctypes.byref(info))
|
||||
source = info.dli_fname.decode() if info.dli_fname else "<unknown>"
|
||||
assert "libhip_stub.so" not in source, source
|
||||
assert "libamdhip64.so" in source, source
|
||||
print("OK")
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,185 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
NUM_TOKENS = [1, 4, 16, 64, 128]
|
||||
NUM_HEADS = [128]
|
||||
NOPE_DIM = [512]
|
||||
ROPE_DIM = [64]
|
||||
DTYPES = [torch.bfloat16, torch.float16]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("nope_dim", NOPE_DIM)
|
||||
@pytest.mark.parametrize("rope_dim", ROPE_DIM)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_concat_mla_q_contiguous(num_tokens, num_heads, nope_dim, rope_dim, dtype):
|
||||
"""Test with contiguous inputs (standard layout)."""
|
||||
torch.manual_seed(42)
|
||||
ql_nope = torch.randn(num_tokens, num_heads, nope_dim, dtype=dtype, device="cuda")
|
||||
q_pe = torch.randn(num_tokens, num_heads, rope_dim, dtype=dtype, device="cuda")
|
||||
|
||||
ref = torch.cat((ql_nope, q_pe), dim=-1)
|
||||
|
||||
q_out = torch.empty(
|
||||
num_tokens, num_heads, nope_dim + rope_dim, dtype=dtype, device="cuda"
|
||||
)
|
||||
ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
torch.testing.assert_close(q_out, ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_concat_mla_q_padded_output_heads():
|
||||
"""The fused concat can write directly into a head-padded workspace."""
|
||||
ql_nope = torch.randn(4, 8, 512, dtype=torch.bfloat16, device="cuda")
|
||||
q_pe = torch.randn(4, 8, 64, dtype=torch.bfloat16, device="cuda")
|
||||
q_out = torch.empty(4, 64, 576, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
torch.testing.assert_close(q_out[:, :8], torch.cat((ql_nope, q_pe), dim=-1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [t for t in NUM_TOKENS if t > 1])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("nope_dim", NOPE_DIM)
|
||||
@pytest.mark.parametrize("rope_dim", ROPE_DIM)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_concat_mla_q_transposed_nope(num_tokens, num_heads, nope_dim, rope_dim, dtype):
|
||||
"""Test with transposed nope input (simulates BMM output after transpose).
|
||||
|
||||
In the real code path, mqa_ql_nope is the result of:
|
||||
torch.bmm(q_nope, W_UK_T) # [N, B, L]
|
||||
.transpose(0, 1) # [B, N, L] — non-contiguous!
|
||||
"""
|
||||
torch.manual_seed(42)
|
||||
nope_raw = torch.randn(num_heads, num_tokens, nope_dim, dtype=dtype, device="cuda")
|
||||
ql_nope = nope_raw.transpose(0, 1) # [B, N, L], non-contiguous
|
||||
assert not ql_nope.is_contiguous()
|
||||
|
||||
q_pe = torch.randn(num_tokens, num_heads, rope_dim, dtype=dtype, device="cuda")
|
||||
|
||||
ref = torch.cat((ql_nope, q_pe), dim=-1)
|
||||
|
||||
q_out = torch.empty(
|
||||
num_tokens, num_heads, nope_dim + rope_dim, dtype=dtype, device="cuda"
|
||||
)
|
||||
ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
torch.testing.assert_close(q_out, ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_concat_mla_q_split_rope(num_tokens, num_heads, dtype):
|
||||
"""Test with rope from a split (simulates the actual code path).
|
||||
|
||||
In the real code path, q_pe comes from:
|
||||
mqa_q.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
|
||||
which creates a non-contiguous view with stride(1) != rope_dim.
|
||||
"""
|
||||
torch.manual_seed(42)
|
||||
nope_dim = 512
|
||||
rope_dim = 64
|
||||
orig_dim = 128 + 64 # original q before absorption: [B, N, 192]
|
||||
|
||||
# Simulate split from original q tensor
|
||||
q_orig = torch.randn(num_tokens, num_heads, orig_dim, dtype=dtype, device="cuda")
|
||||
q_nope_orig, q_pe = q_orig.split([128, 64], dim=-1)
|
||||
|
||||
# q_pe is non-contiguous: stride(1) = 192, not 64
|
||||
assert q_pe.stride(1) == orig_dim
|
||||
assert q_pe.stride(2) == 1 # but innermost is fine
|
||||
|
||||
# Simulate absorbed nope (contiguous, different size)
|
||||
ql_nope = torch.randn(num_tokens, num_heads, nope_dim, dtype=dtype, device="cuda")
|
||||
|
||||
ref = torch.cat((ql_nope, q_pe), dim=-1)
|
||||
|
||||
q_out = torch.empty(
|
||||
num_tokens, num_heads, nope_dim + rope_dim, dtype=dtype, device="cuda"
|
||||
)
|
||||
ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
torch.testing.assert_close(q_out, ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_concat_mla_q_zero_tokens():
|
||||
"""Test with zero tokens (edge case)."""
|
||||
ql_nope = torch.empty(0, 128, 512, dtype=torch.bfloat16, device="cuda")
|
||||
q_pe = torch.empty(0, 128, 64, dtype=torch.bfloat16, device="cuda")
|
||||
q_out = torch.empty(0, 128, 576, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 64])
|
||||
def test_concat_mla_q_values_preserved(num_tokens):
|
||||
"""Verify exact bit-level preservation (no computation, pure copy).
|
||||
|
||||
Compares raw int16 bits to avoid NaN != NaN issues from IEEE 754.
|
||||
"""
|
||||
nope_dim, rope_dim = 512, 64
|
||||
|
||||
# Use specific bit patterns (stay in int16 for bit-exact comparison)
|
||||
ql_nope_bits = torch.arange(
|
||||
num_tokens * 128 * nope_dim, dtype=torch.int16, device="cuda"
|
||||
).view(num_tokens, 128, nope_dim)
|
||||
q_pe_bits = torch.arange(
|
||||
num_tokens * 128 * rope_dim, dtype=torch.int16, device="cuda"
|
||||
).view(num_tokens, 128, rope_dim)
|
||||
|
||||
ql_nope = ql_nope_bits.view(torch.bfloat16)
|
||||
q_pe = q_pe_bits.view(torch.bfloat16)
|
||||
|
||||
q_out = torch.empty(
|
||||
num_tokens, 128, nope_dim + rope_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
ops.concat_mla_q(ql_nope, q_pe, q_out)
|
||||
|
||||
out_bits = q_out.view(torch.int16)
|
||||
|
||||
assert torch.equal(out_bits[..., :nope_dim], ql_nope_bits)
|
||||
|
||||
assert torch.equal(out_bits[..., nope_dim:], q_pe_bits)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rope_dim,transposed", [(0, False), (0, True), (64, False)])
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
def test_concat_mla_q_fp8_nope_and_rope(rope_dim, transposed, enabled):
|
||||
"""FP8 query packing preserves values with empty RoPE and strided BMM output."""
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import _DecodeConcatQuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
torch.manual_seed(42)
|
||||
shape = (8, 17, 512) if transposed else (17, 8, 512)
|
||||
ql = torch.randn(shape, dtype=torch.bfloat16, device="cuda") * 128
|
||||
if transposed:
|
||||
ql = ql.transpose(0, 1)
|
||||
qp = torch.randn(17, 8, rope_dim, dtype=torch.bfloat16, device="cuda") * 128
|
||||
scale = torch.tensor(0.3, device="cuda", dtype=torch.float32)
|
||||
joined = torch.cat((ql, qp), dim=-1)
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
expected = (joined.float() * scale.reciprocal()).clamp(fp8_min, fp8_max)
|
||||
expected = expected.to(current_platform.fp8_dtype())
|
||||
custom_ops = ["none"] + (["+mla_decode_concat_quant_fp8"] if enabled else [])
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(mode=0, custom_ops=custom_ops)
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
quant = _DecodeConcatQuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
actual = quant(ql, qp, scale)
|
||||
assert actual.dtype == current_platform.fp8_dtype()
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=0, rtol=0)
|
||||
assert actual.is_contiguous()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layout plumbing for DeepSeek V4.1 mega attention.
|
||||
|
||||
The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and
|
||||
``wo_a`` columns in their weight loaders, so what has to hold is that the
|
||||
permuted GEMMs agree with a reference that permutes the activation instead.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import (
|
||||
WV_GROUP_SIZE,
|
||||
o_fused_chunk_permutation,
|
||||
o_fused_permutation,
|
||||
permute_on_load,
|
||||
permute_q_to_fused,
|
||||
q_fused_permutation,
|
||||
)
|
||||
|
||||
HEAD_DIM = 512
|
||||
|
||||
|
||||
def _load(checkpoint: torch.Tensor, perm: torch.Tensor, dim: int) -> torch.Tensor:
|
||||
"""Load ``checkpoint`` into a fresh parameter through a permuting loader."""
|
||||
param = torch.nn.Parameter(torch.empty_like(checkpoint), requires_grad=False)
|
||||
param.weight_loader = default_weight_loader # type: ignore[attr-defined]
|
||||
permute_on_load(param, perm, dim)
|
||||
param.weight_loader(param, checkpoint) # type: ignore[attr-defined]
|
||||
return param.data
|
||||
|
||||
|
||||
def test_permuted_wq_b_gemm_matches_permuted_activation():
|
||||
"""Permuting wq_b's rows makes the Q GEMM emit the fused layout directly."""
|
||||
torch.manual_seed(0)
|
||||
num_heads, q_lora_rank, num_tokens = 16, 64, 5
|
||||
weight = torch.randn(num_heads * HEAD_DIM, q_lora_rank)
|
||||
# One scale row per weight row, as an MXFP8 wq_b shard carries.
|
||||
scale = torch.randint(
|
||||
0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8
|
||||
)
|
||||
qr = torch.randn(num_tokens, q_lora_rank)
|
||||
|
||||
standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
expected = permute_q_to_fused(standard)
|
||||
|
||||
perm = q_fused_permutation(num_heads, HEAD_DIM)
|
||||
loaded_weight = _load(weight, perm, dim=0)
|
||||
fused = (qr @ loaded_weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(fused, expected)
|
||||
# The scale has to follow its row, or dequant reads another row's scale.
|
||||
torch.testing.assert_close(loaded_weight, weight[perm])
|
||||
torch.testing.assert_close(_load(scale, perm, dim=0), scale[perm])
|
||||
|
||||
|
||||
def test_permuted_wo_a_consumes_fused_output():
|
||||
"""Permuting wo_a's input columns lets it read the kernel's O layout."""
|
||||
torch.manual_seed(1)
|
||||
out_features, num_tokens = 32, 4
|
||||
in_features = WV_GROUP_SIZE * HEAD_DIM
|
||||
weight = torch.randn(out_features, in_features)
|
||||
scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8)
|
||||
standard_o = torch.randn(num_tokens, in_features)
|
||||
|
||||
expected = standard_o @ weight.T
|
||||
# The kernel emits the same values in the fused chunk order.
|
||||
perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM)
|
||||
fused_o = standard_o[:, perm]
|
||||
|
||||
loaded_weight = _load(weight, perm, dim=1)
|
||||
torch.testing.assert_close(loaded_weight, weight[:, perm])
|
||||
chunk_perm = o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)
|
||||
torch.testing.assert_close(_load(scale, chunk_perm, dim=1), scale[:, chunk_perm])
|
||||
# Unlike wq_b, this permutation sits inside the 4096-term reduction, so the
|
||||
# summation order changes and the result differs in the last few ulps.
|
||||
torch.testing.assert_close(
|
||||
fused_o @ loaded_weight.T, expected, rtol=1e-3, atol=1e-3
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,524 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Integration tests for FlexAttention backend vs default backend."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from packaging import version
|
||||
|
||||
from tests.utils import set_random_seed
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_standard_kv_cache_spec,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import AttentionConfig
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.v1.attention.backends.flex_attention import (
|
||||
BlockSparsityHint,
|
||||
FlexAttentionMetadataBuilder,
|
||||
physical_to_logical_mapping,
|
||||
)
|
||||
|
||||
from ..models.utils import check_embeddings_close, check_logprobs_close
|
||||
|
||||
TORCH_VERSION = version.parse(torch.__version__)
|
||||
MINIMUM_TORCH_VERSION = version.parse("2.7.0")
|
||||
DIRECT_BUILD_VERSION = version.parse("2.9.dev0")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("supports_small_blocks", "uses_paged_kv", "expected"),
|
||||
[
|
||||
(True, True, (16, 16)),
|
||||
(True, False, (128, 128)),
|
||||
(False, True, (128, 128)),
|
||||
(False, False, (128, 128)),
|
||||
],
|
||||
)
|
||||
def test_flex_attention_default_block_sizes(
|
||||
supports_small_blocks: bool,
|
||||
uses_paged_kv: bool,
|
||||
expected: tuple[int, int],
|
||||
):
|
||||
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
|
||||
AttentionConfig(),
|
||||
supports_small_blocks=supports_small_blocks,
|
||||
cache_block_size=16,
|
||||
uses_paged_kv=uses_paged_kv,
|
||||
)
|
||||
assert block_sizes == expected
|
||||
|
||||
|
||||
def test_flex_attention_explicit_block_sizes_override_encoder_defaults():
|
||||
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
|
||||
AttentionConfig(
|
||||
flex_attn_q_block_size=64,
|
||||
flex_attn_kv_block_size=32,
|
||||
),
|
||||
supports_small_blocks=True,
|
||||
cache_block_size=16,
|
||||
uses_paged_kv=False,
|
||||
)
|
||||
assert block_sizes == (64, 32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
|
||||
reason="CUDA not available or PyTorch version < 2.7",
|
||||
)
|
||||
def test_flex_attention_full_cudagraphs(vllm_runner):
|
||||
"""Test the numerics for flex attention full cudagraphs support."""
|
||||
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
seed = 42
|
||||
max_tokens = 24
|
||||
num_logprobs = 5
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
]
|
||||
|
||||
# Run with flex attention eager
|
||||
set_random_seed(seed)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="generate",
|
||||
tensor_parallel_size=1,
|
||||
num_gpu_blocks_override=128,
|
||||
enforce_eager=True,
|
||||
attention_config={"backend": "FLEX_ATTENTION"},
|
||||
) as llm_flex:
|
||||
output_eager = llm_flex.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
# Run with flex attention compiled
|
||||
set_random_seed(seed)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="generate",
|
||||
tensor_parallel_size=1,
|
||||
num_gpu_blocks_override=128,
|
||||
enforce_eager=False,
|
||||
gpu_memory_utilization=0.85,
|
||||
attention_config={"backend": "FLEX_ATTENTION"},
|
||||
) as llm_default:
|
||||
output_compile = llm_default.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=output_eager,
|
||||
outputs_1_lst=output_compile,
|
||||
name_0="eager",
|
||||
name_1="compile",
|
||||
)
|
||||
|
||||
|
||||
def windowed_causal_mask_mod(b, h, q_idx, kv_idx):
|
||||
return (kv_idx <= q_idx) & (q_idx - kv_idx < 4)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
|
||||
reason="CUDA not available or PyTorch version < 2.7",
|
||||
)
|
||||
def test_flex_attention_custom_mask_full_cudagraphs(vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setattr(
|
||||
Attention,
|
||||
"logical_mask_mod",
|
||||
staticmethod(windowed_causal_mask_mod),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
seed = 42
|
||||
max_tokens = 24
|
||||
num_logprobs = 5
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
]
|
||||
|
||||
set_random_seed(seed)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="generate",
|
||||
tensor_parallel_size=1,
|
||||
num_gpu_blocks_override=128,
|
||||
enforce_eager=True,
|
||||
attention_config={"backend": "FLEX_ATTENTION"},
|
||||
) as llm_eager:
|
||||
output_eager = llm_eager.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
set_random_seed(seed)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="generate",
|
||||
tensor_parallel_size=1,
|
||||
num_gpu_blocks_override=128,
|
||||
enforce_eager=False,
|
||||
gpu_memory_utilization=0.85,
|
||||
compilation_config={
|
||||
"cudagraph_mode": "FULL",
|
||||
"cudagraph_capture_sizes": [4],
|
||||
},
|
||||
attention_config={"backend": "FLEX_ATTENTION"},
|
||||
) as llm_cudagraph:
|
||||
output_cudagraph = llm_cudagraph.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=output_eager,
|
||||
outputs_1_lst=output_cudagraph,
|
||||
name_0="eager",
|
||||
name_1="cudagraph",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
|
||||
reason="CUDA not available or PyTorch version < 2.7",
|
||||
)
|
||||
def test_flex_attention_vs_default_backend(vllm_runner):
|
||||
"""Test that FlexAttention produces the same outputs as the default backend.
|
||||
|
||||
This test compares the outputs from the FlexAttention backend with
|
||||
the default backend, ensuring they are similar when using the same seed.
|
||||
"""
|
||||
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
seed = 42
|
||||
max_tokens = 24
|
||||
num_logprobs = 5
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
]
|
||||
|
||||
# Run with flex attention
|
||||
set_random_seed(seed)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="generate",
|
||||
tensor_parallel_size=1,
|
||||
num_gpu_blocks_override=128,
|
||||
enforce_eager=True,
|
||||
attention_config={"backend": "FLEX_ATTENTION"},
|
||||
) as llm_flex:
|
||||
output_flex = llm_flex.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
# Run with default backend
|
||||
set_random_seed(seed)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="generate",
|
||||
tensor_parallel_size=1,
|
||||
num_gpu_blocks_override=128,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.85,
|
||||
) as llm_default:
|
||||
output_default = llm_default.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=output_flex,
|
||||
outputs_1_lst=output_default,
|
||||
name_0="flex",
|
||||
name_1="default",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
|
||||
reason="CUDA not available or PyTorch version < 2.7",
|
||||
)
|
||||
def test_encoder_flex_attention_vs_default_backend(vllm_runner):
|
||||
"""Test that FlexAttention produces the same outputs as the default backend.
|
||||
|
||||
This test compares the outputs from the FlexAttention backend with
|
||||
the default backend for encoder models.
|
||||
"""
|
||||
model_name = "BAAI/bge-base-en-v1.5"
|
||||
# Exercise packed sequence boundaries inside 128-token FlexAttention
|
||||
# blocks, including sequences that span more than one block.
|
||||
prompts = [
|
||||
"hello " * 120,
|
||||
"world " * 130,
|
||||
"attention " * 254,
|
||||
]
|
||||
|
||||
# Run with flex attention
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="pooling",
|
||||
dtype=torch.bfloat16,
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=384,
|
||||
enforce_eager=True,
|
||||
attention_config={"backend": "FLEX_ATTENTION"},
|
||||
) as llm_flex:
|
||||
flex_outputs = llm_flex.embed(prompts)
|
||||
|
||||
# Run with default backend
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="pooling",
|
||||
dtype=torch.bfloat16,
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=384,
|
||||
enforce_eager=True,
|
||||
) as llm_default:
|
||||
default_outputs = llm_default.embed(prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=flex_outputs,
|
||||
embeddings_1_lst=default_outputs,
|
||||
name_0="flex",
|
||||
name_1="default",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or TORCH_VERSION < DIRECT_BUILD_VERSION,
|
||||
reason="CUDA not available or PyTorch version < 2.7",
|
||||
)
|
||||
def test_block_mask_direct_vs_slow_path():
|
||||
"""Test that direct path block mask is a superset of slow path.
|
||||
|
||||
The direct path may include extra blocks for performance (over-estimation),
|
||||
but must include all blocks that the slow path determines are necessary.
|
||||
"""
|
||||
device = torch.device("cuda")
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen2.5-1.5B-Instruct", block_size=16, max_model_len=1024
|
||||
)
|
||||
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
|
||||
|
||||
# Use a mixed batch that will create groups spanning multiple sequences
|
||||
batch_spec = BatchSpec(
|
||||
seq_lens=[35, 64, 128, 256], query_lens=[33, 5, 32, 64], name="test_mixed_batch"
|
||||
)
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, vllm_config.cache_config.block_size, device
|
||||
)
|
||||
|
||||
builder = FlexAttentionMetadataBuilder(kv_cache_spec, [], vllm_config, device)
|
||||
|
||||
metadata_direct = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
builder.direct_build = False
|
||||
metadata_slow = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
assert metadata_direct.block_mask is not None
|
||||
assert metadata_slow.block_mask is not None
|
||||
|
||||
# Extract block indices for comparison, B, H are the same
|
||||
direct_indices = metadata_direct.block_mask.kv_indices[0, 0]
|
||||
slow_indices = metadata_slow.block_mask.kv_indices[0, 0]
|
||||
direct_num = metadata_direct.block_mask.kv_num_blocks[0, 0]
|
||||
slow_num = metadata_slow.block_mask.kv_num_blocks[0, 0]
|
||||
|
||||
# main test: every block needed by slow path must be in direct path
|
||||
num_groups = direct_num.shape[0]
|
||||
all_contained = True
|
||||
missing_details = []
|
||||
|
||||
for group_idx in range(num_groups):
|
||||
direct_blocks = set(direct_indices[group_idx, : direct_num[group_idx]].tolist())
|
||||
slow_blocks = set(slow_indices[group_idx, : slow_num[group_idx]].tolist())
|
||||
|
||||
missing_blocks = slow_blocks - direct_blocks
|
||||
if missing_blocks:
|
||||
all_contained = False
|
||||
missing_details.append(
|
||||
f"Group {group_idx}: missing {sorted(missing_blocks)}"
|
||||
)
|
||||
|
||||
assert all_contained, (
|
||||
"Direct path is missing blocks required by slow path:\n"
|
||||
+ "\n".join(missing_details)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or TORCH_VERSION < DIRECT_BUILD_VERSION,
|
||||
reason="CUDA not available or PyTorch version < 2.9",
|
||||
)
|
||||
@pytest.mark.parametrize("direct_build", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_flex_attention_request_count_changes_reuse_compiled_graph(direct_build):
|
||||
"""Request-count changes preserve attention results and compiled graph reuse."""
|
||||
from torch._dynamo.testing import CompileCounterWithBackend
|
||||
from torch.nn.attention.flex_attention import flex_attention
|
||||
|
||||
from vllm.v1.attention.backends.flex_attention import get_kernel_options
|
||||
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3-0.6B",
|
||||
max_model_len=128,
|
||||
num_gpu_blocks=40,
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=128,
|
||||
)
|
||||
builder = FlexAttentionMetadataBuilder(
|
||||
create_standard_kv_cache_spec(config), [], config, device
|
||||
)
|
||||
builder.direct_build = direct_build
|
||||
query = torch.randn(1, 2, 32, 64, device=device)
|
||||
key = torch.randn(1, 2, 640, 64, device=device)
|
||||
value = torch.randn_like(key)
|
||||
kernel_options = get_kernel_options(query, 16, 16, direct_build)
|
||||
counter = CompileCounterWithBackend("inductor")
|
||||
compiled_attention = torch.compile(flex_attention, backend=counter, fullgraph=True)
|
||||
|
||||
def build_metadata(num_reqs):
|
||||
# Keep Q/K tensor and block-mask sizes fixed to isolate request counts.
|
||||
common = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[64] * num_reqs, query_lens=[32 // num_reqs] * num_reqs),
|
||||
16,
|
||||
device,
|
||||
max_block_idx=40,
|
||||
)
|
||||
common.block_table_tensor.copy_(
|
||||
torch.arange(1, num_reqs * 4 + 1, device=device).view(num_reqs, 4)
|
||||
)
|
||||
return builder.build(0, common)
|
||||
|
||||
def attend(metadata):
|
||||
return compiled_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
block_mask=metadata.block_mask,
|
||||
kernel_options=kernel_options,
|
||||
)
|
||||
|
||||
def reference(num_reqs):
|
||||
query_len = 32 // num_reqs
|
||||
q_idx = torch.arange(32, device=device)[:, None]
|
||||
kv_idx = torch.arange(640, device=device)[None, :]
|
||||
request = q_idx // query_len
|
||||
logical_q = q_idx % query_len + 64 - query_len
|
||||
logical_kv = kv_idx - (request * 4 + 1) * 16
|
||||
mask = (logical_kv >= 0) & (logical_kv <= logical_q)
|
||||
return torch.nn.functional.scaled_dot_product_attention(
|
||||
query, key, value, attn_mask=mask
|
||||
)
|
||||
|
||||
for num_reqs in (4, 2, 1, 8, 1):
|
||||
metadata = build_metadata(num_reqs)
|
||||
torch.testing.assert_close(
|
||||
attend(metadata), reference(num_reqs), atol=1e-4, rtol=1e-4
|
||||
)
|
||||
assert counter.frame_count == 1
|
||||
|
||||
if direct_build:
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = attend(metadata)
|
||||
for num_reqs in (8, 2, 1):
|
||||
metadata = build_metadata(num_reqs)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
output, reference(num_reqs), atol=1e-4, rtol=1e-4
|
||||
)
|
||||
|
||||
|
||||
def test_physical_to_logical_mapping_handles_reused_blocks():
|
||||
"""Regression test: reused physical blocks map to the latest logical block.
|
||||
|
||||
For sliding-window / hybrid attention layers, physical KV-cache blocks can be
|
||||
reused over time. The inverse mapping must therefore select the latest
|
||||
logical block index for a physical block id.
|
||||
"""
|
||||
# Padding should not make physical block 0 look live.
|
||||
block_table = torch.tensor([[6, 0, 0, 0]], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([1 * 16], dtype=torch.int32) # only 1 block valid
|
||||
out = physical_to_logical_mapping(
|
||||
block_table=block_table, seq_lens=seq_lens, block_size=16, total_blocks=10
|
||||
)
|
||||
assert out[0, 0].item() == -1
|
||||
assert out[0, 6].item() == 0
|
||||
|
||||
# If a physical block id appears multiple times (block reuse), mapping should
|
||||
# point to the latest logical block index.
|
||||
block_table2 = torch.tensor([[2, 2, 5]], dtype=torch.int32)
|
||||
seq_lens2 = torch.tensor([3 * 16], dtype=torch.int32)
|
||||
out2 = physical_to_logical_mapping(
|
||||
block_table=block_table2, seq_lens=seq_lens2, block_size=16, total_blocks=8
|
||||
)
|
||||
assert out2[0, 2].item() == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or TORCH_VERSION < DIRECT_BUILD_VERSION,
|
||||
reason="CUDA not available or PyTorch version < 2.9",
|
||||
)
|
||||
def test_block_sparsity_hint_prunes_blocks():
|
||||
"""Test that BlockSparsityHint prunes KV blocks from the direct build path.
|
||||
|
||||
Uses a hint that only keeps the diagonal (q_block == kv_block) to verify
|
||||
that off-diagonal blocks are excluded from the resulting BlockMask.
|
||||
"""
|
||||
device = torch.device("cuda")
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="facebook/opt-125m",
|
||||
block_size=16,
|
||||
max_model_len=1024,
|
||||
)
|
||||
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
|
||||
|
||||
batch_spec = BatchSpec(
|
||||
seq_lens=[256],
|
||||
query_lens=[256],
|
||||
name="test_sparsity_hint",
|
||||
)
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, vllm_config.cache_config.block_size, device
|
||||
)
|
||||
|
||||
builder = FlexAttentionMetadataBuilder(kv_cache_spec, [], vllm_config, device)
|
||||
|
||||
metadata_no_hint = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
metadata_no_hint.block_mask = metadata_no_hint._build_block_mask_direct()
|
||||
assert metadata_no_hint.block_mask.kv_num_blocks.max().item() > 1
|
||||
|
||||
def diagonal_hint(q_block_idx, kv_block_idx, block_size):
|
||||
return q_block_idx == kv_block_idx
|
||||
|
||||
metadata_with_hint = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
metadata_with_hint.block_sparsity_hint = BlockSparsityHint(
|
||||
hint_fn=diagonal_hint,
|
||||
)
|
||||
metadata_with_hint.block_mask = metadata_with_hint._build_block_mask_direct()
|
||||
assert metadata_with_hint.block_mask.kv_num_blocks.max().item() <= 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,620 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit test for fused_indexer_q_rope_quant.
|
||||
|
||||
Compares the fused Triton kernel against the unfused reference flow used by
|
||||
the DeepseekV4 indexer in model_tracking:
|
||||
q_rot = ops.rotary_embedding(positions, q, None, head_dim, cos_sin_cache,
|
||||
is_neox_style=False,
|
||||
rope_dim_offset=head_dim - rope_dim)
|
||||
q_fp8, q_scale = per_token_group_quant_fp8(q_rot, head_dim, use_ue8m0=True)
|
||||
weights_out = weights * q_scale * softmax_scale * head_scale
|
||||
|
||||
Expects bit-exact equality on both q_fp8 and weights_out.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import importlib
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import fused_indexer_q_rope_quant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
|
||||
HEAD_DIM = 128
|
||||
ROPE_DIM = 64
|
||||
MAX_POS = 4096
|
||||
|
||||
CUTEDSL_MODULE = "vllm.models.deepseek_v4.nvidia.ops.fused_indexer_q_cutedsl"
|
||||
|
||||
|
||||
def quantize_to_mxfp4(
|
||||
x: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Reference MXFP4 quantization.
|
||||
|
||||
Args:
|
||||
x: [..., head_dim] where head_dim is divisible by 32
|
||||
Returns:
|
||||
packed: [..., head_dim//2] uint8 2 E2M1 nibbles/byte, low nibble = even index
|
||||
scales: [..., head_dim//32] uint8 1 ue8m0 byte
|
||||
|
||||
"""
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
orig_shape = x.shape
|
||||
head_dim = orig_shape[-1]
|
||||
n_blocks = head_dim // MXFP4_BLOCK_SIZE
|
||||
|
||||
x_f32 = x.float().reshape(-1, n_blocks, MXFP4_BLOCK_SIZE)
|
||||
|
||||
# Per-block ue8m0 scale: 2^ceil(log2(amax / 6.0)), stored as byte = exp + 127
|
||||
# 6 * 2^-126 is from https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/inference/kernel.py#L163
|
||||
amax = x_f32.abs().amax(dim=-1, keepdim=True).clamp(min=6 * (2**-126))
|
||||
log2_ratio = (amax * (1.0 / 6.0)).log2().ceil().clamp(-127.0, 127.0)
|
||||
scale = log2_ratio.exp2()
|
||||
ue8m0 = (log2_ratio + 127.0).to(torch.uint8) # [*, n_blocks]
|
||||
|
||||
# E2M1 round-to-nearest-even: midpoints round to the even code.
|
||||
# E2M1 values: [0.00, 0.50, 1.00, 1.50, 2.00, 3.00, 4.00, 6.00]
|
||||
# boundaries: [ 0.25, 0.75, 1.25, 1.75, 2.50, 3.50, 5.00]
|
||||
x_scaled = (x_f32 / scale).clamp(-6.0, 6.0)
|
||||
abs_x = x_scaled.abs()
|
||||
code = torch.zeros_like(abs_x, dtype=torch.int32)
|
||||
code = torch.where(abs_x > 0.25, 1, code)
|
||||
code = torch.where(abs_x >= 0.75, 2, code)
|
||||
code = torch.where(abs_x > 1.25, 3, code)
|
||||
code = torch.where(abs_x >= 1.75, 4, code)
|
||||
code = torch.where(abs_x > 2.5, 5, code)
|
||||
code = torch.where(abs_x >= 3.5, 6, code)
|
||||
code = torch.where(abs_x > 5.0, 7, code)
|
||||
sign = ((x_scaled.view(torch.int32) >> 31) & 1).to(torch.uint8)
|
||||
nibble = code.to(torch.uint8) | (sign << 3)
|
||||
|
||||
# Pack: even-index element → low nibble, odd-index → high nibble
|
||||
nibble_flat = nibble.reshape(-1, head_dim)
|
||||
packed = (nibble_flat[:, 0::2] | (nibble_flat[:, 1::2] << 4)).contiguous()
|
||||
packed = packed.reshape(*orig_shape[:-1], head_dim // 2)
|
||||
|
||||
scales = ue8m0.view(*orig_shape[:-1], n_blocks)
|
||||
return packed, scales
|
||||
|
||||
|
||||
def _reference(
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
head_scale: float,
|
||||
n_head: int,
|
||||
use_fp4: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q_rot = q.clone()
|
||||
ops.rotary_embedding(
|
||||
positions,
|
||||
q_rot,
|
||||
None,
|
||||
HEAD_DIM,
|
||||
cos_sin_cache,
|
||||
False, # is_neox_style=False → GPT-J interleaved
|
||||
HEAD_DIM - ROPE_DIM, # rope_dim_offset → rotate the tail
|
||||
False,
|
||||
)
|
||||
|
||||
if use_fp4:
|
||||
q_packed, ue8m0 = quantize_to_mxfp4(q_rot.view(-1, n_head, HEAD_DIM))
|
||||
# Pack 4 ue8m0 bytes into 1 int32
|
||||
q_scale = ue8m0.view(torch.int32).squeeze(-1)
|
||||
# FP4 path: q_scale stays separate (cannot be folded into a per-token scalar)
|
||||
weights_out = weights.to(torch.float32) * softmax_scale * head_scale
|
||||
return (q_packed, q_scale), weights_out
|
||||
|
||||
else:
|
||||
q_fp8, q_scale = per_token_group_quant_fp8(
|
||||
q_rot.view(-1, HEAD_DIM).contiguous(),
|
||||
HEAD_DIM,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
q_fp8 = q_fp8.view(-1, n_head, HEAD_DIM)
|
||||
q_scale = q_scale.view(-1, n_head)
|
||||
|
||||
weights_out = weights.to(torch.float32) * q_scale * softmax_scale * head_scale
|
||||
return q_fp8, weights_out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 257, 1023])
|
||||
@pytest.mark.parametrize("cache_dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"use_fp4",
|
||||
[
|
||||
False,
|
||||
pytest.param(
|
||||
True,
|
||||
marks=pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
),
|
||||
reason="MXFP4 indexer cache requires an SM100-family GPU",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_cutedsl", [False, True])
|
||||
@pytest.mark.parametrize("n_head", [32, 64])
|
||||
@torch.inference_mode()
|
||||
def test_fused_indexer_q_rope_quant_matches_unfused(
|
||||
num_tokens, cache_dtype, use_fp4, use_cutedsl, n_head
|
||||
):
|
||||
if use_cutedsl and not has_cutedsl():
|
||||
pytest.skip("cutedsl (cutlass) not installed")
|
||||
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
|
||||
q = torch.randn(num_tokens, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
positions = torch.randint(
|
||||
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=cache_dtype, device=device)
|
||||
weights = torch.randn(num_tokens, n_head, dtype=torch.bfloat16, device=device)
|
||||
softmax_scale = HEAD_DIM**-0.5
|
||||
# head_scale must be an exact power of two for bit-exactness to be a fair
|
||||
# expectation: the triton kernel multiplies by softmax_scale and head_scale
|
||||
# separately (two roundings) while the cutedsl kernels fold them into one
|
||||
# fp32 scalar first (one rounding). Those agree only when one factor is a
|
||||
# pow2. n_head**-0.5 is a pow2 at 64 heads but not at 32.
|
||||
head_scale = 0.125
|
||||
|
||||
q_quant_ref, weights_ref = _reference(
|
||||
positions,
|
||||
q,
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
softmax_scale,
|
||||
head_scale,
|
||||
n_head,
|
||||
use_fp4,
|
||||
)
|
||||
# use_cutedsl=False: force the triton path even when cutedsl is installed
|
||||
# by patching the dispatcher's has_cutedsl() binding to return False.
|
||||
cutedsl_patch = (
|
||||
mock.patch(
|
||||
"vllm.models.deepseek_v4.common.ops.fused_indexer_q.has_cutedsl",
|
||||
return_value=False,
|
||||
)
|
||||
if not use_cutedsl
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
with cutedsl_patch:
|
||||
q_quant_fused, weights_fused = fused_indexer_q_rope_quant(
|
||||
positions,
|
||||
q.clone(),
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
softmax_scale,
|
||||
head_scale,
|
||||
use_fp4,
|
||||
)
|
||||
|
||||
if use_fp4:
|
||||
q_quant_ref, q_scale_ref = q_quant_ref
|
||||
q_quant_fused, q_scale_fused = q_quant_fused
|
||||
|
||||
assert torch.equal(q_scale_ref, q_scale_fused), (
|
||||
f"q_scale mismatch: "
|
||||
f"{(q_scale_ref != q_scale_fused).sum().item()} "
|
||||
f"/ {q_scale_ref.numel()} bytes differ"
|
||||
)
|
||||
|
||||
# fp8 tensors aren't directly comparable via torch.equal — reinterpret as int8.
|
||||
ref_bits = q_quant_ref.view(torch.int8)
|
||||
fused_bits = q_quant_fused.view(torch.int8)
|
||||
assert torch.equal(ref_bits, fused_bits), (
|
||||
f"q_quant_fused mismatch: "
|
||||
f"{(ref_bits != fused_bits).sum().item()} / {ref_bits.numel()} bytes differ"
|
||||
)
|
||||
|
||||
assert weights_fused.dtype == torch.float32
|
||||
assert torch.equal(weights_ref, weights_fused), (
|
||||
f"weights mismatch: max abs diff "
|
||||
f"{(weights_ref - weights_fused).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_cutedsl(), reason="cutedsl (cutlass) not installed")
|
||||
@pytest.mark.parametrize(
|
||||
"use_fp4",
|
||||
[
|
||||
False,
|
||||
pytest.param(
|
||||
True,
|
||||
marks=pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
),
|
||||
reason="MXFP4 indexer cache requires an SM100-family GPU",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("n_head", [32, 64])
|
||||
@torch.inference_mode()
|
||||
def test_cutedsl_indexer_q_writes_stay_within_num_tokens(use_fp4, n_head):
|
||||
"""The cutedsl kernels must not store past ``num_tokens``.
|
||||
|
||||
The launcher rounds ``num_tokens * threads_per_token`` up to whole 128-thread
|
||||
blocks, so the trailing block is only partially in range unless
|
||||
``threads_per_token`` is itself a multiple of the block size. With 32 heads
|
||||
and coarsen=4 (picked for >= 512 tokens) a token needs 64 threads, so an odd
|
||||
token count leaves half of the last block addressing token ``num_tokens``.
|
||||
Every output gets one extra sentinel row that the kernel must leave alone.
|
||||
"""
|
||||
mod = importlib.import_module(CUTEDSL_MODULE)
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 935
|
||||
|
||||
sentinel_u8, sentinel_f32 = 0x55, -12345.0
|
||||
# positions/q are padded too, so an out-of-bounds *load* stays addressable
|
||||
# and the assertions below are about stores only.
|
||||
positions = torch.zeros(num_tokens + 1, dtype=torch.int64, device=device)
|
||||
q = torch.randn(
|
||||
num_tokens + 1, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=torch.float32, device=device)
|
||||
weights = torch.randn(num_tokens + 1, n_head, dtype=torch.bfloat16, device=device)
|
||||
weights_out = torch.full(
|
||||
(num_tokens + 1, n_head), sentinel_f32, dtype=torch.float32, device=device
|
||||
)
|
||||
outputs = {"weights_out": weights_out}
|
||||
|
||||
if use_fp4:
|
||||
outputs["q_packed"] = torch.full(
|
||||
(num_tokens + 1, n_head, HEAD_DIM // 2),
|
||||
sentinel_u8,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
outputs["q_scale"] = torch.full(
|
||||
(num_tokens + 1, n_head, HEAD_DIM // 32),
|
||||
sentinel_u8,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
mod._INDEXER_Q_MXFP4_KERNEL(
|
||||
positions=positions[:num_tokens],
|
||||
q=q[:num_tokens],
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
weights=weights[:num_tokens],
|
||||
weights_softmax_scale=1.0,
|
||||
weights_head_scale=1.0,
|
||||
q_packed=outputs["q_packed"][:num_tokens],
|
||||
q_scale=outputs["q_scale"][:num_tokens],
|
||||
weights_out=weights_out[:num_tokens],
|
||||
)
|
||||
else:
|
||||
outputs["q_fp8"] = torch.full(
|
||||
(num_tokens + 1, n_head, HEAD_DIM),
|
||||
sentinel_u8,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
mod._INDEXER_Q_FP8_KERNEL(
|
||||
positions=positions[:num_tokens],
|
||||
q=q[:num_tokens],
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
weights=weights[:num_tokens],
|
||||
weights_softmax_scale=1.0,
|
||||
weights_head_scale=1.0,
|
||||
q_fp8=outputs["q_fp8"][:num_tokens].view(torch.float8_e4m3fn),
|
||||
weights_out=weights_out[:num_tokens],
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
clobbered = {}
|
||||
for name, tensor in outputs.items():
|
||||
sentinel = sentinel_f32 if tensor.dtype == torch.float32 else sentinel_u8
|
||||
n_changed = (tensor[num_tokens] != sentinel).sum().item()
|
||||
if n_changed:
|
||||
clobbered[name] = n_changed
|
||||
assert not clobbered, (
|
||||
f"kernel wrote past token {num_tokens - 1} into the sentinel row: {clobbered}"
|
||||
)
|
||||
|
||||
|
||||
def _indexer_k_reference(
|
||||
k_pre, positions, cos_sin_cache, rms_norm_weight, compress_ratio, use_fp4
|
||||
):
|
||||
k = k_pre.float()
|
||||
k = k * torch.rsqrt(k.square().mean(dim=-1, keepdim=True) + 1e-20)
|
||||
k = (k * rms_norm_weight.float()).to(torch.bfloat16).float()
|
||||
group_positions = positions // compress_ratio * compress_ratio
|
||||
cos, sin = cos_sin_cache[group_positions].float().chunk(2, dim=-1)
|
||||
even, odd = k[:, 64::2], k[:, 65::2]
|
||||
rotated = torch.stack((even * cos - odd * sin, odd * cos + even * sin), dim=-1)
|
||||
k = torch.cat((k[:, :64], rotated.flatten(1)), dim=-1).to(torch.bfloat16)
|
||||
if use_fp4:
|
||||
return quantize_to_mxfp4(k)
|
||||
exponent = k.float().abs().amax(dim=-1, keepdim=True).clamp(min=1e-4) / 448
|
||||
scale = exponent.log2().ceil().exp2()
|
||||
values = (k.float() / scale).clamp(-448, 448).to(torch.float8_e4m3fn)
|
||||
if mqa_logits_use_mx_scales():
|
||||
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = (scale.view(torch.int32) >> 23).to(torch.uint8)
|
||||
return values.view(torch.uint8), ue8m0.repeat(1, 4)
|
||||
return values.view(torch.uint8), scale.view(torch.uint8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA kernel")
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 257])
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
@pytest.mark.parametrize("use_fp4", [False, True])
|
||||
@pytest.mark.parametrize("cache_dtype", [torch.float32, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_indexer_k_inserts_only_valid_groups_into_padded_pages(
|
||||
num_tokens, compress_ratio, use_fp4, cache_dtype
|
||||
):
|
||||
"""Insert group ends and preserve skipped slots, graph rows, and page padding."""
|
||||
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
|
||||
indexer_k_norm_rope_store,
|
||||
)
|
||||
|
||||
if use_fp4 and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("MXFP4 indexer cache requires an SM100-family GPU")
|
||||
torch.manual_seed(17)
|
||||
device = "cuda"
|
||||
k_pre = torch.randn(num_tokens + 3, 128, device=device, dtype=torch.bfloat16)
|
||||
norm_weight = torch.randn(128, device=device, dtype=torch.bfloat16)
|
||||
positions = torch.arange(7, num_tokens + 10, device=device)
|
||||
slots = torch.randperm(2 * num_tokens, device=device)[:num_tokens].clone()
|
||||
slots[5::7] = -1
|
||||
valid = (slots >= 0) & ((positions[:num_tokens] + 1) % compress_ratio == 0)
|
||||
k_pre[3::7] = 0
|
||||
k_pre[8::11] *= 1e-10
|
||||
k_pre[:num_tokens][~valid] = float("nan")
|
||||
k_pre[num_tokens:] = float("nan")
|
||||
positions[num_tokens:] = MAX_POS + 10
|
||||
angles = torch.randn(MAX_POS, 32, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1).to(cache_dtype)
|
||||
block_size = 16
|
||||
value_bytes = 64 if use_fp4 else 128
|
||||
page_bytes = block_size * (value_bytes + 4)
|
||||
num_blocks = (2 * num_tokens + block_size - 1) // block_size + 1
|
||||
storage = torch.full(
|
||||
(num_blocks, page_bytes + 128), 0xA5, device=device, dtype=torch.uint8
|
||||
)
|
||||
cache = storage.as_strided(
|
||||
(num_blocks, block_size, value_bytes + 4),
|
||||
(storage.stride(0), value_bytes + 4, 1),
|
||||
)
|
||||
expected = storage.clone()
|
||||
values, scales = _indexer_k_reference(
|
||||
k_pre[:num_tokens][valid],
|
||||
positions[:num_tokens][valid],
|
||||
cos_sin,
|
||||
norm_weight,
|
||||
compress_ratio,
|
||||
use_fp4,
|
||||
)
|
||||
block_ids = slots[valid] // block_size
|
||||
block_offsets = slots[valid] % block_size
|
||||
value_offsets = block_offsets[:, None] * value_bytes + torch.arange(
|
||||
value_bytes, device=device
|
||||
)
|
||||
scale_offsets = block_size * value_bytes + block_offsets[:, None] * 4
|
||||
scale_offsets = scale_offsets + torch.arange(4, device=device)
|
||||
expected[block_ids[:, None], value_offsets] = values
|
||||
expected[block_ids[:, None], scale_offsets] = scales
|
||||
|
||||
indexer_k_norm_rope_store(
|
||||
k_pre,
|
||||
positions,
|
||||
cos_sin,
|
||||
norm_weight,
|
||||
1e-20,
|
||||
cache,
|
||||
slots,
|
||||
compress_ratio,
|
||||
use_fp4,
|
||||
)
|
||||
# Fused RoPE can change zero signs without changing their numerical values.
|
||||
actual_values = storage[block_ids[:, None], value_offsets]
|
||||
if use_fp4:
|
||||
for shift in (0, 4):
|
||||
zero_mask = (actual_values >> shift) & 7 == 0
|
||||
actual_values &= ~(zero_mask.to(torch.uint8) << (shift + 3))
|
||||
zero_mask = (values >> shift) & 7 == 0
|
||||
values &= ~(zero_mask.to(torch.uint8) << (shift + 3))
|
||||
else:
|
||||
actual_values[actual_values == 128] = 0
|
||||
values[values == 128] = 0
|
||||
storage[block_ids[:, None], value_offsets] = actual_values
|
||||
expected[block_ids[:, None], value_offsets] = values
|
||||
torch.testing.assert_close(storage, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA kernel")
|
||||
@pytest.mark.parametrize("use_fp4", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_indexer_k_cuda_graph_replay_reads_current_projection(use_fp4):
|
||||
"""Replay must consume updated keys and slot mapping through the captured API."""
|
||||
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
|
||||
indexer_k_norm_rope_store,
|
||||
)
|
||||
|
||||
if use_fp4 and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("MXFP4 indexer cache requires an SM100-family GPU")
|
||||
torch.manual_seed(18)
|
||||
k_pre = torch.randn(19, 128, device="cuda", dtype=torch.bfloat16)
|
||||
norm_weight = torch.ones(128, device="cuda", dtype=torch.bfloat16)
|
||||
positions = torch.arange(19, device="cuda")
|
||||
cos_sin = torch.zeros(32, 64, device="cuda")
|
||||
cos_sin[:, :32] = 1
|
||||
slots = torch.arange(17, device="cuda")
|
||||
cache = torch.full(
|
||||
(2, 16, (64 if use_fp4 else 128) + 4),
|
||||
0xA5,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
|
||||
def run():
|
||||
indexer_k_norm_rope_store(
|
||||
k_pre,
|
||||
positions,
|
||||
cos_sin,
|
||||
norm_weight,
|
||||
1e-20,
|
||||
cache,
|
||||
slots,
|
||||
2,
|
||||
use_fp4,
|
||||
)
|
||||
|
||||
run()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
k_pre.neg_()
|
||||
slots.add_(4)
|
||||
cache.fill_(0xA5)
|
||||
graph.replay()
|
||||
replayed = cache.clone()
|
||||
cache.fill_(0xA5)
|
||||
run()
|
||||
torch.testing.assert_close(cache, replayed, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm cache layout")
|
||||
@pytest.mark.parametrize("block_size", [16, 64, 128])
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
@torch.inference_mode()
|
||||
def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_ratio):
|
||||
"""ROCm reads the indexer K cache back with the 16x16-tiled value layout.
|
||||
|
||||
``cp_gather_indexer_k_quant_cache_triton`` (prefill) and aiter's
|
||||
``Preshuffle=True`` paged kernel (decode) both pick that layout whenever
|
||||
``block_size > 1``, so a row-major store would feed the indexer permuted
|
||||
key bytes and randomize its top-k. Assert the write/read pair is exact.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
|
||||
indexer_k_norm_rope_store,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
cp_gather_indexer_k_quant_cache_triton,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens = 3 * block_size + 5
|
||||
num_blocks = num_tokens // block_size + 2
|
||||
k_pre = torch.randn(num_tokens, HEAD_DIM, device=device, dtype=torch.bfloat16)
|
||||
norm_weight = torch.randn(HEAD_DIM, device=device, dtype=torch.bfloat16)
|
||||
positions = torch.arange(num_tokens, device=device)
|
||||
angles = torch.randn(MAX_POS, ROPE_DIM // 2, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1).float()
|
||||
cache = torch.zeros(
|
||||
num_blocks, block_size, HEAD_DIM + 4, device=device, dtype=torch.uint8
|
||||
)
|
||||
slots = torch.arange(num_tokens, device=device)
|
||||
|
||||
indexer_k_norm_rope_store(
|
||||
k_pre,
|
||||
positions,
|
||||
cos_sin,
|
||||
norm_weight,
|
||||
1e-20,
|
||||
cache,
|
||||
slots,
|
||||
compress_ratio,
|
||||
False,
|
||||
)
|
||||
|
||||
k_fp8 = torch.zeros(num_tokens, HEAD_DIM, device=device, dtype=torch.float8_e4m3fn)
|
||||
k_scale = torch.zeros(num_tokens, 4, device=device, dtype=torch.uint8)
|
||||
cp_gather_indexer_k_quant_cache_triton(
|
||||
cache,
|
||||
k_fp8,
|
||||
k_scale,
|
||||
torch.arange(num_blocks, device=device, dtype=torch.int32).view(1, -1),
|
||||
torch.tensor([0, num_tokens], device=device, dtype=torch.int32),
|
||||
torch.zeros(num_tokens, device=device, dtype=torch.int32),
|
||||
)
|
||||
|
||||
expected_values, expected_scale = _indexer_k_reference(
|
||||
k_pre, positions, cos_sin, norm_weight, compress_ratio, False
|
||||
)
|
||||
emitted = (positions + 1) % compress_ratio == 0
|
||||
gathered = k_fp8.view(torch.uint8)
|
||||
gathered = torch.where(gathered == 128, gathered.new_zeros(()), gathered)
|
||||
expected_values = torch.where(
|
||||
expected_values == 128, expected_values.new_zeros(()), expected_values
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
gathered[emitted], expected_values[emitted], rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
k_scale[emitted], expected_scale[emitted], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_cutedsl", [False, True])
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
),
|
||||
reason="MXFP4 indexer cache requires an SM100-family GPU",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_indexer_q_rope_quant_writes_bf16_weights(use_cutedsl):
|
||||
"""The MXFP4 path can emit the per-head weights in bf16 (what DeepGEMM's
|
||||
sparse MQA-logits kernels take) instead of fp32; the values are the fp32
|
||||
result rounded once, so the scoring kernel needs no cast."""
|
||||
if use_cutedsl and not has_cutedsl():
|
||||
pytest.skip("cutedsl (cutlass) not installed")
|
||||
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
num_tokens, n_head = 257, 32
|
||||
q = torch.randn(num_tokens, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
positions = torch.randint(
|
||||
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=torch.float32, device=device)
|
||||
weights = torch.randn(num_tokens, n_head, dtype=torch.bfloat16, device=device)
|
||||
softmax_scale, head_scale = HEAD_DIM**-0.5, 0.125 # pow2 head_scale, see above
|
||||
|
||||
_, weights_ref = _reference(
|
||||
positions, q, cos_sin_cache, weights, softmax_scale, head_scale, n_head, True
|
||||
)
|
||||
cutedsl_patch = (
|
||||
mock.patch(
|
||||
"vllm.models.deepseek_v4.common.ops.fused_indexer_q.has_cutedsl",
|
||||
return_value=False,
|
||||
)
|
||||
if not use_cutedsl
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
with cutedsl_patch:
|
||||
_, weights_fused = fused_indexer_q_rope_quant(
|
||||
positions,
|
||||
q.clone(),
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
softmax_scale,
|
||||
head_scale,
|
||||
use_fp4=True,
|
||||
weights_out_dtype=torch.bfloat16,
|
||||
)
|
||||
assert weights_fused.dtype == torch.bfloat16
|
||||
assert torch.equal(weights_fused, weights_ref.to(torch.bfloat16))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,310 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
_SHAPES = (
|
||||
(129, 512),
|
||||
(129, 768),
|
||||
(257, 768),
|
||||
(1023, 4224),
|
||||
(1024, 4224),
|
||||
(8191, 4224),
|
||||
)
|
||||
_N = 512
|
||||
|
||||
|
||||
def _reference(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor | None,
|
||||
world_size: int,
|
||||
group: dist.ProcessGroup,
|
||||
all_reduce: bool,
|
||||
) -> torch.Tensor:
|
||||
M = x.shape[0]
|
||||
padded_M = (M + world_size - 1) // world_size * world_size
|
||||
partial = torch.empty(
|
||||
(padded_M, weight.shape[0]),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
if weight_scale is None:
|
||||
torch.mm(x, weight.T, out=partial[:M])
|
||||
else:
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
|
||||
x_q, x_sf = mxfp8_e4m3_quantize(x, is_sf_swizzled_layout=True)
|
||||
partial[:M] = torch._scaled_mm(
|
||||
x_q,
|
||||
weight.T,
|
||||
x_sf.view(torch.float8_e8m0fnu),
|
||||
weight_scale.view(torch.float8_e8m0fnu),
|
||||
out_dtype=x.dtype,
|
||||
)
|
||||
if padded_M > M:
|
||||
partial[M:].zero_()
|
||||
|
||||
if all_reduce:
|
||||
dist.all_reduce(partial, group=group)
|
||||
return partial[:M]
|
||||
|
||||
output = torch.empty(
|
||||
(padded_M // world_size, weight.shape[0]),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
dist.reduce_scatter_single(output, partial, group=group)
|
||||
return output
|
||||
|
||||
|
||||
def _assert_valid_rows_close(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
M: int,
|
||||
rank: int,
|
||||
all_reduce: bool,
|
||||
) -> None:
|
||||
if all_reduce:
|
||||
torch.testing.assert_close(actual, expected, rtol=5e-2, atol=4.0)
|
||||
return
|
||||
rank_start = rank * actual.shape[0]
|
||||
valid_rows = min(max(M - rank_start, 0), actual.shape[0])
|
||||
torch.testing.assert_close(
|
||||
actual[:valid_rows],
|
||||
expected[:valid_rows],
|
||||
rtol=5e-2,
|
||||
atol=4.0,
|
||||
)
|
||||
# Sequence-parallel callers may pass an unpadded M and read the padding
|
||||
# rows back (they flow through per-token kernels before being dropped),
|
||||
# so RS must leave them zero like ``sp_reduce_scatter`` does.
|
||||
assert not actual[valid_rows:].any()
|
||||
|
||||
|
||||
def _run_mode(
|
||||
*,
|
||||
all_reduce: bool,
|
||||
device: torch.device,
|
||||
group: dist.ProcessGroup,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
weights: dict[int, torch.Tensor],
|
||||
backend: str,
|
||||
) -> None:
|
||||
# cute_dsl is unavailable off CUDA, so import it only inside the GPU worker.
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar import GemmRsAr
|
||||
|
||||
gemm_rs_ar = GemmRsAr(
|
||||
max_M=max(M for M, _ in _SHAPES),
|
||||
N=_N,
|
||||
all_reduce=all_reduce,
|
||||
)
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.linear import RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
quant_config = (
|
||||
None
|
||||
if backend == "bf16"
|
||||
else OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
|
||||
)
|
||||
projections = {}
|
||||
operands = {}
|
||||
for K, w in weights.items():
|
||||
with torch.device(device):
|
||||
linear = RowParallelLinear(
|
||||
K * world_size,
|
||||
_N,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
# Select the fused path before online quantization replaces meta weights.
|
||||
assert gemm_rs_ar.can_run(linear)
|
||||
linear.weight = torch.nn.Parameter(w.clone(), requires_grad=False)
|
||||
linear.quant_method.process_weights_after_loading(linear)
|
||||
projections[K] = linear
|
||||
weight = linear.weight
|
||||
if backend == "flashinfer_cutedsl":
|
||||
weight = weight.t()
|
||||
operands[K] = (weight, getattr(linear, "weight_scale", None))
|
||||
|
||||
input_generator = torch.Generator(device=device)
|
||||
|
||||
# Alternating shapes exercise producer-flag reuse across different grids
|
||||
# and CTA-group choices.
|
||||
for M, K in (*_SHAPES, *_SHAPES[::-1]):
|
||||
input_generator.manual_seed(2000 + M + K)
|
||||
x = torch.randn(
|
||||
M,
|
||||
K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
expected = _reference(x, *operands[K], world_size, group, all_reduce)
|
||||
actual = gemm_rs_ar.apply(x, projections[K])
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_valid_rows_close(actual, expected, M, rank, all_reduce)
|
||||
|
||||
if all_reduce:
|
||||
# AR outputs must remain valid after the symmetric workspace is reused.
|
||||
lifetime_M, lifetime_K = 257, 768
|
||||
input_generator.manual_seed(2500)
|
||||
lifetime_x = torch.randn(
|
||||
lifetime_M,
|
||||
lifetime_K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
first_output = gemm_rs_ar.apply(lifetime_x, projections[lifetime_K])
|
||||
first_snapshot = first_output.clone()
|
||||
second_output = gemm_rs_ar.apply(-lifetime_x, projections[lifetime_K])
|
||||
torch.accelerator.synchronize(device)
|
||||
assert first_output.data_ptr() != second_output.data_ptr()
|
||||
torch.testing.assert_close(first_output, first_snapshot, rtol=0, atol=0)
|
||||
|
||||
graph_M, graph_K = 1025, 4224
|
||||
input_generator.manual_seed(3000)
|
||||
graph_x = torch.randn(
|
||||
graph_M,
|
||||
graph_K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
graph_expected = _reference(
|
||||
graph_x,
|
||||
*operands[graph_K],
|
||||
world_size,
|
||||
group,
|
||||
all_reduce,
|
||||
)
|
||||
graph_output = torch.empty_like(graph_expected)
|
||||
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(capture_stream):
|
||||
for _ in range(3):
|
||||
graph_output.copy_(gemm_rs_ar.apply(graph_x, projections[graph_K]))
|
||||
capture_stream.synchronize()
|
||||
dist.barrier(group=group)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
graph_output.copy_(gemm_rs_ar.apply(graph_x, projections[graph_K]))
|
||||
torch.cuda.current_stream().wait_stream(capture_stream)
|
||||
dist.barrier(group=group)
|
||||
|
||||
for _ in range(3):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_valid_rows_close(
|
||||
graph_output,
|
||||
graph_expected,
|
||||
graph_M,
|
||||
rank,
|
||||
all_reduce,
|
||||
)
|
||||
|
||||
dist.barrier(group=group)
|
||||
del gemm_rs_ar, capture_stream, graph, graph_output
|
||||
|
||||
|
||||
def _worker(local_rank: int, world_size: int, master_port: int, backend: str) -> None:
|
||||
# This module pulls in cute_dsl, which is unavailable off CUDA, so importing
|
||||
# it at module level would fail collection. Import it where it is used, as
|
||||
# `kda.py` does.
|
||||
|
||||
device = torch.device("cuda", local_rank)
|
||||
torch.accelerator.set_device_index(device)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(master_port),
|
||||
}
|
||||
)
|
||||
|
||||
init_distributed_environment()
|
||||
config = VllmConfig()
|
||||
config.model_config = ModelConfig(dtype="bfloat16")
|
||||
config.kernel_config.linear_backend = "auto" if backend == "bf16" else backend
|
||||
with set_current_vllm_config(config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
group = dist.group.WORLD
|
||||
rank = dist.get_rank(group)
|
||||
|
||||
weight_generator = torch.Generator(device=device)
|
||||
weights = {}
|
||||
for K in {K for _, K in _SHAPES}:
|
||||
weight_generator.manual_seed(1000 + rank * 10 + K)
|
||||
weights[K] = torch.randn(
|
||||
_N,
|
||||
K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=weight_generator,
|
||||
)
|
||||
|
||||
# Production binds one mode per worker; exercise both mode-bound instances
|
||||
# sequentially in this test without implying that both are initialized.
|
||||
with set_current_vllm_config(config):
|
||||
for all_reduce in (False, True):
|
||||
_run_mode(
|
||||
all_reduce=all_reduce,
|
||||
device=device,
|
||||
group=group,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
weights=weights,
|
||||
backend=backend,
|
||||
)
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend", ["bf16", "flashinfer_cutlass", "flashinfer_cutedsl"]
|
||||
)
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="GEMM-RS/AR requires SM100",
|
||||
)
|
||||
def test_gemm_rs_ar(monkeypatch: pytest.MonkeyPatch, backend: str) -> None:
|
||||
world_size = 2
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip("GEMM-RS/AR requires two GPUs")
|
||||
|
||||
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
|
||||
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
|
||||
try:
|
||||
mp.spawn(
|
||||
_worker,
|
||||
args=(world_size, get_open_port(), backend),
|
||||
nprocs=world_size,
|
||||
)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -1,670 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit test for the batched kpool decode-update kernel.
|
||||
|
||||
Validates ``kpool_decode_update_and_maybe_write_cache_batched`` against an
|
||||
independent pure-torch reference that replicates the per-request, in-position
|
||||
order semantics: stash each token into a paged tail ring; on pool completion
|
||||
(``pos % pool_size == pool_size-1``) softmax(gate+ape)-weighted sum + Hadamard-128
|
||||
+ per-vector fp8 absmax quant + write to the indexer K cache. Covers
|
||||
no-completion, completion-at-end, completion-mid-batch, non-uniform padding,
|
||||
plain decode, plus a randomized fuzz pass.
|
||||
|
||||
The kernel iterates each request's ``next_n`` tokens in position order inside
|
||||
one program (grid = num_requests) to preserve the pool-completion
|
||||
read-after-stash dependency; the reference mirrors that ordering.
|
||||
|
||||
``test_decode_writer_matches_prefill_writer`` is deliberately NOT
|
||||
reference-based: it checks the decode writer against the *prefill* writer
|
||||
(``kpool_compress_and_write_cache``), the invariant that actually matters in
|
||||
production. A hand-written reference can drift to match a buggy kernel -- that
|
||||
is exactly how the stash-gating bug (intra-pool tokens never entering the tail
|
||||
ring, because the stash was gated on the pool-granular ``slot_mapping``) stayed
|
||||
green here.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.glm5next.amd.ops.kpool_compress import (
|
||||
kpool_compress_and_write_cache,
|
||||
kpool_decode_update_and_maybe_write_cache_batched,
|
||||
kpool_seed_tail_cache,
|
||||
)
|
||||
else:
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import (
|
||||
kpool_compress_and_write_cache,
|
||||
kpool_decode_update_and_maybe_write_cache_batched,
|
||||
kpool_seed_tail_cache,
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
POOL_SIZE = 16
|
||||
PAGE_SIZE = 64
|
||||
NUM_BLOCKS = 32
|
||||
ROUND_SCALE = True
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP8_MAX = torch.finfo(FP8_DTYPE).max
|
||||
|
||||
|
||||
def _make_caches():
|
||||
kv = torch.zeros(
|
||||
NUM_BLOCKS, PAGE_SIZE, HEAD_DIM + 4, dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
tail = torch.zeros(
|
||||
NUM_BLOCKS, 2, POOL_SIZE, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
return kv, tail
|
||||
|
||||
|
||||
def _tail_slot_for(blocks, pos):
|
||||
"""tail_slot = block*POOL + pos%POOL; each request owns a distinct tail block."""
|
||||
blk = torch.tensor(blocks, device=pos.device, dtype=torch.int32).unsqueeze(1)
|
||||
return (blk * POOL_SIZE + pos % POOL_SIZE).to(torch.int32)
|
||||
|
||||
|
||||
def _seed_prior(tail, blocks, n_prior, seed=42):
|
||||
if n_prior <= 0:
|
||||
return
|
||||
g = torch.Generator(device=tail.device).manual_seed(seed)
|
||||
prior_k = torch.randn(
|
||||
len(blocks),
|
||||
n_prior,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=tail.device,
|
||||
generator=g,
|
||||
)
|
||||
prior_s = torch.randn(
|
||||
len(blocks),
|
||||
n_prior,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=tail.device,
|
||||
generator=g,
|
||||
)
|
||||
for i, blk in enumerate(blocks):
|
||||
tail[blk, 0, :n_prior, :] = prior_k[i]
|
||||
tail[blk, 1, :n_prior, :] = prior_s[i]
|
||||
|
||||
|
||||
def _hadamard128_torch(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Reference Hadamard-128 on the last dim (must be 128)."""
|
||||
n = x.shape[-1]
|
||||
assert n == 128
|
||||
h = torch.tensor([[1.0, 1.0], [1.0, -1.0]], dtype=torch.float32, device=x.device)
|
||||
while h.shape[0] < n:
|
||||
h = torch.cat([torch.cat([h, h], dim=1), torch.cat([h, -h], dim=1)], dim=0)
|
||||
h = h / math.sqrt(n)
|
||||
return x @ h
|
||||
|
||||
|
||||
def _torch_reference(
|
||||
kv: torch.Tensor,
|
||||
tail: torch.Tensor,
|
||||
tail_slot: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
slot_map: torch.Tensor,
|
||||
pos: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Independent reference for the batched decode-update kernel.
|
||||
|
||||
For each request, iterate its next_n tokens in order. On each token:
|
||||
- if pos%POOL == POOL-1 and pos_valid: compress the pool (slots
|
||||
[pool_start..pool_start+POOL-1], current token via is_current) and write
|
||||
fp8 K + fp32 scale to kv_cache at cache_loc.
|
||||
- always stash the current token's K/score into tail[block, pos%POOL].
|
||||
"""
|
||||
kv = kv.clone()
|
||||
tail = tail.clone()
|
||||
B, next_n = pos.shape
|
||||
# The indexer K cache is [num_blocks, PAGE_SIZE, HEAD_DIM+4] uint8 but the
|
||||
# kernels interpret each page as [HEAD_DIM*PAGE_SIZE bytes of K (token-major)
|
||||
# | 4*PAGE_SIZE bytes of fp32 scale (token-major)]. Operate on a flat byte
|
||||
# view so the reference writes K and scale at the exact offsets the kernel
|
||||
# uses (page_base + tok*HEAD_DIM for K; page_base + HEAD_DIM*PAGE_SIZE +
|
||||
# tok*4 for scale).
|
||||
page_bytes = PAGE_SIZE * (HEAD_DIM + 4)
|
||||
k_region = HEAD_DIM * PAGE_SIZE
|
||||
tail_slot_cpu = tail_slot.cpu().tolist()
|
||||
slot_map_cpu = slot_map.cpu().tolist()
|
||||
pos_cpu = pos.cpu().tolist()
|
||||
key_cpu = key.float().cpu()
|
||||
score_cpu = score.float().cpu()
|
||||
ape_cpu = ape.cpu()
|
||||
tail_cpu = tail.float().cpu()
|
||||
kv_flat = kv.view(torch.uint8).reshape(-1).cpu()
|
||||
|
||||
for b in range(B):
|
||||
for t in range(next_n):
|
||||
cache_loc = slot_map_cpu[b][t]
|
||||
p = pos_cpu[b][t]
|
||||
pos_valid = cache_loc >= 0 and p >= 0
|
||||
safe_pos = max(p, 0)
|
||||
slot = safe_pos % POOL_SIZE
|
||||
phys_slot = safe_pos % POOL_SIZE
|
||||
# Per-token block derivation (a leading invalid sentinel must not
|
||||
# poison the base for the rest of the request); clamped like the
|
||||
# kernel so an invalid entry can't form a negative base.
|
||||
block = max(tail_slot_cpu[b][t], 0) // POOL_SIZE
|
||||
|
||||
cur_key = key_cpu[b, t]
|
||||
cur_score = score_cpu[b, t]
|
||||
|
||||
if pos_valid and slot == POOL_SIZE - 1:
|
||||
pool_logical_start = safe_pos - slot
|
||||
pool_scores = []
|
||||
pool_ks = []
|
||||
for ps in range(POOL_SIZE):
|
||||
is_current = ps == slot
|
||||
phys = (pool_logical_start + ps) % POOL_SIZE
|
||||
if is_current:
|
||||
s = cur_score
|
||||
k = cur_key
|
||||
else:
|
||||
s = tail_cpu[block, 1, phys]
|
||||
k = tail_cpu[block, 0, phys]
|
||||
s = s + ape_cpu[ps]
|
||||
pool_scores.append(s)
|
||||
pool_ks.append(k)
|
||||
pool_scores = torch.stack(pool_scores) # [POOL, D]
|
||||
pool_ks = torch.stack(pool_ks) # [POOL, D]
|
||||
max_score = pool_scores.max(dim=0).values
|
||||
prob = torch.exp(pool_scores - max_score)
|
||||
denom = prob.sum(dim=0)
|
||||
acc = (pool_ks * prob).sum(dim=0)
|
||||
x = (acc / denom).to(torch.bfloat16).to(torch.float32)
|
||||
x = _hadamard128_torch(x).to(torch.bfloat16).to(torch.float32)
|
||||
absmax = torch.clamp(x.abs().max(), min=1e-4)
|
||||
if ROUND_SCALE:
|
||||
scale = torch.exp2(torch.ceil(torch.log2(absmax / FP8_MAX)))
|
||||
else:
|
||||
scale = absmax / FP8_MAX
|
||||
quantized = torch.clamp(x / scale, -FP8_MAX, FP8_MAX).to(FP8_DTYPE)
|
||||
# write K and scale at the separated-layout offsets
|
||||
loc = cache_loc
|
||||
loc_page_index = loc // PAGE_SIZE
|
||||
loc_tok = loc % PAGE_SIZE
|
||||
page_base = loc_page_index * page_bytes
|
||||
if current_platform.is_rocm():
|
||||
dims = torch.arange(HEAD_DIM)
|
||||
k_off = (
|
||||
page_base
|
||||
+ (loc_tok // 16) * 16 * HEAD_DIM
|
||||
+ (dims // 16) * 16 * 16
|
||||
+ (loc_tok % 16) * 16
|
||||
+ dims % 16
|
||||
)
|
||||
else:
|
||||
k_off = page_base + loc_tok * HEAD_DIM + torch.arange(HEAD_DIM)
|
||||
s_off = page_base + k_region + loc_tok * 4
|
||||
kv_flat[k_off] = quantized.view(torch.uint8)
|
||||
scale_bytes = scale.detach().reshape(1).view(torch.uint8)
|
||||
if mqa_logits_use_mx_scales():
|
||||
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = scale.reshape(1).view(torch.int32) >> 23
|
||||
scale_bytes = ue8m0.to(torch.uint8).repeat(4)
|
||||
kv_flat[s_off : s_off + 4] = scale_bytes
|
||||
|
||||
# stash -- gated on the TOKEN-granular tail slot, not on pos_valid.
|
||||
# pos_valid keys off the pool-granular cache_loc, which is -1 for
|
||||
# every token that is not the pool's last, so gating the stash on it
|
||||
# would drop all intra-pool tokens.
|
||||
if p >= 0 and tail_slot_cpu[b][t] >= 0:
|
||||
tail_cpu[block, 0, phys_slot] = cur_key
|
||||
tail_cpu[block, 1, phys_slot] = cur_score
|
||||
|
||||
kv_out = kv_flat.view(NUM_BLOCKS, PAGE_SIZE, HEAD_DIM + 4).to(device="cuda")
|
||||
return kv_out, tail_cpu.to(torch.bfloat16).to(device="cuda")
|
||||
|
||||
|
||||
def _assert_eq(r_ref, r_kern, k_byte_tol=0):
|
||||
kv_ref, tail_ref = r_ref
|
||||
kv_kern, tail_kern = r_kern
|
||||
# Each page holds the fp8 K bytes followed by the fp32 scales.
|
||||
k_region = PAGE_SIZE * HEAD_DIM
|
||||
pages_ref = kv_ref.view(kv_ref.shape[0], -1)
|
||||
pages_kern = kv_kern.view(kv_kern.shape[0], -1)
|
||||
k_diff = (pages_ref[:, :k_region].int() - pages_kern[:, :k_region].int()).abs()
|
||||
assert k_diff.max().item() <= k_byte_tol, (
|
||||
f"kv_cache K differs: max diff {k_diff.max().item()}"
|
||||
)
|
||||
assert torch.equal(pages_ref[:, k_region:], pages_kern[:, k_region:]), (
|
||||
"kv_cache scales differ"
|
||||
)
|
||||
assert torch.equal(tail_ref, tail_kern), (
|
||||
"tail_kv_cache differs: max diff "
|
||||
f"{(tail_ref.float() - tail_kern.float()).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
def _cache_pool_bytes(kv_cache: torch.Tensor, pool_slot: int) -> torch.Tensor:
|
||||
"""Read a logical pool's K and scale bytes from the platform cache layout."""
|
||||
page_size = kv_cache.shape[1]
|
||||
head_dim = kv_cache.shape[2] - 4
|
||||
page_idx, token_offset = divmod(pool_slot, page_size)
|
||||
flat = kv_cache[page_idx].reshape(-1)
|
||||
dims = torch.arange(head_dim, device=kv_cache.device)
|
||||
if current_platform.is_rocm() and page_size > 1:
|
||||
k_offsets = (
|
||||
(token_offset // 16) * 16 * head_dim
|
||||
+ (dims // 16) * 16 * 16
|
||||
+ (token_offset % 16) * 16
|
||||
+ dims % 16
|
||||
)
|
||||
else:
|
||||
k_offsets = token_offset * head_dim + dims
|
||||
scale_offset = page_size * head_dim + 4 * token_offset
|
||||
return torch.cat((flat[k_offsets], flat[scale_offset : scale_offset + 4]))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_amd_prefill_writer_uses_preshuffled_cache_layout():
|
||||
from vllm.models.glm5next.amd.ops.kpool_compress import (
|
||||
kpool_compress_and_write_cache as amd_kpool_compress,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
token_offset = 17
|
||||
kv = torch.zeros(1, PAGE_SIZE, HEAD_DIM + 4, dtype=torch.uint8, device="cuda")
|
||||
key = torch.randn(1, POOL_SIZE, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
score = torch.randn_like(key)
|
||||
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
|
||||
compressed_k, compressed_scale = amd_kpool_compress(
|
||||
kv,
|
||||
key,
|
||||
score,
|
||||
ape,
|
||||
torch.tensor([token_offset], dtype=torch.int64, device="cuda"),
|
||||
pool_size=POOL_SIZE,
|
||||
head_dim=HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
return_compressed=True,
|
||||
)
|
||||
|
||||
dim = torch.arange(HEAD_DIM, device="cuda")
|
||||
offsets = (
|
||||
(token_offset // 16) * 16 * HEAD_DIM
|
||||
+ (dim // 16) * 16 * 16
|
||||
+ (token_offset % 16) * 16
|
||||
+ dim % 16
|
||||
)
|
||||
flat = kv.view(torch.uint8).reshape(-1)
|
||||
stored_k = flat[offsets].view(compressed_k.dtype)
|
||||
scale_offset = PAGE_SIZE * HEAD_DIM + token_offset * 4
|
||||
stored_scale = flat[scale_offset : scale_offset + 4].view(torch.float32)
|
||||
|
||||
assert torch.equal(stored_k, compressed_k[0])
|
||||
assert torch.equal(stored_scale, compressed_scale)
|
||||
|
||||
|
||||
def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos):
|
||||
kv = kv.clone()
|
||||
tail = tail.clone()
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv,
|
||||
tail,
|
||||
tail_slot,
|
||||
key,
|
||||
score,
|
||||
ape,
|
||||
slot_map,
|
||||
pos,
|
||||
POOL_SIZE,
|
||||
HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
return kv, tail
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pool_size", [4, 16])
|
||||
@pytest.mark.parametrize("ring_pools", [1, 2])
|
||||
def test_decode_writer_matches_prefill_writer(pool_size, ring_pools):
|
||||
ring = ring_pools * pool_size
|
||||
n_pools, page, nblk = 8, 64, 4
|
||||
n_tok = n_pools * pool_size
|
||||
dev = "cuda"
|
||||
torch.manual_seed(0)
|
||||
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
ape = torch.randn(pool_size, HEAD_DIM, dtype=torch.float32, device=dev)
|
||||
|
||||
kv_prefill = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
|
||||
kpool_compress_and_write_cache(
|
||||
kv_prefill,
|
||||
k.view(n_pools, pool_size, HEAD_DIM),
|
||||
score.view(n_pools, pool_size, HEAD_DIM),
|
||||
ape,
|
||||
torch.arange(n_pools, dtype=torch.int64, device=dev),
|
||||
pool_size=pool_size,
|
||||
head_dim=HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
# One request owning tail block 0, fed one token per decode step.
|
||||
kv_decode = torch.zeros_like(kv_prefill)
|
||||
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
for t in range(n_tok):
|
||||
completes = t % pool_size == pool_size - 1
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv_decode,
|
||||
tail,
|
||||
# token-granular: every token has a valid tail slot
|
||||
torch.tensor([[t % ring]], dtype=torch.int32, device=dev),
|
||||
k[t].view(1, 1, HEAD_DIM),
|
||||
score[t].view(1, 1, HEAD_DIM),
|
||||
ape,
|
||||
# pool-granular: only the pool's last token carries a cache slot
|
||||
torch.tensor(
|
||||
[[t // pool_size if completes else -1]], dtype=torch.int32, device=dev
|
||||
),
|
||||
torch.tensor([[t]], dtype=torch.int32, device=dev),
|
||||
pool_size,
|
||||
HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
differing = [
|
||||
p
|
||||
for p in range(n_pools)
|
||||
if not torch.equal(
|
||||
_cache_pool_bytes(kv_prefill, p), _cache_pool_bytes(kv_decode, p)
|
||||
)
|
||||
]
|
||||
assert not differing, (
|
||||
f"decode-written pools differ from prefill-written pools: "
|
||||
f"{len(differing)}/{n_pools} (pool_size={pool_size}, first={differing[:5]})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ring_pools", [1, 2])
|
||||
def test_rejected_draft_redo_needs_ring_slots(ring_pools):
|
||||
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
|
||||
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
|
||||
pool, spec, page, nblk = 4, 3, 64, 2
|
||||
ring = ring_pools * pool
|
||||
dev = "cuda"
|
||||
torch.manual_seed(1)
|
||||
n_tok = 3 * pool
|
||||
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
ape = torch.randn(pool, HEAD_DIM, dtype=torch.float32, device=dev)
|
||||
kv_ref = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
|
||||
kpool_compress_and_write_cache(
|
||||
kv_ref,
|
||||
k.view(3, pool, HEAD_DIM),
|
||||
score.view(3, pool, HEAD_DIM),
|
||||
ape,
|
||||
torch.arange(3, dtype=torch.int64, device=dev),
|
||||
pool_size=pool,
|
||||
head_dim=HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
kv = torch.zeros_like(kv_ref)
|
||||
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
|
||||
def step(positions, keys, scores):
|
||||
pos = torch.tensor([positions], dtype=torch.int32, device=dev)
|
||||
slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions]
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv,
|
||||
tail,
|
||||
pos % ring,
|
||||
keys.view(1, -1, HEAD_DIM),
|
||||
scores.view(1, -1, HEAD_DIM),
|
||||
ape,
|
||||
torch.tensor([slots], dtype=torch.int32, device=dev),
|
||||
pos,
|
||||
pool,
|
||||
HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
for t in range(7):
|
||||
step([t], k[t], score[t])
|
||||
# Control: verified token 7 completes pool 1 before drafts 8..10 are stashed.
|
||||
drafts = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
draft_scores = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
step(
|
||||
[7, 8, 9, 10],
|
||||
torch.cat([k[7:8], drafts]),
|
||||
torch.cat([score[7:8], draft_scores]),
|
||||
)
|
||||
step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected
|
||||
for p in (1, 2):
|
||||
assert torch.equal(_cache_pool_bytes(kv, p), _cache_pool_bytes(kv_ref, p)), p
|
||||
|
||||
# Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts
|
||||
# 8 and 9 overwrite the slots of positions 4 and 5, which are read by the
|
||||
# redo of 7.
|
||||
kv.zero_()
|
||||
tail.zero_()
|
||||
for t in range(6):
|
||||
step([t], k[t], score[t])
|
||||
step(
|
||||
[6, 7, 8, 9],
|
||||
torch.cat([k[6:7], drafts]),
|
||||
torch.cat([score[6:7], draft_scores]),
|
||||
)
|
||||
step([7, 8, 9, 10], k[7:11], score[7:11])
|
||||
pool1_ok = torch.equal(_cache_pool_bytes(kv, 1), _cache_pool_bytes(kv_ref, 1))
|
||||
if ring_pools == 1:
|
||||
assert not pool1_ok, "expected a one-pool ring to corrupt pool 1"
|
||||
else:
|
||||
assert pool1_ok
|
||||
|
||||
|
||||
def test_leading_invalid_tail_slot():
|
||||
"""A request whose FIRST token carries an invalid (-1) tail slot while a
|
||||
later token is a real pool completion.
|
||||
|
||||
The tail block must be derived per token, not from token 0: a leading
|
||||
invalid sentinel would otherwise poison the base address for the whole
|
||||
request (out-of-bounds tail reads on the completion).
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
B, next_n, blocks = 2, 4, [3, 5]
|
||||
# req 0: token 0 invalid (pos -1), tokens 1..3 valid, completion at pos 15
|
||||
# req 1: all valid, no completion
|
||||
pos = torch.tensor(
|
||||
[[-1, 13, 14, 15], [4, 5, 6, 7]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
safe_pos = torch.where(pos >= 0, pos, 0)
|
||||
tail_slot = _tail_slot_for(blocks, safe_pos)
|
||||
# leading invalid entry carries the -1 sentinel, as the scatter path emits
|
||||
tail_slot[0, 0] = -1
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
slot_map[0, 3] = 15 # req 0 completes its pool on the last verify token
|
||||
|
||||
key = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
score = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
|
||||
|
||||
kv, tail = _make_caches()
|
||||
_seed_prior(tail, blocks, 13)
|
||||
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
_assert_eq(r_ref, r_kern)
|
||||
|
||||
|
||||
def test_prefill_seed_honors_padded_tail_block_stride():
|
||||
"""The tail shares a padded indexer allocation in production.
|
||||
|
||||
``get_kv_cache_config_from_groups`` aliases each tail tensor onto its
|
||||
indexer tensor with the indexer's block stride (38016 B for GLM-5.3-Flash
|
||||
vs a dense 2048 B tail block), so a seed kernel that addresses blocks
|
||||
densely writes into an unrelated indexer block and leaves the request's
|
||||
tail block untouched. Runs on every platform; the NVIDIA kernel had this
|
||||
bug while the AMD kernel did not.
|
||||
"""
|
||||
kpool = 4
|
||||
num_blocks = 6
|
||||
logical_block_elems = 2 * kpool * HEAD_DIM
|
||||
padded_block_elems = logical_block_elems + 256
|
||||
sentinel = -123.0
|
||||
backing = torch.full(
|
||||
(num_blocks * padded_block_elems,),
|
||||
sentinel,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
tail = torch.as_strided(
|
||||
backing,
|
||||
size=(num_blocks, 2, kpool, HEAD_DIM),
|
||||
stride=(padded_block_elems, kpool * HEAD_DIM, HEAD_DIM, 1),
|
||||
)
|
||||
|
||||
block = 3
|
||||
ring_slot = 2
|
||||
key = torch.arange(HEAD_DIM, dtype=torch.bfloat16, device="cuda").unsqueeze(0)
|
||||
score = (key + 256).to(torch.bfloat16)
|
||||
tail_slot = torch.tensor(
|
||||
[block * kpool + ring_slot], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
kpool_seed_tail_cache(tail, key, score, tail_slot, kpool, HEAD_DIM)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(tail[block, 0, ring_slot], key[0])
|
||||
assert torch.equal(tail[block, 1, ring_slot], score[0])
|
||||
|
||||
compact_offset = (block * 2 * kpool + ring_slot) * HEAD_DIM
|
||||
assert torch.all(backing[compact_offset : compact_offset + HEAD_DIM] == sentinel)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case_id",
|
||||
[
|
||||
"no_completion",
|
||||
"completion_at_end",
|
||||
"completion_mid_batch",
|
||||
"non_uniform_padding",
|
||||
"plain_decode",
|
||||
],
|
||||
)
|
||||
def test_batched_matches_reference(case_id):
|
||||
torch.manual_seed(0)
|
||||
if case_id == "no_completion":
|
||||
B, next_n, blocks = 3, 4, [0, 1, 2]
|
||||
pos = (
|
||||
torch.arange(next_n, device="cuda", dtype=torch.int32)
|
||||
.unsqueeze(0)
|
||||
.expand(B, -1)
|
||||
.contiguous()
|
||||
)
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
n_prior = 0
|
||||
elif case_id == "completion_at_end":
|
||||
B, next_n, blocks = 2, 4, [0, 1]
|
||||
pos = torch.tensor(
|
||||
[[12, 13, 14, 15], [12, 13, 14, 15]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
slot_map[:, 3] = torch.tensor(
|
||||
[15, PAGE_SIZE + 15], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
n_prior = POOL_SIZE - next_n
|
||||
elif case_id == "completion_mid_batch":
|
||||
B, next_n, blocks = 3, 4, [0, 1, 2]
|
||||
pos = torch.tensor([[13, 14, 15, 16]] * B, dtype=torch.int32, device="cuda")
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
slot_map[:, 2] = torch.tensor(
|
||||
[15, PAGE_SIZE + 15, 2 * PAGE_SIZE + 15], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
n_prior = 13
|
||||
elif case_id == "non_uniform_padding":
|
||||
B, next_n, blocks = 2, 4, [0, 1]
|
||||
pos = torch.tensor(
|
||||
[[12, 13, 14, 15], [12, 13, -1, -1]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
slot_map[0, 3] = 15
|
||||
n_prior = POOL_SIZE - 4
|
||||
else: # plain_decode
|
||||
B, next_n, blocks = 4, 1, [0, 1, 2, 3]
|
||||
pos = torch.tensor([[5], [6], [7], [8]], dtype=torch.int32, device="cuda")
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
n_prior = 0
|
||||
|
||||
if case_id == "non_uniform_padding":
|
||||
safe_pos = torch.where(pos >= 0, pos, 0)
|
||||
tail_slot = torch.where(pos >= 0, _tail_slot_for(blocks, safe_pos), 0)
|
||||
else:
|
||||
tail_slot = _tail_slot_for(blocks, pos)
|
||||
|
||||
key = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
score = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
|
||||
|
||||
kv, tail = _make_caches()
|
||||
_seed_prior(tail, blocks, n_prior)
|
||||
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
_assert_eq(r_ref, r_kern)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seed", list(range(20)))
|
||||
def test_batched_matches_reference_fuzz(seed):
|
||||
"""Random B / next_n / start positions; covers 0, 1, and multi completion."""
|
||||
g = torch.Generator(device="cuda").manual_seed(seed)
|
||||
B = int(torch.randint(1, 6, (1,), generator=g, device="cuda").item())
|
||||
next_n = int(torch.randint(1, 8, (1,), generator=g, device="cuda").item())
|
||||
blocks = list(range(B))
|
||||
|
||||
starts = torch.randint(0, 33, (B,), generator=g, device="cuda", dtype=torch.int32)
|
||||
pos = starts.unsqueeze(1) + torch.arange(
|
||||
next_n, device="cuda", dtype=torch.int32
|
||||
).unsqueeze(0)
|
||||
tail_slot = _tail_slot_for(blocks, pos)
|
||||
|
||||
is_completion = pos % POOL_SIZE == POOL_SIZE - 1
|
||||
blk = torch.tensor(blocks, device="cuda", dtype=torch.int32).unsqueeze(1)
|
||||
pool_slot = blk * PAGE_SIZE + (POOL_SIZE - 1)
|
||||
slot_map = torch.where(is_completion, pool_slot, torch.full_like(pos, -1))
|
||||
|
||||
key = torch.randn(
|
||||
B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda", generator=g
|
||||
)
|
||||
score = torch.randn(
|
||||
B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda", generator=g
|
||||
)
|
||||
ape = torch.randn(
|
||||
POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda", generator=g
|
||||
)
|
||||
|
||||
kv, tail = _make_caches()
|
||||
prior_g = torch.Generator(device="cuda").manual_seed(seed + 1000)
|
||||
for b in range(B):
|
||||
n_prior = int(starts[b].item()) % POOL_SIZE
|
||||
if n_prior > 0:
|
||||
pk = torch.randn(
|
||||
n_prior,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
generator=prior_g,
|
||||
)
|
||||
ps = torch.randn(
|
||||
n_prior,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
generator=prior_g,
|
||||
)
|
||||
tail[blocks[b], 0, :n_prior, :] = pk
|
||||
tail[blocks[b], 1, :n_prior, :] = ps
|
||||
|
||||
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
# Random inputs can land the pooled value on a bf16 rounding midpoint,
|
||||
# where the kernel and the CPU reference may round apart by one fp8 step.
|
||||
_assert_eq(r_ref, r_kern, k_byte_tol=1)
|
||||
@@ -1,609 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for cuteDSL low-latency router GEMM (dot-product + split-K)."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _require_sm90_and_cutedsl():
|
||||
if torch.cuda.get_device_capability()[0] < 9:
|
||||
pytest.skip("Requires SM90+ (Hopper/Blackwell)")
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
is_available,
|
||||
)
|
||||
|
||||
if not is_available():
|
||||
pytest.skip("cuteDSL (CUTLASS Python) not installed")
|
||||
|
||||
|
||||
# ===== Helpers =====
|
||||
|
||||
|
||||
def _ref(a, b):
|
||||
return torch.mm(a.float(), b.float().T)
|
||||
|
||||
|
||||
def _assert_close(out, ref, *, min_cos_sim=0.99, context=""):
|
||||
assert out.device.type == "cuda", f"{context}: not on CUDA"
|
||||
assert torch.isfinite(out).all(), f"{context}: NaN/Inf"
|
||||
cos = F.cosine_similarity(
|
||||
out.reshape(-1).float(), ref.reshape(-1).float(), dim=0
|
||||
).item()
|
||||
assert cos > min_cos_sim, (
|
||||
f"{context}: cos_sim {cos:.6f} < {min_cos_sim} "
|
||||
f"(abs_err={(out.float() - ref.float()).abs().max().item():.2e})"
|
||||
)
|
||||
|
||||
|
||||
def _can_precompile(a, b):
|
||||
return (
|
||||
a.dim() == 2
|
||||
and b.dim() == 2
|
||||
and a.dtype == torch.bfloat16
|
||||
and b.dtype == torch.bfloat16
|
||||
and a.device.type == "cuda"
|
||||
and b.device.type == "cuda"
|
||||
and a.device == b.device
|
||||
and a.shape[1] == b.shape[1]
|
||||
and a.is_contiguous()
|
||||
and b.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _gemm(a, b):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
ll_bf16_gemm,
|
||||
ll_bf16_gemm_kernel,
|
||||
)
|
||||
|
||||
if _can_precompile(a, b):
|
||||
compile_key = ll_bf16_gemm_kernel.dispatch(
|
||||
M=a.shape[0], K=a.shape[1], N=b.shape[0]
|
||||
)
|
||||
ll_bf16_gemm_kernel.compile(compile_key)
|
||||
return ll_bf16_gemm(a, b)
|
||||
|
||||
|
||||
def test_c1_pdl_kernel_is_selected_automatically(monkeypatch):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
|
||||
|
||||
calls = []
|
||||
|
||||
def default_kernel(hidden_states, router_weight, output_dtype):
|
||||
calls.append(("default", hidden_states.shape[0]))
|
||||
return hidden_states
|
||||
|
||||
def c1_pdl_kernel(hidden_states, router_weight, output_dtype):
|
||||
calls.append(("c1_pdl", hidden_states.shape[0]))
|
||||
return hidden_states
|
||||
|
||||
monkeypatch.setattr(ll_bf16, "ll_bf16_gemm_kernel", default_kernel)
|
||||
monkeypatch.setattr(ll_bf16, "ll_bf16_gemm_c1_pdl_kernel", c1_pdl_kernel)
|
||||
weight = torch.empty(4, 8, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
ll_bf16.ll_bf16_gemm(torch.empty(1, 8, device="cuda", dtype=torch.bfloat16), weight)
|
||||
ll_bf16.ll_bf16_gemm(torch.empty(2, 8, device="cuda", dtype=torch.bfloat16), weight)
|
||||
|
||||
assert calls == [("c1_pdl", 1), ("default", 2)]
|
||||
|
||||
|
||||
# ===== Shapes =====
|
||||
|
||||
SHAPES = [
|
||||
(256, 7168, "DSV3"),
|
||||
(256, 14400, "DSV4-Flash"),
|
||||
(128, 5120, "DeepSeek-V2"),
|
||||
(8, 4096, "Mixtral-8x7B"),
|
||||
(64, 2880, "non-tile-aligned-K"),
|
||||
(256, 2048, "split-K-boundary"),
|
||||
]
|
||||
|
||||
SHAPES_SPLITK = [(n, k, d) for n, k, d in SHAPES if k >= 2048]
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Dot-product kernel (M<=4 or K<2048)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 2, 3, 4])
|
||||
@pytest.mark.parametrize("N,K,desc", SHAPES, ids=[s[2] for s in SHAPES])
|
||||
def test_dotprod(M, N, K, desc):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.bfloat16, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.dtype == torch.float32
|
||||
assert out.shape == (M, N)
|
||||
_assert_close(out, _ref(a, b), context=f"dotprod {M}x{N}x{K}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 4, 8, 16])
|
||||
def test_dotprod_small_K(M):
|
||||
"""K<2048 forces dot-product regardless of M."""
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, 1024, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 1024, dtype=torch.bfloat16, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"small_K M={M}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("K", [16, 32, 64, 128, 256, 512, 1024, 1536])
|
||||
def test_dotprod_K_sweep(K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(32, K, dtype=torch.bfloat16, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"K={K}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Split-K kernel (M>4 and K>=2048)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [5, 6, 8, 12, 16])
|
||||
@pytest.mark.parametrize("N,K,desc", SHAPES_SPLITK, ids=[s[2] for s in SHAPES_SPLITK])
|
||||
def test_splitk(M, N, K, desc):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.bfloat16, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.dtype == torch.float32
|
||||
assert out.shape == (M, N)
|
||||
_assert_close(out, _ref(a, b), context=f"splitk {M}x{N}x{K}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("K", [2048, 2304, 2880, 3072, 4096, 5120, 7168, 14400])
|
||||
def test_splitk_K_sweep(K):
|
||||
"""Includes non-tile-aligned K (2880) and uneven split (2304)."""
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(8, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"splitk K={K}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Dispatch boundary (M=4/5, K=2032/2048)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [4, 5])
|
||||
@pytest.mark.parametrize("K", [2032, 2048])
|
||||
def test_dispatch_boundary(M, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
|
||||
path = "splitk" if M > 4 and K >= 2048 else "dotprod"
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"M={M} K={K} ({path})")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Arbitrary N
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("N", [1, 3, 7, 16, 17, 64, 128, 256, 384])
|
||||
def test_arbitrary_N_dotprod(N):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.shape == (4, N)
|
||||
_assert_close(out, _ref(a, b), context=f"dotprod N={N}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("N", [1, 8, 16, 17, 64, 128, 256])
|
||||
def test_arbitrary_N_splitk(N):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(8, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.shape == (8, N)
|
||||
_assert_close(out, _ref(a, b), context=f"splitk N={N}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Single token (M=1, decode path)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"N,K",
|
||||
[(256, 7168), (256, 14400), (8, 4096), (384, 7168), (264, 6144)],
|
||||
ids=["DSV3", "DSV4-Flash", "Mixtral", "DSV4-Pro", "Inkling"],
|
||||
)
|
||||
def test_single_token(N, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(1, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.bfloat16, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.shape == (1, N)
|
||||
_assert_close(out, _ref(a, b), context=f"M=1 {N}x{K}")
|
||||
|
||||
|
||||
def test_inkling_max_tokens():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(64, 6144, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(264, 6144, dtype=torch.bfloat16, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.shape == (64, 264)
|
||||
_assert_close(out, _ref(a, b), context="Inkling M=64")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Numerical robustness
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M,K", [(4, 2048), (8, 4096)], ids=["dotprod", "splitk"])
|
||||
def test_large_values(M, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda") * 100
|
||||
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda") * 100
|
||||
out = _gemm(a, b)
|
||||
assert torch.isfinite(out).all()
|
||||
_assert_close(out, _ref(a, b), context=f"large M={M}")
|
||||
|
||||
|
||||
def test_near_zero():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda") * 1e-4
|
||||
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda") * 1e-4
|
||||
out = _gemm(a, b)
|
||||
assert torch.isfinite(out).all()
|
||||
assert out.abs().max() < 1.0
|
||||
|
||||
|
||||
def test_zeros():
|
||||
a = torch.zeros(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
assert (_gemm(a, b) == 0).all()
|
||||
|
||||
|
||||
def test_ones():
|
||||
a = torch.ones(1, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(32, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context="ones")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Output dtype
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M,K", [(4, 2048), (8, 4096)], ids=["dotprod", "splitk"])
|
||||
def test_output_fp32(M, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
|
||||
assert _gemm(a, b).dtype == torch.float32
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Determinism
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"M,K", [(1, 4096), (4, 4096), (5, 4096), (8, 4096), (16, 4096)]
|
||||
)
|
||||
def test_deterministic(M, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(128, K, dtype=torch.bfloat16, device="cuda")
|
||||
torch.testing.assert_close(_gemm(a, b), _gemm(a, b), atol=0, rtol=0)
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Cross-kernel consistency
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("K", [2048, 4096, 7168])
|
||||
def test_dotprod_vs_splitk(K):
|
||||
"""First 4 rows from split-K (M=5) match dot-product (M=4)."""
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(5, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
|
||||
_assert_close(
|
||||
_gemm(a, b)[:4],
|
||||
_gemm(a[:4], b),
|
||||
min_cos_sim=0.999,
|
||||
context=f"cross-kernel K={K}",
|
||||
)
|
||||
|
||||
|
||||
# =================================================================
|
||||
# CUDA graph
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M,K", [(4, 2048), (8, 4096)], ids=["dotprod", "splitk"])
|
||||
def test_cudagraph(M, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
for _ in range(5):
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
_assert_close(out, _ref(a, b), context=f"cudagraph M={M}")
|
||||
|
||||
|
||||
def test_cudagraph_20x_replay():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(256, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
results = []
|
||||
for _ in range(20):
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
results.append(out.clone())
|
||||
for i in range(1, len(results)):
|
||||
torch.testing.assert_close(
|
||||
results[0], results[i], atol=0, rtol=0, msg=f"Replay {i} differs"
|
||||
)
|
||||
|
||||
|
||||
def test_cudagraph_input_update():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
a.copy_(torch.randn_like(a))
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
_assert_close(out, _ref(a, b), context="cudagraph input update")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# GateLinear dispatch integration
|
||||
# =================================================================
|
||||
|
||||
|
||||
def _make_gate_linear(monkeypatch, *, params_dtype, out_dtype=torch.float32):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.is_available",
|
||||
lambda: True,
|
||||
)
|
||||
return GateLinear(
|
||||
input_size=2048,
|
||||
output_size=64,
|
||||
bias=False,
|
||||
out_dtype=out_dtype,
|
||||
params_dtype=params_dtype,
|
||||
).cuda()
|
||||
|
||||
|
||||
def test_gate_linear_uses_ll_bf16_for_bf16_fast_path(monkeypatch):
|
||||
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16)
|
||||
x = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
calls = []
|
||||
|
||||
def fake_ll_bf16_gemm(hidden_states, router_weight):
|
||||
calls.append((hidden_states, router_weight))
|
||||
return torch.full(
|
||||
(hidden_states.shape[0], router_weight.shape[0]),
|
||||
1.0,
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.ll_bf16_gemm",
|
||||
fake_ll_bf16_gemm,
|
||||
)
|
||||
out, bias = gate(x)
|
||||
assert bias is None
|
||||
assert out.shape == (4, 64)
|
||||
assert out.dtype == torch.float32
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] is x
|
||||
assert calls[0][1] is gate.weight
|
||||
|
||||
|
||||
def test_gate_linear_fp32_weight_falls_back(monkeypatch):
|
||||
gate = _make_gate_linear(monkeypatch, params_dtype=torch.float32)
|
||||
assert not gate.allow_ll_bf16_gemm
|
||||
x = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def fail_ll_bf16_gemm(hidden_states, router_weight):
|
||||
raise AssertionError("ll_bf16_gemm should not run for fp32 weights")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.ll_bf16_gemm",
|
||||
fail_ll_bf16_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (4, 64)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_linear_non_bf16_activation_falls_back(monkeypatch):
|
||||
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16)
|
||||
x = torch.randn(4, 2048, dtype=torch.float16, device="cuda")
|
||||
|
||||
def fail_ll_bf16_gemm(hidden_states, router_weight):
|
||||
raise AssertionError("ll_bf16_gemm should not run for non-bf16 activations")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.ll_bf16_gemm",
|
||||
fail_ll_bf16_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (4, 64)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_linear_set_out_dtype_enables_ll_bf16(monkeypatch):
|
||||
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16, out_dtype=None)
|
||||
assert not gate.allow_ll_bf16_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_ll_bf16_gemm
|
||||
|
||||
|
||||
def test_gate_linear_non_fp32_out_dtype_disables_ll_bf16(monkeypatch):
|
||||
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16, out_dtype=None)
|
||||
gate.set_out_dtype(torch.bfloat16)
|
||||
assert not gate.allow_ll_bf16_gemm
|
||||
|
||||
|
||||
def test_gate_linear_m_gt_16_falls_back(monkeypatch):
|
||||
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16)
|
||||
x = torch.randn(17, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def fail_ll_bf16_gemm(hidden_states, router_weight):
|
||||
raise AssertionError("ll_bf16_gemm should not run for M > 16")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.ll_bf16_gemm",
|
||||
fail_ll_bf16_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (17, 64)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Negative tests — invalid inputs
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,dtype",
|
||||
[
|
||||
pytest.param(4, 2048, 64, torch.float32, id="fp32_input"),
|
||||
pytest.param(4, 2048, 64, torch.float16, id="fp16_input"),
|
||||
pytest.param(8, 4096, 64, torch.float32, id="fp32_splitk_path"),
|
||||
],
|
||||
)
|
||||
def test_invalid_dtype(M, K, N, dtype):
|
||||
a = torch.randn(M, K, device="cuda", dtype=dtype)
|
||||
b = torch.randn(N, K, device="cuda", dtype=dtype)
|
||||
with pytest.raises(ValueError, match="dtype=bfloat16"):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_device_cpu():
|
||||
a = torch.randn(4, 2048, dtype=torch.bfloat16)
|
||||
b = torch.randn(64, 2048, dtype=torch.bfloat16)
|
||||
with pytest.raises(ValueError, match="device_type=cuda"):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_1d_input():
|
||||
a = torch.randn(2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
with pytest.raises(ValueError, match="2D tensors"):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_mismatched_K():
|
||||
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 1024, dtype=torch.bfloat16, device="cuda")
|
||||
with pytest.raises(ValueError, match="matching K dimensions"):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_K_divisibility():
|
||||
a = torch.randn(4, 2049, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2049, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
with pytest.raises(ValueError, match="K to be divisible by 8"):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_non_contiguous_input():
|
||||
a = torch.randn(2048, 4, dtype=torch.bfloat16, device="cuda").T
|
||||
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
assert not a.is_contiguous()
|
||||
with pytest.raises(ValueError, match="contiguous row-major"):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_output_dtype():
|
||||
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
with pytest.raises(ValueError, match="output_dtype=torch.float32"):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
ll_bf16_gemm(a, b, output_dtype=torch.float16)
|
||||
|
||||
|
||||
def test_cache_miss_compiles_dotprod():
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import LLBf16Gemm
|
||||
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(3, 64, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(17, 64, dtype=torch.bfloat16, device="cuda")
|
||||
kernel = LLBf16Gemm()
|
||||
out = kernel(a, b)
|
||||
assert out.shape == (3, 17)
|
||||
_assert_close(out, _ref(a, b), context="cache miss dotprod")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 2, 8, 16])
|
||||
@pytest.mark.parametrize("N", [1792, 3584])
|
||||
@torch.inference_mode()
|
||||
def test_bf16_output_and_fp32_cache_coexist(M, N):
|
||||
"""BF16 epilogues retain FP32 accumulation and can replay with new inputs."""
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(M, 5120, device="cuda", dtype=torch.bfloat16)
|
||||
w = torch.randn(N, 5120, device="cuda", dtype=torch.bfloat16) * 0.01
|
||||
for dtype in (torch.float32, torch.bfloat16):
|
||||
actual = ll_bf16_gemm(x, w, output_dtype=dtype)
|
||||
assert actual.dtype == dtype
|
||||
reference = (x.double() @ w.double().T).to(dtype)
|
||||
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = ll_bf16_gemm(x, w, output_dtype=torch.bfloat16)
|
||||
x.normal_()
|
||||
graph.replay()
|
||||
reference = (x.double() @ w.double().T).bfloat16()
|
||||
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -1,544 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for cuteDSL low-latency GEMM with fp32 weights."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.accelerator.is_available(), reason="CUDA required"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _require_sm90_and_cutedsl():
|
||||
if not current_platform.has_device_capability(90):
|
||||
pytest.skip("Requires SM90+ (Hopper/Blackwell)")
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
is_available,
|
||||
)
|
||||
|
||||
if not is_available():
|
||||
pytest.skip("cuteDSL (CUTLASS Python) not installed")
|
||||
|
||||
|
||||
def _ref(a, b):
|
||||
return torch.mm(a.float(), b.float().T)
|
||||
|
||||
|
||||
def _assert_close(out, ref, *, min_cos_sim=0.99, context=""):
|
||||
assert out.device.type == "cuda", f"{context}: not on CUDA"
|
||||
assert torch.isfinite(out).all(), f"{context}: NaN/Inf"
|
||||
cos = F.cosine_similarity(
|
||||
out.reshape(-1).float(), ref.reshape(-1).float(), dim=0
|
||||
).item()
|
||||
assert cos > min_cos_sim, (
|
||||
f"{context}: cos_sim {cos:.6f} < {min_cos_sim} "
|
||||
f"(abs_err={(out.float() - ref.float()).abs().max().item():.2e})"
|
||||
)
|
||||
|
||||
|
||||
def _gemm(a, b, output_dtype=torch.float32):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
LL_FP32W_GEMM_KERNEL,
|
||||
)
|
||||
|
||||
return LL_FP32W_GEMM_KERNEL(a, b, output_dtype)
|
||||
|
||||
|
||||
SHAPES = [
|
||||
(256, 3072, "MiniMax-M2"),
|
||||
(256, 7168, "DSV3-sized"),
|
||||
(128, 5120, "DeepSeek-V2"),
|
||||
(64, 2880, "non-aligned-K"),
|
||||
(8, 4096, "Mixtral"),
|
||||
]
|
||||
|
||||
A_DTYPES = [
|
||||
pytest.param(torch.bfloat16, id="bf16"),
|
||||
pytest.param(torch.float16, id="fp16"),
|
||||
pytest.param(torch.float32, id="fp32"),
|
||||
]
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Correctness: all dtype combos x shapes x M values
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a_dtype", A_DTYPES)
|
||||
@pytest.mark.parametrize("M", [1, 2, 4, 8, 16])
|
||||
@pytest.mark.parametrize("N,K,desc", SHAPES, ids=[s[2] for s in SHAPES])
|
||||
def test_correctness(M, N, K, desc, a_dtype):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, device="cuda").to(a_dtype)
|
||||
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.dtype == torch.float32
|
||||
assert out.shape == (M, N)
|
||||
_assert_close(out, _ref(a, b), context=f"{a_dtype} M={M} {desc}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Small K (k_main=0, only tail/scalar paths)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("K", [16, 32, 64, 128, 256, 512, 1024, 1536])
|
||||
def test_small_K(K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"small_K={K}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# K sweep (covers main loop + various tail combinations)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("K", [2048, 2304, 2880, 3072, 4096, 5120, 7168])
|
||||
def test_K_sweep(K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(8, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"K={K}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# MiniMax-M2 specific shapes
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 4, 8, 16, 32])
|
||||
def test_minimax_m2(M):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, 3072, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(256, 3072, dtype=torch.float32, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"MiniMax-M2 M={M}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Single token (M=1, decode path)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"N,K",
|
||||
[(256, 3072), (256, 7168), (8, 4096)],
|
||||
ids=["MiniMax-M2", "DSV3-sized", "Mixtral"],
|
||||
)
|
||||
def test_single_token(N, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(1, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.shape == (1, N)
|
||||
_assert_close(out, _ref(a, b), context=f"M=1 {N}x{K}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Arbitrary N
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("N", [1, 3, 16, 64, 256])
|
||||
def test_arbitrary_N(N):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 3072, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, 3072, dtype=torch.float32, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.shape == (4, N)
|
||||
_assert_close(out, _ref(a, b), context=f"N={N}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Numerical robustness
|
||||
# =================================================================
|
||||
|
||||
|
||||
def test_large_values():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda") * 100
|
||||
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda") * 100
|
||||
out = _gemm(a, b)
|
||||
assert torch.isfinite(out).all()
|
||||
_assert_close(out, _ref(a, b), context="large_values")
|
||||
|
||||
|
||||
def test_near_zero():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda") * 1e-4
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda") * 1e-4
|
||||
out = _gemm(a, b)
|
||||
assert torch.isfinite(out).all()
|
||||
assert out.abs().max() < 1.0
|
||||
|
||||
|
||||
def test_zeros():
|
||||
a = torch.zeros(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2048, dtype=torch.float32, device="cuda")
|
||||
assert (_gemm(a, b) == 0).all()
|
||||
|
||||
|
||||
def test_ones():
|
||||
a = torch.ones(1, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(32, 4096, dtype=torch.float32, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context="ones")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Output dtype
|
||||
# =================================================================
|
||||
|
||||
|
||||
def test_output_fp32():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
|
||||
assert _gemm(a, b).dtype == torch.float32
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Determinism
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 4, 8, 16])
|
||||
@pytest.mark.parametrize("K", [1024, 3072, 7168])
|
||||
def test_deterministic(M, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
|
||||
torch.testing.assert_close(_gemm(a, b), _gemm(a, b), atol=0, rtol=0)
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Cross-dtype consistency
|
||||
# =================================================================
|
||||
|
||||
|
||||
def test_bf16_vs_fp32_activations():
|
||||
"""bf16 and fp32 activations on same data should roughly agree."""
|
||||
torch.manual_seed(42)
|
||||
K, N = 4096, 128
|
||||
a_bf16 = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
|
||||
out_bf16 = _gemm(a_bf16, b)
|
||||
a_fp32 = a_bf16.float()
|
||||
out_fp32 = _gemm(a_fp32, b)
|
||||
_assert_close(
|
||||
out_bf16, out_fp32, min_cos_sim=0.999, context="bf16_vs_fp32_activations"
|
||||
)
|
||||
|
||||
|
||||
# =================================================================
|
||||
# CUDA graph
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N", [(4, 3072, 256), (16, 5120, 128)], ids=["MiniMax", "DeepSeek"]
|
||||
)
|
||||
def test_cudagraph(M, K, N):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
for _ in range(5):
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
_assert_close(out, _ref(a, b), context=f"cudagraph M={M} K={K}")
|
||||
|
||||
|
||||
def test_cudagraph_20x_replay():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
results = []
|
||||
for _ in range(20):
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
results.append(out.clone())
|
||||
for i in range(1, len(results)):
|
||||
torch.testing.assert_close(
|
||||
results[0], results[i], atol=0, rtol=0, msg=f"Replay {i} differs"
|
||||
)
|
||||
|
||||
|
||||
def test_cudagraph_input_update():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
a.copy_(torch.randn_like(a))
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
_assert_close(out, _ref(a, b), context="cudagraph input update")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Negative tests
|
||||
# =================================================================
|
||||
|
||||
|
||||
def test_invalid_weight_dtype():
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_device_cpu():
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16)
|
||||
b = torch.randn(64, 4096, dtype=torch.float32)
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_1d_input():
|
||||
a = torch.randn(4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_activation_dtype():
|
||||
a = torch.ones(4, 4096, dtype=torch.int32, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_output_dtype():
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b, torch.bfloat16)
|
||||
|
||||
|
||||
def test_invalid_k_mismatch():
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2048, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_non_contiguous():
|
||||
base = torch.randn(4, 8192, dtype=torch.bfloat16, device="cuda")
|
||||
a = base[:, ::2]
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
# =================================================================
|
||||
# GateLinear dispatch integration
|
||||
# =================================================================
|
||||
|
||||
|
||||
def _make_gate_linear_fp32w(
|
||||
monkeypatch,
|
||||
*,
|
||||
params_dtype,
|
||||
out_dtype=torch.float32,
|
||||
input_size=2048,
|
||||
output_size=64,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.is_available",
|
||||
lambda: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.is_available",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
return GateLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=False,
|
||||
out_dtype=out_dtype,
|
||||
params_dtype=params_dtype,
|
||||
).cuda()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("x_dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
def test_gate_linear_uses_ll_fp32w_for_fp32_weight_fast_path(monkeypatch, x_dtype):
|
||||
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.float32)
|
||||
x = torch.randn(4, 2048, dtype=x_dtype, device="cuda")
|
||||
calls = []
|
||||
|
||||
def fake_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
calls.append((hidden_states, router_weight))
|
||||
return torch.full(
|
||||
(hidden_states.shape[0], router_weight.shape[0]),
|
||||
1.0,
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fake_ll_fp32w_gemm,
|
||||
)
|
||||
out, bias = gate(x)
|
||||
assert bias is None
|
||||
assert out.shape == (4, 64)
|
||||
assert out.dtype == torch.float32
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] is x
|
||||
assert calls[0][1] is gate.weight
|
||||
|
||||
|
||||
PREFERRED_FP32W_GATE_CASES = [
|
||||
*((6144, 128, M) for M in range(1, 33)),
|
||||
*((6144, 256, M) for M in (1, 8, 12, 14, 16, 20, 22, 25, 27, 32)),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("input_size", "output_size", "M"), PREFERRED_FP32W_GATE_CASES)
|
||||
def test_gate_linear_uses_ll_fp32w_when_cpp_tier_disabled(
|
||||
monkeypatch, input_size, output_size, M
|
||||
):
|
||||
gate = _make_gate_linear_fp32w(
|
||||
monkeypatch,
|
||||
params_dtype=torch.float32,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
)
|
||||
gate.allow_fp32_router_gemm = False
|
||||
x = torch.randn(M, input_size, dtype=torch.bfloat16, device="cuda")
|
||||
calls = []
|
||||
|
||||
def fake_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
calls.append((hidden_states, router_weight))
|
||||
return torch.full(
|
||||
(hidden_states.shape[0], router_weight.shape[0]),
|
||||
2.0,
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fake_ll_fp32w_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] is x
|
||||
assert calls[0][1] is gate.weight
|
||||
assert torch.all(out == 2.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [8, 16, 32])
|
||||
def test_gate_linear_uses_cpp_fp32_router_for_supported_cpp_shape(monkeypatch, M):
|
||||
gate = _make_gate_linear_fp32w(
|
||||
monkeypatch,
|
||||
params_dtype=torch.float32,
|
||||
input_size=3072,
|
||||
output_size=256,
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
x = torch.randn(M, 3072, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def fail_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
raise AssertionError(
|
||||
"ll_fp32w_gemm should not bypass fp32_router_gemm_dispatch"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fail_ll_fp32w_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (M, 256)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_linear_bf16_weight_skips_ll_fp32w(monkeypatch):
|
||||
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.bfloat16)
|
||||
assert not gate.allow_ll_fp32w_gemm
|
||||
x = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def fail_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
raise AssertionError("ll_fp32w_gemm should not run for bf16 weights")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fail_ll_fp32w_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (4, 64)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_linear_set_out_dtype_enables_ll_fp32w(monkeypatch):
|
||||
gate = _make_gate_linear_fp32w(
|
||||
monkeypatch, params_dtype=torch.float32, out_dtype=None
|
||||
)
|
||||
assert not gate.allow_ll_fp32w_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_ll_fp32w_gemm
|
||||
|
||||
|
||||
def test_gate_linear_non_fp32_out_dtype_disables_ll_fp32w(monkeypatch):
|
||||
gate = _make_gate_linear_fp32w(
|
||||
monkeypatch, params_dtype=torch.float32, out_dtype=None
|
||||
)
|
||||
gate.set_out_dtype(torch.bfloat16)
|
||||
assert not gate.allow_ll_fp32w_gemm
|
||||
|
||||
|
||||
def test_gate_linear_m_gt_32_skips_ll_fp32w(monkeypatch):
|
||||
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.float32)
|
||||
x = torch.randn(33, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def fail_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
raise AssertionError("ll_fp32w_gemm should not run for M > 32")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fail_ll_fp32w_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (33, 64)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -1,688 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reference-vs-optimized unit tests for the MiniMax-M3 AMD/ROCm fused kernels.
|
||||
|
||||
Each optimized kernel added for the ROCm port has a slow PyTorch reference; the
|
||||
tests assert the two agree within tolerance:
|
||||
|
||||
* Gemma RMSNorm (plain + fused-add-residual) -> fp32 PyTorch normalize
|
||||
* SwiGLU-OAI (split layout) -> fp32 PyTorch elementwise
|
||||
* Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch
|
||||
* Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul
|
||||
* Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math
|
||||
* Strided K/V sparse-PA cache insert -> contiguous-copy insert
|
||||
|
||||
The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the
|
||||
scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T``
|
||||
makes the shape ``[K//32, N]`` and Triton raises before producing output, so any
|
||||
regression there fails these tests loudly.
|
||||
|
||||
Hardware scope: the whole module is ROCm-only (these are the AMD path; NVIDIA
|
||||
uses the FlashInfer kernels). The norm/activation/quant kernels run on any ROCm
|
||||
arch; the native MXFP8 ``dot_scaled`` linear/MoE tests are additionally gated to
|
||||
CDNA4 gfx95x (``@requires_gfx950``) since gfx942 uses the BF16 emulation path.
|
||||
|
||||
Run: pytest tests/kernels/test_minimax_m3_amd_ops.py -v
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("MiniMax-M3 AMD fused ops require ROCm.", allow_module_level=True)
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("Requires a GPU.", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa: E402
|
||||
_mxfp8_e4m3_quantize_torch,
|
||||
_mxfp8_e4m3_quantize_triton,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.model import _kv_insert_operand # noqa: E402
|
||||
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
|
||||
gemma_fused_add_rmsnorm,
|
||||
gemma_rmsnorm,
|
||||
swiglu_oai_quantize_mxfp8,
|
||||
swiglu_oai_split,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
|
||||
|
||||
DEVICE = "cuda"
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
def _gcn_arch() -> str:
|
||||
try:
|
||||
return torch.cuda.get_device_properties(0).gcnArchName
|
||||
except Exception: # pragma: no cover - no device / non-AMD
|
||||
return ""
|
||||
|
||||
|
||||
# The pure-Triton norm/activation/quant kernels run on any ROCm arch (CDNA3
|
||||
# gfx942 and CDNA4 gfx950). The native MXFP8 ``dot_scaled`` GEMMs (linear + MoE)
|
||||
# use CDNA4 hardware microscaling and are gated to gfx95x in the source
|
||||
# (``RocmDotScaledMxfp8LinearKernel.is_supported``; the MoE oracle routes gfx942
|
||||
# to the BF16 emulation path instead) — so those tests are gfx950-only.
|
||||
requires_gfx950 = pytest.mark.skipif(
|
||||
"gfx95" not in _gcn_arch(),
|
||||
reason="native MXFP8 dot_scaled is a CDNA4 (gfx95x) feature; "
|
||||
"gfx942 uses the BF16 emulation path instead.",
|
||||
)
|
||||
|
||||
|
||||
def _relerr(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
a = a.float()
|
||||
b = b.float()
|
||||
return ((a - b).norm() / (b.norm() + 1e-8)).item()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Gemma RMSNorm
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _ref_gemma_rmsnorm(x, w, eps, residual=None):
|
||||
orig_dtype = x.dtype
|
||||
xf = x.float()
|
||||
res_out = None
|
||||
if residual is not None:
|
||||
xf = xf + residual.float()
|
||||
res_out = xf.to(orig_dtype)
|
||||
xf = xf * torch.rsqrt(xf.pow(2).mean(dim=-1, keepdim=True) + eps)
|
||||
xf = xf * (1.0 + w.float())
|
||||
out = xf.to(orig_dtype)
|
||||
return out if residual is None else (out, res_out)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 4096), (37, 6144), (128, 2048)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("seed", [0, 1234])
|
||||
@torch.inference_mode()
|
||||
def test_gemma_rmsnorm(shape, dtype, seed):
|
||||
torch.manual_seed(seed)
|
||||
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
|
||||
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
|
||||
got = gemma_rmsnorm(x, w, EPS)
|
||||
ref = _ref_gemma_rmsnorm(x, w, EPS)
|
||||
assert got.shape == x.shape
|
||||
assert _relerr(got, ref) < 5e-3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 6144), (64, 4096)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_gemma_fused_add_rmsnorm(shape, dtype):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
|
||||
res = torch.randn(*shape, device=DEVICE, dtype=dtype)
|
||||
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
|
||||
got_out, got_res = gemma_fused_add_rmsnorm(x, res, w, EPS)
|
||||
ref_out, ref_res = _ref_gemma_rmsnorm(x, w, EPS, residual=res)
|
||||
assert _relerr(got_out, ref_out) < 5e-3
|
||||
# residual_out is the pre-norm sum (x + res): bit-for-bit identical cast.
|
||||
assert torch.equal(got_res, ref_res)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gemma_rmsnorm_per_head_strided():
|
||||
"""q_norm/k_norm normalize a non-contiguous ``qkv.split`` slice over head_dim."""
|
||||
torch.manual_seed(0)
|
||||
T, H, D, kv = 7, 48, 128, 8
|
||||
total = (H + 2 * kv) * D
|
||||
qkv = torch.randn(T, total, device=DEVICE, dtype=torch.bfloat16)
|
||||
q = qkv[..., : H * D] # non-contiguous view (row stride == total)
|
||||
q_by_head = q.view(T, H, D)
|
||||
assert not q_by_head.is_contiguous()
|
||||
w = torch.randn(D, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
got = gemma_rmsnorm(q_by_head, w, EPS)
|
||||
ref = _ref_gemma_rmsnorm(q_by_head, w, EPS)
|
||||
assert got.shape == q_by_head.shape
|
||||
assert _relerr(got, ref) < 5e-3
|
||||
|
||||
|
||||
def test_num_warps_monotonic():
|
||||
assert _num_warps(128) <= _num_warps(2048) <= _num_warps(8192)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SwiGLU-OAI (split layout)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _ref_swiglu(gate_up, alpha, beta, limit):
|
||||
d = gate_up.shape[-1] // 2
|
||||
gate = gate_up[..., :d].float()
|
||||
up = gate_up[..., d:].float()
|
||||
if limit is not None:
|
||||
gate = gate.clamp(max=limit)
|
||||
up = up.clamp(min=-limit, max=limit)
|
||||
return (gate * torch.sigmoid(alpha * gate) * (up + beta)).to(gate_up.dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,inter", [(1, 768), (64, 1536), (128, 1024)])
|
||||
@pytest.mark.parametrize("limit", [7.0, None])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_swiglu_oai_split(m, inter, limit, dtype):
|
||||
torch.manual_seed(0)
|
||||
gate_up = torch.randn(m, 2 * inter, device=DEVICE, dtype=dtype)
|
||||
got = swiglu_oai_split(gate_up, alpha=1.702, beta=1.0, limit=limit)
|
||||
ref = _ref_swiglu(gate_up, 1.702, 1.0, limit)
|
||||
assert got.shape == (m, inter)
|
||||
assert _relerr(got, ref) < 5e-3
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_swiglu_oai_quantize_mxfp8_uses_e4m3_range_for_scale():
|
||||
# Keep a small value in two MX blocks next to their maxima. The scale must
|
||||
# use E4M3's finite range (448), otherwise that value underflows to zero.
|
||||
# Keep the third block empty to cover the reference's finite tiny clamp.
|
||||
m, inter = 3, 96
|
||||
gate = torch.full((m, inter), 2.0, device=DEVICE, dtype=torch.float16)
|
||||
up = torch.full((m, inter), 2**-10, device=DEVICE, dtype=torch.float16)
|
||||
up[:, :64:32] = 1.0
|
||||
up[:, 64:] = 0.0
|
||||
gate_up = torch.cat((gate, up), dim=-1)
|
||||
|
||||
got_q, got_s = swiglu_oai_quantize_mxfp8(gate_up, alpha=0.0, beta=0.0, limit=None)
|
||||
ref_q, ref_s = _mxfp8_e4m3_quantize_torch(up, is_sf_swizzled_layout=False)
|
||||
|
||||
assert torch.equal(got_s, ref_s)
|
||||
assert torch.equal(got_q, ref_q)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fused MXFP8 activation quant (Triton vs torch reference)
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("shape", [(64, 4096), (1, 6144), (333, 2048)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_quant_triton_matches_torch(shape, dtype):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
|
||||
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
|
||||
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
|
||||
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
|
||||
# Both paths use the E4M3-aware scale calculation; allow a 1-step
|
||||
# difference at exact powers of two due to floating-point rounding.
|
||||
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
|
||||
# Dequantized values agree to fp8 granularity.
|
||||
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
|
||||
deq_k = dequant_mxfp8_to_bf16(xq_k, s_k)
|
||||
assert _relerr(deq_k, deq_t) < 1e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Native MXFP8 linear (dot_scaled) vs dequant-to-bf16 matmul
|
||||
# --------------------------------------------------------------------------- #
|
||||
@requires_gfx950
|
||||
@pytest.mark.parametrize("m,n,k", [(64, 256, 128), (37, 512, 256), (1, 6144, 4096)])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_native_linear(m, n, k):
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
_mxfp8_dot_scaled_linear,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
w_bf16 = torch.randn(n, k, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
|
||||
x = torch.randn(m, k, device=DEVICE, dtype=torch.bfloat16) * 0.5
|
||||
|
||||
got = _mxfp8_dot_scaled_linear(x, w_fp8, w_scale)
|
||||
# Reference: consume the SAME quantized weights (isolates activation-quant
|
||||
# noise) -> dequant to bf16, plain matmul.
|
||||
w_deq = dequant_mxfp8_to_bf16(w_fp8, w_scale)
|
||||
ref = torch.nn.functional.linear(x, w_deq).to(x.dtype)
|
||||
assert got.shape == (m, n)
|
||||
# Only the activation is re-quantized inside the kernel -> small MX noise.
|
||||
assert _relerr(got, ref) < 5e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Native MXFP8 MoE (dot_scaled grouped GEMM) vs dequant-to-bf16 MoE math
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _ref_moe(x, w13, w2, topk_weights, topk_ids, alpha, beta, limit):
|
||||
T, H = x.shape
|
||||
inter = w2.shape[-1]
|
||||
top_k = topk_ids.shape[1]
|
||||
out = torch.zeros(T, H, device=x.device, dtype=torch.float32)
|
||||
for t in range(T):
|
||||
for j in range(top_k):
|
||||
e = int(topk_ids[t, j].item())
|
||||
g1 = x[t].float() @ w13[e].float().T # [2I]
|
||||
gate = g1[:inter]
|
||||
up = g1[inter:]
|
||||
if limit is not None:
|
||||
gate = gate.clamp(max=limit)
|
||||
up = up.clamp(min=-limit, max=limit)
|
||||
act = gate * torch.sigmoid(alpha * gate) * (up + beta)
|
||||
g2 = act @ w2[e].float().T # [H]
|
||||
out[t] += topk_weights[t, j].float() * g2
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@pytest.mark.parametrize(
|
||||
"T,H,inter,E,top_k", [(8, 256, 512, 8, 2), (1, 512, 256, 16, 4)]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_native_moe(T, H, inter, E, top_k):
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
|
||||
fused_moe_mxfp8_native,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
alpha, beta, limit = 1.702, 1.0, 7.0
|
||||
w13_bf16 = torch.randn(E, 2 * inter, H, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
w2_bf16 = torch.randn(E, H, inter, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
w13_fp8, w13_scale = _mxfp8_e4m3_quantize_torch(
|
||||
w13_bf16, is_sf_swizzled_layout=False
|
||||
)
|
||||
w2_fp8, w2_scale = _mxfp8_e4m3_quantize_torch(w2_bf16, is_sf_swizzled_layout=False)
|
||||
|
||||
x = torch.randn(T, H, device=DEVICE, dtype=torch.bfloat16) * 0.5
|
||||
logits = torch.randn(T, E, device=DEVICE, dtype=torch.float32)
|
||||
topk_weights, topk_ids = logits.softmax(dim=-1).topk(top_k, dim=-1)
|
||||
topk_weights = topk_weights.to(torch.float32)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
got = fused_moe_mxfp8_native(
|
||||
x,
|
||||
w13_fp8,
|
||||
w13_scale,
|
||||
w2_fp8,
|
||||
w2_scale,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
alpha=alpha,
|
||||
beta=beta,
|
||||
limit=limit,
|
||||
global_num_experts=E,
|
||||
expert_map=None,
|
||||
)
|
||||
# Reference consumes the dequantized weights (same bits the kernel reads).
|
||||
w13_deq = dequant_mxfp8_to_bf16(w13_fp8, w13_scale)
|
||||
w2_deq = dequant_mxfp8_to_bf16(w2_fp8, w2_scale)
|
||||
ref = _ref_moe(x, w13_deq, w2_deq, topk_weights, topk_ids, alpha, beta, limit)
|
||||
assert got.shape == (T, H)
|
||||
assert _relerr(got, ref) < 5e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Native MXFP8 grouped GEMM (dot_scaled) vs pure-PyTorch grouped matmul
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _ref_grouped_gemm(a_deq, w_deq, topk_ids, a_div, num_valid, mul_weight=None):
|
||||
"""Pure-PyTorch reference for ``_grouped_gemm_mxfp8``.
|
||||
|
||||
For each routed (expanded) token ``tid in [0, num_valid)`` the kernel writes
|
||||
``out[tid] = a[tid // a_div] @ w[expert(tid)].T`` (fp32 accumulate), optionally
|
||||
scaled by ``mul_weight[tid]``. The expert for ``tid`` is ``topk_ids.flatten()
|
||||
[tid]`` (row-major expansion: ``tid = token*top_k + slot``). This is computed
|
||||
here with plain ``torch.matmul`` on the dequantized operands — independent of
|
||||
the Triton ``dot_scaled`` path and of the (separate) aiter backend.
|
||||
"""
|
||||
eids = topk_ids.reshape(-1)
|
||||
n = w_deq.shape[1]
|
||||
out = torch.empty(num_valid, n, dtype=torch.float32, device=a_deq.device)
|
||||
for tid in range(num_valid):
|
||||
e = int(eids[tid].item())
|
||||
out[tid] = a_deq[tid // a_div].float() @ w_deq[e].float().T
|
||||
if mul_weight is not None:
|
||||
out[tid] *= float(mul_weight[tid].item())
|
||||
return out
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@pytest.mark.parametrize("T,N,K,E,top_k", [(8, 256, 128, 8, 2), (5, 512, 256, 16, 4)])
|
||||
@pytest.mark.parametrize("weighted", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_grouped_gemm_native(T, N, K, E, top_k, weighted):
|
||||
"""Directly exercise ``_grouped_gemm_mxfp8`` against a non-Triton reference.
|
||||
|
||||
Covers both call modes used by ``fused_moe_mxfp8_native``:
|
||||
* ``weighted=False`` -> g1: ``a_div=top_k`` (a-row shared across the top_k
|
||||
expansions of a token), no per-token weight.
|
||||
* ``weighted=True`` -> g2: ``a_div=1`` (one a-row per expansion), output
|
||||
scaled by ``topk_weights``.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
|
||||
_grouped_gemm_mxfp8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
block_m = 64
|
||||
a_div = 1 if weighted else top_k
|
||||
m_routed = T * top_k
|
||||
# a-rows: g1 reads one row per token (a_div=top_k); g2 one per expansion.
|
||||
a_rows = m_routed if weighted else T
|
||||
a_bf16 = torch.randn(a_rows, K, device=DEVICE, dtype=torch.bfloat16) * 0.5
|
||||
w_bf16 = torch.randn(E, N, K, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
a_fp8, a_scale = _mxfp8_e4m3_quantize_torch(a_bf16, is_sf_swizzled_layout=False)
|
||||
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
|
||||
|
||||
logits = torch.randn(T, E, device=DEVICE, dtype=torch.float32)
|
||||
topk_weights, topk_ids = logits.softmax(dim=-1).topk(top_k, dim=-1)
|
||||
topk_weights = topk_weights.to(torch.float32)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
mul = topk_weights.reshape(-1) if weighted else None
|
||||
|
||||
sorted_ids, expert_ids, num_post = moe_align_block_size(
|
||||
topk_ids, block_m, E, None, ignore_invalid_experts=False
|
||||
)
|
||||
got = _grouped_gemm_mxfp8(
|
||||
a_fp8,
|
||||
a_scale,
|
||||
w_fp8,
|
||||
w_scale,
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_post,
|
||||
m_routed,
|
||||
top_k,
|
||||
block_m,
|
||||
torch.bfloat16,
|
||||
a_div=a_div,
|
||||
mul_weight_by=mul,
|
||||
)
|
||||
# Reference: dequant the SAME bits the kernel reads, plain torch matmul.
|
||||
a_deq = dequant_mxfp8_to_bf16(a_fp8, a_scale)
|
||||
w_deq = dequant_mxfp8_to_bf16(w_fp8, w_scale)
|
||||
ref = _ref_grouped_gemm(a_deq, w_deq, topk_ids, a_div, m_routed, mul)
|
||||
assert got.shape == (m_routed, N)
|
||||
assert _relerr(got, ref) < 5e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# MXFP8 linear emulation: BF16-at-load (default) vs per-step dequant + switch
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("shape", [(512, 2048), (1, 6144)])
|
||||
@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("dequant_at_load", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_linear_emulation_bf16_at_load(
|
||||
shape, act_dtype, dequant_at_load, monkeypatch
|
||||
):
|
||||
"""EmulationMxfp8LinearKernel load-time BF16 dequant (default) and the
|
||||
``VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0`` per-step fallback must produce the
|
||||
same result; the dtype-match (BF16/FP16 activations) must also hold."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
|
||||
EmulationMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
|
||||
monkeypatch.setenv(
|
||||
"VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD", "1" if dequant_at_load else "0"
|
||||
)
|
||||
N, K = shape
|
||||
torch.manual_seed(0)
|
||||
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16)
|
||||
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
|
||||
assert w_scale.shape == (N, K // 32)
|
||||
|
||||
# Reference: dequant once, plain linear in the activation dtype.
|
||||
w_ref = dequant_mxfp8_to_bf16(w_fp8, w_scale).to(act_dtype)
|
||||
x = torch.randn(7, K, device=DEVICE, dtype=act_dtype)
|
||||
out_ref = torch.nn.functional.linear(x, w_ref)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
|
||||
|
||||
kernel = EmulationMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(N, K)))
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
if dequant_at_load:
|
||||
# weights converted to BF16 at load (>= 2-byte)
|
||||
assert layer.weight.element_size() >= 2
|
||||
else:
|
||||
# opt-out: weights stay 1-byte MXFP8, dequant happens per-step
|
||||
assert layer.weight.element_size() == 1
|
||||
|
||||
out = kernel.apply_weights(layer, x)
|
||||
assert out.dtype == act_dtype # dtype-match preserved (no tl.dot/F.linear crash)
|
||||
assert _relerr(out.float(), out_ref.float()) < 2e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Native MXFP8 linear: the K % 128 != 0 weight is dequantized once at load
|
||||
# --------------------------------------------------------------------------- #
|
||||
@requires_gfx950
|
||||
# (5120, 576) and (5120, 288): DSv4.1 shared-expert down_proj at TP4 and TP8.
|
||||
@pytest.mark.parametrize("shape", [(5120, 576), (5120, 288), (256, 2048)])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_rocm_native_unaligned_k_dequantizes_at_load(shape):
|
||||
"""``dot_scaled`` tiles K by 128, so an unaligned weight must not reach it.
|
||||
|
||||
``process_weights_after_loading`` converts those to BF16 once; a weight that
|
||||
never went through it has to fall back per step rather than take the
|
||||
invalid path, which is what keying the dispatch on the weight dtype alone
|
||||
would do.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.mxfp8 import rocm_native
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
|
||||
N, K = shape
|
||||
aligned = K % 128 == 0
|
||||
torch.manual_seed(0)
|
||||
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
|
||||
x = torch.randn(7, K, device=DEVICE, dtype=torch.bfloat16) * 0.5
|
||||
out_ref = torch.nn.functional.linear(x, dequant_mxfp8_to_bf16(w_fp8, w_scale))
|
||||
# dot_scaled re-quantizes the activation, the dequant paths do not.
|
||||
tol = 5e-2 if aligned else 2e-3
|
||||
|
||||
def make_layer():
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
|
||||
return layer
|
||||
|
||||
kernel = rocm_native.RocmDotScaledMxfp8LinearKernel(
|
||||
Mxfp8LinearLayerConfig(weight_shape=(N, K))
|
||||
)
|
||||
layer = make_layer()
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert (layer.weight.element_size() >= 2) is not aligned
|
||||
assert _relerr(kernel.apply_weights(layer, x).float(), out_ref.float()) < tol
|
||||
|
||||
# An unaligned weight that is still FP8 must not reach dot_scaled.
|
||||
unprocessed = make_layer()
|
||||
unprocessed.weight_scale = torch.nn.Parameter(
|
||||
w_scale[:N, : K // 32].contiguous(), requires_grad=False
|
||||
)
|
||||
calls = []
|
||||
real = rocm_native._mxfp8_dot_scaled_linear
|
||||
|
||||
def _spy(*a, **k):
|
||||
calls.append(1)
|
||||
return real(*a, **k)
|
||||
|
||||
rocm_native._mxfp8_dot_scaled_linear = _spy
|
||||
try:
|
||||
out = kernel.apply_weights(unprocessed, x)
|
||||
finally:
|
||||
rocm_native._mxfp8_dot_scaled_linear = real
|
||||
assert bool(calls) is aligned
|
||||
assert _relerr(out.float(), out_ref.float()) < tol
|
||||
|
||||
|
||||
# ── EP expert_mask handling for the FlyDSL (AITER_MXFP8) MoE ────────────────
|
||||
# The map/mask choice lives in ``RoutedExperts.expert_map``: it hands AITER
|
||||
# experts (``consumes_expert_mask``) the precomputed 0/1 ``expert_mask`` and
|
||||
# everyone else the canonical -1 index map, keyed on the resolved experts kernel
|
||||
# rather than the global VLLM_ROCM_USE_AITER switch. ``AiterMxfp8Experts.apply``
|
||||
# forwards that mask to aiter unchanged (it no longer rebuilds it via
|
||||
# ``(expert_map >= 0)``, which collapsed an already-0/1 mask to all-ones and
|
||||
# produced EP garbage).
|
||||
def _capture_expert_mask(expert_mask, *, global_num_experts):
|
||||
"""Drive the real ``AiterMxfp8Experts.apply`` mask branch and capture the
|
||||
``expert_mask`` it forwards to ``rocm_aiter_ops.fused_moe``."""
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import (
|
||||
AiterMxfp8Experts,
|
||||
)
|
||||
|
||||
experts = object.__new__(AiterMxfp8Experts) # bypass heavy __init__
|
||||
experts.quant_config = SimpleNamespace(gemm1_clamp_limit=None)
|
||||
experts.w1_scale_val = None
|
||||
experts.w2_scale_val = None
|
||||
|
||||
captured = {}
|
||||
|
||||
def _fake_fused_moe(hidden_states, w1, w2, tw, ti, *, expert_mask, **kw):
|
||||
captured["expert_mask"] = expert_mask
|
||||
return torch.zeros_like(hidden_states)
|
||||
|
||||
w1 = torch.zeros(1, device=DEVICE)
|
||||
w2 = torch.zeros(1, device=DEVICE)
|
||||
out = torch.zeros(4, 8, device=DEVICE, dtype=torch.bfloat16)
|
||||
hidden = torch.zeros(4, 8, device=DEVICE, dtype=torch.bfloat16)
|
||||
tw = torch.ones(4, 2, device=DEVICE)
|
||||
ti = torch.zeros(4, 2, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
with mock.patch.object(rocm_aiter_ops, "fused_moe", side_effect=_fake_fused_moe):
|
||||
experts.apply(
|
||||
output=out,
|
||||
hidden_states=hidden,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=tw,
|
||||
topk_ids=ti,
|
||||
activation=None,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_mask,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=None,
|
||||
workspace2=None,
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
return captured["expert_mask"]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_aiter_mxfp8_apply_forwards_expert_mask_unchanged():
|
||||
"""AiterMxfp8Experts.apply forwards the precomputed 0/1 mask to aiter as-is
|
||||
(no re-derivation that would collapse an already-0/1 mask to all-ones)."""
|
||||
# 0/1 local-expert mask over global ids + trailing sentinel (rank owns 0..3).
|
||||
ep_mask = torch.tensor(
|
||||
[1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
got = _capture_expert_mask(ep_mask, global_num_experts=8)
|
||||
assert torch.equal(got.cpu().to(torch.int32), ep_mask.cpu().to(torch.int32))
|
||||
assert got.sum().item() == 4 # the 4 local experts, not all 9
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_routed_experts_expert_map_delegates_to_kernel():
|
||||
"""RoutedExperts.expert_map returns the 0/1 mask only for kernels that set
|
||||
``consumes_expert_mask`` (AITER), and the canonical -1 map otherwise -- keyed
|
||||
on the resolved kernel, not the global aiter switch."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
|
||||
canonical = torch.tensor([0, 1, 2, 3, -1, -1, -1, -1], dtype=torch.int32)
|
||||
mask = torch.tensor([1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32)
|
||||
|
||||
def resolve(consumes_mask, *, has_moe_kernel=True):
|
||||
moe_kernel = (
|
||||
SimpleNamespace(
|
||||
fused_experts=SimpleNamespace(consumes_expert_mask=consumes_mask)
|
||||
)
|
||||
if has_moe_kernel
|
||||
else None
|
||||
)
|
||||
layer = SimpleNamespace(
|
||||
_expert_map=canonical,
|
||||
expert_mask=mask,
|
||||
quant_method=SimpleNamespace(moe_kernel=moe_kernel),
|
||||
)
|
||||
return RoutedExperts.expert_map.fget(layer)
|
||||
|
||||
assert torch.equal(resolve(True), mask) # AITER kernel -> 0/1 mask
|
||||
assert torch.equal(resolve(False), canonical) # non-AITER -> canonical map
|
||||
assert torch.equal(resolve(False, has_moe_kernel=False), canonical) # non-modular
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Strided K/V operand for the AITER sparse-PA cache insert
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim, num_q_heads=16):
|
||||
"""K/V as the sparse-PA insert sees them: column slices of a fused qkv."""
|
||||
kv = num_kv_heads * head_dim
|
||||
row = num_q_heads * head_dim + 2 * kv + 2 * head_dim
|
||||
qkv = torch.randn((num_tokens, row), dtype=torch.bfloat16, device=DEVICE)
|
||||
k_start = num_q_heads * head_dim
|
||||
k = qkv[:, k_start : k_start + kv].view(num_tokens, num_kv_heads, head_dim)
|
||||
v = qkv[:, k_start + kv : k_start + 2 * kv].view(num_tokens, num_kv_heads, head_dim)
|
||||
return k, v
|
||||
|
||||
|
||||
def _asm_kv_cache(pages, num_kv_heads, head_dim):
|
||||
"""Page-16 K/V cache views matching ``_ensure_aiter_sparse_pa_kv_cache``."""
|
||||
x = 16 // torch.tensor([], dtype=torch.bfloat16).element_size()
|
||||
key = torch.zeros(
|
||||
(pages, num_kv_heads, head_dim // x, 16, x),
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
value = torch.zeros(
|
||||
(pages, num_kv_heads, 16 // x, head_dim, x),
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
return key, value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 8, 129, 1024])
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 2])
|
||||
def test_kv_insert_operand_matches_contiguous(num_tokens, num_kv_heads):
|
||||
"""A strided K/V slice must insert bit-identically to a contiguous copy."""
|
||||
reshape_and_cache = pytest.importorskip("aiter").reshape_and_cache
|
||||
head_dim = 128
|
||||
pages = num_tokens // 16 + 4
|
||||
slots = torch.randperm(pages * 16, device=DEVICE)[:num_tokens].to(torch.int32)
|
||||
|
||||
torch.manual_seed(num_tokens)
|
||||
k, v = _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim)
|
||||
assert num_tokens == 1 or not k.is_contiguous()
|
||||
|
||||
caches = []
|
||||
for operand in (lambda t: t.contiguous(), _kv_insert_operand):
|
||||
key_cache, value_cache = _asm_kv_cache(pages, num_kv_heads, head_dim)
|
||||
reshape_and_cache(
|
||||
operand(k),
|
||||
operand(v),
|
||||
key_cache,
|
||||
value_cache,
|
||||
slots,
|
||||
kv_cache_dtype="auto",
|
||||
asm_layout=True,
|
||||
)
|
||||
caches.append((key_cache, value_cache))
|
||||
|
||||
(ref_k, ref_v), (got_k, got_v) = caches
|
||||
assert torch.equal(ref_k, got_k)
|
||||
assert torch.equal(ref_v, got_v)
|
||||
|
||||
|
||||
def test_kv_insert_operand_copies_unsupported_layouts():
|
||||
"""Only a row-strided slice passes through; anything else is copied."""
|
||||
k, _ = _fused_qkv_kv_slices(32, 2, 128)
|
||||
assert _kv_insert_operand(k) is k
|
||||
|
||||
inner_strided = k[:, :, ::2]
|
||||
assert _kv_insert_operand(inner_strided).is_contiguous()
|
||||
|
||||
flat = k.reshape(32, -1)
|
||||
assert _kv_insert_operand(flat).is_contiguous()
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not torch.cuda.is_available() or not current_platform.supports_fp8():
|
||||
pytest.skip(
|
||||
"MiniMax-M3 FP8 sparse attention scale tests require an FP8 GPU.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.minimax_m3.amd.ops.sparse_attn import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
else:
|
||||
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
|
||||
|
||||
DEVICE = "cuda"
|
||||
DTYPE = torch.bfloat16
|
||||
HEAD_DIM = 128
|
||||
NUM_KV_HEADS = 1
|
||||
NUM_HEADS = 2
|
||||
K_SCALE = 0.25
|
||||
V_SCALE = 0.5
|
||||
|
||||
|
||||
def _scale_tensors(mode: str, num_blocks: int):
|
||||
if mode == "scalar":
|
||||
k_scale = torch.tensor(K_SCALE, dtype=torch.float32, device=DEVICE)
|
||||
v_scale = torch.tensor(V_SCALE, dtype=torch.float32, device=DEVICE)
|
||||
else:
|
||||
shape = (NUM_KV_HEADS, num_blocks * SPARSE_BLOCK_SIZE)
|
||||
k_scale = torch.full(shape, K_SCALE, dtype=torch.float32, device=DEVICE)
|
||||
v_scale = torch.full(shape, V_SCALE, dtype=torch.float32, device=DEVICE)
|
||||
return k_scale, v_scale
|
||||
|
||||
|
||||
def _make_kv_cache(num_blocks: int, seed: int):
|
||||
torch.manual_seed(seed)
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
kv_ref = torch.randn(
|
||||
num_blocks,
|
||||
NUM_KV_HEADS,
|
||||
SPARSE_BLOCK_SIZE,
|
||||
2 * HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
device=DEVICE,
|
||||
)
|
||||
kv_fp8 = torch.empty_like(kv_ref, dtype=fp8_dtype)
|
||||
kv_fp8[..., :HEAD_DIM] = (kv_ref[..., :HEAD_DIM].float() / K_SCALE).to(fp8_dtype)
|
||||
kv_fp8[..., HEAD_DIM:] = (kv_ref[..., HEAD_DIM:].float() / V_SCALE).to(fp8_dtype)
|
||||
|
||||
kv_dequant = torch.empty_like(kv_ref)
|
||||
kv_dequant[..., :HEAD_DIM] = (kv_fp8[..., :HEAD_DIM].float() * K_SCALE).to(DTYPE)
|
||||
kv_dequant[..., HEAD_DIM:] = (kv_fp8[..., HEAD_DIM:].float() * V_SCALE).to(DTYPE)
|
||||
return kv_fp8, kv_dequant
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scale_mode", ["scalar", "per_token_head"])
|
||||
@torch.inference_mode()
|
||||
def test_minimax_m3_sparse_prefill_fp8_kv_scales(scale_mode: str):
|
||||
total_q = 17
|
||||
num_blocks = 1
|
||||
torch.manual_seed(0)
|
||||
q = torch.randn(total_q, NUM_HEADS, HEAD_DIM, dtype=DTYPE, device=DEVICE) * 0.1
|
||||
kv_fp8, kv_dequant = _make_kv_cache(num_blocks, seed=1)
|
||||
k_scale, v_scale = _scale_tensors(scale_mode, num_blocks)
|
||||
|
||||
topk = torch.zeros(NUM_KV_HEADS, total_q, 1, dtype=torch.int32, device=DEVICE)
|
||||
block_table = torch.zeros(1, 1, dtype=torch.int32, device=DEVICE)
|
||||
cu_seqlens = torch.tensor([0, total_q], dtype=torch.int32, device=DEVICE)
|
||||
seq_lens = torch.tensor([total_q], dtype=torch.int32, device=DEVICE)
|
||||
prefix_lens = torch.zeros(1, dtype=torch.int32, device=DEVICE)
|
||||
got = torch.empty_like(q)
|
||||
ref = torch.empty_like(q)
|
||||
unscaled = torch.empty_like(q)
|
||||
|
||||
minimax_m3_sparse_attn(
|
||||
q,
|
||||
kv_fp8,
|
||||
topk,
|
||||
block_table,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
total_q,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
got,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
minimax_m3_sparse_attn(
|
||||
q,
|
||||
kv_dequant,
|
||||
topk,
|
||||
block_table,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
total_q,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
ref,
|
||||
)
|
||||
minimax_m3_sparse_attn(
|
||||
q,
|
||||
kv_fp8,
|
||||
topk,
|
||||
block_table,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
total_q,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
unscaled,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2)
|
||||
assert not torch.allclose(unscaled, ref, rtol=1e-1, atol=1e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scale_mode", ["scalar", "per_token_head"])
|
||||
@torch.inference_mode()
|
||||
def test_minimax_m3_sparse_decode_fp8_kv_scales(scale_mode: str):
|
||||
total_q = 2
|
||||
num_blocks = 1
|
||||
torch.manual_seed(2)
|
||||
q = torch.randn(total_q, NUM_HEADS, HEAD_DIM, dtype=DTYPE, device=DEVICE) * 0.1
|
||||
kv_fp8, kv_dequant = _make_kv_cache(num_blocks, seed=3)
|
||||
k_scale, v_scale = _scale_tensors(scale_mode, num_blocks)
|
||||
|
||||
topk = torch.zeros(NUM_KV_HEADS, total_q, 1, dtype=torch.int32, device=DEVICE)
|
||||
block_table = torch.zeros(total_q, 1, dtype=torch.int32, device=DEVICE)
|
||||
seq_lens = torch.tensor([64, 128], dtype=torch.int32, device=DEVICE)
|
||||
got = torch.empty_like(q)
|
||||
ref = torch.empty_like(q)
|
||||
unscaled = torch.empty_like(q)
|
||||
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_fp8,
|
||||
topk,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
got,
|
||||
decode_query_len=1,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_dequant,
|
||||
topk,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
ref,
|
||||
decode_query_len=1,
|
||||
)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_fp8,
|
||||
topk,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
unscaled,
|
||||
decode_query_len=1,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2)
|
||||
assert not torch.allclose(unscaled, ref, rtol=1e-1, atol=1e-1)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,36 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only integration tests for the UltraQuant cache dtype."""
|
||||
|
||||
from vllm.v1.attention.backends.ultraquant_attn import (
|
||||
UltraQuantAttentionBackend,
|
||||
)
|
||||
from vllm.v1.attention.ops.flydsl_ultraquant_decode import (
|
||||
ultraquant_flydsl_decode_eligible,
|
||||
)
|
||||
|
||||
|
||||
def test_ultraquant_prefers_block_size_64():
|
||||
# FlyDSL D=256 decode geometry requires a 64-token KV block.
|
||||
assert UltraQuantAttentionBackend.get_preferred_block_size(16) == 64
|
||||
assert UltraQuantAttentionBackend.get_preferred_block_size(128) == 64
|
||||
|
||||
|
||||
def test_ultraquant_flydsl_decode_routing():
|
||||
"""CPU check of FlyDSL vs unified-Triton decode eligibility."""
|
||||
eligible = dict(
|
||||
head_size=256,
|
||||
num_kv_groups=8,
|
||||
has_sinks=False,
|
||||
sliding_window=None,
|
||||
flydsl_loaded=True,
|
||||
)
|
||||
assert ultraquant_flydsl_decode_eligible(**eligible)
|
||||
for gqa in (6, 8, 16):
|
||||
assert ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": gqa})
|
||||
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "flydsl_loaded": False})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "head_size": 128})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": 4})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "has_sinks": True})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "sliding_window": 4096})
|
||||
@@ -1,106 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""UltraQuant inherits TurboQuant's forward/do_kv_cache_update orchestration.
|
||||
|
||||
UltraQuant only overrides the store/decode/prefill seams, so a TurboQuant-side
|
||||
change to a seam signature, to an inherited helper, or to the ``layer._tq_*``
|
||||
attributes ``forward`` reads will break UltraQuant at serve time without
|
||||
touching any UltraQuant kernel test. These checks pin that contract.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
from vllm.v1.attention.backends import ultraquant_attn
|
||||
from vllm.v1.attention.backends.turboquant_attn import TurboQuantAttentionImpl
|
||||
from vllm.v1.attention.backends.ultraquant_attn import UltraQuantAttentionImpl
|
||||
|
||||
_SEAMS = ("_ensure_on_device", "_store_kv", "_decode_attention", "_prefill_attention")
|
||||
|
||||
|
||||
def _method_ast(cls: type, name: str) -> ast.AST:
|
||||
return ast.parse(textwrap.dedent(inspect.getsource(getattr(cls, name))))
|
||||
|
||||
|
||||
def _tq_attrs_read(cls: type, name: str) -> set[str]:
|
||||
"""``_tq_*`` attribute names read (not assigned) in ``cls.name``."""
|
||||
tree = _method_ast(cls, name)
|
||||
assigned = {
|
||||
t.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
for t in node.targets
|
||||
if isinstance(t, ast.Attribute)
|
||||
}
|
||||
read = {
|
||||
node.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Attribute) and node.attr.startswith("_tq_")
|
||||
}
|
||||
return read - assigned
|
||||
|
||||
|
||||
def _tq_attrs_written(cls: type, name: str) -> set[str]:
|
||||
tree = _method_ast(cls, name)
|
||||
return {
|
||||
t.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
for t in node.targets
|
||||
if isinstance(t, ast.Attribute) and t.attr.startswith("_tq_")
|
||||
}
|
||||
|
||||
|
||||
def test_ultraquant_matches_turboquant_seam_contract():
|
||||
# 1. Each overridden seam accepts every parameter the base declares, so any
|
||||
# call the inherited orchestration makes stays valid for UltraQuant.
|
||||
for name in _SEAMS:
|
||||
base = list(
|
||||
inspect.signature(getattr(TurboQuantAttentionImpl, name)).parameters
|
||||
)
|
||||
override = list(
|
||||
inspect.signature(getattr(UltraQuantAttentionImpl, name)).parameters
|
||||
)
|
||||
assert override[: len(base)] == base, (
|
||||
f"UltraQuantAttentionImpl.{name} no longer accepts the arguments "
|
||||
f"TurboQuantAttentionImpl passes: base={base} override={override}"
|
||||
)
|
||||
|
||||
# 2. Keyword arguments UltraQuant passes to *inherited* TurboQuant helpers
|
||||
# must exist on those helpers (e.g. _flash_attn_varlen).
|
||||
own = vars(UltraQuantAttentionImpl)
|
||||
module = ast.parse(inspect.getsource(ultraquant_attn))
|
||||
for node in ast.walk(module):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
fn = node.func
|
||||
if not (
|
||||
isinstance(fn, ast.Attribute)
|
||||
and isinstance(fn.value, ast.Name)
|
||||
and fn.value.id == "self"
|
||||
):
|
||||
continue
|
||||
target = getattr(TurboQuantAttentionImpl, fn.attr, None)
|
||||
if target is None or fn.attr in own or not callable(target):
|
||||
continue
|
||||
params = inspect.signature(target).parameters
|
||||
if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
|
||||
continue
|
||||
for kw in node.keywords:
|
||||
if kw.arg is None:
|
||||
continue
|
||||
assert kw.arg in params, (
|
||||
f"UltraQuant calls inherited {fn.attr}(..., {kw.arg}=...) but "
|
||||
f"TurboQuantAttentionImpl.{fn.attr} accepts {list(params)}"
|
||||
)
|
||||
|
||||
# 3. Every layer._tq_* attribute the inherited forward reads is set by
|
||||
# UltraQuant's _ensure_on_device.
|
||||
required = _tq_attrs_read(TurboQuantAttentionImpl, "forward")
|
||||
provided = _tq_attrs_written(UltraQuantAttentionImpl, "_ensure_on_device")
|
||||
assert required, "expected inherited forward to read layer._tq_* attributes"
|
||||
assert required <= provided, (
|
||||
f"UltraQuant._ensure_on_device does not set {sorted(required - provided)}, "
|
||||
f"which the inherited TurboQuant forward reads"
|
||||
)
|
||||
Reference in New Issue
Block a user